crawler.ignore.robots.tags=false
crawler.ignore.content.exception=true
crawler.failure.url.status.codes=404
crawler.system.monitor.interval=60
crawler.hotthread.ignore_idle_threads=true
crawler.hotthread.interval=500ms
crawler.hotthread.snapshots=10
crawler.hotthread.threads=3
crawler.hotthread.timeout=30s
crawler.hotthread.type=cpu
crawler.metadata.content.excludes=resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*,X-TIKA.*

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.EsAccessResult;
import org.codelibs.fess.crawler.entity.EsUrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;

    public static final String CRAWLER_START_TIME = "CrawlerStartTime";

    public static final String CRAWLER_END_TIME = "CrawlerEndTime";

    public static final String CRAWLER_EXEC_TIME = "CrawlerExecTime";

    public static final String WEB_FS_CRAWLER_START_TIME = "WebFsCrawlStartTime";

    public static final String WEB_FS_CRAWLER_END_TIME = "WebFsCrawlEndTime";

    /** The key of the configuration. e.g. 500ms */
    String CRAWLER_HOTTHREAD_INTERVAL = "crawler.hotthread.interval";

    /** The key of the configuration. e.g. 10 */
    String CRAWLER_HOTTHREAD_SNAPSHOTS = "crawler.hotthread.snapshots";

    /** The key of the configuration. e.g. 3 */
    String CRAWLER_HOTTHREAD_THREADS = "crawler.hotthread.threads";

            <td>Fri Jan 13 13:53:34 UTC 2017</td>
            <td align="right">
                              &nbsp;
                          </td>
            <td></td>
          </tr>
                  <tr>
            <td><a href="https://oss.sonatype.org/content/repositories/snapshots/org/codelibs/fess/fess-crawler-db-mysql/">fess-crawler-db-mysql/</a></td>

import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.log.LogType;
import org.codelibs.fess.es.client.SearchEngineClient;

import org.codelibs.fess.app.service.WebConfigService;
import org.codelibs.fess.crawler.client.http.Authentication;
import org.codelibs.fess.crawler.client.http.form.FormScheme;
import org.codelibs.fess.crawler.client.http.impl.AuthenticationImpl;
import org.codelibs.fess.crawler.client.http.ntlm.JcifsEngine;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.es.config.bsentity.BsWebAuthentication;

Martin Taillefer <******@****.***> 1568244741 -0700

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/transformer_basic.xml"/>

	<component name="fessXpathTransformer" class="org.codelibs.fess.crawler.transformer.FessXpathTransformer" instance="singleton">
		<property name="name">"fessXpathTransformer"</property>
		<property name="featureMap">defaultFeatureMap</property>

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;