try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

        for (String userAgent : new String[] { "FessCrawler", "FessCrawler/1.0", "Mozilla FessCrawler" }) {
            assertTrue(robotsTxt.allows("/aaa", userAgent));
            assertTrue(robotsTxt.allows("/private/", userAgent));
            assertTrue(robotsTxt.allows("/private/index.html", userAgent));

	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<component name="crawlerConfig" class="org.codelibs.fess.crawler.util.FessCrawlerConfig">
		<property name="queueIndex">"fess_crawler.queue"</property>
		<property name="queueShards">5</property>
		<property name="queueReplicas">1</property>
		<property name="dataIndex">"fess_crawler.data"</property>
		<property name="dataShards">5</property>

				</component>
			</arg>
		</postConstruct>
		 -->
	</component>
	<component name="crawlerStatsHelper"
		class="org.codelibs.fess.helper.CrawlerStatsHelper">
	</component>
	<component name="fessCrawler" class="org.codelibs.fess.exec.Crawler"
		instance="prototype">
	</component>

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN" 
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<component name="esClient"
		class="org.codelibs.fess.crawler.client.FesenClient">
	</component>

		<property name="queueIndex">"fess_crawler.queue"</property>
		<property name="queueShards">5</property>
		<property name="queueReplicas">1</property>
		<property name="dataIndex">"fess_crawler.data"</property>
		<property name="dataShards">5</property>
		<property name="dataReplicas">1</property>
		<property name="filterIndex">"fess_crawler.filter"</property>
		<property name="filterShards">5</property>

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />
	<include path="crawler/robotstxt.xml" />
	<include path="crawler/contentlength.xml" />
	<include path="crawler/mimetype.xml" />

	<component name="internalHttpClient" class="org.codelibs.fess.crawler.client.http.HcHttpClient"

        urlQueueService.insert(urlQueue);
        assertTrue(fesenClient.prepareSearch("fess_crawler.queue").setQuery(QueryBuilders.termQuery("sessionId", "sessionId")).setSize(0)
                .execute().actionGet().getHits().getTotalHits().value > 0);

        urlQueueService.delete("sessionId");
        assertFalse(fesenClient.prepareSearch("fess_crawler.queue").setQuery(QueryBuilders.termQuery("sessionId", "sessionId")).setSize(0)

response.inline.mimetypes=application/pdf,text/plain

# document index
index.document.search.index=fess.search
index.document.update.index=fess.update
index.document.suggest.index=fess
index.document.crawler.index=fess_crawler
index.document.crawler.queue.number_of_shards=10
index.document.crawler.data.number_of_shards=10
index.document.crawler.filter.number_of_shards=10
index.document.crawler.queue.number_of_replicas=1

    /** The key of the configuration. e.g. fess */
    String INDEX_DOCUMENT_SUGGEST_INDEX = "index.document.suggest.index";

    /** The key of the configuration. e.g. fess_crawler */
    String INDEX_DOCUMENT_CRAWLER_INDEX = "index.document.crawler.index";

    /** The key of the configuration. e.g. 10 */