client.setConnectionTimeout(30000);
    client.setMaxConnections(100);
});

// Configure URL filtering
container.singleton("urlFilter", UrlFilterImpl.class, filter -> {
    filter.addInclude("https://example.com/.*");
    filter.addExclude(".*\\.(css|js|png|jpg|gif)$");
});

// Configure content extraction
container.singleton("tikaExtractor", TikaExtractor.class);

				"application/vnd.shana.informed.package",
				"application/vnd.simtech-mindmapper",
				"application/vnd.smaf",
				"application/vnd.smart.teacher",
				"application/vnd.software602.filler.form+xml",
				"application/vnd.software602.filler.form-xml-zip",
				"application/vnd.solent.sdkm+xml",
				"application/vnd.spotfire.dxp",
				"application/vnd.spotfire.sfs",
				"application/vnd.sss-cod",
				"application/vnd.sss-dtf",

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.filter.impl;

import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.impl.UrlFilterServiceImpl;

	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml"/>
	<include path="crawler/client.xml"/>
	<include path="crawler/rule.xml"/>
	<include path="crawler/filter.xml"/>
	<include path="crawler/interval.xml"/>
	<include path="crawler/extractor.xml"/>
	<include path="crawler/mimetype.xml"/>
	<include path="crawler/encoding.xml"/>
	<include path="crawler/urlconverter.xml"/>

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.filter.UrlFilter;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.helper.RobotsTxtHelper;
import org.codelibs.fess.crawler.helper.impl.MimeTypeHelperImpl;

        if (crawlerContext.getMaxDepth() >= 0 && depth > crawlerContext.getMaxDepth()) {
            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {

                getAliasesResponse.getAliases()
                        .entrySet()
                        .forEach(x -> x.getValue()
                                .stream()
                                .filter(y -> updateAlias.equals(y.alias()))
                                .forEach(y -> updateIndices.add(x.getKey())));
            }
            if (updateIndices.size() != 1) {
                if (logger.isDebugEnabled()) {

a===b!==c});if("string"==typeof b){if(w.test(b))return n.filter(b,a,c);b=n.filter(b,a)}return n.grep(a,function(a){return g.call(b,a)>=0!==c})}n.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?n.find.matchesSelector(d,a)?[d]:[]:n.find.matches(a,n.grep(b,function(a){return 1===a.nodeType}))},n.fn.extend({find:function(a){var b,c=this.length,d=[],e=this;if("string"!=typeof a)return this.pushStack(n(a).filter(function(){for(b=0;c>b;b++)if(n.contains(e[b],this))...

    /**
     * Test urlFilter getter and setter
     */
    public void test_urlFilter() {
        assertNull(crawlerContext.getUrlFilter());

        TestUrlFilter filter = new TestUrlFilter();
        crawlerContext.setUrlFilter(filter);
        assertSame(filter, crawlerContext.getUrlFilter());

        crawlerContext.setUrlFilter(null);
        assertNull(crawlerContext.getUrlFilter());
    }

    /**

        SearchResponse response = getClient().get(c -> c.prepareSearch(index)
                .setScroll(new TimeValue(scrollTimeout))
                .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery(SESSION_ID, oldSessionId)))
                .setSize(scrollSize)
                .execute());
        String scrollId = response.getScrollId();
        try {
            while (scrollId != null) {