}

    /**
     * Adds a tag to filter by.
     * @param tag The tag.
     */
    public void addTag(final String tag) {
        tags.add(tag);
    }

    /**
     * Adds a role to filter by.
     * @param role The role.
     */
    public void addRole(final String role) {
        roles.add(role);
    }

    /**
     * Adds a field to filter by.
     * @param field The field name.

    public void setStatus(final CrawlerStatus status) {
        this.status = status;
    }

    /**
     * Returns the URL filter.
     * @return The UrlFilter.
     */
    public UrlFilter getUrlFilter() {
        return urlFilter;
    }

    /**
     * Sets the URL filter.
     * @param urlFilter The UrlFilter.
     */
    public void setUrlFilter(final UrlFilter urlFilter) {

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.filter.impl;

import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.impl.UrlFilterServiceImpl;

	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml"/>
	<include path="crawler/client.xml"/>
	<include path="crawler/rule.xml"/>
	<include path="crawler/filter.xml"/>
	<include path="crawler/interval.xml"/>
	<include path="crawler/extractor.xml"/>
	<include path="crawler/mimetype.xml"/>
	<include path="crawler/encoding.xml"/>
	<include path="crawler/urlconverter.xml"/>

        if (crawlerContext.getMaxDepth() >= 0 && depth > crawlerContext.getMaxDepth()) {
            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.filter.UrlFilter;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.helper.RobotsTxtHelper;
import org.codelibs.fess.crawler.helper.impl.MimeTypeHelperImpl;

                getAliasesResponse.getAliases()
                        .entrySet()
                        .forEach(x -> x.getValue()
                                .stream()
                                .filter(y -> updateAlias.equals(y.alias()))
                                .forEach(y -> updateIndices.add(x.getKey())));
            }
            if (updateIndices.size() != 1) {
                if (logger.isDebugEnabled()) {

    /**
     * Test urlFilter getter and setter
     */
    public void test_urlFilter() {
        assertNull(crawlerContext.getUrlFilter());

        TestUrlFilter filter = new TestUrlFilter();
        crawlerContext.setUrlFilter(filter);
        assertSame(filter, crawlerContext.getUrlFilter());

        crawlerContext.setUrlFilter(null);
        assertNull(crawlerContext.getUrlFilter());
    }

    /**

            final int depth, final String encoding) {
        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.getUrlFilter().match(d.getUrl()))
                .map(d -> {

        SearchResponse response = getClient().get(c -> c.prepareSearch(index)
                .setScroll(new TimeValue(scrollTimeout))
                .setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery(SESSION_ID, oldSessionId)))
                .setSize(scrollSize)
                .execute());
        String scrollId = response.getScrollId();
        try {
            while (scrollId != null) {