request.setSize(size);
        return this;
    }

    /**
     * Adds a tag to filter by.
     * @param tag The tag.
     * @return This builder instance.
     */
    public PopularWordsRequestBuilder addTag(final String tag) {
        request.addTag(tag);
        return this;
    }

    /**
     * Adds a role to filter by.
     * @param role The role.
     * @return This builder instance.
     */

    }

    /**
     * Adds a tag to filter by.
     * @param tag The tag.
     */
    public void addTag(final String tag) {
        tags.add(tag);
    }

    /**
     * Adds a role to filter by.
     * @param role The role.
     */
    public void addRole(final String role) {
        roles.add(role);
    }

    /**
     * Adds a field to filter by.
     * @param field The field name.

    public void setStatus(final CrawlerStatus status) {
        this.status = status;
    }

    /**
     * Returns the URL filter.
     * @return The UrlFilter.
     */
    public UrlFilter getUrlFilter() {
        return urlFilter;
    }

    /**
     * Sets the URL filter.
     * @param urlFilter The UrlFilter.
     */
    public void setUrlFilter(final UrlFilter urlFilter) {

import java.util.logging.Logger;
import java.util.logging.SimpleFormatter;

import org.codelibs.fess.crawler.client.FesenClient;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;
import org.codelibs.fess.crawler.service.UrlQueueService;

        }
        return new File(baseDir, filePath);
    }

    /**
     * Gets the base path for storing files.
     *
     * @return the base path
     */
    public String getPath() {
        return path;
    }

    /**
     * Sets the base path for storing files.
     *
     * @param path the base path to set
     */
    public void setPath(final String path) {

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.filter.impl;

import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.impl.UrlFilterServiceImpl;

	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml"/>
	<include path="crawler/client.xml"/>
	<include path="crawler/rule.xml"/>
	<include path="crawler/filter.xml"/>
	<include path="crawler/interval.xml"/>
	<include path="crawler/extractor.xml"/>
	<include path="crawler/mimetype.xml"/>
	<include path="crawler/encoding.xml"/>
	<include path="crawler/urlconverter.xml"/>

                }
                final Set<RequestData> requestDataSet = new HashSet<>(100);
                if (includeContent) {
                    final SmbFile[] files = file.listFiles();
                    if (files != null) {
                        for (final SmbFile f : files) {
                            final String childUri = f.getURL().toExternalForm();

        if (crawlerContext.getMaxDepth() >= 0 && depth > crawlerContext.getMaxDepth()) {
            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.filter.UrlFilter;
import org.codelibs.fess.crawler.filter.impl.UrlFilterImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.helper.RobotsTxtHelper;
import org.codelibs.fess.crawler.helper.impl.MimeTypeHelperImpl;