labels.wizard_button_register_again=Continu aanmaken
labels.wizard_button_register_next=Maken
labels.wizard_start_crawling_title=Start crawlen
labels.wizard_start_crawler_title=Crawler
labels.wizard_start_crawling_desc=U kunt nu beginnen met crawlen door op de knop "Start crawlen" te klikken.
labels.wizard_button_start_crawling=Start crawlen
labels.wizard_button_finish=Overslaan
labels.search_list_configuration=Zoeken
labels.search_list_button_delete=Verwijderen

     * Datetime format.
     *
     * By providing the last modification timestamp, you enable search engine
     * crawlers to retrieve only a subset of the Sitemaps in the index i.e. a
     * crawler may only retrieve Sitemaps that were modified since a certain
     * date. This incremental Sitemap fetching mechanism allows for the rapid
     * discovery of new URLs on very large sites.

 */
package org.codelibs.fess.crawler.processor;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor;

 */
package org.codelibs.fess.crawler;

import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;

import org.codelibs.core.collection.LruHashSet;
import org.codelibs.fess.crawler.filter.UrlFilter;
import org.codelibs.fess.crawler.interval.IntervalController;
import org.codelibs.fess.crawler.rule.RuleManager;

/**

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * Exception thrown during the extraction process in the crawler.
 * This exception indicates a failure or error that occurred while extracting content from a crawled resource.
 * It extends {@link org.codelibs.fess.crawler.exception.CrawlerSystemException} and provides constructors

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

    /**
     * Add an url pattern as a target.
     *
     * @param urlPattern Regular expression that is crawled
     */
    void addInclude(String urlPattern);

    /**
     * Add an url pattern as a non-target.
     *
     * @param urlPattern Regular expression that is not crawled
     */
    void addExclude(String urlPattern);

    /**
     * Process an url when it's added as a seed url.
     *

import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.log.LogType;
import org.codelibs.fess.exception.ContainerNotAvailableException;

import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.entity.UrlQueueImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * Implementation of the {@link UrlQueueService} interface.
 * This class provides methods for managing a queue of URLs to be crawled,

    /**
     * Interval in milliseconds between crawler thread executions.
     * Used to control the rate at which new crawler threads are started
     * and the frequency of status checks.
     */
    protected long crawlingExecutionInterval = Constants.DEFAULT_CRAWLING_EXECUTION_INTERVAL;

    /**
     * Thread priority for crawler threads.
     * Defaults to normal thread priority.
     */