* This job launches a separate crawler process that can crawl web sites, file systems,
 * and data sources based on the configured crawling settings.
 *
 * <p>The job supports selective crawling by specifying configuration IDs for different
 * types of crawlers (web, file, data). It manages the crawler process lifecycle,
 * handles timeout scenarios, and ensures proper cleanup of resources.</p>
 *

        // Default constructor
    }

    /**
     * Enable or disable incremental crawling.
     * When enabled, only new or modified documents are crawled.
     */
    @Size(max = 10)
    public String incrementalCrawling;

    /**
     * Number of days to keep crawled documents before cleanup.
     * Set to -1 to disable automatic cleanup.
     */
    @Required
    @Min(-1)
    @Max(1000)
    @ValidateTypeFailure

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

    public void test_urlQueueKeyObject() {
        TestUrlQueue urlQueue = new TestUrlQueue("queue_123", "http://example.com/test");
        crawlerStatsHelper.begin(urlQueue);
        crawlerStatsHelper.record(urlQueue, "crawled");
        crawlerStatsHelper.done(urlQueue);

        String[] values = localLogMsg.get().split("\t");
        assertEquals(4, values.length);
        assertEquals("url:http://example.com/test", values[0]);

    /**
     * Interval in milliseconds between crawler thread executions.
     * Used to control the rate at which new crawler threads are started
     * and the frequency of status checks.
     */
    protected long crawlingExecutionInterval = Constants.DEFAULT_CRAWLING_EXECUTION_INTERVAL;

    /**
     * Thread priority for crawler threads.
     * Defaults to normal thread priority.
     */

import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.log.LogType;
import org.codelibs.fess.exception.ContainerNotAvailableException;

import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
import org.codelibs.fess.crawler.util.FieldConfigs;
import org.codelibs.fess.mylasta.direction.FessConfig;

 * </pre>
 */
public class Crawler {

    /**
     * Creates a new instance of Crawler.
     */
    public Crawler() {
        // Default constructor
    }

    /** Logger instance for this class. */
    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /** Thread name for web and file system crawling process. */

labels.wizard_crawling_config_path=Crawlpad
labels.wizard_button_register_again=Continu aanmaken
labels.wizard_button_register_next=Maken
labels.wizard_start_crawling_title=Start crawlen
labels.wizard_start_crawler_title=Crawler
labels.wizard_start_crawling_desc=U kunt nu beginnen met crawlen door op de knop "Start crawlen" te klikken.
labels.wizard_button_start_crawling=Start crawlen
labels.wizard_button_finish=Overslaan