* This job launches a separate crawler process that can crawl web sites, file systems,
 * and data sources based on the configured crawling settings.
 *
 * <p>The job supports selective crawling by specifying configuration IDs for different
 * types of crawlers (web, file, data). It manages the crawler process lifecycle,
 * handles timeout scenarios, and ensures proper cleanup of resources.</p>
 *

/**
 * Integration tests for the Favorites API (/api/v1/favorites and /api/v1/documents/{docId}/favorite)
 * This test requires crawled documents and favorites feature to be properly configured
 */
@Tag("it")
@Disabled("Requires favorites feature to be enabled and crawled documents")
public class FavoritesApiTests extends CrawlTestBase {
    private static final Logger logger = LogManager.getLogger(FavoritesApiTests.class);

    public void test_urlQueueKeyObject() {
        TestUrlQueue urlQueue = new TestUrlQueue("queue_123", "http://example.com/test");
        crawlerStatsHelper.begin(urlQueue);
        crawlerStatsHelper.record(urlQueue, "crawled");
        crawlerStatsHelper.done(urlQueue);

        String[] values = localLogMsg.get().split("\t");
        assertEquals(4, values.length);
        assertEquals("url:http://example.com/test", values[0]);

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;

    /**
     * Interval in milliseconds between crawler thread executions.
     * Used to control the rate at which new crawler threads are started
     * and the frequency of status checks.
     */
    protected long crawlingExecutionInterval = Constants.DEFAULT_CRAWLING_EXECUTION_INTERVAL;

    /**
     * Thread priority for crawler threads.
     * Defaults to normal thread priority.
     */

import io.restassured.RestAssured;
import io.restassured.path.json.JsonPath;
import io.restassured.response.Response;

/**
 * Integration tests for the Scroll Search API (/api/v1/documents/all)
 * This test requires crawled documents and api.search.scroll to be enabled
 */
@Tag("it")
@Disabled("Requires api.search.scroll to be enabled in fess_config.properties")
public class ScrollSearchApiTests extends CrawlTestBase {

import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.log.LogType;
import org.codelibs.fess.exception.ContainerNotAvailableException;

 * This class defines tag patterns that match HTML elements based on tag name, CSS class, ID, or custom attributes.
 * It is used to identify and remove unwanted HTML elements from crawled documents.
 */
public class PrunedTag {
    /** The HTML tag name to match (e.g., "div", "span", "p") */
    private final String tag;
    /** The ID attribute value to match */
    private String id;

/**
 * Helper class for managing and validating URL protocols in Fess crawling system.
 * This class handles the initialization and validation of web and file protocols
 * used by the crawler to determine which URLs can be crawled.
 */
public class ProtocolHelper {
    private static final Logger logger = LogManager.getLogger(ProtocolHelper.class);