// Default constructor
    }

    /**
     * Enable or disable incremental crawling.
     * When enabled, only new or modified documents are crawled.
     */
    @Size(max = 10)
    public String incrementalCrawling;

    /**
     * Number of days to keep crawled documents before cleanup.
     * Set to -1 to disable automatic cleanup.
     */
    @Required
    @Min(-1)
    @Max(1000)
    @ValidateTypeFailure

    }

    /**
     * Name of the crawling configuration.
     */
    @Required
    @Size(max = 200)
    public String crawlingConfigName;

    /**
     * Path or URL to be crawled by this configuration.
     */
    @Required
    @Size(max = 1000)
    public String crawlingConfigPath;

    /**
     * Maximum depth for crawling (how many levels deep to follow links).
     */

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * Exception thrown during the extraction process in the crawler.
 * This exception indicates a failure or error that occurred while extracting content from a crawled resource.
 * It extends {@link org.codelibs.fess.crawler.exception.CrawlerSystemException} and provides constructors

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.entity;

/**
 * Implementation of the {@link UrlQueue} interface.
 * This class represents a URL to be crawled, storing its ID, session ID,
 * HTTP method, URL, metadata, encoding, parent URL, depth, last modified time,
 * creation time, and weight.
 *

import org.lastaflute.web.response.StreamResponse;

/**
 * Action class for handling cached document content requests.
 * Provides functionality to retrieve and display cached versions of crawled documents.
 */
public class CacheAction extends FessSearchAction {

    /**
     * Creates a new instance of CacheAction.
     */
    public CacheAction() {
        super();
    }

 * This class extends ArrayList to hold document maps while tracking content size
 * and processing time metrics. It's used throughout the Fess search system to
 * manage collections of search results and crawled documents.
 *
 */
public class DocList extends ArrayList<Map<String, Object>> {

    /** Serial version UID for serialization */
    private static final long serialVersionUID = 1L;

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.util;

import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlQueueService;

/**

import org.apache.tika.metadata.Message;
import org.apache.tika.metadata.TIFF;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.metadata.TikaMimeKeys;

/**
 * Represents extracted data from a crawled resource, including content and metadata.
 */
public class ExtractData
        implements TikaCoreProperties, CreativeCommons, Geographic, HttpHeaders, Message, ClimateForcast, TIFF, TikaMimeKeys, Serializable {

    public void test_urlQueueKeyObject() {
        TestUrlQueue urlQueue = new TestUrlQueue("queue_123", "http://example.com/test");
        crawlerStatsHelper.begin(urlQueue);
        crawlerStatsHelper.record(urlQueue, "crawled");
        crawlerStatsHelper.done(urlQueue);

        String[] values = localLogMsg.get().split("\t");
        assertEquals(4, values.length);
        assertEquals("url:http://example.com/test", values[0]);

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;