}

    /**
     * Adds a document to the search index by crawling the specified URL and processing the content.
     * This method handles recursive crawling with depth and access count limits, follows redirects,
     * and processes child URLs discovered during crawling.
     *
     * @param paramMap the data store parameters containing crawling configuration

labels.createdTime=Created Time
labels.depth=Depth
labels.excludedPaths=Excluded Paths for Crawling
labels.excludedUrls=Excluded URLs for Crawling
labels.excludedDocPaths=Excluded Paths for Searching
labels.excludedDocUrls=Excluded URLs for Searching
labels.hostname=Hostname
labels.id=ID
labels.includedPaths=Included Paths for Crawling
labels.includedUrls=Included URLs for Crawling
labels.includedDocPaths=Included Paths for Searching

    /**
     * Filter for URLs to control which URLs are processed.
     */
    protected UrlFilter urlFilter;

    /**
     * Manager for crawling rules and configurations.
     */
    protected RuleManager ruleManager;

    /**
     * Controller for managing crawling intervals and delays.
     */
    protected IntervalController intervalController;

    /**
     * Set of robots.txt URLs that have been processed.

    /**
     * Default constructor.
     */
    public MailForm() {
        // Default constructor
    }

    /** The incremental crawling setting. */
    public String incrementalCrawling;

    /** The day for cleanup setting. */
    public String dayForCleanup;

    /** The crawling thread count setting. */
    public String crawlingThreadCount;

    /** The search log setting. */
    public String searchLog;

    @CustomSize(maxKey = "form.admin.max.input.size")
    public String urls;

    /**
     * URL patterns to include during crawling.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String includedUrls;

    /**
     * URL patterns to exclude during crawling.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String excludedUrls;

    /**

        return ComponentUtil.getIngestFactory();
    }

    /**
     * Processes a result data object for web/file crawling.
     * Default implementation returns the target unchanged.
     *
     * @param target the result data to process
     * @param responseData the response data from crawling
     * @return the processed result data
     */

        final StringBuilder resultBuf = new StringBuilder();

        // purge crawling sessions
        try {
            crawlingInfoService.deleteBefore(systemHelper.getCurrentTimeAsLong());
        } catch (final Exception e) {
            logger.error("Failed to purge crawling sessions.", e);
            resultBuf.append(e.getMessage()).append("\n");
        }

        // purge search logs

        }

        if (cae.isDebugEnabled()) {
            logger.debug("[{}] Crawling Access Exception at {}", failureUrlId, urlQueue.getUrl(), cae);
        } else if (cae.isInfoEnabled()) {
            logger.info("[{}] {}", failureUrlId, cae.getMessage());
        } else if (cae.isWarnEnabled()) {
            logger.warn("[{}] Crawling Access Exception at {}", failureUrlId, urlQueue.getUrl(), cae);

    public void setSessionIdList(final List<String> sessionIdList) {
        this.sessionIdList = sessionIdList;
    }

    /**
     * Sets the flag indicating whether crawling should be finished.
     *
     * @param finishCrawling true if crawling should be finished, false otherwise
     */
    public void setFinishCrawling(final boolean finishCrawling) {
        this.finishCrawling = finishCrawling;
    }

    /**

import org.codelibs.fess.app.web.api.admin.BaseSearchBody;

/**
 * Search request body for file crawling configuration administration API.
 */
public class SearchBody extends BaseSearchBody {

    /**
     * Default constructor.
     */
    public SearchBody() {
        super();
    }

    /** Name of the file crawling configuration */
    public String name;

    /** File paths to crawl */
    public String paths;