*/
package org.codelibs.fess.crawler.processor;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor;

 * This class extends ArrayList to hold document maps while tracking content size
 * and processing time metrics. It's used throughout the Fess search system to
 * manage collections of search results and crawled documents.
 *
 */
public class DocList extends ArrayList<Map<String, Object>> {

    /** Serial version UID for serialization */
    private static final long serialVersionUID = 1L;

    }

    /**
     * Name of the crawling configuration.
     */
    @Required
    @Size(max = 200)
    public String crawlingConfigName;

    /**
     * Path or URL to be crawled by this configuration.
     */
    @Required
    @Size(max = 1000)
    public String crawlingConfigPath;

    /**
     * Maximum depth for crawling (how many levels deep to follow links).
     */

import org.lastaflute.web.response.StreamResponse;

/**
 * Action class for handling cached document content requests.
 * Provides functionality to retrieve and display cached versions of crawled documents.
 */
public class CacheAction extends FessSearchAction {

    /**
     * Creates a new instance of CacheAction.
     */
    public CacheAction() {
        super();
    }

 * This class defines tag patterns that match HTML elements based on tag name, CSS class, ID, or custom attributes.
 * It is used to identify and remove unwanted HTML elements from crawled documents.
 */
public class PrunedTag {
    /** The HTML tag name to match (e.g., "div", "span", "p") */
    private final String tag;
    /** The ID attribute value to match */
    private String id;

    @Size(max = 200)
    public String name;

    /**
     * The description of the web configuration.
     */
    @Size(max = 1000)
    public String description;

    /**
     * The URLs to be crawled by this web configuration.
     */
    @Required
    @UriType(protocolType = ProtocolType.WEB)
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String urls;

    /**

     *
     * @param crawlingConfig the crawling configuration associated with the failure
     * @param errorName the name/type of the error that occurred
     * @param url the URL that failed to be crawled
     * @param e the exception that caused the failure
     * @return the stored or updated FailureUrl entity, or null if the exception should be ignored
     */

/**
 * Helper class for managing and validating URL protocols in Fess crawling system.
 * This class handles the initialization and validation of web and file protocols
 * used by the crawler to determine which URLs can be crawled.
 */
public class ProtocolHelper {
    private static final Logger logger = LogManager.getLogger(ProtocolHelper.class);

    OkHttpClient client = new OkHttpClient.Builder()
        .cache(cache)
        .callTimeout(5, TimeUnit.SECONDS)
        .build();

    Crawler crawler = new Crawler(client, queueLimit, hostLimit);
    crawler.queue.add(HttpUrl.get(args[1]));
    crawler.parallelDrainQueue(threadCount);
  }

/*
 [Crawler Notification]
 Crawler notification mail.
*/
subject: [FESS] Crawler completed: /*pmb.hostname*/
>>>
--- Server Info ---
Host Name: /*pmb.hostname:orElse('Unknown')*/
Job Name: /*pmb.jobname:orElse('Unknown')*/

--- Web/FileSystem Crawler ---
Start Time: /*pmb.webFsCrawlStartTime:orElse('-')*/
End Time:   /*pmb.webFsCrawlEndTime:orElse('-')*/
Exec Time:  /*pmb.webFsCrawlExecTime:orElse('-')*/ ms

--- Web/FileSystem Indexer ---