import com.google.common.cache.LoadingCache;

import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;

/**
 * Helper class for managing crawler statistics and performance metrics.
 * This class provides functionality to track, record, and report statistics
 * about crawler operations including timing data, performance metrics, and
 * operational events. It uses an internal cache to maintain statistics

 */
package org.codelibs.fess.exception;

import org.codelibs.fess.crawler.exception.CrawlingAccessException;

/**
 * Exception thrown when an error occurs during data store crawling operations.
 * This exception provides information about the URL where the error occurred
 * and whether the crawling process should be aborted.
 */
public class DataStoreCrawlingException extends CrawlingAccessException {

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.exception.ChildUrlsException;

import jakarta.annotation.PostConstruct;

/**
 * Helper class for managing and validating URL protocols in Fess crawling system.
 * This class handles the initialization and validation of web and file protocols
 * used by the crawler to determine which URLs can be crawled.
 */
public class ProtocolHelper {
    private static final Logger logger = LogManager.getLogger(ProtocolHelper.class);

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.interval.impl.DefaultIntervalController;
import org.codelibs.fess.helper.IntervalControlHelper;
import org.codelibs.fess.util.ComponentUtil;

/**

# Encoding for crawling data.
crawler.crawling.data.encoding=UTF-8
# Supported web protocols for crawling.
crawler.web.protocols=http,https
# Supported file protocols for crawling.
crawler.file.protocols=file,smb,smb1,ftp,storage,s3,gcs
# Pattern for environment variable keys in crawling data.
crawler.data.env.param.key.pattern=^FESS_ENV_.*
# Whether to ignore robots.txt during crawling.
crawler.ignore.robots.txt=false

/**
 * Form class for crawling configuration wizard in the admin interface.
 * Contains validation constraints for creating new crawling configurations.
 */
public class CrawlingConfigForm {

    /**
     * Creates a new form instance.
     */
    public CrawlingConfigForm() {
        // Default constructor
    }

    /**
     * Name of the crawling configuration.
     */
    @Required

        assertEquals(0, logListAfter.size(), "All crawling info logs should be deleted after calling delete all endpoint");

        // Log the result
        if (sizeBeforeDeletion > 0) {
            logger.info("Successfully deleted {} crawling info log(s) using bulk delete", sizeBeforeDeletion);
        } else {
            logger.info("No crawling info logs to delete (may have been deleted by previous test)");
        }

    public String id;

    /**
     * The URL that failed during crawling.
     * This is a required field containing the actual URL that encountered an error.
     */
    @Required
    public String url;

    /**
     * The name of the crawler thread that encountered the failure.
     * This is a required field used for identifying which crawler process failed.
     */
    @Required
    public String threadName;

 * including crawling behavior, authentication, logging, and various system parameters.
 *
 */
public class EditForm {

    /**
     * Creates a new EditForm instance.
     */
    public EditForm() {
        // Default constructor
    }

    /**
     * Enable or disable incremental crawling.
     * When enabled, only new or modified documents are crawled.
     */
    @Size(max = 10)