import org.codelibs.fess.crawler.entity.Sitemap;
import org.codelibs.fess.crawler.entity.SitemapSet;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.helper.SitemapsHelper;
import org.codelibs.fess.crawler.processor.ResponseProcessor;

import jakarta.annotation.Resource;

/**
 * A response processor implementation that handles sitemaps.

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.entity;

/**
 * Implementation of the {@link UrlQueue} interface.
 * This class represents a URL to be crawled, storing its ID, session ID,
 * HTTP method, URL, metadata, encoding, parent URL, depth, last modified time,
 * creation time, and weight.
 *

/**
 * Form class for crawling configuration wizard in the admin interface.
 * Contains validation constraints for creating new crawling configurations.
 */
public class CrawlingConfigForm {

    /**
     * Creates a new form instance.
     */
    public CrawlingConfigForm() {
        // Default constructor
    }

    /**
     * Name of the crawling configuration.
     */
    @Required

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.ingest;

import java.util.Map;

import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.entity.DataStoreParams;
import org.codelibs.fess.util.ComponentUtil;

/**

import org.codelibs.core.io.CopyUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * <p>
 * FileTransformer stores the content of a crawled resource as a file on the file system.

labels.crawling_info_CrawlerStartTime=Crawler Start Time
labels.crawling_info_CrawlerEndTime=Crawler End Time
labels.crawling_info_CrawlerExecTime=Crawler Execution Time
labels.crawling_info_CrawlerStatus=Crawler Status
labels.crawling_info_WebFsCrawlExecTime=Crawl Execution Time (Web/File)
labels.crawling_info_WebFsCrawlStartTime=Crawl Start Time (Web/File)
labels.crawling_info_WebFsCrawlEndTime=Crawl End Time (Web/File)

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.interval.impl.DefaultIntervalController;
import org.codelibs.fess.helper.IntervalControlHelper;
import org.codelibs.fess.util.ComponentUtil;

/**

# Encoding for crawling data.
crawler.crawling.data.encoding=UTF-8
# Supported web protocols for crawling.
crawler.web.protocols=http,https
# Supported file protocols for crawling.
crawler.file.protocols=file,smb,smb1,ftp,storage,s3,gcs
# Pattern for environment variable keys in crawling data.
crawler.data.env.param.key.pattern=^FESS_ENV_.*
# Whether to ignore robots.txt during crawling.
crawler.ignore.robots.txt=false

 */
package org.codelibs.fess.exception;

import org.codelibs.fess.crawler.exception.CrawlingAccessException;

/**
 * Exception thrown when an error occurs during data store crawling operations.
 * This exception provides information about the URL where the error occurred
 * and whether the crawling process should be aborted.
 */
public class DataStoreCrawlingException extends CrawlingAccessException {

import jakarta.annotation.PostConstruct;

/**
 * Helper class for managing and validating URL protocols in Fess crawling system.
 * This class handles the initialization and validation of web and file protocols
 * used by the crawler to determine which URLs can be crawled.
 */
public class ProtocolHelper {
    private static final Logger logger = LogManager.getLogger(ProtocolHelper.class);