*/
package org.codelibs.fess.crawler.helper;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.regex.Pattern;

import org.codelibs.fess.crawler.entity.AccessResultImpl;
import org.codelibs.fess.crawler.entity.UrlQueueImpl;

/**

 * including crawling behavior, authentication, logging, and various system parameters.
 *
 */
public class EditForm {

    /**
     * Creates a new EditForm instance.
     */
    public EditForm() {
        // Default constructor
    }

    /**
     * Enable or disable incremental crawling.
     * When enabled, only new or modified documents are crawled.
     */
    @Size(max = 10)

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.ingest;

import java.util.Map;

import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.entity.DataStoreParams;
import org.codelibs.fess.util.ComponentUtil;

/**

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.interval.impl.DefaultIntervalController;
import org.codelibs.fess.helper.IntervalControlHelper;
import org.codelibs.fess.util.ComponentUtil;

/**

import jakarta.annotation.PostConstruct;

/**
 * Helper class for managing and validating URL protocols in Fess crawling system.
 * This class handles the initialization and validation of web and file protocols
 * used by the crawler to determine which URLs can be crawled.
 */
public class ProtocolHelper {
    private static final Logger logger = LogManager.getLogger(ProtocolHelper.class);

import java.util.concurrent.atomic.AtomicInteger;

import org.codelibs.core.collection.LruHashSet;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.filter.UrlFilter;
import org.codelibs.fess.crawler.interval.IntervalController;
import org.codelibs.fess.crawler.rule.Rule;
import org.codelibs.fess.crawler.rule.RuleManager;
import org.dbflute.utflute.core.PlainTestCase;

/**
 * Test class for CrawlerContext.

import org.codelibs.core.lang.SystemUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;

# Encoding for crawling data.
crawler.crawling.data.encoding=UTF-8
# Supported web protocols for crawling.
crawler.web.protocols=http,https
# Supported file protocols for crawling.
crawler.file.protocols=file,smb,smb1,ftp,storage,s3,gcs
# Pattern for environment variable keys in crawling data.
crawler.data.env.param.key.pattern=^FESS_ENV_.*
# Whether to ignore robots.txt during crawling.
crawler.ignore.robots.txt=false

import org.codelibs.core.io.CopyUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * <p>
 * FileTransformer stores the content of a crawled resource as a file on the file system.

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * RobotsTxtException is an exception class that represents an error related to robots.txt processing during web crawling.
 * It extends CrawlerSystemException and provides constructors to create instances with a message and/or a cause.
 *
 */