import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.MaxLengthExceededException;

import jakarta.annotation.Resource;

/**
 * Abstract base class for CrawlerClient implementations.
 * Provides common functionality for handling initialization parameters,
 * content length checks, and default method implementations.

import org.codelibs.fess.crawler.transformer.impl.FileTransformer;
import org.codelibs.fess.crawler.util.CrawlerWebServer;
import org.dbflute.utflute.lastadi.LastaDiTestCase;

import jakarta.annotation.Resource;

public class CrawlerTest extends LastaDiTestCase {

    @Resource
    private Crawler crawler;

    @Resource
    private DataService dataService;

    @Resource

import org.codelibs.fess.crawler.exception.MaxLengthExceededException;
import org.codelibs.fess.crawler.helper.ContentLengthHelper;
import org.codelibs.fess.crawler.helper.MimeTypeHelper;

import jakarta.annotation.Resource;

/**
 * FileSystemClient is CrawlerClient implementation to crawl files on a file
 * system.
 *
 * @author shinsuke
 *
 */
public class FileSystemClient extends AbstractCrawlerClient {

import org.codelibs.fess.crawler.service.UrlQueueService;
import org.codelibs.fess.crawler.transformer.Transformer;
import org.codelibs.fess.crawler.util.CrawlingParameterUtil;

import jakarta.annotation.Resource;

/**
 * <p>
 * {@link DefaultResponseProcessor} is a default implementation of {@link ResponseProcessor}.
 * It processes the response data based on the HTTP status code and configured transformer.
 * </p>

import org.opensearch.search.SearchHit;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.client.AdminClient;
import org.opensearch.transport.client.Client;

import jakarta.annotation.PreDestroy;

/**
 * FesenClient is a client for OpenSearch.
 *
 * @author shinsuke
 *
 */
public class FesenClient implements Client {
    /**
     * HTTP address for OpenSearch.
     */

import io.minio.Result;
import io.minio.StatObjectArgs;
import io.minio.StatObjectResponse;
import io.minio.errors.ErrorResponseException;
import io.minio.messages.Item;
import io.minio.messages.Tags;
import jakarta.annotation.Resource;

/**
 * A crawler client implementation for accessing and retrieving content from storage systems using MinIO.
 * This client supports operations on object storage systems compatible with S3 protocol.
 *

import org.codelibs.fess.crawler.rule.RuleManager;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * The Crawler class is the main class for web crawling. It manages the crawling process,
 * including adding URLs to the queue, filtering URLs, managing crawler threads,

import org.codelibs.fess.crawler.util.XPathAPI;
import org.codelibs.nekohtml.parsers.DOMParser;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;

import jakarta.annotation.Resource;

/**
 * The {@code HtmlTransformer} class is responsible for transforming HTML responses
 * during the crawling process. It extracts data, identifies child URLs, and handles
 * character set encoding.
 * <p>

import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlQueueService;
import org.codelibs.fess.crawler.util.CrawlingParameterUtil;

import jakarta.annotation.Resource;

/**
 * The {@code CrawlerThread} class represents a thread that executes the crawling process.
 * It is responsible for fetching URLs from the queue, accessing the content,

import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;

import jakarta.annotation.Resource;

/**
 * <p>
 * XmlTransformer is a class that extends AbstractTransformer to transform XML documents into a specific format for indexing.