*/
package org.codelibs.fess.crawler.exception;

/**
 * Exception thrown during the extraction process in the crawler.
 * This exception indicates a failure or error that occurred while extracting content from a crawled resource.
 * It extends {@link org.codelibs.fess.crawler.exception.CrawlerSystemException} and provides constructors
 * to handle different scenarios such as wrapping another exception or providing a specific error message.
 */

        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {
            @SuppressWarnings("resource")
            final org.apache.poi.hssf.extractor.ExcelExtractor excelExtractor =
                    new org.apache.poi.hssf.extractor.ExcelExtractor(new HSSFWorkbook(in));

import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.extractor.Extractor;
import org.codelibs.fess.crawler.extractor.ExtractorFactory;

import jakarta.annotation.Resource;

/**
 * TextTransformer is a class that transforms a ResponseData object into a ResultData object containing the extracted text content.
 * It uses an Extractor to extract the text from the response body based on the MIME type.

import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * <p>
 * FileTransformer stores the content of a crawled resource as a file on the file system.
 * It extends HtmlTransformer and provides functionality to:
 * </p>
 * <ul>
 *     <li>Specify a base directory for storing files.</li>

import org.opensearch.search.sort.SortBuilders;
import org.opensearch.search.sort.SortOrder;

import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import jakarta.annotation.Resource;

/**
 * OpenSearchUrlQueueService is an implementation of {@link UrlQueueService} for OpenSearch.
 *
 * @author shinsuke
 *
 */

import org.w3c.dom.Node;
import org.xml.sax.InputSource;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;

import jakarta.annotation.Resource;

/**
 * {@link HtmlXpathExtractor} is an implementation of the {@link org.codelibs.fess.crawler.extractor.Extractor} interface.
 * It uses XPath expressions to extract text content from HTML documents.
 * <p>

import org.codelibs.fess.crawler.entity.UrlQueueImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * Implementation of the {@link UrlQueueService} interface.
 * This class provides methods for managing a queue of URLs to be crawled,
 * including adding, deleting, and retrieving URLs from the queue.

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.MaxLengthExceededException;

import jakarta.annotation.Resource;

/**
 * Abstract base class for CrawlerClient implementations.
 * Provides common functionality for handling initialization parameters,
 * content length checks, and default method implementations.

     */
    public RobotsTxt parse(final InputStream stream, final String charsetName) {
        if (!enabled) {
            return null;
        }

        try {
            @SuppressWarnings("resource")
            final BufferedReader reader = new BufferedReader(new InputStreamReader(new BOMInputStream(stream), charsetName));

            String line;
            final RobotsTxt robotsTxt = new RobotsTxt();

import org.codelibs.fess.crawler.exception.MaxLengthExceededException;
import org.codelibs.fess.crawler.helper.ContentLengthHelper;
import org.codelibs.fess.crawler.helper.MimeTypeHelper;

import jakarta.annotation.Resource;

/**
 * FileSystemClient is CrawlerClient implementation to crawl files on a file
 * system.
 *
 * @author shinsuke
 *
 */
public class FileSystemClient extends AbstractCrawlerClient {