import org.codelibs.fess.crawler.service.UrlQueueService;
import org.codelibs.fess.crawler.transformer.Transformer;
import org.codelibs.fess.crawler.util.CrawlingParameterUtil;

import jakarta.annotation.Resource;

/**
 * <p>
 * {@link DefaultResponseProcessor} is a default implementation of {@link ResponseProcessor}.
 * It processes the response data based on the HTTP status code and configured transformer.
 * </p>
 *

    protected String getEncoding(final BufferedInputStream bis) {
        final byte[] b = new byte[preloadSizeForCharset];
        try {
            bis.mark(preloadSizeForCharset);
            @SuppressWarnings("resource")
            final BOMInputStream bomIn = new BOMInputStream(bis, false, ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE,
                    ByteOrderMark.UTF_32BE, ByteOrderMark.UTF_32LE, BOM_UTF_7);

import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;

import jakarta.annotation.Resource;

/**
 * <p>
 * XmlTransformer is a class that extends AbstractTransformer to transform XML documents into a specific format for indexing.

import org.codelibs.fess.crawler.util.XPathAPI;
import org.codelibs.nekohtml.parsers.DOMParser;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;

import jakarta.annotation.Resource;

/**
 * The {@code HtmlTransformer} class is responsible for transforming HTML responses
 * during the crawling process. It extracts data, identifies child URLs, and handles
 * character set encoding.
 * <p>

				"application/prs.plucker",
				"application/qsig",
				"application/reginfo+xml",
				"application/relax-ng-compact-syntax",
				"application/remote-printing",
				"application/resource-lists+xml",
				"application/resource-lists-diff+xml",
				"application/riscos",
				"application/rlmi+xml",
				"application/rls-services+xml",
				"application/rsd+xml",
				"application/rss+xml",
				"application/rtf",

     * desired, and use YYYY-MM-DD.
     *
     * Note that this tag is separate from the If-Modified-Since (304) header
     * the server can return, and search engines may use the information from
     * both sources differently.
     */
    private String lastmod;

    /**
     * How frequently the page is likely to change. This value provides general
     * information to search engines and may not correlate exactly to how often

import org.opensearch.search.SearchHits;
import org.opensearch.search.sort.SortBuilder;

import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;

import jakarta.annotation.Resource;

/**
 * Abstract base class for crawler services that interact with OpenSearch.
 *
 * @author shinsuke
 *
 */
public abstract class AbstractCrawlerService {
    /**

        // Verify
        assertEquals("serializeRule", deserializedRule.getRuleId());
        assertNotNull(deserializedRule.getResponseProcessor());

        // Note: crawlerContainer is transient (marked with @Resource)
        // so it won't be serialized
        assertNull(deserializedRule.crawlerContainer);
    }

    /**
     * Test with null CrawlerContainer
     */
    public void test_nullCrawlerContainer() {

        tuples.add(new Tuple<>(DefaultKeys.SUPPORTED_FIELDS, "content"));
        return tuples;
    }

    /**
     * Loads the index settings from a resource file.
     * @return The index settings as a string.
     * @throws IOException If an I/O error occurs.
     */
    protected String loadIndexSettings() throws IOException {

    <sub-class-of type="text/plain"/>
    <glob pattern="*.rnc"/>
  </mime-type>
  <mime-type type="application/remote-printing"/>
  <mime-type type="application/resource-lists+xml">
    <glob pattern="*.rl"/>
  </mime-type>
  <mime-type type="application/resource-lists-diff+xml">
    <glob pattern="*.rld"/>
  </mime-type>
  <mime-type type="application/riscos"/>
  <mime-type type="application/rlmi+xml"/>