* The {@code IdnDnsResolver} class implements the {@code DnsResolver} interface
 * to provide DNS resolution with support for Internationalized Domain Names (IDN).
 * It converts Unicode domain names to ASCII Compatible Encoding (ACE) using the
 * {@link java.net.IDN} class.
 *
 * <p>This class allows setting a custom flag for the IDN conversion and a custom
 * encoding for URL decoding.</p>
 *
 * <p>Example usage:</p>
 * <pre>
 * {@code

 *   <li>{@link #convert(String, String, String...)}: Converts the given text using the specified field and languages.</li>
 * </ul>
 *
 * <p>Inner Class:
 * <ul>
 *   <li>LangAnayzerConverter: A protected inner class that implements the ReadingConverter interface.
 *       It is responsible for converting text using a specific language analyzer.
 *       <ul>

 * </p>
 * <ol>
 *   <li>Fetching a URL from the queue using {@link UrlQueueService#poll(String)}.</li>
 *   <li>Checking if the URL is valid using {@link #isValid(UrlQueue)}.</li>
 *   <li>Accessing the content using a {@link CrawlerClient} obtained from {@link CrawlerClientFactory}.</li>
 *   <li>Processing the response using a {@link ResponseProcessor} associated with a {@link Rule}.</li>

     * Parses a robots.txt file from the given input stream using UTF-8 encoding.
     * @param stream the input stream to parse
     * @return the parsed RobotsTxt object, or null if disabled
     */
    public RobotsTxt parse(final InputStream stream) {
        return parse(stream, Constants.UTF_8);
    }

    /**
     * Parses a robots.txt file from the given input stream using the specified character encoding.

 *
 * <p>The extractor supports two types of password management:
 * <ul>
 *   <li>Static passwords configured via {@link #addPassword(String, String)}</li>
 *   <li>Dynamic passwords provided through extraction parameters</li>
 * </ul>
 *
 * <p>Passwords are matched against URLs or resource names using regular expression patterns.

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.RequestData.Method;

/**
 * Builder class for creating RequestData objects using a fluent interface.
 * This class provides a simple way to construct RequestData objects with method chaining.
 *
 * <p>Usage example:</p>
 * <pre>
 * RequestData request = RequestDataBuilder.newRequestData()

     * @return the detected content type
     */
    String getContentType(InputStream is, String filename);

    /**
     * Determines the content type of the given input stream using the provided parameters.
     * @param is the input stream to analyze
     * @param params the parameters containing additional information like filename
     * @return the detected content type
     */

 */
package org.codelibs.fess.suggest.normalizer;

import com.ibm.icu.text.Transliterator;

/**
 * ICUNormalizer is a class that implements the Normalizer interface and provides
 * functionality to normalize text using ICU4J's Transliterator.
 *
 * <p>This class uses a specified Transliterator to perform text normalization.
 * The Transliterator is initialized with a given ID during the construction of
 * the ICUNormalizer instance.</p>
 *

                    if (logger.isDebugEnabled()) {
                        logger.debug("Using default extractor {} for MIME type {}", extractorName, mimeType);
                    }
                    extractor = crawlerContainer.getComponent(extractorName);
                } else if (logger.isDebugEnabled()) {
                    logger.debug("Using {} for detected MIME type {}, not {}", extractor.getClass().getName(), detectedMimeType, mimeType);

import org.opensearch.search.SearchHit;
import org.opensearch.search.sort.SortBuilder;
import org.opensearch.transport.client.Client;

/**
 * <p>
 * {@link ESSourceReader} reads documents from Elasticsearch using the scroll API.
 * It implements the {@link DocumentReader} interface to provide a way to iterate over documents
 * in a large index without loading all of them into memory at once.
 * </p>
 *
 * <p>