package org.codelibs.fess.crawler.entity;

import java.io.IOException;

import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

/**
 * OpenSearchUrlQueue is an implementation of {@link UrlQueue} for OpenSearch.
 */
public class OpenSearchUrlQueue extends UrlQueueImpl<String> implements ToXContent {

    /**
     * Creates a new instance of OpenSearchUrlQueue.
     */

     * @return an ExtractData object containing the extracted text
     */
    ExtractData getText(InputStream in, Map<String, String> params);

    /**
     * Returns the weight of the extractor.
     * The default implementation returns a weight of 1.
     *
     * @return the weight of the extractor
     */
    default int getWeight() {
        return 1;
    }

        assertNotNull(readings);
        assertEquals(1, readings.size());
        assertEquals("TEST", readings.get(0));
    }

    /**
     * Test implementation of ReadingConverter for testing purposes
     */
    private static class TestReadingConverter implements ReadingConverter {

        private boolean initialized = false;
        private int initCount = 0;

import org.dbflute.utflute.core.PlainTestCase;

/**
 * Test class for Rule interface.
 * Tests the contract and behavior of Rule implementations.
 */
public class RuleTest extends PlainTestCase {

    /**
     * Test implementation of Rule interface for testing purposes
     */
    public static class TestRule implements Rule {
        private static final long serialVersionUID = 1L;

```

### Key Components

#### Core Engine
- **Crawler**: Main orchestrator managing crawl execution
- **CrawlerContext**: Execution context and configuration
- **CrawlerThread**: Individual crawler thread implementation

#### Client Architecture
- **HcHttpClient**: HTTP/HTTPS client using Apache HttpComponents
- **FileSystemClient**: File system access
- **FtpClient**: FTP protocol support

import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.helper.SitemapsHelper;

/**
 * SitemapsRule is a rule implementation that extends RegexRule to determine if a given response data
 * represents a valid sitemap. It uses a SitemapsHelper to validate the response body as an InputStream.

import org.apache.http.auth.AuthScheme;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.impl.auth.NTLMScheme;
import org.apache.http.protocol.HttpContext;

/**
 * This class is AuthSchemeFactory implementation for NTLM.
 *
 * @author shinsuke
 *
 */
public class NTLMSchemeProvider implements AuthSchemeProvider {

    /**
     * Creates a new NTLMSchemeProvider instance.
     */

 */
package org.codelibs.fess.crawler.transformer.impl;

import org.codelibs.fess.crawler.transformer.Transformer;

/**
 * An abstract base class for transformers.
 * Provides basic implementation for managing the transformer's name.
 *
 */
public abstract class AbstractTransformer implements Transformer {

    /**
     * Creates a new instance of AbstractTransformer.
     */

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.interval.impl;

import java.util.Map;

import org.codelibs.core.lang.ThreadUtil;

/**
 * Default implementation of the IntervalController.
 * This class provides a default way to manage delays between crawler operations.
 * It allows setting delays before processing, after processing, when no URLs are in the queue,

import org.opensearch.index.query.QueryBuilders;
import org.opensearch.search.SearchHit;
import org.opensearch.search.SearchHits;

import jakarta.annotation.PostConstruct;

/**
 * OpenSearchDataService is an implementation of {@link DataService} for OpenSearch.
 */
public class OpenSearchDataService extends AbstractCrawlerService implements DataService<OpenSearchAccessResult> {

    /**