final int threadCount = 10;
        final int matchesPerThread = 100;
        final AtomicInteger matchCount = new AtomicInteger(0);
        final AtomicInteger noMatchCount = new AtomicInteger(0);
        final List<Exception> exceptions = new ArrayList<>();

        Thread[] threads = new Thread[threadCount];
        for (int i = 0; i < threadCount; i++) {

        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final int maxCount = 50;
            final int numOfThread = 10;

            final File file = File.createTempFile("crawler-", "");
            file.delete();
            file.mkdirs();
            file.deleteOnExit();

                        for (int i = 0; i < nnm.getLength(); i++) {
                            final Node attr = nnm.item(i);
                            final String aname = attr.getNodeName();
                            final boolean isPrefix = aname.startsWith("xmlns:");

                            if (isPrefix || "xmlns".equals(aname)) {
                                final int index = aname.indexOf(':');

    public void test_isSuccessful() {
        DefaultResponseProcessor processor = new DefaultResponseProcessor();
        processor.setSuccessfulHttpCodes(new int[] { 200 });
        processor.setNotModifiedHttpCodes(new int[] { 304 });

        ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        assertTrue(processor.isSuccessful(responseData));

 * and the time taken to complete the operation.
 */
public class SuggestIndexResponse implements Response {
    /** The number of suggest documents. */
    protected final int numberOfSuggestDocs;
    /** The number of input documents. */
    protected final int numberOfInputDocs;
    /** Flag indicating if there are errors. */
    protected final boolean hasError;
    /** List of errors. */

 * @author shinsuke
 *
 */
public class CrawlerWebServer {
    private int port = 8080;

    private File docRoot;

    private Server server;

    private boolean tempDocRoot = false;

    public CrawlerWebServer(final int port) {
        this(port, createDocRoot(3));
        tempDocRoot = true;
    }

    public CrawlerWebServer(final int port, final File docRoot) {
        this.port = port;

import java.io.Closeable;
import java.util.Map;

/**
 * Interface for reading documents and extracting their contents into a map.
 * Implementations of this interface should provide the logic for reading
 * documents and converting them into a key-value structure.
 *
 * <p>This interface extends {@link java.io.Closeable}, so implementations

    /** Constant representing the pre-processing state. */
    int PRE_PROCESSING = 1;

    /** Constant representing the post-processing state. */
    int POST_PROCESSING = 2;

    /** Constant indicating that there are no URLs in the queue. */
    int NO_URL_IN_QUEUE = 4;

    /** Constant indicating that the crawler is waiting for new URLs. */
    int WAIT_NEW_URL = 8;

    /**

        SuggestSettings settings = suggester.settings();
        int num = 10000;

        addDocument(indexName, client, num);

        ESSourceReader reader = new ESSourceReader(client, settings, indexName);
        reader.setScrollSize(1000);
        reader.addSort(SortBuilders.fieldSort("field2"));
        int count = 0;
        int prev = -1;
        Map<String, Object> source;

                    processor.setTransformer(container.getComponent("fileTransformer"));
                    processor.setSuccessfulHttpCodes(new int[] { 200 });
                    processor.setNotModifiedHttpCodes(new int[] { 304 });
                })//
                .<RegexRule> singleton("fileRule", RegexRule.class, rule -> {
                    rule.setRuleId("fileRule");