/**
 * NullResponseProcessor is a class that implements the ResponseProcessor interface.
 * It provides a no-op implementation for processing response data.
 * This class can be used when you want to skip the response processing step in a crawler.
 *
 * @author kuma
 */
public class NullResponseProcessor implements ResponseProcessor {

    /**
     * Creates a new NullResponseProcessor instance.
     */

package org.codelibs.fess.crawler.util;

import java.io.IOException;
import java.io.InputStream;

/**
 * A wrapper for an {@link InputStream} that ignores the {@link #close()} method call.
 * This can be useful when you want to prevent the underlying input stream from being closed.
 *
 * <p>All other methods delegate to the wrapped input stream.</p>
 *
 */
public class IgnoreCloseInputStream extends InputStream {

    /** Default character encoding to use when none is specified. */
    protected String defaultEncoding;

    /** Number of bytes to read from input stream to determine character set encoding.
     * If you want to follow a html spec, use 512. */
    protected int preloadSizeForCharset = 2048;

    /**
     * Pattern for invalid URLs.
     */
    protected Pattern invalidUrlPattern = Pattern.compile("^\\s*javascript:|" //

        <match value="[\x72|\x33|\x36|\x35]" type="regex" offset="3">
          <match value="0x00000000000000" offset="5"/>
        </match>
      </match>
    </magic>
    <!-- do we want to use this for such a rare format?   <glob pattern="*.rpy"/> -->
  </mime-type>

  <!-- =================================================================== -->

        } catch (final ExecutionException e) {
            // no thing to do
        }

        long elapsed = (System.nanoTime() - time) / 1000000;
        long wait = waittime * (numTasks - 1);
        assertTrue(elapsed + " >= " + wait, elapsed + 1L >= wait);
    }

// Check crawling status
while (crawler.crawlerContext.getStatus() == CrawlerStatus.RUNNING) {
    Thread.sleep(1000);
    System.out.println("Crawling in progress...");
}

// Wait for completion
crawler.awaitTermination();
System.out.println("Crawling completed");
```

### Custom Content Extractor

```java
public class CustomExtractor extends AbstractExtractor {
    @Override

            crawler.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler.getCrawlerContext().setNumOfThread(numOfThread);
            final String sessionId = crawler.execute();

            // Wait for crawler to start running with polling
            long startTime = System.currentTimeMillis();
            while (crawler.crawlerContext.getStatus() != CrawlerStatus.RUNNING && System.currentTimeMillis() - startTime < 5000) {

            settingsBuilder.put("http.cors.enabled", true);
            settingsBuilder.put("discovery.type", "single-node");
        }).build(newConfigs().clusterName(clusterName).numOfNode(1));

        // wait for yellow status
        runner.ensureYellow();

        System.setProperty(FesenClient.HTTP_ADDRESS, "localhost:" + runner.node().settings().get("http.port", "9201"));

        super.setUp();
    }

    @Override

                    }
                }
            });
        }

        // Start all threads
        for (Thread thread : threads) {
            thread.start();
        }

        // Wait for all threads to complete
        for (Thread thread : threads) {
            thread.join();
        }

        // Verify results
        assertTrue(exceptions.isEmpty());

            crawler.getCrawlerContext().setMaxAccessCount(maxCount);
            crawler.getCrawlerContext().setNumOfThread(numOfThread);
            final String sessionId = crawler.execute();

            // Wait for crawler to start running with polling
            long startTime = System.currentTimeMillis();
            while (crawler.crawlerContext.getStatus() != CrawlerStatus.RUNNING && System.currentTimeMillis() - startTime < 5000) {