* It extends CrawlingAccessException and provides methods to print the stack traces of all causes.
 *
 */
public class MultipleCrawlingAccessException extends CrawlingAccessException {

    private static final long serialVersionUID = 1L;

    /**
     * The array of throwables that caused this exception.
     */
    private final Throwable[] throwables;

    /**

 *
 */
public class SitemapsRule extends RegexRule {
    /**
     * Serial version UID for serialization.
     */
    private static final long serialVersionUID = 1L;

    private static final Logger logger = LogManager.getLogger(SitemapsRule.class);

    /**
     * Creates a new SitemapsRule instance.
     */
    public SitemapsRule() {
        super();

    private int cacheFileSize = 1_000_000;

    /** Name of the extractor to use */
    private String extractorName = "tikaExtractor";

    /** Maximum content length allowed */
    private long maxContentLength = -1;

    /**
     * Constructs a new ExtractorBuilder.
     *
     * @param crawlerContainer the crawler container
     * @param in the input stream to extract data from

            crawler.getCrawlerContext().setNumOfThread(numOfThread);
            final String sessionId = crawler.execute();

            // Wait for crawler to start running with polling
            long startTime = System.currentTimeMillis();
            while (crawler.crawlerContext.getStatus() != CrawlerStatus.RUNNING && System.currentTimeMillis() - startTime < 5000) {
                try {
                    Thread.sleep(50);

 */
public class RuleTest extends PlainTestCase {

    /**
     * Test implementation of Rule interface for testing purposes
     */
    public static class TestRule implements Rule {
        private static final long serialVersionUID = 1L;

        private final String ruleId;
        private final ResponseProcessor responseProcessor;
        private final boolean matchResult;
        private int matchCallCount = 0;

     * @return The total number of words.
     */
    public long getAllWordsNum() {
        return getNum(QueryBuilders.matchAllQuery());
    }

    /**
     * Returns the number of document words in the suggestion index.
     * @return The number of document words.
     */
    public long getDocumentWordsNum() {
        return getNum(QueryBuilders.rangeQuery(FieldNames.DOC_FREQ).gte(1));

 * defined in the {@link Rule} interface to provide specific rule logic.
 * </p>
 *
 */
public abstract class AbstractRule implements Rule {

    private static final long serialVersionUID = 1L;

    /** The rule ID. */
    protected String ruleId;

    /** The response processor. */
    protected ResponseProcessor responseProcessor;

    /** The crawler container. */

    }

    /**
     * Retrieves a setting value as a long.
     * @param key The key of the setting.
     * @param defaultValue The default value if the setting is not found.
     * @return The setting value as a long.
     */
    public long getAsLong(final String key, final long defaultValue) {
        final Object obj = get(key);

        final long value;
        if (obj == null) {
            value = defaultValue;

     * Returns the retry interval.
     * @return The retry interval in milliseconds.
     */
    public long getRetryInterval() {
        return retryInterval;
    }

    /**
     * Sets the retry interval.
     * @param retryInterval The retry interval in milliseconds.
     */
    public void setRetryInterval(final long retryInterval) {
        this.retryInterval = retryInterval;
    }

    /**

/**
 * This exception is thrown when an error occurs during OpenSearch access.
 *
 * @author shinsuke
 *
 */
public class OpenSearchAccessException extends CrawlerSystemException {

    private static final long serialVersionUID = 1L;

    /**
     * Creates a new instance of OpenSearchAccessException.
     *
     * @param message the detail message
     */
    public OpenSearchAccessException(final String message) {