}

    /**
     * Returns the set of robots.txt URLs.
     * @return The set of robots.txt URLs.
     */
    public Set<String> getRobotsTxtUrlSet() {
        return robotsTxtUrlSet;
    }

    /**
     * Sets the set of robots.txt URLs.
     * @param robotsTxtUrlSet The set of robots.txt URLs.
     */
    public void setRobotsTxtUrlSet(final Set<String> robotsTxtUrlSet) {

    /**
     * The URL field for searching failure URLs.
     */
    public String url;

    /**
     * The minimum error count field for filtering failure URLs.
     */
    public String errorCountMin;

    /**
     * The maximum error count field for filtering failure URLs.
     */
    public String errorCountMax;

    /**
     * The error name field for searching failure URLs by error type.
     */

    }

    /**
     * Gets the delay time in milliseconds when there are no URLs in the queue.
     *
     * @return the delay time in milliseconds when no URLs are available
     */
    public long getDelayMillisAtNoUrlInQueue() {
        return delayMillisAtNoUrlInQueue;
    }

    /**
     * Sets the delay time in milliseconds when there are no URLs in the queue.
     *

        }
        return true;
    }

    /**
     * Stores child URLs found in the response data.
     *
     * @param crawlerContext the crawler context
     * @param childUrlList the set of child URLs
     * @param url the parent URL
     * @param depth the depth of the child URLs
     * @param encoding the encoding of the child URLs
     */

/**
 * {@link ChildUrlsException} is thrown when child URLs are found during crawling.
 * It extends {@link CrawlerSystemException} and holds a set of {@link RequestData}
 * representing the child URLs that caused the exception.
 *
 */
public class ChildUrlsException extends CrawlerSystemException {

    private static final long serialVersionUID = 1L;

    /**
     * The list of child URLs.
     */

        } catch (final ChildUrlsException e) {
            String[] urls = e.getChildUrlList().stream().map(r -> r.getUrl()).sorted().toArray(String[]::new);
            assertEquals(3, urls.length);
            assertEquals(baseUrl + "dir1/", urls[0]);
            assertEquals(baseUrl + "dir3/", urls[1]);
            assertEquals(baseUrl + "file1.txt", urls[2]);
        }
        try {
            smbClient.doGet(baseUrl + "dir1/");

import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * The Crawler class is the main class for web crawling. It manages the crawling process,
 * including adding URLs to the queue, filtering URLs, managing crawler threads,
 * and handling the overall crawling lifecycle.
 *
 * <p>It implements the Runnable interface to be executed in a separate thread,

## OpenAPI Custom URL { #openapi-custom-url }

To customize the OpenAPI URL (or remove it), read the docs for [Tutorial - Metadata and Docs URLs](../tutorial/metadata.md#openapi-url){.internal-link target=_blank}.

## OpenAPI Docs URLs { #openapi-docs-urls }

            cb.query().setErrorName_Wildcard(failureUrlPager.errorName);
        }

    }

    /**
     * Deletes all failure URLs associated with a specific configuration ID.
     *
     * @param configId the configuration ID to delete failure URLs for
     */
    public void deleteByConfigId(final String configId) {
        failureUrlBhv.queryDelete(cb -> {

 * This class provides methods for managing a queue of URLs to be crawled,
 * including adding, deleting, and retrieving URLs from the queue.
 * It uses a {@link MemoryDataHelper} to store the URL queue data in memory.
 *
 * <p>
 * The class is responsible for:
 * </p>
 * <ul>
 *   <li>Updating session IDs for URL queues.</li>
 *   <li>Adding new URLs to the queue.</li>