} catch (final ChildUrlsException e) {
            String[] urls = e.getChildUrlList().stream().map(r -> r.getUrl()).sorted().toArray(String[]::new);
            assertEquals(3, urls.length);
            assertEquals(baseUrl + "dir1/", urls[0]);
            assertEquals(baseUrl + "dir3/", urls[1]);
            assertEquals(baseUrl + "file1.txt", urls[2]);
        }
        try {
            smbClient.doGet(baseUrl + "dir1/");

import jakarta.annotation.Resource;

/**
 * A response processor implementation that handles sitemaps.
 * It parses the response body as a SitemapSet, extracts URLs from the sitemaps,
 * and adds them as child URLs to be crawled.
 *
 * <p>
 * This class uses a {@link SitemapsHelper} to parse the sitemap XML or text.
 * It then iterates through the sitemaps in the SitemapSet, extracts the URL

        }
        return null;
    }

    /**
     * Extracts URLs from HTML tag attributes using XPath.
     *
     * @param url the base URL for resolving relative URLs
     * @param document the document to extract URLs from
     * @param xpath the XPath expression to select elements
     * @param attr the attribute name to extract URLs from
     * @param encoding the character encoding to use

 * Default implementation of the IntervalController.
 * This class provides a default way to manage delays between crawler operations.
 * It allows setting delays before processing, after processing, when no URLs are in the queue,
 * and when waiting for new URLs.
 * The delays are configurable via constructor parameters.
 *
 */
public class DefaultIntervalController extends AbstractIntervalController {

        this.path = path;
    }

    /**
     * Gets the replacement string for question marks in URLs.
     *
     * @return the question mark replacement string
     */
    public String getQuestionStr() {
        return questionStr;
    }

    /**
     * Sets the replacement string for question marks in URLs.
     *
     * @param questionStr the question mark replacement string to set
     */

/**
 * {@link ChildUrlsException} is thrown when child URLs are found during crawling.
 * It extends {@link CrawlerSystemException} and holds a set of {@link RequestData}
 * representing the child URLs that caused the exception.
 *
 */
public class ChildUrlsException extends CrawlerSystemException {

    private static final long serialVersionUID = 1L;

    /**
     * The list of child URLs.
     */

 *   <li>{@code POST_PROCESSING} - Represents the post-processing state.</li>
 *   <li>{@code NO_URL_IN_QUEUE} - Indicates that there are no URLs in the queue.</li>
 *   <li>{@code WAIT_NEW_URL} - Indicates that the crawler is waiting for new URLs.</li>
 * </ul>
 */
public interface IntervalController {
    /** Constant representing the pre-processing state. */
    int PRE_PROCESSING = 1;

        }
    }

    /**
     * Registers a client component with a regular expression.
     * The component will be loaded into all registered CrawlerClientFactories.
     * @param regex The regular expression to match URLs.
     * @param componentName The name of the component to register.
     */
    public synchronized void register(final String regex, final String componentName) {
        clientMap.put(regex, componentName);

     */
    void addIncludeUrlFilter(String sessionId, String url);

    /**
     * Adds a list of URLs to the include filter for a given session.
     *
     * @param sessionId the ID of the session for which the URLs should be included
     * @param urlList the list of URLs to be added to the include filter
     */
    void addIncludeUrlFilter(String sessionId, List<String> urlList);

    /**

    }

    /**
     * Returns the set of robots.txt URLs.
     * @return The set of robots.txt URLs.
     */
    public Set<String> getRobotsTxtUrlSet() {
        return robotsTxtUrlSet;
    }

    /**
     * Sets the set of robots.txt URLs.
     * @param robotsTxtUrlSet The set of robots.txt URLs.
     */
    public void setRobotsTxtUrlSet(final Set<String> robotsTxtUrlSet) {