/**
 * {@link ChildUrlsException} is thrown when child URLs are found during crawling.
 * It extends {@link CrawlerSystemException} and holds a set of {@link RequestData}
 * representing the child URLs that caused the exception.
 *
 */
public class ChildUrlsException extends CrawlerSystemException {

    private static final long serialVersionUID = 1L;

    /**
     * The list of child URLs.
     */

 *   <li>{@code POST_PROCESSING} - Represents the post-processing state.</li>
 *   <li>{@code NO_URL_IN_QUEUE} - Indicates that there are no URLs in the queue.</li>
 *   <li>{@code WAIT_NEW_URL} - Indicates that the crawler is waiting for new URLs.</li>
 * </ul>
 */
public interface IntervalController {
    /** Constant representing the pre-processing state. */
    int PRE_PROCESSING = 1;

        }
    }

    /**
     * Registers a client component with a regular expression.
     * The component will be loaded into all registered CrawlerClientFactories.
     * @param regex The regular expression to match URLs.
     * @param componentName The name of the component to register.
     */
    public synchronized void register(final String regex, final String componentName) {
        clientMap.put(regex, componentName);

     */
    void addIncludeUrlFilter(String sessionId, String url);

    /**
     * Adds a list of URLs to the include filter for a given session.
     *
     * @param sessionId the ID of the session for which the URLs should be included
     * @param urlList the list of URLs to be added to the include filter
     */
    void addIncludeUrlFilter(String sessionId, List<String> urlList);

    /**

                            InputStream in = null;
                            try {
                                if (isByteStream) {
                                    inputStream.reset();
                                    in = inputStream;
                                } else {
                                    in = new FileInputStream(tempFile);
                                }

    }

    /**
     * Returns the set of robots.txt URLs.
     * @return The set of robots.txt URLs.
     */
    public Set<String> getRobotsTxtUrlSet() {
        return robotsTxtUrlSet;
    }

    /**
     * Sets the set of robots.txt URLs.
     * @param robotsTxtUrlSet The set of robots.txt URLs.
     */
    public void setRobotsTxtUrlSet(final Set<String> robotsTxtUrlSet) {

    protected String transformerName;

    /** The data of the result. */
    protected byte[] data;

    /** The encoding of the result. */
    protected String encoding;

    /** The set of the child URLs. */
    protected Set<RequestData> childUrlSet = new LinkedHashSet<>();

    /** The raw data. */
    protected Object rawData = null;

    /** The serializer. */
    protected Function<Object, byte[]> serializer;

        assertEquals("http://example.com/test", transformer.getLastResponseData().getUrl());

        // Reset and verify
        transformer.reset();
        assertEquals(0, transformer.getTransformCallCount());
        assertNull(transformer.getLastResponseData());

        // Name should persist after reset
        assertEquals("workflowTransformer", transformer.getName());
    }

    REDIRECT_LOCATION,
    /** Indicates processing a response. */
    PROCESS_RESPONSE,
    /** Indicates the crawling process has finished. */
    FINISHED_CRAWLING,
    /** Indicates processing child URLs due to an exception. */
    PROCESS_CHILD_URLS_BY_EXCEPTION,
    /** Indicates processing a child URL due to an exception. */
    PROCESS_CHILD_URL_BY_EXCEPTION,
    /** Indicates an access exception during crawling. */

     * influence the position of your URLs in a search engine's result pages.
     * Search engines may use this information when selecting between URLs on
     * the same site, so you can use this tag to increase the likelihood that
     * your most important pages are present in a search index.
     *
     * Also, please note that assigning a high priority to all of the URLs on