}
        return true;
    }

    /**
     * Stores child URLs found in the response data.
     *
     * @param crawlerContext the crawler context
     * @param childUrlList the set of child URLs
     * @param url the parent URL
     * @param depth the depth of the child URLs
     * @param encoding the encoding of the child URLs
     */

        } catch (final ChildUrlsException e) {
            String[] urls = e.getChildUrlList().stream().map(r -> r.getUrl()).sorted().toArray(String[]::new);
            assertEquals(3, urls.length);
            assertEquals(baseUrl + "dir1/", urls[0]);
            assertEquals(baseUrl + "dir3/", urls[1]);
            assertEquals(baseUrl + "file1.txt", urls[2]);
        }
        try {
            smbClient.doGet(baseUrl + "dir1/");

import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * The Crawler class is the main class for web crawling. It manages the crawling process,
 * including adding URLs to the queue, filtering URLs, managing crawler threads,
 * and handling the overall crawling lifecycle.
 *
 * <p>It implements the Runnable interface to be executed in a separate thread,

        }
        return null;
    }

    /**
     * Extracts URLs from HTML tag attributes using XPath.
     *
     * @param url the base URL for resolving relative URLs
     * @param document the document to extract URLs from
     * @param xpath the XPath expression to select elements
     * @param attr the attribute name to extract URLs from
     * @param encoding the character encoding to use

        this.path = path;
    }

    /**
     * Gets the replacement string for question marks in URLs.
     *
     * @return the question mark replacement string
     */
    public String getQuestionStr() {
        return questionStr;
    }

    /**
     * Sets the replacement string for question marks in URLs.
     *
     * @param questionStr the question mark replacement string to set
     */

        this.updatedTime = value;
    }

    public String getUrls() {
        checkSpecifiedProperty("urls");
        return convertEmptyToNull(urls);
    }

    public void setUrls(String value) {
        registerModifiedProperty("urls");
        this.urls = value;
    }

    public String getUserAgent() {
        checkSpecifiedProperty("userAgent");

    public void addSitemap(final String url) {
        if (!sitemapList.contains(url)) {
            sitemapList.add(url);
        }
    }

    /**
     * Returns an array of sitemap URLs.
     *
     * @return an array of sitemap URLs
     */
    public String[] getSitemaps() {
        return sitemapList.toArray(new String[sitemapList.size()]);
    }

    /**
     * Represents a directive in a robots.txt file.

        } catch (Exception e) {
            fail("sendToSlack() should not throw an exception with blank URLs: " + e.getMessage());
        }
    }

    public void test_sendToSlack_withNullWebhookUrls() {
        // Setup mock configuration with null webhook URLs
        ComponentUtil.setFessConfig(new MockFessConfig() {
            @Override
            public String getSlackWebhookUrls() {

            if (StringUtil.isNotBlank(getIncludedDocUrls())) {
                final List<Pattern> urlPatterList = new ArrayList<>();
                final String[] urls = getIncludedDocUrls().split("[\r\n]");
                for (final String u : urls) {
                    final String v = systemHelper.normalizeConfigPath(u);
                    if (StringUtil.isNotBlank(v)) {

        }
    }

    /**
     * Extracts anchor URLs from the given object and converts them to RequestData objects.
     * The input object can be either a single string or a list of strings representing URLs.
     *
     * @param obj the object containing anchor URLs (String or List of Strings)
     * @return a set of RequestData objects for the anchor URLs, or null if no valid URLs found
     */