* Sets the type of this sitemap set.
     * @param type the type to set (URLSET or INDEX)
     */
    public void setType(final String type) {
        this.type = type;
    }

    /**
     * Checks if this sitemap set is of type URLSET.
     * @return true if this is a URLSET, false otherwise
     */
    public boolean isUrlSet() {
        return URLSET.equals(type);
    }

    /**

            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            assertEquals(1, urlSet.size());
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                assertTrue(url.contains("dir1/test3.txt"));
            }

     */
    public void test_robotsTxtUrlSet() {
        Set<String> urlSet = crawlerContext.getRobotsTxtUrlSet();
        assertNotNull(urlSet);
        assertTrue(urlSet instanceof LruHashSet);
        assertTrue(urlSet.isEmpty());

        // Add URLs to default set
        urlSet.add("http://example.com/robots.txt");
        urlSet.add("http://test.com/robots.txt");

        try {
            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                if (url.indexOf(".svn") < 0) {

        }
        return true;
    }

    /**
     * Stores child URLs found in the response data.
     *
     * @param crawlerContext the crawler context
     * @param childUrlList the set of child URLs
     * @param url the parent URL
     * @param depth the depth of the child URLs
     * @param encoding the encoding of the child URLs
     */

            if (preloadDate.indexOf("<urlset") >= 0) {
                // XML Sitemaps
                bis.reset();
                return parseXmlSitemaps(bis);
            }
            if (preloadDate.indexOf("<sitemapindex") >= 0) {
                // XML Sitemaps Index
                bis.reset();
                return parseXmlSitemapsIndex(bis);
            }

        }
    }

    /**
     * Extracts anchor URLs from the given object and converts them to RequestData objects.
     * The input object can be either a single string or a list of strings representing URLs.
     *
     * @param obj the object containing anchor URLs (String or List of Strings)
     * @return a set of RequestData objects for the anchor URLs, or null if no valid URLs found
     */

            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()
                .filter(d -> StringUtil.isNotBlank(d.getUrl()) && urlSet.add(d.getUrl()) && crawlerContext.urlFilter.match(d.getUrl()))
                .map(d -> {

	urlStr := endPoint + SlashSeparator
	if bucketName != "" {
		urlStr = urlStr + bucketName + SlashSeparator
	}
	if objectName != "" {
		urlStr += s3utils.EncodePath(objectName)
	}
	if len(queryValues) > 0 {
		urlStr = urlStr + "?" + queryValues.Encode()
	}
	return urlStr
}

// return URL for uploading object into the bucket.

     *
     * @param urlStr the URL string to validate
     * @return true if the URL is valid, false otherwise
     */
    protected boolean isValidUrl(final String urlStr) {
        if (StringUtil.isBlank(urlStr)) {
            return false;
        }
        final String value;
        if (urlStr.startsWith("://")) {
            value = "http" + urlStr;
        } else if (urlStr.startsWith("//")) {