* Sets the type of this sitemap set.
     * @param type the type to set (URLSET or INDEX)
     */
    public void setType(final String type) {
        this.type = type;
    }

    /**
     * Checks if this sitemap set is of type URLSET.
     * @return true if this is a URLSET, false otherwise
     */
    public boolean isUrlSet() {
        return URLSET.equals(type);
    }

    /**

            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            assertEquals(1, urlSet.size());
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                assertTrue(url.contains("dir1/test3.txt"));
            }

     */
    public void test_robotsTxtUrlSet() {
        Set<String> urlSet = crawlerContext.getRobotsTxtUrlSet();
        assertNotNull(urlSet);
        assertTrue(urlSet instanceof LruHashSet);
        assertTrue(urlSet.isEmpty());

        // Add URLs to default set
        urlSet.add("http://example.com/robots.txt");
        urlSet.add("http://test.com/robots.txt");

<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <url>
      <loc>http://www.example.com/</loc>
      <lastmod>2005-01-01</lastmod>
      <changefreq>monthly</changefreq>
      <priority>0.8</priority>
   </url>
   <url>
      <loc>http://www.example.com/catalog?item=12&amp;desc=vacation_hawaii</loc>
      <changefreq>weekly</changefreq>
   </url>
   <url>

        try {
            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            for (final RequestData requestData : urlSet.toArray(new RequestData[urlSet.size()])) {
                String url = requestData.getUrl();
                if (url.indexOf(".svn") < 0) {

        }
        return true;
    }

    /**
     * Stores child URLs found in the response data.
     *
     * @param crawlerContext the crawler context
     * @param childUrlList the set of child URLs
     * @param url the parent URL
     * @param depth the depth of the child URLs
     * @param encoding the encoding of the child URLs
     */

            if (preloadDate.indexOf("<urlset") >= 0) {
                // XML Sitemaps
                bis.reset();
                return parseXmlSitemaps(bis);
            }
            if (preloadDate.indexOf("<sitemapindex") >= 0) {
                // XML Sitemaps Index
                bis.reset();
                return parseXmlSitemapsIndex(bis);
            }

            robotTxtFile.deleteOnExit();

            // sitemaps.xml
            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');
            buf.append("<loc>http://localhost:7070/index.html</loc>").append('\n');

            robotTxtFile.deleteOnExit();

            // sitemaps.xml
            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');
            buf.append("<loc>http://localhost:7070/index.html</loc>").append('\n');

        }
    }

    /**
     * Extracts anchor URLs from the given object and converts them to RequestData objects.
     * The input object can be either a single string or a list of strings representing URLs.
     *
     * @param obj the object containing anchor URLs (String or List of Strings)
     * @return a set of RequestData objects for the anchor URLs, or null if no valid URLs found
     */