final InputStream in = ResourceUtil.getResourceAsStream("sitemaps/sitemap1.xml");
        final SitemapSet sitemapSet = sitemapsHelper.parse(in);
        final Sitemap[] sitemaps = sitemapSet.getSitemaps();
        assertEquals(5, sitemaps.length);
        assertTrue(sitemapSet.isUrlSet());
        assertFalse(sitemapSet.isIndex());

        assertEquals("2005-01-01", sitemaps[0].getLastmod());

        responseData.setResponseBody(file, false);
        return responseData;
    }

    private ResponseData getTestData3_OK() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/sitemap.txt");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.txt");
        responseData.setResponseBody(file, false);
        return responseData;
    }

    }

    public void addSitemaps(final String[] sitemaps) {
        sitemapsLocal.set(sitemaps);
    }

    public String[] removeSitemaps() {
        final String[] sitemaps = sitemapsLocal.get();
        if (sitemaps != null) {
            sitemapsLocal.remove();
        }
        return sitemaps;
    }

    }

    public void test_getRule_sitemaps1() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://www.example.com/sitemap1.xml");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml");
        responseData.setResponseBody(file, false);
        final Rule rule = ruleManager.getRule(responseData);
        assertNotNull(rule);

        } catch (final Exception e) {
            throw new CrawlingAccessException("Could not parse Sitemaps: " + preloadDate, e);
        }
    }

    protected SitemapSet parseTextSitemaps(final InputStream in) {
        final SitemapSet sitemapSet = new SitemapSet();
        sitemapSet.setType(SitemapSet.URLSET);

        try {

            robotTxtFile.deleteOnExit();

            // sitemaps.xml
            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');

        assertFalse(robotsTxt.allows("/ddd", "Hoge Crawler"));

        String[] sitemaps = robotsTxt.getSitemaps();
        assertEquals(2, sitemaps.length);
        assertEquals("http://www.example.com/sitmap.xml", sitemaps[0]);
        assertEquals("http://www.example.net/sitmap.xml", sitemaps[1]);

    }

    public void testParse_disable() {

            robotTxtFile.deleteOnExit();

            // sitemaps.xml
            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');

     * Sitemap were changed. The value for the lastmod tag should be in W3C
     * Datetime format.
     *
     * By providing the last modification timestamp, you enable search engine
     * crawlers to retrieve only a subset of the Sitemaps in the index i.e. a
     * crawler may only retrieve Sitemaps that were modified since a certain
     * date. This incremental Sitemap fetching mechanism allows for the rapid

    }

    protected void addSitemapsFromRobotsTxt(final UrlQueue<?> urlQueue) {
        final String[] sitemaps = crawlerContext.removeSitemaps();
        if (sitemaps != null) {
            for (final String childUrl : sitemaps) {
                try {
                    storeChildUrl(childUrl, urlQueue.getUrl(), null, urlQueue.getDepth() == null ? 1 : urlQueue.getDepth() + 1);