String[] sitemaps = robotsTxt.getSitemaps();
        assertEquals(2, sitemaps.length);
        assertEquals("http://www.example.com/sitmap.xml", sitemaps[0]);
        assertEquals("http://www.example.net/sitmap.xml", sitemaps[1]);

    }

    public void testParse_disable() {
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler
Disallow: /ddd

sitemap: http://www.example.com/sitmap.xml

 * </p>
 *
 * <p>
 * Example usage:
 * </p>
 *
 * <pre>
 * XpathTransformer transformer = new XpathTransformer();
 * transformer.addFieldRule("title", "//title/text()");
 * transformer.addFieldRule("body", "//body/p/text()");
 *
 * ResponseData responseData = new ResponseData();

sitemap2.xml http://www.example.com/sitemap1.xml.gz 2004-10-01T18:23:17+00:00 http://www.example.com/sitemap2.xml.gz 2005-01-01...

        responseData.setUrl("http://example.com/sitemap.xml");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml");
        responseData.setResponseBody(file, false);
        return responseData;
    }

    private ResponseData getTestData2_OK() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/sitemap.xml.gz");

        Crawler crawler = container.getComponent("crawler");
        
        // Configure crawling parameters
        crawler.addUrl("https://example.com");
        crawler.crawlerContext.setMaxAccessCount(100);
        crawler.crawlerContext.setNumOfThread(5);
        crawler.urlFilter.addInclude("https://example.com/.*");
        
        // Execute crawling
        String sessionId = crawler.execute();

/**
 * Exception thrown when there is an issue with the suggestion index.
 * This exception extends {@link SuggesterException}.
 *
 * <p>This exception can be thrown with a message, a cause, or both.</p>
 *
 * <pre>
 * Example usage:
 * throw new SuggestIndexException("Indexing error occurred");
 * throw new SuggestIndexException(new IOException("IO error"));
 * throw new SuggestIndexException("Indexing error", new IOException("IO error"));
 * </pre>

 * encoding for URL decoding.</p>
 *
 * <p>Example usage:</p>
 * <pre>
 * {@code
 * IdnDnsResolver resolver = new IdnDnsResolver();
 * resolver.setFlag(IDN.ALLOW_UNASSIGNED);
 * resolver.setEncoding("UTF-8");
 * InetAddress[] addresses = resolver.resolve("example.com");
 * }
 * </pre>
 *
 */
public class IdnDnsResolver implements DnsResolver {

        String[] sitemaps = new String[] { "http://example.com/sitemap.xml", "http://test.com/sitemap.xml" };
        crawlerContext.addSitemaps(sitemaps);

        // Remove and verify
        String[] removedSitemaps = crawlerContext.removeSitemaps();
        assertNotNull(removedSitemaps);
        assertEquals(2, removedSitemaps.length);
        assertEquals("http://example.com/sitemap.xml", removedSitemaps[0]);

			<groupId>org.mortbay.jetty</groupId>
			<artifactId>jetty</artifactId>
			<version>6.1.26</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>org.slf4j</groupId>
			<artifactId>slf4j-simple</artifactId>
			<version>${slf4j.version}</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>org.apache.ftpserver</groupId>
			<artifactId>ftpserver-core</artifactId>