}

    /**
     * Test sitemaps add and remove operations
     */
    public void test_sitemaps() {
        // Initial state
        assertNull(crawlerContext.removeSitemaps());

        // Add sitemaps
        String[] sitemaps = new String[] { "http://example.com/sitemap.xml", "http://test.com/sitemap.xml" };
        crawlerContext.addSitemaps(sitemaps);

        // Remove and verify

    controller.setDefaultIntervalTime(1000);
});
```

### Sitemap Support

```java
// Enable sitemap processing
container.singleton("sitemapsRule", SitemapsRule.class, rule -> {
    rule.addRule("url", ".*sitemap.*");
});

// Add sitemap URL
crawler.addUrl("https://example.com/sitemap.xml");
```

## Data Access and Storage

### Accessing Crawled Data

```java

                    rule.setResponseProcessor(container.getComponent("sitemapsResponseProcessor"));
                    rule.setRuleId("sitemapsRule");
                    rule.addRule("url", ".*sitemap.*");
                })//
                .<DefaultResponseProcessor> singleton("defaultResponseProcessor", DefaultResponseProcessor.class, processor -> {

                    if (robotsTxt != null) {
                        final String[] sitemaps = robotsTxt.getSitemaps();
                        if (sitemaps.length > 0) {
                            crawlerContext.addSitemaps(sitemaps);
                        }

                        final RobotsTxt.Directive directive = robotsTxt.getMatchedDirective(userAgent);

    private static final int ST_EXT = 5;

    private final byte[] in;
    private int inLen;

    /**
     * Creates a new MimeMap instance by loading MIME type mappings from the resource file.
     *
     * @throws IOException if there is an error reading the mime.map resource file
     */
    public MimeMap() throws IOException {
        int n;

        in = new byte[IN_SIZE];

            assertEquals("application/pdf", mimeMap.getMimeType("pdf"));
            assertEquals("application/msword", mimeMap.getMimeType("doc"));
            assertEquals("application/vnd.ms-excel", mimeMap.getMimeType("xls"));
            assertEquals("text/html", mimeMap.getMimeType("html"));
            assertEquals("text/html", mimeMap.getMimeType("htm"));
            assertEquals("image/jpeg", mimeMap.getMimeType("jpg"));

            final Map<String, Object> itemMap = new HashMap<>();
            itemMap.put("result", item.status().name());
            if (item.isFailed()) {
                itemMap.put("message", item.getFailureMessage());
            } else {
                itemMap.put("id", item.getId());
            }
            return itemMap;

            return fileList.stream().map(fileMap -> {
                try {
                    final String path = fileMap.get("path").toString();
                    final Date timestamp =
                            new SimpleDateFormat(Constants.DATE_FORMAT_ISO_8601_EXTEND_UTC).parse(fileMap.get("@timestamp").toString());
                    for (final DictionaryCreator creator : creatorList) {

    }

    /**
     * Adds sitemaps from robots.txt to the crawling queue.
     * @param urlQueue The URL queue to add sitemaps to.
     */
    protected void addSitemapsFromRobotsTxt(final UrlQueue<?> urlQueue) {
        final String[] sitemaps = crawlerContext.removeSitemaps();
        if (sitemaps != null) {
            for (final String childUrl : sitemaps) {
                try {

     * Default constructor.
     */
    public NetworkExplorer() {
        super();
    }

    private static LogStream log = LogStream.getInstance();

    /** The MIME type mapping */
    private MimeMap mimeMap;
    /** The CSS style for HTML rendering */
    private String style;
    /** The NTLM SSP handler */
    private NtlmSsp ntlmSsp;
    /** Flag indicating if credentials were supplied */