RobotsTxt robotsTxt = new RobotsTxt();

        robotsTxt.addSitemap("https://example.com/sitemap.xml");
        robotsTxt.addSitemap("https://example.com/sitemap2.xml");

        String[] sitemaps = robotsTxt.getSitemaps();
        assertEquals(2, sitemaps.length);
        assertEquals("https://example.com/sitemap.xml", sitemaps[0]);
        assertEquals("https://example.com/sitemap2.xml", sitemaps[1]);
    }

Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler
Disallow: /ddd

sitemap: http://www.example.com/sitmap.xml

      throws Exception {
    File fileA = newFile("file-a");
    File dir1 = newDir("dir-1");
    newFile("dir-1/file-b");
    newFile("dir-1/dir-2");

    assertThat(Iterables.limit(Files.fileTraverser().breadthFirst(rootDir), 3))
        .containsExactly(rootDir, fileA, dir1);
  }

  public void testFileTraverser_multipleDirectoryLayers_traversalReturnsAll() throws Exception {
    File fileA = newFile("file-a");

	var lockType uint32 = lockFileFailImmediately | lockFileExclusiveLock
	if flag == syscall.O_RDONLY {
		// https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-lockfileex
		//lint:ignore SA4016 Reasons
		lockType = lockFileFailImmediately // Set this to enable shared lock and fail immediately.
	}
	return lockedOpenFile(path, flag, perm, lockType)
}

 */
package org.codelibs.fess.crawler.entity;

import java.io.Serializable;

/**
 * Represents a news entry within a sitemap URL.
 * This class encapsulates the properties of a news article as defined in the Google News Sitemap extension.
 *
 * <p>
 * The news extension allows you to provide additional information about news articles on your site.

     * Default constructor.
     */
    public NetworkExplorer() {
        super();
    }

    private static LogStream log = LogStream.getInstance();

    /** The MIME type mapping */
    private MimeMap mimeMap;
    /** The CSS style for HTML rendering */
    private String style;
    /** The NTLM SSP handler */
    private NtlmSsp ntlmSsp;
    /** Flag indicating if credentials were supplied */

 */
package org.codelibs.fess.crawler.entity;

import java.io.Serializable;

/**
 * Represents an image entry within a sitemap URL.
 * This class encapsulates the properties of an image as defined in the Google Image Sitemap extension.
 *
 * <p>
 * The image extension allows you to provide additional information about images on your pages.

    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */
    protected static final Pattern SITEMAP_RECORD = Pattern.compile("^sitemap:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Whether robots.txt processing is enabled. */
    protected boolean enabled = true;

    /**

		// Recognize "fixed", "removable", "remote" and "ramdisk" drives as proper drives
		// which can be treated as an actual mount-point, rest can be ignored.
		// https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-getdrivetypew
		mountPointCache.Store(path, true)
		return true
	}
	mountPointCache.Store(path, false)
	return false

    controller.setDefaultIntervalTime(1000);
});
```

### Sitemap Support

```java
// Enable sitemap processing
container.singleton("sitemapsRule", SitemapsRule.class, rule -> {
    rule.addRule("url", ".*sitemap.*");
});

// Add sitemap URL
crawler.addUrl("https://example.com/sitemap.xml");
```

## Data Access and Storage

### Accessing Crawled Data

```java