}

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.

</components>
```

### Crawler Context Configuration

```java
// Set maximum number of URLs to crawl
crawler.crawlerContext.setMaxAccessCount(1000);

// Set number of crawler threads
crawler.crawlerContext.setNumOfThread(10);

// Set maximum crawl depth
crawler.crawlerContext.setMaxDepth(3);

// Set request interval (politeness)

import org.codelibs.fess.crawler.helper.ContentLengthHelper;
import org.codelibs.fess.crawler.helper.MimeTypeHelper;

import jakarta.annotation.Resource;

/**
 * FileSystemClient is CrawlerClient implementation to crawl files on a file
 * system.
 *
 * @author shinsuke
 *
 */
public class FileSystemClient extends AbstractCrawlerClient {

    /** Logger instance for this class */

     */
    protected CrawlerClient getClient(final String url) {
        return clientFactory.getClient(url);
    }

    /**
     * Checks if the content has been updated since the last crawl.
     * @param client The crawler client.
     * @param urlQueue The URL queue entry.
     * @return true if content is updated, false otherwise.
     */