}

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

</components>
```

### Crawler Context Configuration

```java
// Set maximum number of URLs to crawl
crawler.crawlerContext.setMaxAccessCount(1000);

// Set number of crawler threads
crawler.crawlerContext.setNumOfThread(10);

// Set maximum crawl depth
crawler.crawlerContext.setMaxDepth(3);

// Set request interval (politeness)

     * command. Even though search engine crawlers may consider this information
     * when making decisions, they may crawl pages marked "hourly" less
     * frequently than that, and they may crawl pages marked "yearly" more
     * frequently than that. Crawlers may periodically crawl pages marked
     * "never" so that they can handle unexpected changes to those pages.
     */
    private String changefreq;

    /**

import java.util.LinkedHashSet;
import java.util.Set;
import java.util.function.Function;

import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * This class represents the result data of a crawl.
 */
public class ResultData implements Serializable {
    private static final long serialVersionUID = 1L;

    /** The name of the transformer. */
    protected String transformerName;

     */
    private boolean initializedData = false;

    /**
     * Initializes the access result with response data and result data.
     *
     * @param responseData The response data from the crawl operation.
     * @param resultData The result data from content processing.
     */
    @Override
    public void init(final ResponseData responseData, final ResultData resultData) {

import org.codelibs.fess.crawler.helper.ContentLengthHelper;
import org.codelibs.fess.crawler.helper.MimeTypeHelper;

import jakarta.annotation.Resource;

/**
 * FileSystemClient is CrawlerClient implementation to crawl files on a file
 * system.
 *
 * @author shinsuke
 *
 */
public class FileSystemClient extends AbstractCrawlerClient {

    /** Logger instance for this class */

     */
    protected CrawlerClient getClient(final String url) {
        return clientFactory.getClient(url);
    }

    /**
     * Checks if the content has been updated since the last crawl.
     * @param client The crawler client.
     * @param urlQueue The URL queue entry.
     * @return true if content is updated, false otherwise.
     */