User-agent: GoodBot
Disallow: /admin/
InvalidDirective: some-value
unknown-field: test
Disallow /missing-colon
Disallow:    # empty value (should be treated as allowing all)
Allow: /public/
Crawl-delay: invalid-number
Crawl-delay: -10
Crawl-delay: 5.5
Crawl-delay:
Allow:    # empty value

# Case 3: Multiple colons in directive
User-agent: MultiColonBot
Disallow: http://example.com:8080/path
Allow: /path:with:colons

 * monitoring, and cleanup operations.
 *
 * <p>The crawler can operate in different modes based on command-line options:
 * <ul>
 * <li>Web crawling - crawls web sites and web content</li>
 * <li>File system crawling - crawls file systems and documents</li>
 * <li>Data store crawling - crawls databases and other data sources</li>
 * <li>Combined crawling - runs multiple crawling types simultaneously</li>
 * </ul>
 *
 * <p>Command line usage:

    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */
    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */

    }

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.

     *
     * @param sessionId The session ID for this crawling operation
     * @param webConfigIdList List of web configuration IDs to crawl, null for all
     * @param fileConfigIdList List of file configuration IDs to crawl, null for all
     */
    public void crawl(final String sessionId, final List<String> webConfigIdList, final List<String> fileConfigIdList) {

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

labels.crawling_info_WebFsCrawlExecTime=Crawl-Ausführungszeit (Web/Datei)
labels.crawling_info_WebFsCrawlStartTime=Crawl-Startzeit (Web/Datei)
labels.crawling_info_WebFsCrawlEndTime=Crawl-Endzeit (Web/Datei)
labels.crawling_info_WebFsIndexExecTime=Indizierungs-Ausführungszeit (Web/Datei)
labels.crawling_info_WebFsIndexSize=Indexgröße (Web/Datei)
labels.crawling_info_DataCrawlExecTime=Crawl-Ausführungszeit (Datenspeicher)

labels.crawling_info_WebFsCrawlExecTime=Crawl Execution Time (Web/File)
labels.crawling_info_WebFsCrawlStartTime=Crawl Start Time (Web/File)
labels.crawling_info_WebFsCrawlEndTime=Crawl End Time (Web/File)
labels.crawling_info_WebFsIndexExecTime=Indexing Execution Time (Web/File)
labels.crawling_info_WebFsIndexSize=Index Size (Web/File)
labels.crawling_info_DataCrawlExecTime=Crawl Execution Time (Data Store)

labels.crawling_info_WebFsCrawlExecTime=Crawl Execution Time (Web/File)
labels.crawling_info_WebFsCrawlStartTime=Crawl Start Time (Web/File)
labels.crawling_info_WebFsCrawlEndTime=Crawl End Time (Web/File)
labels.crawling_info_WebFsIndexExecTime=Indexing Execution Time (Web/File)
labels.crawling_info_WebFsIndexSize=Index Size (Web/File)
labels.crawling_info_DataCrawlExecTime=Crawl Execution Time (Data Store)

        } finally {
            CloseableUtil.closeQuietly(in);
        }

        // Should not throw exception for invalid crawl-delay
        assertNotNull(robotsTxt);
        // Invalid crawl-delay should be ignored (default 0)
        assertEquals(0, robotsTxt.getCrawlDelay("TestBot"));
        // Other directives should still work
        assertFalse(robotsTxt.allows("/test/", "TestBot"));
    }