User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

     *
     * @return the {@link CrawlerContext} for the current thread, or {@code null} if no context is set.
     */
    public static CrawlerContext getCrawlerContext() {
        return ROBOT_CONTEXT_THREAD_LOCAL.get();
    }

    /**
     * Sets the current {@link CrawlerContext} for the current thread.
     * If the provided {@code crawlerContext} is {@code null}, the context is removed from the thread-local storage.

     *
     * @return the {@link AuthScope} object representing the authentication scope.
     */
    AuthScope getAuthScope();

    /**
     * Retrieves the credentials associated with the current authentication.
     *
     * @return the credentials object containing authentication details.
     */
    Credentials getCredentials();

    /**

         * @param method The HTTP method.
         * @return The current RequestDataContext instance.
         */
        public RequestDataContext method(final Method method) {
            data.setMethod(method);
            return this;
        }

        /**
         * Sets the HTTP method to GET.
         * @return The current RequestDataContext instance.
         */
        public RequestDataContext get() {

 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

    public void close() throws IOException {
        try {
            fileInputStream.close();
        } finally {
            FileUtil.deleteInBackground(tempFile);
        }
    }

    /**
     * Marks the current position in this input stream. A subsequent call to the reset method repositions this stream at the last marked position so that subsequent reads re-read the same bytes.
     * This method delegates to {@link FileInputStream#mark(int)}.

        }

        /**
         * Sets the maximum size for alphanumeric terms.
         *
         * @param maxAlphanumTermSize the maximum size for alphanumeric terms
         * @return the current instance of {@code TextNormalizeContext} for method chaining
         */
        public TextNormalizeContext maxAlphanumTermSize(final int maxAlphanumTermSize) {
            this.maxAlphanumTermSize = maxAlphanumTermSize;

        }
        matcher.appendTail(sb);
        return sb.toString().replaceAll("\\s+", " ").trim();
    }

    /**
     * Returns the current encoding setting.
     * @return The current encoding.
     */
    public String getEncoding() {
        return encoding;
    }

    /**
     * Sets the encoding for content extraction.

            tempDir.delete();
            tempDir.mkdirs();

            // robots.txt
            StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");

     */
    public void setCookieSpec(final String cookieSpec) {
        this.cookieSpec = cookieSpec;
    }

    /**
     * Sets the user agent string.
     *
     * @param userAgent The user agent string
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the proxy host name.
     *