User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

     * command. Even though search engine crawlers may consider this information
     * when making decisions, they may crawl pages marked "hourly" less
     * frequently than that, and they may crawl pages marked "yearly" more
     * frequently than that. Crawlers may periodically crawl pages marked
     * "never" so that they can handle unexpected changes to those pages.
     */
    private String changefreq;

    /**

import java.util.LinkedHashSet;
import java.util.Set;
import java.util.function.Function;

import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * This class represents the result data of a crawl.
 */
public class ResultData implements Serializable {
    private static final long serialVersionUID = 1L;

    /** The name of the transformer. */
    protected String transformerName;

     */
    private boolean initializedData = false;

    /**
     * Initializes the access result with response data and result data.
     *
     * @param responseData The response data from the crawl operation.
     * @param resultData The result data from content processing.
     */
    @Override
    public void init(final ResponseData responseData, final ResultData resultData) {