User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

    public static final String EMPTY_STRING = "";

    /** The system property name for user dictionary encoding. */
    public static final String USER_DICT_ENCODING = "fess.user.dict.encoding";

    /** The system property name for user dictionary path. */
    public static final String USER_DICT_PATH = "fess.user.dict.path";

    /** The text separator. */
    public static final String TEXT_SEPARATOR = " ";

 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */

        String[] values = CSVUtil.parse(rule);
        if (dup.add(values[0]) == false) {
            throw new IllegalArgumentException("Found duplicate term [" + values[0] + "] in user dictionary.");
        }
        return rule;
    }

    public static UserDictionary getUserDictionary(Environment env, Settings settings) {

 * <li>{@link #SCORE} - The score field.</li>
 * <li>{@link #QUERY_FREQ} - The query frequency field.</li>
 * <li>{@link #DOC_FREQ} - The document frequency field.</li>
 * <li>{@link #USER_BOOST} - The user boost field.</li>
 * <li>{@link #KINDS} - The kinds field.</li>
 * <li>{@link #TIMESTAMP} - The timestamp field.</li>
 * <li>{@link #TAGS} - The tags field.</li>
 * <li>{@link #ROLES} - The roles field.</li>

            final File tempDir = File.createTempFile("crawlerDocRoot", "");
            tempDir.delete();
            tempDir.mkdirs();

            final StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");

            tempDir.delete();
            tempDir.mkdirs();

            // robots.txt
            StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");

 * </p>
 *
 * <p>
 * Example usage:
 * </p>
 * <pre>
 * {@code
 * AuthScope authScope = new AuthScope("example.com", 80);
 * Credentials credentials = new UsernamePasswordCredentials("user", "password");
 * AuthenticationImpl auth = new AuthenticationImpl(authScope, credentials);
 * }
 * </pre>
 *
 * @see org.codelibs.fess.crawler.client.http.Authentication
 */

                this.getBoost(), this.getTags().toArray(new String[this.getTags().size()]),
                this.getRoles().toArray(new String[this.getRoles().size()]), null, SuggestItem.Kind.USER);
    }

            tempDir.delete();
            tempDir.mkdirs();

            // robots.txt
            StringBuilder buf = new StringBuilder();
            buf.append("User-agent: *").append('\n');
            buf.append("Disallow: /admin/").append('\n');
            buf.append("Disallow: /websvn/").append('\n');
            final File robotTxtFile = new File(tempDir, "robots.txt");