labels.excludedDocUrls=URL da escludere dalla ricerca
labels.hostname=Nome host
labels.id=ID
labels.includedPaths=Percorsi da includere nella scansione
labels.includedUrls=URL da includere nella scansione
labels.includedDocPaths=Percorsi da includere nella ricerca
labels.includedDocUrls=URL da includere nella ricerca
labels.maxAccessCount=Numero massimo di accessi
labels.name=Nome
labels.numOfThread=Numero di thread

        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

        for (String userAgent : new String[] { "FessCrawler", "FessCrawler/1.0", "Mozilla FessCrawler" }) {
            assertTrue(robotsTxt.allows("/aaa", userAgent));
            assertTrue(robotsTxt.allows("/private/", userAgent));
            assertTrue(robotsTxt.allows("/private/index.html", userAgent));

    // User Agent Configuration
    // ============================================================

    /** Prefix for Fess crawler user agent string. */
    public static final String CRAWLING_USER_AGENT_PREFIX = "Mozilla/5.0 (compatible; Fess/";

    /** Suffix for Fess crawler user agent string with bot information URL. */
    public static final String CRAWLING_USER_AGENT_SUFFIX = "; +http://fess.codelibs.org/bot.html)";