aurons
aurez
auront
aurais
aurait
aurions
auriez
auraient
avais
avait
avions
aviez
avaient
eut
eûmes
eûtes
eurent
aie
aies
ait
ayons
ayez
aient
eusse
eusses
eût
eussions
eussiez
eussent
ceci
cela
celà
cet
cette
ici
ils
les
leurs
quel
quels
quelle
quelles
sans

        },
        "french_keywords": {

        },
        "french_keywords": {

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: PriorityBot
Disallow: /store
Allow: /store/public
Disallow: /store/public/sale

# Test Allow vs Disallow with same length (Allow wins)
User-agent: SameLengthBot
Disallow: /page
Allow: /page

# Test multiple wildcards
User-agent: MultiWildcardBot
Disallow: /*.cgi*
Disallow: /*?*id=*

# Test literal $ in middle of pattern
User-agent: DollarBot
Disallow: /price$info

 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

NoValue:
:::
   :

# Case 13: Numeric crawl-delay edge cases
User-agent: NumericBot
Crawl-delay: 0
Crawl-delay: 999999999
Crawl-delay: 1.23e10

# Case 14: Tab characters instead of spaces
User-agent:	TabBot
Disallow:	/tab1/
Allow:	/tab2/

# Case 15: Unicode and special characters in user-agent
User-agent: Bot™
Disallow: /trademark/

User-agent: Bot®
Disallow: /registered/

    }

    /** The HTTP header name for User-Agent */
    protected static final String USER_AGENT = "user-agent";

    /** The request attribute key for storing cached user agent type */
    protected static final String USER_AGENT_TYPE = "ViewHelper.UserAgent";

    /**
     * Determines the user agent type from the current HTTP request.
     * The method analyzes the User-Agent header to categorize the browser type

        RobotsTxt robotsTxt;
        final InputStream in = new java.io.ByteArrayInputStream(robotsTxtContent.getBytes());
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

        // Directives before User-agent should be ignored

 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */