User-agent: PriorityBot
Disallow: /store
Allow: /store/public
Disallow: /store/public/sale

# Test Allow vs Disallow with same length (Allow wins)
User-agent: SameLengthBot
Disallow: /page
Allow: /page

# Test multiple wildcards
User-agent: MultiWildcardBot
Disallow: /*.cgi*
Disallow: /*?*id=*

# Test literal $ in middle of pattern
User-agent: DollarBot
Disallow: /price$info

 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

NoValue:
:::
   :

# Case 13: Numeric crawl-delay edge cases
User-agent: NumericBot
Crawl-delay: 0
Crawl-delay: 999999999
Crawl-delay: 1.23e10

# Case 14: Tab characters instead of spaces
User-agent:	TabBot
Disallow:	/tab1/
Allow:	/tab2/

# Case 15: Unicode and special characters in user-agent
User-agent: Bot™
Disallow: /trademark/

User-agent: Bot®
Disallow: /registered/

    }

    /** The HTTP header name for User-Agent */
    protected static final String USER_AGENT = "user-agent";

    /** The request attribute key for storing cached user agent type */
    protected static final String USER_AGENT_TYPE = "ViewHelper.UserAgent";

    /**
     * Determines the user agent type from the current HTTP request.
     * The method analyzes the User-Agent header to categorize the browser type

        RobotsTxt robotsTxt;
        final InputStream in = new java.io.ByteArrayInputStream(robotsTxtContent.getBytes());
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

        // Directives before User-agent should be ignored

 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */

        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

            logger.debug("replace: {} -> {}", url, newUrl);
        }
        return newUrl;
    }

    /**
     * Checks if the user agent matches the path mapping.
     *
     * @param pathMapping the path mapping
     * @return true if the user agent matches
     */
    protected boolean matchUserAgent(final PathMapping pathMapping) {
        if (!pathMapping.hasUAMathcer()) {
            return true;

    // ============================================================
    // User Agent Configuration
    // ============================================================

    /** Prefix for Fess crawler user agent string. */
    public static final String CRAWLING_USER_AGENT_PREFIX = "Mozilla/5.0 (compatible; Fess/";

    /** Suffix for Fess crawler user agent string with bot information URL. */

     */
    public void setCookieSpec(final String cookieSpec) {
        this.cookieSpec = cookieSpec;
    }

    /**
     * Sets the user agent string.
     *
     * @param userAgent The user agent string
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the proxy host name.
     *