* </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {
    private static final String ALL_BOTS = "*";

    /** Map of user agent patterns to their corresponding directives. */

        assertTrue(robotsTxt.allows("/public/", "ComplexBot"));
        assertTrue(robotsTxt.allows("/public/page.html", "ComplexBot"));

        // Test PriorityBot - longest match wins
        // Disallow: /store, Allow: /store/public, Disallow: /store/public/sale
        assertFalse(robotsTxt.allows("/store", "PriorityBot"));
        assertFalse(robotsTxt.allows("/store/items", "PriorityBot"));