User-agent: PriorityBot
Disallow: /store
Allow: /store/public
Disallow: /store/public/sale

# Test Allow vs Disallow with same length (Allow wins)
User-agent: SameLengthBot
Disallow: /page
Allow: /page

# Test multiple wildcards
User-agent: MultiWildcardBot
Disallow: /*.cgi*
Disallow: /*?*id=*

# Test literal $ in middle of pattern
User-agent: DollarBot
Disallow: /price$info

        assertFalse(robotsTxt.allows("/store/public/sale", "PriorityBot")); // Most specific disallow
        assertFalse(robotsTxt.allows("/store/public/sale/item", "PriorityBot"));

        // Test SameLengthBot - Allow wins when same length as Disallow
        // Disallow: /page, Allow: /page
        assertTrue(robotsTxt.allows("/page", "SameLengthBot")); // Allow takes precedence

Disallow: /test/

# Case 9: Special characters in paths
User-agent: SpecialCharBot
Disallow: /path with spaces/
Disallow: /path%20encoded/
Disallow: /path?query=value
Disallow: /path#fragment
Allow: /unicode/日本語/

# Case 10: Multiple User-agents in sequence
User-agent: Bot1
User-agent: Bot2
User-agent: Bot3
Disallow: /shared/

# Case 11: Sitemap with various formats
Sitemap: http://example.com/sitemap.xml

 *
 * <p>Key features:</p>
 * <ul>
 *   <li>Supports multiple user-agent directives with pattern matching</li>
 *   <li>Handles Allow and Disallow rules for path-based access control</li>
 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *

 * <li>Disallow and Allow directives with pattern matching</li>
 * <li>Wildcard (*) in paths - matches any sequence of characters</li>
 * <li>End-of-path ($) matching - matches the end of URL path</li>
 * <li>Crawl-delay directive</li>
 * <li>Sitemap directive</li>
 * <li>Comment support (#)</li>
 * <li>Priority-based matching (longest match wins, Allow beats Disallow at equal length)</li>
 * </ul>
 *

{* ../../docs_src/header_param_models/tutorial003_an_py310.py hl[19] *}

/// warning

Before setting `convert_underscores` to `False`, bear in mind that some HTTP proxies and servers disallow the usage of headers with underscores.

///

## Summary { #summary }

        directive.addDisallow("/admin/");
        directive.addDisallow("/private/");

        String[] disallows = directive.getDisallows();
        assertEquals(2, disallows.length);
        assertEquals("/admin/", disallows[0]);
        assertEquals("/private/", disallows[1]);
    }

    public void test_directiveAddDisallowNoDuplicates() {
        // Test that addDisallow doesn't add duplicates

{* ../../docs_src/header_params/tutorial002_an_py310.py hl[10] *}

/// warning

Before setting `convert_underscores` to `False`, bear in mind that some HTTP proxies and servers disallow the usage of headers with underscores.

///

## Duplicate headers { #duplicate-headers }

It is possible to receive duplicate headers. That means, the same header with multiple values.

     */
    protected void initDisallowCompound(final String prop) {
        if (prop == null) {
            return;
        }
        final Set<String> disallow = new HashSet<>();
        final StringTokenizer st = new StringTokenizer(prop, ",");
        while (st.hasMoreTokens()) {
            disallow.add(st.nextToken().trim());
        }

        }
    }

    protected
    fun fetchVersionsFromMavenMetadata(url: String): List<String> =
        DocumentBuilderFactory.newInstance().apply {
            setFeature("http://apache.org/xml/features/disallow-doctype-decl", true)
            setFeature("http://xml.org/sax/features/external-general-entities", false)
            setFeature("http://xml.org/sax/features/external-parameter-entities", false)