Crawl-delay: -10
Crawl-delay: 5.5
Crawl-delay:
Allow:    # empty value

# Case 3: Multiple colons in directive
User-agent: MultiColonBot
Disallow: http://example.com:8080/path
Allow: /path:with:colons

# Case 4: Extra whitespace
User-agent:    ExtraSpaceBot
Disallow:     /spaced/
   Allow:   /also-spaced/

# Case 5: Mixed case directives (should still work)
UsEr-AgEnT: MixedCaseBot
DiSaLlOw: /test1/
AlLoW: /test2/

        Thread.sleep(100);

        // Create additional update index (making 2 total)
        final String updateAlias = suggester.getIndex() + ".update";
        final String extraIndexName = suggester.getIndex() + ".extra";

        client.admin().indices().prepareCreate(extraIndexName).execute().actionGet();
        client.admin().indices().prepareAliases().addAlias(extraIndexName, updateAlias).execute().actionGet();
        Thread.sleep(100);

        assertFalse(robotsTxt.allows("http://example.com:8080/path", "MultiColonBot"));
        assertTrue(robotsTxt.allows("/path:with:colons", "MultiColonBot"));

        // Test ExtraSpaceBot - extra whitespace should be handled
        assertFalse(robotsTxt.allows("/spaced/", "ExtraSpaceBot"));
        assertTrue(robotsTxt.allows("/also-spaced/", "ExtraSpaceBot"));

labels.suggest_word_number=Aantal woorden
labels.suggest_word_type_all=Alles
labels.suggest_word_type_document=Document
labels.suggest_word_type_query=Zoekterm
labels.elevate_word_configuration=Extra woord
labels.elevate_word_title_details=Extra woord
labels.elevate_word_link_list=Lijst
labels.elevate_word_link_create=Nieuw maken
labels.elevate_word_link_edit=Bewerken
labels.elevate_word_link_delete=Verwijderen

        assertEquals("2025-01-01", sitemaps[0].getLastmod());
    }

    public void test_parseXmlSitemaps_withExtraWhitespace() {
        // Sitemap with extra whitespace should be trimmed
        final String xml =
                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n"