assertTrue(robotsTxt.allows("/help/faq.html", userAgent));
            assertTrue(robotsTxt.allows("/foo/bar/", userAgent));
            assertTrue(robotsTxt.allows("/foo/bar/index.html", userAgent));
            assertEquals(0, robotsTxt.getCrawlDelay(userAgent));
        }

        for (String userAgent : new String[] { "BruteBot", "FOO BruteBot/1.0" }) {
            assertFalse(robotsTxt.allows("/aaa", userAgent));

                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">\n"
                        + "  <url>\n" + "    <loc><![CDATA[http://www.example.com/page?foo=bar&baz=qux]]></loc>\n"
                        + "    <lastmod>2025-01-01</lastmod>\n" + "  </url>\n" + "</urlset>";
        final InputStream in = new ByteArrayInputStream(xml.getBytes());