}

        // Empty Disallow means allow all
        assertNotNull(robotsTxt);
        assertTrue(robotsTxt.allows("/anything", "TestBot"));
    }

    public void testParse_unicodeContent() {
        String robotsTxtContent =
                "# コメント\n" + "User-agent: 日本語Bot\n" + "Disallow: /日本語/\n" + "User-agent: TestBot\n" + "Disallow: /test/\n";

        RobotsTxt robotsTxt;