User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler
Disallow: /ddd

sitemap: http://www.example.com/sitmap.xml

        assertEquals("aaa bbb ccc", normalizeText("aaa bbb aaa ccc aaa", 100, -1, -1, true));
        assertEquals("aaa? bbb ccc", normalizeText("aaa? bbb aaa? ccc", 100, -1, -1, true));
        assertEquals("aaa #bbb ccc?", normalizeText("aaa #bbb# aaa ccc? aaa", 100, -1, -1, true));
        assertEquals("123 abc", normalizeText(" 123 abc 123", 100, -1, -1, true));

        }

        assertFalse(robotsTxt.allows("/aaa", "Crawler"));
        assertTrue(robotsTxt.allows("/bbb", "Crawler"));
        assertTrue(robotsTxt.allows("/ccc", "Crawler"));
        assertTrue(robotsTxt.allows("/ddd", "Crawler"));
        assertTrue(robotsTxt.allows("/aaa", "Crawler/1.0"));
        assertFalse(robotsTxt.allows("/bbb", "Crawler/1.0"));
        assertTrue(robotsTxt.allows("/ccc", "Crawler/1.0"));

        assertEquals("test.txt", ftpInfo.getName());
        assertEquals("ftp://123.123.123.123/", ftpInfo.toUrl("/"));
        assertEquals("ftp://123.123.123.123/aaa/bbb/ccc.txt", ftpInfo.toUrl("/aaa//bbb/ccc.txt"));
        assertEquals("ftp://123.123.123.123/ccc.txt", ftpInfo.toUrl("/aaa/../ccc.txt"));

        value = "ftp://123.123.123.123/test test.txt";

            settings.badword().add("");
            fail();
        } catch (IllegalArgumentException e) {
            assertTrue(true);
        }

        try {
            settings.badword().add("aaaa bbb");
            fail();
        } catch (IllegalArgumentException e) {
            assertTrue(true);
        }
    }

    @Test
    public void test_setAndGetAsArray() {
        String value1 = "a";

        url = "http://hoge/aaa/../index.html";
        assertEquals("http://hoge/index.html", htmlTransformer.normalizeUrl(url));

        url = "http://hoge/aaa/bbb/../../index.html";
        assertEquals("http://hoge/index.html", htmlTransformer.normalizeUrl(url));

        url = "/../index.html";
        assertEquals(url, htmlTransformer.normalizeUrl(url));