assertFalse(robotsTxt.allows("/aaa", userAgent));
            assertFalse(robotsTxt.allows("/private/", userAgent));
            assertFalse(robotsTxt.allows("/private/index.html", userAgent));
            assertFalse(robotsTxt.allows("/help/", userAgent));
            assertFalse(robotsTxt.allows("/help.html", userAgent));
            assertFalse(robotsTxt.allows("/help/faq.html", userAgent));

        regexRule.addRule("mimeType", Pattern.compile("text/html"));

        assertTrue(regexRule.match(getTestData1()));
        assertFalse(regexRule.match(getTestData2()));
        assertFalse(regexRule.match(getTestData3()));
        assertFalse(regexRule.match(getTestData4()));
    }

    public void test_match_or() {
        final RegexRule regexRule = new RegexRule();

        assertTrue(processor.isSuccessful(responseData));

        responseData.setHttpStatusCode(100);
        assertFalse(processor.isSuccessful(responseData));
        responseData.setHttpStatusCode(304);
        assertFalse(processor.isSuccessful(responseData));
        responseData.setHttpStatusCode(404);
        assertFalse(processor.isSuccessful(responseData));
    }

    public void test_isNotModified() {

        assertFalse(auth.matches("ftp://hostname:111/"));
        assertFalse(auth.matches("ftp://hostname:xx/"));
        assertFalse(auth.matches("ftp://hoge/test/aaa.html"));
        assertFalse(auth.matches("ftp://hoge/test"));
        assertFalse(auth.matches("ftp://hoge/"));
        assertFalse(auth.matches("ftp://hoge"));
        assertFalse(auth.matches("ftp://"));

        assertTrue(request.isHiraganaQuery("ぁぃぅぇぉっゃゅょ"));

        assertFalse(request.isHiraganaQuery("こ犬"));
        assertFalse(request.isHiraganaQuery("abc"));
        assertFalse(request.isHiraganaQuery("カキク"));
        assertFalse(request.isHiraganaQuery("あカ"));
        assertFalse(request.isHiraganaQuery("アか"));
        assertFalse(request.isHiraganaQuery("abcあ"));
    }

        final String sessionId = "id1";
        urlFilter.init(sessionId);

        assertFalse(urlFilter.match("http://example.com/"));
        assertFalse(urlFilter.match("http://example.com/a"));
        assertFalse(urlFilter.match("http://test.com/"));
        assertFalse(urlFilter.match("http://test.com/a"));
    }

    public void test_match_both() {

        final RegexRule rule3 = new RegexRule();
        rule3.addRule("url", "http:.*");

        assertFalse(ruleManager.hasRule(rule2));
        assertFalse(ruleManager.hasRule(rule3));

        ruleManager.addRule(rule2);

        assertTrue(ruleManager.hasRule(rule2));
        assertFalse(ruleManager.hasRule(rule3));

        ruleManager.addRule(rule3);

        assertTrue(ruleManager.hasRule(rule2));

        urlQueueService.deleteAll();
        assertFalse(fesenClient.prepareSearch("fess_crawler.queue")
                .setQuery(QueryBuilders.termQuery("sessionId", "id1"))
                .execute()
                .actionGet()
                .getHits()
                .getTotalHits()
                .value() > 0);
        assertFalse(fesenClient.prepareSearch("fess_crawler.queue")

        final SitemapSet sitemapSet = sitemapsHelper.parse(in);
        final Sitemap[] sitemaps = sitemapSet.getSitemaps();
        assertEquals(5, sitemaps.length);
        assertTrue(sitemapSet.isUrlSet());
        assertFalse(sitemapSet.isIndex());

        assertEquals("2005-01-01", sitemaps[0].getLastmod());
        assertEquals("http://www.example.com/", sitemaps[0].getLoc());

        assertFalse(sitemapsRule.match(responseData));
        InputStream is = responseData.getResponseBody();
        if (is instanceof TemporaryFileInputStream) {
            File temporaryFile = ((TemporaryFileInputStream) is).getTemporaryFile();
            assertTrue(temporaryFile.exists());
            CloseableUtil.closeQuietly(is);
            assertFalse(temporaryFile.exists());
        } else {