assertNotNull(robotsTxt);
        // Everything should be allowed by default
        assertTrue(robotsTxt.allows("/anything", "AnyBot"));
    }

    public void testParse_onlyWhitespace() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots_only_whitespace.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);