import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

public class RobotsTxt {
    private static final String ALL_BOTS = "*";

    protected final Map<Pattern, Directive> directiveMap = new LinkedHashMap<>();

    private final List<String> sitemapList = new ArrayList<>();

Shinsuke Sugaya <******@****.***> 1444529815 +0900

        assertTrue(robotsTxt.allows("/bbb", "Crawler"));
        assertTrue(robotsTxt.allows("/ccc", "Crawler"));
        assertTrue(robotsTxt.allows("/ddd", "Crawler"));
        assertTrue(robotsTxt.allows("/aaa", "Crawler/1.0"));
        assertFalse(robotsTxt.allows("/bbb", "Crawler/1.0"));
        assertTrue(robotsTxt.allows("/ccc", "Crawler/1.0"));
        assertTrue(robotsTxt.allows("/ddd", "Crawler/1.0"));

            final BufferedReader reader = new BufferedReader(new InputStreamReader(new BOMInputStream(stream), charsetName));

            String line;
            final RobotsTxt robotsTxt = new RobotsTxt();
            final List<Directive> currentDirectiveList = new ArrayList<>();
            boolean isGroupRecodeStarted = false;
            while ((line = reader.readLine()) != null) {
                line = stripComment(line).trim();

                    }
                }

                if (httpEntity != null) {
                    final RobotsTxt robotsTxt = robotsTxtHelper.parse(httpEntity.getContent());
                    if (robotsTxt != null) {
                        final String[] sitemaps = robotsTxt.getSitemaps();
                        if (sitemaps.length > 0) {
                            crawlerContext.addSitemaps(sitemaps);

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />
	<include path="crawler/robotstxt.xml" />
	<include path="crawler/contentlength.xml" />
	<include path="crawler/mimetype.xml" />

	<component name="internalHttpClient" class="org.codelibs.fess.crawler.client.http.HcHttpClient"
		instance="prototype">

github.com/tedsuo/rata,v1.0.0,h1:Sf9aZrYy6ElSTncjnGkyC2yuVvz5YJetBIUKJ4CmeKE=,f6745fd8ef8ee098410b31b1219def2c4e86c337ba6ff1319f086419b928f134
github.com/temoto/robotstxt,v1.1.1,h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=,c37f16f826a27512b7ae683ed32be5124a0252d1c7a8c4a00fd4e27d01c563d4