private final List<String> sitemapList = new ArrayList<>();

    /**
     * Creates a new RobotsTxt instance.
     */
    public RobotsTxt() {
        // Default constructor
    }

    /**
     * Checks if access to a given path is allowed for a specific user agent according to robots.txt rules.
     *
     * @param path The path to check for access permission

        assertTrue(robotsTxt.allows("/bbb", "Crawler"));
        assertTrue(robotsTxt.allows("/ccc", "Crawler"));
        assertTrue(robotsTxt.allows("/ddd", "Crawler"));
        assertTrue(robotsTxt.allows("/aaa", "Crawler/1.0"));
        assertFalse(robotsTxt.allows("/bbb", "Crawler/1.0"));
        assertTrue(robotsTxt.allows("/ccc", "Crawler/1.0"));
        assertTrue(robotsTxt.allows("/ddd", "Crawler/1.0"));

        Directive directive = new Directive("MyBot");
        robotsTxt.addDirective(directive);

        Directive retrieved = robotsTxt.getDirective("OtherBot");
        assertNull(retrieved);
    }

    public void test_getDirectiveWithNull() {
        // Test getDirective with null user agent
        RobotsTxt robotsTxt = new RobotsTxt();

        try {
            reader = new BufferedReader(new InputStreamReader(new BOMInputStream(stream), charsetName));

            String line;
            final RobotsTxt robotsTxt = new RobotsTxt();
            final List<Directive> currentDirectiveList = new ArrayList<>();
            boolean isGroupRecordStarted = false;

            while ((line = reader.readLine()) != null) {
                try {

                    }
                }

                if (httpEntity != null) {
                    final RobotsTxt robotsTxt = robotsTxtHelper.parse(httpEntity.getContent());
                    if (robotsTxt != null) {
                        final String[] sitemaps = robotsTxt.getSitemaps();
                        if (sitemaps.length > 0) {
                            crawlerContext.addSitemaps(sitemaps);

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />
	<include path="crawler/robotstxt.xml" />
	<include path="crawler/contentlength.xml" />
	<include path="crawler/mimetype.xml" />

	<component name="internalHttpClient" class="org.codelibs.fess.crawler.client.http.HcHttpClient"
		instance="prototype">

# Case 16: Multiple wildcards user agent
User-agent: *
Disallow: /default/

# Case 17: Empty file handling (just comments and whitespace after this)