robotsTxt.addDirective(directive);

        Directive retrieved = robotsTxt.getDirective("MyBot");
        assertNotNull(retrieved);
        assertEquals("MyBot", retrieved.getUserAgent());
    }

    public void test_getDirectiveNotFound() {
        // Test getDirective when directive doesn't exist
        RobotsTxt robotsTxt = new RobotsTxt();

            if (entry.getKey().matcher(target).find()) {
                final Directive directive = entry.getValue();
                final String ua = directive.getUserAgent();
                int uaLength = 0;
                if (!ALL_BOTS.equals(ua)) {
                    uaLength = ua.length();
                }
                if (uaLength > maxUaLength) {

        setupEpg(_epgMap, et -> ((PathMapping) et).getUpdatedTime(), (et, vl) -> ((PathMapping) et).setUpdatedTime(DfTypeUtil.toLong(vl)),
                "updatedTime");
        setupEpg(_epgMap, et -> ((PathMapping) et).getUserAgent(), (et, vl) -> ((PathMapping) et).setUserAgent(DfTypeUtil.toString(vl)),
                "userAgent");
    }

    @Override
    public PropertyGateway findPropertyGateway(final String prop) {

        if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
            paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
        }

        final String userAgent = getUserAgent();
        if (!StringUtil.isNotBlank(userAgent) || (userAgent.startsWith(Constants.CRAWLING_USER_AGENT_PREFIX)
                && userAgent.endsWith(Constants.CRAWLING_USER_AGENT_SUFFIX))) {

                "updatedTime");
        setupEpg(_epgMap, et -> ((WebConfig) et).getUrls(), (et, vl) -> ((WebConfig) et).setUrls(DfTypeUtil.toString(vl)), "urls");
        setupEpg(_epgMap, et -> ((WebConfig) et).getUserAgent(), (et, vl) -> ((WebConfig) et).setUserAgent(DfTypeUtil.toString(vl)),
                "userAgent");
        setupEpg(_epgMap, et -> ((WebConfig) et).getVirtualHosts(), (et, vl) -> ((WebConfig) et).setVirtualHosts((String[]) vl),

        while (!queue.isEmpty()) {
            final SearchLog searchLog = queue.poll();
            if (searchLog != null) {
                final String userAgent = searchLog.getUserAgent();
                final boolean isBot =
                        userAgent != null && stream(botNames).get(stream -> stream.anyMatch(botName -> userAgent.indexOf(botName) >= 0));
                if (!isBot) {

                    appendJson("languages", entity.getLanguages(), buf).append(',');
                    appendJson("roles", entity.getRoles(), buf).append(',');
                    appendJson("user-agent", entity.getUserAgent(), buf).append(',');
                    appendJson("client-ip", entity.getClientIp(), buf).append(',');
                    appendJson("access-type", entity.getAccessType(), buf).append(',');