*
     * @param userAgent The user agent string to look up in the directives
     * @return The Directive object matching the user agent, or null if no matching directive is found or if userAgent is null
     */
    public Directive getDirective(final String userAgent) {
        if (userAgent == null) {
            return null;
        }

            assertFalse(robotsTxt.allows("/private/index.html", userAgent));
            assertFalse(robotsTxt.allows("/help/", userAgent));
            assertFalse(robotsTxt.allows("/help.html", userAgent));
            assertFalse(robotsTxt.allows("/help/faq.html", userAgent));
            assertTrue(robotsTxt.allows("/foo/bar/", userAgent));
            assertTrue(robotsTxt.allows("/foo/bar/index.html", userAgent));

        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.
     * @param credentialsProvider The credentials provider.
     */

            }
        }

        // user agent
        userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent, String.class);
        if (StringUtil.isNotBlank(userAgent)) {
            httpClientBuilder.setUserAgent(userAgent);
        }

        final HttpRoutePlanner planner = buildRoutePlanner();
        if (planner != null) {

            false, false, "Long", 0, 0, null, null, false, null, null, null, null, null, false);
    protected final ColumnInfo _columnUserAgent = cci("userAgent", "userAgent", null, null, String.class, "userAgent", null, false, false,
            false, "keyword", 0, 0, null, null, false, null, null, null, null, null, false);

    public ColumnInfo columnCreatedBy() {
        return _columnCreatedBy;

        }

        final String userAgent = getUserAgent();
        if (!StringUtil.isNotBlank(userAgent) || (userAgent.startsWith(Constants.CRAWLING_USER_AGENT_PREFIX)
                && userAgent.endsWith(Constants.CRAWLING_USER_AGENT_SUFFIX))) {
            paramMap.put(Client.USER_AGENT, fessConfig.getUserAgentName());
        } else {
            paramMap.put(Client.USER_AGENT, userAgent);
        }

            return LaRequestUtil.getOptionalRequest().map(request -> {
                final String userAgent = request.getHeader("user-agent");
                if (StringUtil.isBlank(userAgent)) {
                    return false;
                }

                return pathMapping.getUAMatcher(userAgent).find();
            }).orElse(false);
        }
        return false;
    }

            0, null, null, false, null, null, null, null, null, false);
    protected final ColumnInfo _columnUserAgent = cci("userAgent", "userAgent", null, null, String.class, "userAgent", null, false, false,
            false, "keyword", 0, 0, null, null, false, null, null, null, null, null, false);

            }
        }

        // user agent
        final String userAgent = paramMap.get(CRAWLER_USERAGENT);
        if (StringUtil.isNotBlank(userAgent)) {
            factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
        }

        // web auth
        final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);

            final SearchLog searchLog = queue.poll();
            if (searchLog != null) {
                final String userAgent = searchLog.getUserAgent();
                final boolean isBot =
                        userAgent != null && stream(botNames).get(stream -> stream.anyMatch(botName -> userAgent.indexOf(botName) >= 0));
                if (!isBot) {
                    searchLog.getUserInfo().ifPresent(userInfo -> {