authSchemeProviderBuilder.register(entry.getKey(), entry.getValue());
            }
        }

        // user agent
        if (StringUtil.isNotBlank(userAgent)) {
            httpClientBuilder.setUserAgent(userAgent);
        }

        // Authentication
        final Authentication[] siteCredentialList = {};
        for (final Authentication authentication : siteCredentialList) {

                "updatedTime");
        setupEpg(_epgMap, et -> ((PathMapping) et).getUserAgent(), (et, vl) -> ((PathMapping) et).setUserAgent(DfTypeUtil.toString(vl)),
                "userAgent");
    }

    @Override
    public PropertyGateway findPropertyGateway(final String prop) {
        return doFindEpg(_epgMap, prop);
    }

        setupEpg(_epgMap, et -> ((WebConfig) et).getUrls(), (et, vl) -> ((WebConfig) et).setUrls(DfTypeUtil.toString(vl)), "urls");
        setupEpg(_epgMap, et -> ((WebConfig) et).getUserAgent(), (et, vl) -> ((WebConfig) et).setUserAgent(DfTypeUtil.toString(vl)),
                "userAgent");
        setupEpg(_epgMap, et -> ((WebConfig) et).getVirtualHosts(), (et, vl) -> ((WebConfig) et).setVirtualHosts((String[]) vl),
                "virtualHosts");

StandardCrawlerContainer container = new StandardCrawlerContainer();

// Configure HTTP client with custom settings
container.singleton("httpClient", HcHttpClient.class, client -> {
    client.setUserAgent("MyBot/1.0");
    client.setConnectionTimeout(30000);
    client.setMaxConnections(100);
});

// Configure URL filtering
container.singleton("urlFilter", UrlFilterImpl.class, filter -> {

                wConfig.setUpdatedBy(username);
                wConfig.setUpdatedTime(now);
                wConfig.setUrls(configPath);
                wConfig.setUserAgent(getDefaultString("default.config.web.userAgent", fessConfig.getUserAgentName()));
                wConfig.setPermissions(ComponentUtil.getFessConfig().getSearchDefaultDisplayEncodedPermissions());

            }
        }

        // user agent
        userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent, String.class);
        if (StringUtil.isNotBlank(userAgent)) {
            httpClientBuilder.setUserAgent(userAgent);
        }

        final HttpRoutePlanner planner = buildRoutePlanner();
        if (planner != null) {
            httpClientBuilder.setRoutePlanner(planner);
        }

                    webConfig.setExcludedUrls(parseFilterPaths(globalParams.get(BAD_URLS), true, false));
                    webConfig.setExcludedDocUrls(StringUtil.EMPTY);
                    webConfig.setUserAgent(userAgent);
                    webConfig.setPermissions(new String[] { "Rguest" });
                    webConfig.setCreatedBy(Constants.SYSTEM_USER);
                    webConfig.setCreatedTime(now);

            searchLog.setClientIp(StringUtils.abbreviate(ComponentUtil.getViewHelper().getClientIp(req), 100));
            searchLog.setReferer(StringUtils.abbreviate(req.getHeader("referer"), 1000));
            searchLog.setUserAgent(StringUtils.abbreviate(req.getHeader("user-agent"), 255));
            final Object accessType = req.getAttribute(Constants.SEARCH_LOG_ACCESS_TYPE);
            if (Constants.SEARCH_LOG_ACCESS_TYPE_JSON.equals(accessType)) {