authSchemeProviderBuilder.register(entry.getKey(), entry.getValue());
            }
        }

        // user agent
        if (StringUtil.isNotBlank(userAgent)) {
            httpClientBuilder.setUserAgent(userAgent);
        }

        // Authentication
        final Authentication[] siteCredentialList = {};
        for (final Authentication authentication : siteCredentialList) {

StandardCrawlerContainer container = new StandardCrawlerContainer();

// Configure HTTP client with custom settings
container.singleton("httpClient", HcHttpClient.class, client -> {
    client.setUserAgent("MyBot/1.0");
    client.setConnectionTimeout(30000);
    client.setMaxConnections(100);
});

// Configure URL filtering
container.singleton("urlFilter", UrlFilterImpl.class, filter -> {

            }
        }

        // user agent
        userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent, String.class);
        if (StringUtil.isNotBlank(userAgent)) {
            httpClientBuilder.setUserAgent(userAgent);
        }

        final HttpRoutePlanner planner = buildRoutePlanner();
        if (planner != null) {
            httpClientBuilder.setRoutePlanner(planner);
        }

                    webConfig.setExcludedUrls(parseFilterPaths(globalParams.get(BAD_URLS), true, false));
                    webConfig.setExcludedDocUrls(StringUtil.EMPTY);
                    webConfig.setUserAgent(userAgent);
                    webConfig.setPermissions(new String[] { "Rguest" });
                    webConfig.setCreatedBy(Constants.SYSTEM_USER);
                    webConfig.setCreatedTime(now);

            searchLog.setClientIp(StringUtils.abbreviate(ComponentUtil.getViewHelper().getClientIp(req), 100));
            searchLog.setReferer(StringUtils.abbreviate(req.getHeader("referer"), 1000));
            searchLog.setUserAgent(StringUtils.abbreviate(req.getHeader("user-agent"), 255));
            final Object accessType = req.getAttribute(Constants.SEARCH_LOG_ACCESS_TYPE);
            if (Constants.SEARCH_LOG_ACCESS_TYPE_JSON.equals(accessType)) {