excludedUrls = fessConfig.getCrawlerDocumentHtmlDefaultExcludeIndexPatterns();
        includedDocUrls = fessConfig.getCrawlerDocumentHtmlDefaultIncludeSearchPatterns();
        excludedDocUrls = fessConfig.getCrawlerDocumentHtmlDefaultExcludeSearchPatterns();
        boost = 1.0f;
        if (StringUtil.isBlank(userAgent)) {
            userAgent = fessConfig.getUserAgentName();
        }
        numOfThread = Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB;