csvReader.readValues(); // ignore header
            while ((list = csvReader.readValues()) != null) {
                String targetWord = getValue(list, 0);
                if (StringUtil.isBlank(targetWord)) {
                    // skip
                    continue;
                }
                try {
                    boolean isDelete = false;

        if (xpath == null) {
            xpath = fessConfig.getCrawlerDocumentHtmlCanonicalXpath();
        }
        if (StringUtil.isBlank(xpath)) {
            return null;
        }
        final String canonicalUrl = getSingleNodeValue(document, xpath, node -> node);
        if (StringUtil.isBlank(canonicalUrl)) {
            return null;
        }
        return normalizeCanonicalUrl(responseData.getUrl(), canonicalUrl);
    }

            }
            if (entity.getTerm().startsWith(regexPrefix)) {
                final String regex = entity.getTerm().substring(regexPrefix.length());
                if (StringUtil.isBlank(regex)) {
                    logger.warn("Unknown regex pattern: {}", entity.getTerm());
                } else {
                    pair.getSecond().add(new Pair<>(Pattern.compile(regex), entity.getContent()));

                return null;
            }

            userCode = getUserCodeFromCookie(request);
            if (StringUtil.isBlank(userCode)) {
                userCode = getUserCodeFromUserBean(request);
                if (StringUtil.isBlank(userCode)) {
                    userCode = getId();
                }
            }

            if (StringUtil.isNotBlank(userCode)) {

     * @throws CrawlerSystemException if data deserialization fails
     */
    public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String crawlingInfoId, final String url) {
        if (StringUtil.isBlank(crawlingInfoId)) {
            throw new CrawlingAccessException("sessionId is null. Cannot access document without a valid session ID.");
        }

,[0-9]{1,3}%,[0-9]{1,3}%,[0,1]{1}.?[0-9]*\)/i;if(e.match(f)){var g=e.replace(/\(/g,"").replace(/\)/g,""),h=g.split(","),i=!0;return h.forEach(function(a,c){var e=b(a);Number.isInteger(e)?0===c?(d=0<=e&&e<=360,d||(i=!1)):(d=0<=e&&e<=100,d||(i=!1)):isNaN(e)?(e=parseInt(a),d=0<=e&&e<=100,d||(i=!1)):(d=0<=e&&e<=1,d||(i=!1))}),i}return!1},errorMessage:"",errorMessageKey:"badHsla"})}(a)});...

    public RescorerBuilder<?> evaluate(final Map<String, Object> params) {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        final String modelName = fessConfig.getLtrModelName();
        if (StringUtil.isBlank(modelName)) {
            return null;
        }
        return new QueryRescorerBuilder(new StoredLtrQueryBuilder().modelName(modelName).params(params))
                .windowSize(fessConfig.getLtrWindowSize());
    }

     *
     * @param url the URL to validate
     * @return true if the URL is valid for redirection, false otherwise
     */
    protected boolean isValidRedirectUrl(final String url) {
        if (StringUtil.isBlank(url)) {
            return false;
        }
        final String lowerUrl = url.toLowerCase();
        if (lowerUrl.startsWith("http://") || lowerUrl.startsWith("https://")) {
            return true;
        }

        includedDocUrls = fessConfig.getCrawlerDocumentHtmlDefaultIncludeSearchPatterns();
        excludedDocUrls = fessConfig.getCrawlerDocumentHtmlDefaultExcludeSearchPatterns();
        boost = 1.0f;
        if (StringUtil.isBlank(userAgent)) {
            userAgent = fessConfig.getUserAgentName();
        }
        numOfThread = Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB;
        intervalTime = Constants.DEFAULT_INTERVAL_TIME_FOR_WEB;

     * @param regex The regex pattern for the condition.
     */
    public void addCondition(final String key, final String regex) {
        final String value = conditionMap.get(key);
        if (StringUtil.isBlank(value)) {
            conditionMap.put(key, regex);
        } else {
            conditionMap.put(key, value + "|" + regex);
        }
    }

    @Override