import java.io.IOException;
import java.io.InputStream;
import java.util.Map;

import org.apache.commons.io.output.DeferredFileOutputStream;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;

        }
        split(fessConfig.getQueryAdditionalAnalyzedFields(), ",")
                .of(stream -> stream.map(String::trim).filter(StringUtil::isNotBlank).forEach(s -> notAnalyzedFieldSet.remove(s)));
        split(fessConfig.getQueryAdditionalDefaultFields(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).map(s -> {
            final Pair<String, Float> pair = new Pair<>();
            final String[] values = s.split(":");

    public void addSearchFieldLogValue(final String name, final String value) {
        if (StringUtil.isNotBlank(name) && StringUtil.isNotBlank(value)) {
            searchFieldLogList.add(new Pair<>(name, value));
        }
    }

    public void addRequestHeaderValue(final String name, final String value) {
        if (StringUtil.isNotBlank(name) && StringUtil.isNotBlank(value)) {
            headerList.add(new Pair<>(name, value));
        }

            final String resourceName = params != null ? params.get(ExtractData.RESOURCE_NAME_KEY) : null;

            String value = null;
            if (StringUtil.isNotEmpty(url)) {
                value = url;
            } else if (StringUtil.isNotEmpty(resourceName)) {
                value = resourceName;
            }

            if (value != null) {

        final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
        if (StringUtil.isBlank(form.queryId) || StringUtil.isBlank(url) || !thumbnailSupport) {
            // 404
            throw responseManager.new404("Thumbnail for " + form.docId + " is not found.");
        }

    }

    protected void setupListCondition(final DuplicateHostCB cb, final DuplicateHostPager duplicateHostPager) {
        if (StringUtil.isNotBlank(duplicateHostPager.regularName)) {
            cb.query().setRegularName_Wildcard(wrapQuery(duplicateHostPager.regularName));
        }
        if (StringUtil.isNotBlank(duplicateHostPager.duplicateHostName)) {

    }

    protected void setupListCondition(final RelatedQueryCB cb, final RelatedQueryPager relatedQueryPager) {
        if (StringUtil.isNotBlank(relatedQueryPager.term)) {
            cb.query().setTerm_Wildcard(wrapQuery(relatedQueryPager.term));
        }
        if (StringUtil.isNotBlank(relatedQueryPager.queries)) {
            cb.query().setQueries_Wildcard(wrapQuery(relatedQueryPager.queries));
        }

    protected String getDecodeText(final String value) {
        if (value == null) {
            return StringUtil.EMPTY;
        }
        try {
            return MimeUtility.decodeText(value);
        } catch (final UnsupportedEncodingException e) {
            logger.warn("Invalid encoding.", e);
            return StringUtil.EMPTY;
        }
    }

    public Properties getMailProperties() {

        if (StringUtil.isNotBlank(webConfigPager.name)) {
            cb.query().setName_Wildcard(wrapQuery(webConfigPager.name));
        }
        if (StringUtil.isNotBlank(webConfigPager.urls)) {
            cb.query().setUrls_Wildcard(wrapQuery(webConfigPager.urls));
        }
        if (StringUtil.isNotBlank(webConfigPager.description)) {

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.helper;

import java.util.HashMap;
import java.util.Map;

import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

/**
 * @author shinsuke
 *
 */
public class ContentLengthHelper {

    protected long defaultMaxLength = 10L * 1024L * 1024L;// 10M