import java.util.Set;

import org.codelibs.core.exception.IORuntimeException;
import org.codelibs.core.io.FileUtil;
import org.codelibs.fess.crawler.Constants;

/**
 * @author shinsuke
 *
 */
public class ResponseData implements Closeable {

    private String url;

    private int httpStatusCode;

    private String charSet;

    private long contentLength;

    private String mimeType;

    protected Map<String, String> metaContentMapping;

    protected FessConfig fessConfig;

    protected abstract Extractor getExtractor(ResponseData responseData);

    @Override
    public ResultData transform(final ResponseData responseData) {
        if (responseData == null || !responseData.hasResponseBody()) {
            throw new CrawlingAccessException("No response body.");
        }

    @Override
    public ResultData transform(final ResponseData responseData) {
        if (responseData == null || !responseData.hasResponseBody()) {
            throw new CrawlingAccessException("No response body.");
        }

        // encoding
        updateCharset(responseData);

        final ResultData resultData = new ResultData();

            ResponseData responseData = null;
            try {
                // head method
                responseData = client.execute(RequestDataBuilder.newRequestData().head().url(urlQueue.getUrl()).build());
                if (responseData != null && responseData.getLastModified() != null
                        && responseData.getLastModified().getTime() <= urlQueue.getLastModified().longValue()

            final Document document) {
        // canonical
        final String canonicalUrl = getCanonicalUrl(responseData, document);
        if (canonicalUrl != null && !canonicalUrl.equalsIgnoreCase(responseData.getUrl()) && isValidUrl(canonicalUrl)
                && isValidCanonicalUrl(responseData.getUrl(), canonicalUrl)) {

     * @see org.codelibs.fess.crawler.rule.impl.AbstractRule#match(org.codelibs.fess.crawler.entity.ResponseData)
     */
    @Override
    public boolean match(final ResponseData responseData) {
        if (defaultRule) {
            return true;
        }

        final Map<String, Object> map = new HashMap<>();
        BeanUtil.copyBeanToMap(responseData, map, CopyOptions::excludeWhitespace);

            }
        }
    }

    protected ResponseData getResponseData(final String uri, final boolean includeContent) {
        final ResponseData responseData = new ResponseData();
        FTPClient client = null;
        try {
            responseData.setMethod(Constants.GET_METHOD);

            final FtpInfo ftpInfo = new FtpInfo(uri);
            responseData.setUrl(ftpInfo.toUrl());

        final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
        if (responseData.getHttpStatusCode() == 404) {
            response.httpStatus(responseData.getHttpStatusCode());
            CloseableUtil.closeQuietly(responseData);
            return response;
        }
        writeFileName(response, responseData);

            }
            ResponseData responseData = new ResponseData();
            responseData.setCharSet("UTF-8");
            responseData.setContentLength(data.length());
            responseData.setExecutionTime(1000L);
            responseData.setHttpStatusCode(200);
            responseData.setLastModified(new Date());
            responseData.setMethod("GET");

    @Override
    protected void storeData(final ResponseData responseData, final ResultData resultData) {
        final DOMParser parser = getDomParser();
        try (final InputStream in = responseData.getResponseBody()) {
            final InputSource is = new InputSource(in);
            if (responseData.getCharSet() != null) {
                is.setEncoding(responseData.getCharSet());
            }
            parser.parse(is);