response.setHeaders(con.getHeaderFields());
            } catch (final Exception e) {
                throw new CurlException("Failed to access the response.", e);
            }
            writeContent(() -> {
                try {
                    if (con.getResponseCode() < 400) {
                        if (GZIP.equals(con.getContentEncoding())) {

        if (client == null) {
            throw new FessSystemException("No CrawlerClient: " + configId + ", url: " + url);
        }
        return writeContent(configId, url, client);
    }

    protected StreamResponse writeContent(final String configId, final String url, final CrawlerClient client) {
        final StreamResponse response = new StreamResponse(StringUtil.EMPTY);