* @return the character set
     */
    protected String getCharSet(final File file) {
        return charset;
    }

    /**
     * Gets the character encoding used for files.
     *
     * @return the character encoding
     */
    public String getCharset() {
        return charset;
    }

    /**
     * Sets the character encoding used for files.

     * @return The character set name
     */
    protected String getCharSet(final File file) {
        return charset;
    }

    /**
     * Gets the character encoding used for FTP operations.
     *
     * @return The character encoding
     */
    public String getCharset() {
        return charset;
    }

    /**

            if (size < 3 || !isUtf8BomBytes(bomBytes)) {
                bis.reset();
            }
            final InputSource is = new InputSource(bis);
            if (responseData.getCharSet() != null) {
                is.setEncoding(responseData.getCharSet());
            }
            parser.parse(is);
        } catch (final Exception e) {
            throw new CrawlingAccessException("Could not parse " + responseData.getUrl(), e);

     * @param responseData the response data
     */
    protected void writeFileName(final StreamResponse response, final ResponseData responseData) {
        String charset = responseData.getCharSet();
        if (charset == null) {
            charset = Constants.UTF_8;
        }
        final String name;
        final String url = responseData.getUrl();
        final int pos = url.lastIndexOf('/');

            if (i % 1000 == 0) {
                logger.info("count:" + i + ", " + MemoryUtil.getMemoryUsageLog());
            }
            ResponseData responseData = new ResponseData();
            responseData.setCharSet("UTF-8");
            responseData.setContentLength(data.length());
            responseData.setExecutionTime(1000L);
            responseData.setHttpStatusCode(200);
            responseData.setLastModified(new Date());

            StandardJavadocDocletOptions options = (StandardJavadocDocletOptions) task.getOptions();
            options.setEncoding("utf-8");
            options.setDocEncoding("utf-8");
            options.setCharSet("utf-8");

            options.addBooleanOption("-allow-script-in-comments", true);
            options.setHeader(

                }
            }

            responseData.setUrl(url);
            responseData.setCharSet(contentEncoding);
            if (httpRequest instanceof HttpHead) {
                responseData.setMethod(Constants.HEAD_METHOD);
            } else {
                responseData.setMethod(Constants.GET_METHOD);

                }
            }

            responseData.setUrl(url);
            responseData.setCharSet(contentEncoding);
            if (httpRequest instanceof HttpHead) {
                responseData.setMethod(Constants.HEAD_METHOD);
            } else {
                responseData.setMethod(Constants.GET_METHOD);