}

    protected void checkMaxContentLength(final ResponseData responseData) {
        if (maxContentLength != null && responseData.getContentLength() > maxContentLength.longValue()) {
            throw new MaxLengthExceededException("The content length (" + responseData.getContentLength() + " byte) is over "
                    + maxContentLength.longValue() + " byte. The url is " + responseData.getUrl());
        }
    }

    public List<String> getSmbRoleTypeList(final ResponseData responseData) {
        final List<String> roleTypeList = new ArrayList<>();
        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        if (fessConfig.isSmbRoleFromFile()) {
            final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
            final Map<String, Object> metaDataMap = responseData.getMetaDataMap();

    }

    private ResponseData getTestData1() {
        final ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        responseData.setUrl("http://example.com/");
        responseData.setMimeType("text/html");
        return responseData;
    }

    private ResponseData getTestData2() {
        final ResponseData responseData = new ResponseData();

    protected void putAdditionalData(final Map<String, Object> dataMap, final ResponseData responseData, final Document document) {
        // canonical
        final String canonicalUrl = getCanonicalUrl(responseData, document);
        if (canonicalUrl != null && !canonicalUrl.equals(responseData.getUrl()) && isValidUrl(canonicalUrl)
                && isValidCanonicalUrl(responseData.getUrl(), canonicalUrl)) {

            }
        }
    }

    protected ResponseData getResponseData(final String uri, final boolean includeContent) {
        final ResponseData responseData = new ResponseData();
        FTPClient client = null;
        try {
            responseData.setMethod(Constants.GET_METHOD);

            final FtpInfo ftpInfo = new FtpInfo(uri);
            responseData.setUrl(ftpInfo.toUrl());

            }
            ResponseData responseData = new ResponseData();
            responseData.setCharSet("UTF-8");
            responseData.setContentLength(data.length());
            responseData.setExecutionTime(1000L);
            responseData.setHttpStatusCode(200);
            responseData.setLastModified(new Date());
            responseData.setMethod("GET");

            path = "/" + path.replace('\\', '/');
        }
        final ResponseData responseData = fsClient.doGet("file:" + path);
        assertEquals(200, responseData.getHttpStatusCode());
        assertEquals("UTF-8", responseData.getCharSet());
        assertTrue(6 == responseData.getContentLength() || 7 == responseData.getContentLength());
        assertNotNull(responseData.getLastModified());

    private static final Logger logger = LoggerFactory.getLogger(SitemapsRule.class);

    @Override
    public boolean match(final ResponseData responseData) {
        if (super.match(responseData)) {
            try (final InputStream is = responseData.getResponseBody()) {
                final SitemapsHelper sitemapsHelper = crawlerContainer.getComponent("sitemapsHelper");
                return sitemapsHelper.isValid(is);

    public ResultData transform(final ResponseData responseData) {
        if (responseData == null || !responseData.hasResponseBody()) {
            throw new CrawlingAccessException("No response body.");
        }

        final ResultData resultData = new ResultData();
        resultData.setTransformerName(getName());

        try (BufferedInputStream bis = new BufferedInputStream(responseData.getResponseBody())) {

                if (responseData == null) {
                    // head method
                    responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build());
                    if (responseData == null) {
                        return true;
                    }
                }

                final int httpStatusCode = responseData.getHttpStatusCode();