throw new MaxLengthExceededException("The content length (" + responseData.getContentLength()
                                    + " byte) is over " + maxLength + " byte. The url is " + filePath);
                        }
                    }
                } else {
                    // Forbidden

    }

    private ResponseData getTestData2_FAIL() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/sitemap.xml");
        responseData.setResponseBody(new byte[0]);
        return responseData;
    }

                            throw new MaxLengthExceededException("The content length (" + responseData.getContentLength()
                                    + " byte) is over " + maxLength + " byte. The url is " + filePath);
                        }
                    }

                    if (includeContent) {
                        if (file.length() < maxCachedContentSize) {

        BufferedInputStream bis = null;
        String encoding = null;
        try {
            bis = new BufferedInputStream(inputStream);
            final byte[] buffer = new byte[preloadSizeForCharset];
            final int size = bis.read(buffer);
            if (size != -1) {
                final String content = new String(buffer, 0, size);

    public void test_getRule_sitemaps1_nocontent() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://www.example.com/sitemap1.xml");
        responseData.setResponseBody(new byte[0]);
        final Rule rule = ruleManager.getRule(responseData);
        assertNotNull(rule);
        assertEquals("fileRule", rule.getRuleId());
        CloseableUtil.closeQuietly(responseData);
    }

                        if (contentLength > maxLength) {
                            throw new MaxLengthExceededException("The content length (" + contentLength + " byte) is over " + maxLength
                                    + " byte. The url is " + robotTxtUrl);
                        }
                    }
                }

                if (httpEntity != null) {

                if (responseData.getContentLength() > maxLength) {
                    throw new MaxLengthExceededException("The content length (" + responseData.getContentLength() + " byte) is over "
                            + maxLength + " byte. The url is " + filePath);
                }
            }

            if (includeContent) {
                final Tags objectTags = getObjectTags(bucketName, path);

                            throw new MaxLengthExceededException("The content length (" + responseData.getContentLength()
                                    + " byte) is over " + maxLength + " byte. The url is " + uri);
                        }
                    }

                    responseData.setCharSet(getCharSet(tempFile));

            return file;
        } catch (final IOException e) {
            throw new IORuntimeException(e);
        }
    }

    private File createContentFile(final String extention, final byte[] data) {
        try {
            final File file = File.createTempFile("content", extention);
            file.deleteOnExit();
            FileUtil.writeBytes(file.getAbsolutePath(), data);
            return file;

        if (isNotModified(responseData)) {
            final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
            final ResultData resultData = new ResultData();
            resultData.setData(new byte[0]);
            resultData.setEncoding(Constants.UTF_8);
            resultData.setTransformerName(Constants.NO_TRANSFORMER);
            processResult(urlQueue, responseData, resultData);