/**
     * Constructs a new ContentLengthHelper.
     */
    public ContentLengthHelper() {
        // Default constructor
    }

    /** Default maximum content length set to 10MB */
    protected long defaultMaxLength = 10L * 1024L * 1024L;

    /** Map to store maximum content lengths for specific MIME types */
    protected Map<String, Long> maxLengthMap = new HashMap<>();

    /**

 * </p>
 *
 */
public abstract class AbstractExtractor implements Extractor {

    /** The crawler container. */
    @Resource
    protected CrawlerContainer crawlerContainer;

    /**
     * Constructs a new AbstractExtractor.
     */
    public AbstractExtractor() {
        // NOP
    }

    /**
     * Registers this extractor with the ExtractorFactory.

        super.setUp();
        binaryTransformer = new BinaryTransformer();
        binaryTransformer.setName("binaryTransformer");
    }

    public void test_name() {
        assertEquals("binaryTransformer", binaryTransformer.getName());
    }

    public void test_transform() {
        final byte[] data = new String("xyz").getBytes();
        final ResponseData responseData = new ResponseData();

     */
    protected static final ByteOrderMark BOM_UTF_7 = new ByteOrderMark("UTF-7", 0x2B, 0x2F, 0x76);

    /**
     * HTML4 unescape translator.
     */
    protected static final CharSequenceTranslator UNESCAPE_HTML4 = new AggregateTranslator(
            new LookupTranslator(EntityArrays.BASIC_UNESCAPE), new LookupTranslator(EntityArrays.ISO8859_1_UNESCAPE),

    protected Map<String, String> metadataXpathMap = new HashMap<>();

    /** Thread-local instance of XPathAPI for thread-safe XPath evaluation. */
    private final ThreadLocal<XPathAPI> xpathAPI = new ThreadLocal<>();

    /**
     * Creates a new HtmlExtractor instance.
     */
    public HtmlExtractor() {
        super();
    }

    @Override

        final ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        responseData.setUrl("http://example.com/");
        responseData.setMimeType("text/html");
        return responseData;
    }

    private ResponseData getTestData2() {
        final ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);

        final List<Exception> exceptions = new ArrayList<>();

        Thread[] threads = new Thread[threadCount];
        for (int i = 0; i < threadCount; i++) {
            final int threadIndex = i;
            threads[i] = new Thread(new Runnable() {
                @Override
                public void run() {
                    try {
                        for (int j = 0; j < matchesPerThread; j++) {

        this.docRoot = docRoot;

        server = new Server(port);

        final ResourceHandler resource_handler = new ResourceHandler();
        resource_handler.setWelcomeFiles(new String[] { "index.html" });
        resource_handler.setResourceBase(docRoot.getAbsolutePath());
        Log.info("serving " + resource_handler.getBaseResource());
        final HandlerList handlers = new HandlerList();

        final BufferedInputStream bis = new BufferedInputStream(in);
        final String encoding = htmlExtractor.getEncoding(bis);
        CloseableUtil.closeQuietly(bis);
        assertEquals("UTF-8", encoding);
    }

    public void test_getEncoding_sjis() {
        final InputStream in = ResourceUtil.getResourceAsStream("extractor/test_sjis.html");
        final BufferedInputStream bis = new BufferedInputStream(in);

    public void test_isSuccessful() {
        DefaultResponseProcessor processor = new DefaultResponseProcessor();
        processor.setSuccessfulHttpCodes(new int[] { 200 });
        processor.setNotModifiedHttpCodes(new int[] { 304 });

        ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        assertTrue(processor.isSuccessful(responseData));