String getCrawlerDocumentHtmlPrunedTags();

    /**
     * Get the value for the key 'crawler.document.html.max.digest.length'. <br>
     * The value is, e.g. 120 <br>
     * comment: Maximum length of digest extracted from HTML documents.
     * @return The value of found property. (NotNull: if not found, exception but basically no way)
     */
    String getCrawlerDocumentHtmlMaxDigestLength();

    /**

            op.setRefreshPolicy(Constants.TRUE);
        });
        return failureUrl;
    }

    /**
     * Extracts and returns the stack trace from a throwable as a string.
     * The stack trace is abbreviated if it exceeds the configured maximum length.
     *
     * @param t the throwable to extract the stack trace from
     * @return the stack trace as a string, or empty string if extraction fails
     */

    // InputStream in =
    // ResourceUtil.getResourceAsStream("extractor/test_pass.pdf");
    // TikaExtractor extractor =
    // (TikaExtractor) getContainer().getComponent(
    // "tikaExtractorForPdfPassword");
    // Map<String, String> params = new HashMap<String, String>();
    // params.put(ExtractData.URL, "http://example.com/test_pass.pdf");
    // ExtractData extractData = extractor.getText(in, params);

    private TestApiExtractorServer server;

    private ApiExtractor extractor;

    @Override
    protected void setUp() throws Exception {
        super.setUp();

        server = new TestApiExtractorServer(port);
        server.start();

        extractor = new ApiExtractor();
        extractor.setUrl("http://127.0.0.1:" + port + "/");
        extractor.init();
    }

    @Override

        final CommandExtractor extractor = new CommandExtractor();
        extractor.executionTimeout = 1000L;
        extractor.command = getCommand(scriptFile);
        final Map<String, String> params = new HashMap<String, String>();
        try {
            final ExtractData data = extractor.getText(new FileInputStream(contentFile), params);
            fail(data.toString());

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.extractor.impl;

import java.io.IOException;
import java.io.InputStream;

import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.compressors.CompressorStreamFactory;

        final InputStream in = new ByteArrayInputStream("test".getBytes());

        extractor.resetTestState();
        assertFalse("Should start with validate not called", extractor.isValidateCalled());

        extractor.getText(in, null);

        assertTrue("Should be called after getText", extractor.isValidateCalled());
    }

    /**
     * Test validateInputStream with edge case: empty stream.

        final String content = htmlXpathExtractor.getText(in, null).getContent();
        CloseableUtil.closeQuietly(in);
        logger.info(content);
        assertTrue(content.contains("テスト"));
    }

    public void test_getHtml_sjis() {
        final InputStream in = ResourceUtil.getResourceAsStream("extractor/test_sjis.html");

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.extractor.impl;

import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.core.io.CloseableUtil;

        final InputStream in = ResourceUtil.getResourceAsStream("extractor/test_sjis.xml");
        final String content = xmlExtractor.getText(in, null).getContent();
        CloseableUtil.closeQuietly(in);
        logger.info(content);
        assertTrue(content.contains("テスト"));
    }

    public void test_getXml_entity() {
        final InputStream in = ResourceUtil.getResourceAsStream("extractor/test_entity.xml");