<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="tikaExtractor"
		class="org.codelibs.fess.crawler.extractor.impl.TikaExtractor">
		<property name="maxCompressionRatio">1</property>
		<property name="maxUncompressionSize">10000000</property>
	</component>

    }

    /**
     * Checks if a path is valid for crawling (not a JavaScript, mailto, or other invalid URL).
     *
     * @param path the path to validate
     * @return true if the path is valid, false otherwise
     */
    protected boolean isValidPath(final String path) {
        if (StringUtil.isBlank(path)) {
            return false;
        }

    }

    private void assertContentType(final String expect, final String path, final String name) throws IOException {
        final MimeTypeHelper mimeTypeHelper = container.getComponent("mimeTypeHelper");
        if (path != null) {
            try (final InputStream is = ResourceUtil.getResourceAsStream(path)) {
                assertEquals(expect, mimeTypeHelper.getContentType(is, name));
            }

     * @param document the DOM document to extract strings from
     * @param path the XPath expression to evaluate
     * @return an array of strings extracted from the document
     */
    protected String[] getStringsByXPath(final Document document, final String path) {
        try {
            final XPathEvaluationResult<?> xObj = getXPathAPI().eval(document, path);
            switch (xObj.type()) {
            case BOOLEAN:

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="binaryTransformer"
		class="org.codelibs.fess.crawler.transformer.impl.BinaryTransformer"
		instance="singleton">
		<property name="name">"binaryTransformer"</property>
	</component>

/**
 * The class CrawlerSystemException is a runtime exception that is thrown when a system error occurs during crawling.
 * It extends the RuntimeException class and provides constructors for creating exceptions with a message, a cause, or both.
 * It also provides a protected constructor that allows to specify whether or not suppression is enabled or stack trace is writable.
 */
public class CrawlerSystemException extends RuntimeException {

 *
 * <p>
 * This exception can be thrown when there are problems accessing URLs, files, or any other resources needed for crawling.
 * It includes constructors to handle messages, causes, or both.
 * </p>
 *
 * <p>
 * The log level can be set to DEBUG, INFO, WARN, or ERROR, and the class provides methods to check if a specific log level is enabled.
 * </p>
 *
 * <p>
 * Example usage:
 * </p>
 *

            ftpUser.setName(username);
            ftpUser.setPassword(password);
            final File file = ResourceUtil.getResourceAsFile("test");
            String path = file.getAbsolutePath();
            ftpUser.setHomeDirectory(path);

            userManager.save(ftpUser);
        }

        FtpServer server = factory.createServer();
        server.start();
        return server;
    }

 * It is used to indicate exceptions related to MIME type handling during the crawling process.
 * This exception can be thrown with a message, a cause, or both.
 */
public class MimeTypeException extends CrawlerSystemException {

    private static final long serialVersionUID = 1L;

    /**
     * Creates a new MimeTypeException with the specified detail message and cause.

    public Tokenizer create() {
        JapaneseTokenizer t = new JapaneseTokenizer(userDictionary, discartPunctuation, mode);
        int nBestCost = this.nBestCost;
        if (nBestExamples != null) {
            nBestCost = Math.max(nBestCost, t.calcNBestCost(nBestExamples));
        }
        t.setNBestCost(nBestCost);
        return t;
    }