// check url
        if (crawlerContext.getRobotsTxtUrlSet().contains(robotTxtUrl)) {
            if (logger.isDebugEnabled()) {
                logger.debug("{} is already visited.", robotTxtUrl);
            }
            return;
        }

        if (logger.isInfoEnabled()) {
            logger.info("Checking URL: {}", robotTxtUrl);
        }
        // add url to a set

            } catch (final Exception e) {
                logger.debug("tikaConfig component is not found.", e);
            }
        }

        if (tikaConfig == null) {
            tikaConfig = TikaConfig.getDefaultConfig();
        }

        if (logger.isDebugEnabled()) {
            final Parser parser = tikaConfig.getParser();
            logger.debug("supportedTypes: {}", parser.getSupportedTypes(new ParseContext()));

                } catch (OpenSearchStatusException | IllegalStateException e) {
                    if (logger.isDebugEnabled()) {
                        logger.warn("[{}][{}] Failed to analyze a text(size:{}).", field, lang, buf.length(), e);
                    } else {
                        logger.warn("[{}][{}] Failed to analyze a text(size:{}). {}", field, lang, buf.length(), e.getMessage());
                    }

import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author shinsuke
 *
 */
public class EmlExtractorTest extends PlainTestCase {
    private static final Logger logger = LogManager.getLogger(EmlExtractorTest.class);

    public EmlExtractor emlExtractor;

    @Override
    protected void setUp() throws Exception {
        super.setUp();

import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author shinsuke
 *
 */
public class HtmlExtractorTest extends PlainTestCase {
    private static final Logger logger = LogManager.getLogger(HtmlExtractorTest.class);

    public HtmlExtractor htmlExtractor;

    @Override
    protected void setUp() throws Exception {
        super.setUp();

 */
public class UrlFilterImpl implements UrlFilter {

    /**
     * Creates a new UrlFilterImpl instance.
     */
    public UrlFilterImpl() {
        // NOP
    }

    private static final Logger logger = LogManager.getLogger(UrlFilterImpl.class);

    /**
     * The crawler container.
     */
    @Resource
    protected CrawlerContainer crawlerContainer;

    /**
     * The URL pattern.
     */

import org.w3c.dom.Node;
import org.xml.sax.InputSource;

/**
 * Extracts text content from HTML documents.
 */
public class HtmlExtractor extends AbstractXmlExtractor {
    /** Logger for this class. */
    protected static final Logger logger = LogManager.getLogger(HtmlExtractor.class);

    /** Pattern for extracting charset from meta tags. */

 * System.out.println(extractedData);
 * </pre>
 */
public class XmlTransformer extends AbstractTransformer {
    private static final Logger logger = LogManager.getLogger(XmlTransformer.class);

    private static final Pattern SPACE_PATTERN = Pattern.compile("\\s+", Pattern.MULTILINE);

    /**
     * If true, the parser will be namespace aware.
     */

                        "-g", "log level = 3");
        logger.info("Starting Samba container with image {}", IMAGE_NAME);
        sambaServer.start();
        logger.info("Samba container started");

        String host = sambaServer.getContainerIpAddress();
        Integer port = sambaServer.getMappedPort(445);
        baseUrl = "smb://" + host + ":" + port + "/users/";
        logger.info("Base URL: {}", baseUrl);

import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author shinsuke
 *
 */
public class MsWordExtractorTest extends PlainTestCase {
    private static final Logger logger = LogManager.getLogger(MsWordExtractorTest.class);

    public MsWordExtractor msWordExtractor;

    @Override
    protected void setUp() throws Exception {
        super.setUp();