*
 * @author shinsuke
 */
public class FtpAuthentication {
    private static final Logger logger = LogManager.getLogger(FtpAuthentication.class);

    /**
     * Constructs a new FtpAuthentication.
     */
    public FtpAuthentication() {
        // Default constructor
    }

    private String server;

    private int port;

    private String username;

 * allowing custom setup and cleanup operations for components.
 *
 */
public class StandardCrawlerContainer implements CrawlerContainer {

    private static final Logger logger = LogManager.getLogger(StandardCrawlerContainer.class);

    private final Map<String, ComponentHolder<?>> singletonMap = new ConcurrentHashMap<>();

        // check url
        if (crawlerContext.getRobotsTxtUrlSet().contains(robotTxtUrl)) {
            if (logger.isDebugEnabled()) {
                logger.debug("{} is already visited.", robotTxtUrl);
            }
            return;
        }

        if (logger.isInfoEnabled()) {
            logger.info("Checking URL: {}", robotTxtUrl);
        }
        // add url to a set

            } catch (final Exception e) {
                logger.debug("tikaConfig component is not found.", e);
            }
        }

        if (tikaConfig == null) {
            tikaConfig = TikaConfig.getDefaultConfig();
        }

        if (logger.isDebugEnabled()) {
            final Parser parser = tikaConfig.getParser();
            logger.debug("supportedTypes: {}", parser.getSupportedTypes(new ParseContext()));

                } catch (OpenSearchStatusException | IllegalStateException e) {
                    if (logger.isDebugEnabled()) {
                        logger.warn("[{}][{}] Failed to analyze a text(size:{}).", field, lang, buf.length(), e);
                    } else {
                        logger.warn("[{}][{}] Failed to analyze a text(size:{}). {}", field, lang, buf.length(), e.getMessage());
                    }

import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author shinsuke
 *
 */
public class EmlExtractorTest extends PlainTestCase {
    private static final Logger logger = LogManager.getLogger(EmlExtractorTest.class);

    public EmlExtractor emlExtractor;

    @Override
    protected void setUp() throws Exception {
        super.setUp();

import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author shinsuke
 *
 */
public class HtmlExtractorTest extends PlainTestCase {
    private static final Logger logger = LogManager.getLogger(HtmlExtractorTest.class);

    public HtmlExtractor htmlExtractor;

    @Override
    protected void setUp() throws Exception {
        super.setUp();

 */
public class UrlFilterImpl implements UrlFilter {

    /**
     * Creates a new UrlFilterImpl instance.
     */
    public UrlFilterImpl() {
        // NOP
    }

    private static final Logger logger = LogManager.getLogger(UrlFilterImpl.class);

    /**
     * The crawler container.
     */
    @Resource
    protected CrawlerContainer crawlerContainer;

    /**
     * The URL pattern.
     */

import org.w3c.dom.Node;
import org.xml.sax.InputSource;

/**
 * Extracts text content from HTML documents.
 */
public class HtmlExtractor extends AbstractXmlExtractor {
    /** Logger for this class. */
    protected static final Logger logger = LogManager.getLogger(HtmlExtractor.class);

    /** Pattern for extracting charset from meta tags. */

 * System.out.println(extractedData);
 * </pre>
 */
public class XmlTransformer extends AbstractTransformer {
    private static final Logger logger = LogManager.getLogger(XmlTransformer.class);

    private static final Pattern SPACE_PATTERN = Pattern.compile("\\s+", Pattern.MULTILINE);

    /**
     * If true, the parser will be namespace aware.
     */