* @param text The text to analyze.
         * @param field The field name.
         * @param lang The language.
         * @return A list of analyzed tokens.
         */
        @Override
        public List<AnalyzeToken> analyze(final String text, final String field, final String lang) {
            if (text == null || text.length() > maxContentLenth) {
                return Collections.emptyList();

 * @author shinsuke
 *
 */
public class XmlExtractorTest extends PlainTestCase {
    private static final Logger logger = LogManager.getLogger(XmlExtractorTest.class);

    public XmlExtractor xmlExtractor;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("xmlExtractor", XmlExtractor.class);

/**
 * @author shinsuke
 *
 */
public class UrlFilterImplTest extends PlainTestCase {
    public UrlFilterImpl urlFilter;

    public MemoryDataHelper dataHelper;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        StandardCrawlerContainer container = new StandardCrawlerContainer().singleton("dataHelper", MemoryDataHelper.class)//

    /**
     * Returns the pattern used to identify tags in the content.
     * @return The tag pattern.
     */
    protected abstract Pattern getTagPattern();

    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {

        urlFilter.clear();
    }

    /**
     * Closes the crawler, releasing resources.
     * This method is called automatically when the crawler is used in a try-with-resources statement.
     */
    @Override
    public void close() {
        clientFactory.close();
    }

    /**
     * Adds an include filter for URLs.
     * Only URLs matching this regular expression will be crawled.

        super();
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.extractor.Extractor#getText(java.io.InputStream,
     * java.util.Map)
     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }

import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author shinsuke
 *
 */
public class MimeTypeHelperImplTest extends PlainTestCase {

    private StandardCrawlerContainer container;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        container = new StandardCrawlerContainer().singleton("mimeTypeHelper", MimeTypeHelperImpl.class);
    }

public class XpathTransformerTest extends PlainTestCase {
    public XpathTransformer xpathTransformer;

    public XpathTransformer xpathMapTransformer;

    public XpathTransformer xpathEntityTransformer;

    @Override
    protected void setUp() throws Exception {
        super.setUp();
        {
            xpathTransformer = new XpathTransformer();
            xpathTransformer.setName("xpathTransformer");

    * user agent, proxy settings, request headers, cookie store, and connection manager.
    * It also processes form-based authentication schemes and sets up the HTTP client context.
    */
    @Override
    public synchronized void init() {
        if (httpClient != null) {
            return;
        }

        if (logger.isDebugEnabled()) {

     */
    public void setClientMap(final Map<Pattern, CrawlerClient> clientMap) {
        this.clientMap = clientMap;
    }

    /**
     * Closes all clients in the factory.
     */
    @Override
    public void close() {
        clientMap.values().stream().distinct().forEach(client -> {
            try {
                client.close();
            } catch (final Exception e) {