private static final Logger logger = LogManager.getLogger(FessCrawlerThread.class);

    protected static final String CRAWLER_CLIENTS = "crawlerClients";

    protected ConcurrentHashMap<String, Pair<String, Pattern>> clientRuleCache = new ConcurrentHashMap<>();

    @Override
    protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?> urlQueue) {
        if (ComponentUtil.getFessConfig().isIncrementalCrawling()) {

import java.util.Map;

import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;

/**
 * @author shinsuke
 *
 */
public interface CrawlerClient extends AutoCloseable {

    void setInitParameterMap(Map<String, Object> params);

    ResponseData execute(RequestData data);

    @Override
    default void close() throws Exception {
        // nothing

        if (StringUtil.isBlank(regex)) {
            throw new CrawlerSystemException("A regular expression is null.");
        }
        if (client == null) {
            throw new CrawlerSystemException("CrawlerClient is null.");
        }
        int current = 0;
        boolean added = false;
        final Map<Pattern, CrawlerClient> newClientMap = new LinkedHashMap<>();

    }

    @Override
    public void addClient(final List<String> regexList, final CrawlerClient client) {
        factory.addClient(regexList, client);
    }

    @Override
    public CrawlerClient getClient(final String url) {
        return factory.getClient(url);
    }

    @Override
    public void setClientMap(final Map<Pattern, CrawlerClient> clientMap) {
        factory.setClientMap(clientMap);
    }

    @Override

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * @author shinsuke
 *
 */
public class FaultTolerantClient implements CrawlerClient {

    private static final Logger logger = LoggerFactory.getLogger(FaultTolerantClient.class);

    protected CrawlerClient client;

    protected int maxRetryCount = 5;

    protected long retryInterval = 500;

    protected RequestListener listener;

                    config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
            final CrawlerClient client = crawlerClientFactory.getClient(url);
            if (client == null) {
                throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
            }
            String u = url;
            for (int i = 0; i < maxRedirectCount; i++) {

        clientFactory.addClient("storage:.*", container.getComponent("storageClient"));
    }

    public void test_getClient() {
        String url;
        CrawlerClient client;

        url = "http://hoge.com/";
        client = clientFactory.getClient(url);
        assertNotNull(client);
        assertTrue(client instanceof FaultTolerantClient);

                }
            }
        }
    }

    protected CrawlerClient getClient(final String url) {
        return clientFactory.getClient(url);
    }

    protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?> urlQueue) {
        if (urlQueue.getLastModified() != null) {

import org.codelibs.fess.crawler.helper.ContentLengthHelper;
import org.codelibs.fess.crawler.helper.MimeTypeHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * FileSystemClient is CrawlerClient implementation to crawl files on a file
 * system.
 *
 * @author shinsuke
 *
 */
public class FileSystemClient extends AbstractCrawlerClient {

                return;
            }

            final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
            final CrawlerClient client = crawlerClientFactory.getClient(url);
            if (client == null) {
                logger.warn("CrawlerClient is null. Data: {}", dataMap);
                return;
            }