protected Map<Pattern, CrawlerClient> clientMap = new LinkedHashMap<>();

    @PostConstruct
    public void init() {
        try {
            final CrawlerClientCreator creator = crawlerContainer.getComponent("crawlerClientCreator");
            if (creator != null) {
                creator.register(this);
            }
        } catch (final Exception e) {
            if (logger.isDebugEnabled()) {

        taskList.clear();
        if (logger.isDebugEnabled()) {
            logger.debug("Storing {} thumbnail tasks.", list.size());
        }
        final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
        thumbnailQueueBhv.batchInsert(list);
    }

    public int generate(final ExecutorService executorService, final boolean cleanup) {

        if (logger.isDebugEnabled()) {
            logger.debug("Initialize {}", this.getClass().getSimpleName());
        }
        try {
            final TikaExtractor tikaExtractor = ComponentUtil.getComponent("tikaExtractor");
            if (tikaExtractor != null) {
                tikaExtractor.setMaxAlphanumTermSize(getMaxAlphanumTermSize());
                tikaExtractor.setMaxSymbolTermSize(getMaxSymbolTermSize());

                && urlSet.add(d.getUrl() + "\n" + d.getMetaData()) && crawlerContext.urlFilter.match(d.getUrl())).map(d -> {
                    final UrlQueue<?> uq = crawlerContainer.getComponent("urlQueue");
                    uq.setCreateTime(SystemUtil.currentTimeMillis());
                    uq.setDepth(depth);
                    uq.setMethod(Constants.GET_METHOD);
                    uq.setParentUrl(url);

            flush();
        }
        return (long) ids.length;
    };

    public abstract long process();

    protected void enable() {
        final ScoreUpdater scoreUpdater = ComponentUtil.getComponent("scoreUpdater");
        scoreUpdater.addScoreBooster(this);
    }

    protected long updateScore(final Map<String, Object> params) {
        return requestHandler.apply(params);
    }

        if (responseData == null || !responseData.hasResponseBody()) {
            throw new CrawlingAccessException("No response body.");
        }

        final ExtractorFactory extractorFactory = crawlerContainer.getComponent("extractorFactory");
        if (extractorFactory == null) {
            throw new CrawlerSystemException("Could not find extractorFactory.");
        }

        asDocMeta().version(version);
    }

    public OptionalEntity<CrawlingInfo> getCrawlingInfo() {
        if (crawlingInfo != null) {
            final CrawlingInfoBhv crawlingInfoBhv = ComponentUtil.getComponent(CrawlingInfoBhv.class);
            crawlingInfo = crawlingInfoBhv.selectEntity(cb -> {
                cb.query().docMeta().setId_Equal(getCrawlingInfoId());
            });
        }
        return crawlingInfo;

        asDocMeta().version(version);
    }

    public FileConfig getFileConfig() {
        if (fileConfig == null) {
            final FileConfigService fileConfigService = ComponentUtil.getComponent(FileConfigService.class);
            try {
                fileConfig = fileConfigService.getFileConfig(getFileConfigId()).get();
            } catch (final Exception e) {

        }
        Extractor extractor = extractorFactory.getExtractor(responseData.getMimeType());
        if (extractor == null) {
            extractor = ComponentUtil.getComponent("tikaExtractor");
            if (extractor == null) {
                throw new FessSystemException("Could not find tikaExtractor.");
            }
        }

        if (logger.isDebugEnabled()) {

    /*
     * (non-Javadoc)
     *
     * @see org.apache.commons.pool2.BasePooledObjectFactory#create()
     */
    @Override
    public T create() throws Exception {
        return (T) crawlerContainer.getComponent(componentName);
    }

    /*
     * (non-Javadoc)
     *
     * @see
     * org.apache.commons.pool2.BasePooledObjectFactory#wrap(java.lang.Object)
     */
    @Override