protected void deleteCrawlData(final String sid) {
        final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
        final EsUrlQueueService urlQueueService = ComponentUtil.getComponent(EsUrlQueueService.class);
        final EsDataService dataService = ComponentUtil.getComponent(EsDataService.class);

        try {
            // clear url filter
            urlFilterService.delete(sid);

        this.cacheFileSize = cacheFileSize;
        return this;
    }

    public ExtractData extract() {
        final ExtractorFactory extractorFactory = crawlerContainer.getComponent("extractorFactory");

        DeferredFileOutputStream dfos = null;
        try (DeferredFileOutputStream out = new DeferredFileOutputStream(cacheFileSize, "fess-extractor-", ".out", null)) {
            dfos = out;

        }
    }

    public Suggester suggester() {
        return suggester;
    }

    public void storeSearchLog() {
        final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);

        searchLogBhv.selectBulk(cb -> {
            final String from = LocalDateTime.now().minusDays(fessConfig.getPurgeSuggestSearchLogDay()).format(DateTimeFormatter.ISO_DATE);

        } finally {
            deleteTempFile(tempFile);
        }
    }

    private void importGsaXml(final String fileName, final File tempFile) {
        final GsaConfigParser configParser = ComponentUtil.getComponent(GsaConfigParser.class);
        try (final InputStream in = new FileInputStream(tempFile)) {
            configParser.parse(new InputSource(in));
        } catch (final IOException e) {

            file.delete();
            file.mkdirs();
            file.deleteOnExit();
            fileTransformer.setPath(file.getAbsolutePath());

            final Crawler crawler1 = getComponent(Crawler.class);
            crawler1.setBackground(true);
            ((UrlFilterImpl) crawler1.urlFilter).setIncludeFilteringPattern("$1$2$3.*");
            crawler1.addUrl(url1);

    private boolean available = true;

    public StandardCrawlerContainer() {
        initialize();
    }

    @Override
    public <T> T getComponent(final String name) {
        if ("crawlerContainer".equals(name)) {
            return (T) this;
        }

        final ComponentDef<?> componentDef = prototypeMap.get(name);
        if (componentDef != null) {

        }

        final Object redirectUrlObj = responseData.getMetaDataMap().get(LOCATION_HEADER);
        if (redirectUrlObj instanceof String) {
            final UrlConvertHelper urlConvertHelper = crawlerContainer.getComponent("urlConvertHelper");
            resultData.addUrl(RequestDataBuilder.newRequestData().get().url(urlConvertHelper.convert(redirectUrlObj.toString())).build());
        }

        return resultData;
    }

        }
        final CrawlerClientFactory factory = creator.get();

        final WebAuthenticationService webAuthenticationService = ComponentUtil.getComponent(WebAuthenticationService.class);
        final RequestHeaderService requestHeaderService = ComponentUtil.getComponent(RequestHeaderService.class);
        final FessConfig fessConfig = ComponentUtil.getFessConfig();

        // HttpClient Parameters

        taskList.clear();
        if (logger.isDebugEnabled()) {
            logger.debug("Storing {} thumbnail tasks.", list.size());
        }
        final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
        thumbnailQueueBhv.batchInsert(list);
    }

    public int generate(final ExecutorService executorService, final boolean cleanup) {

                && urlSet.add(d.getUrl() + "\n" + d.getMetaData()) && crawlerContext.urlFilter.match(d.getUrl())).map(d -> {
                    final UrlQueue<?> uq = crawlerContainer.getComponent("urlQueue");
                    uq.setCreateTime(SystemUtil.currentTimeMillis());
                    uq.setDepth(depth);
                    uq.setMethod(Constants.GET_METHOD);
                    uq.setParentUrl(url);