dataCrawlingThreadStatusList.clear();

        // put cralwing info
        final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();

        final long execTime = systemHelper.getCurrentTimeAsLong() - startTime;
        crawlingInfoHelper.putToInfoMap(Constants.DATA_CRAWLING_EXEC_TIME, Long.toString(execTime));
        if (logger.isInfoEnabled()) {

        return asEsIndexType();
    }

    @Override
    protected String asEsIndex() {
        return "fess_config.crawling_info";
    }

    @Override
    public String asEsIndexType() {
        return "crawling_info";
    }

    @Override
    public String asEsSearchType() {
        return "crawling_info";
    }

    @Override
    public CrawlingInfoDbm asDBMeta() {
        return CrawlingInfoDbm.getInstance();

    public static final String ERRORS_design_file_is_unsupported_type = "{errors.design_file_is_unsupported_type}";

    /** The key of the message: Failed to create a crawling config. */
    public static final String ERRORS_failed_to_create_crawling_config_at_wizard = "{errors.failed_to_create_crawling_config_at_wizard}";

    /** The key of the message: This feature is disabled. */

    @Override
    public CrawlingInfoParamDbm asDBMeta() {
        return CrawlingInfoParamDbm.getInstance();
    }

    @Override
    public String asTableDbName() {
        return "crawling_info_param";
    }

    // ===================================================================================
    //                                                                              Source

                fileConfigService.store(fConfig);
            }
            return configName;
        } catch (final Exception e) {
            logger.warn("Failed to create crawling config: {}", form.crawlingConfigPath, e);
            throwValidationError(messages -> messages.addErrorsFailedToCreateCrawlingConfigAtWizard(GLOBAL),
                    () -> asHtml(path_AdminWizard_AdminWizardConfigJsp));

    //                                                                       =============
    @Override
    public String asTableDbName() {
        return "crawling_info";
    }

    @Override
    public String xgetAliasName() {
        return "crawling_info";
    }

    // ===================================================================================

                    if (logger.isInfoEnabled()) {
                        logger.info("Terminating indexUpdater. emptyListCount is over {}.", maxEmptyListCount);
                    }
                    // terminate crawling
                    finishCrawling = true;
                    forceStop();
                    if (fessConfig.getIndexerThreadDumpEnabledAsBoolean()) {
                        ThreadDumpUtil.printThreadDump();

    /** The key of the configuration. e.g. 100 */
    String PAGE_CRAWLING_INFO_PARAM_MAX_FETCH_SIZE = "page.crawling.info.param.max.fetch.size";

    /** The key of the configuration. e.g. 1000 */
    String PAGE_CRAWLING_INFO_MAX_FETCH_SIZE = "page.crawling.info.max.fetch.size";

    /** The key of the configuration. e.g. 100 */

    public static final String INCREMENTAL_CRAWLING_PROPERTY = "crawling.incremental";

    public static final String CRAWLING_THREAD_COUNT_PROPERTY = "crawling.thread.count";

    public static final String CRAWLING_USER_AGENT_PROPERTY = "crawling.user.agent";

    public static final String DAY_FOR_CLEANUP_PROPERTY = "day.for.cleanup";

public class Crawler {

    private static final Logger logger = LogManager.getLogger(Crawler.class);

    private static final String WEB_FS_CRAWLING_PROCESS = "WebFsCrawler";

    private static final String DATA_CRAWLING_PROCESS = "DataStoreCrawler";

    private static AtomicBoolean running = new AtomicBoolean(false);

    private static Queue<String> errors = new ConcurrentLinkedQueue<>();