}

        if (cae.isDebugEnabled()) {
            logger.debug("[{}] Crawling Access Exception at {}", failureUrlId, urlQueue.getUrl(), cae);
        } else if (cae.isInfoEnabled()) {
            logger.info("[{}] {}", failureUrlId, cae.getMessage());
        } else if (cae.isWarnEnabled()) {
            logger.warn("[{}] Crawling Access Exception at {}", failureUrlId, urlQueue.getUrl(), cae);

        cb.query().addOrderBy_CreatedTime_Desc();
    }

    protected void setupStoreCondition(final CrawlingInfo crawlingInfo) {
        if (crawlingInfo == null) {
            throw new FessSystemException("Crawling Session is null.");
        }
        final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
        if (crawlingInfo.getCreatedTime() == null) {
            crawlingInfo.setCreatedTime(now);
        }

        final StringBuilder resultBuf = new StringBuilder();

        // purge crawling sessions
        try {
            crawlingInfoService.deleteBefore(systemHelper.getCurrentTimeAsLong());
        } catch (final Exception e) {
            logger.error("Failed to purge crawling sessions.", e);
            resultBuf.append(e.getMessage()).append("\n");
        }

        // purge search logs

            logger.warn("Failed to store crawling information.", e);
        }

        try {
            return crawler.doCrawl(options);
        } finally {
            try {
                crawlingInfoHelper.store(options.sessionId, false);
            } catch (final Exception e) {
                logger.warn("Failed to store crawling information.", e);
            }

You can register crawling targets in the Admin UI on the (Web, File, Data Store) crawler configuration pages, and then start the Crawler manually on the [Scheduler page](https://fess.codelibs.org/14.17/admin/scheduler-guide.html).

## Migration from another search provider

Please see [MIGRATION.md](MIGRATION.md).

## Data Store

errors.design_file_name_is_invalid=Der Dateiname ist ungültig.
errors.design_file_is_unsupported_type=Die Dateiart wird nicht unterstützt.
errors.failed_to_create_crawling_config_at_wizard=Erstellung der Crawling-Konfiguration fehlgeschlagen.
errors.design_editor_disabled=Diese Funktion ist deaktiviert.
errors.not_found_on_file_system=Nicht gefunden: {0}

    protected ThreadLocal<String[]> sitemapsLocal = new ThreadLocal<>();

    /** The number of a thread */
    protected int numOfThread = 10;

    protected int maxThreadCheckCount = 20;

    /** a max depth for crawling. -1 is no depth check. */
    protected int maxDepth = -1;

    /** a max count to access urls. 0 is no limit to access it. */
    protected long maxAccessCount = 0;

    public String getSessionId() {

errors.design_file_name_is_invalid=The file name is invalid.
errors.design_file_is_unsupported_type=The kind of file is unsupported.
errors.failed_to_create_crawling_config_at_wizard=Failed to create a crawling config.
errors.design_editor_disabled=This feature is disabled.
errors.not_found_on_file_system=Not Found: {0}
errors.could_not_open_on_system=Could not open {0}. <br/>Please check if the file is associated with an application.

    public static final String INCREMENTAL_CRAWLING_PROPERTY = "crawling.incremental";

    public static final String CRAWLING_THREAD_COUNT_PROPERTY = "crawling.thread.count";

    public static final String CRAWLING_USER_AGENT_PROPERTY = "crawling.user.agent";

    public static final String DAY_FOR_CLEANUP_PROPERTY = "day.for.cleanup";

        if (sessionId == null) {
            return null;
        }
        return pathMappingMap.get(sessionId);
    }

    public String replaceUrl(final String sessionId, final String url) { // for crawling
        final List<PathMapping> pathMappingList = getPathMappingList(sessionId);
        if (pathMappingList == null) {
            return url;
        }
        return replaceUrl(pathMappingList, url);
    }