return getSystemProperty(Constants.PURGE_BY_BOTS_PROPERTY, Constants.DEFAULT_PURGE_BY_BOTS);
    }

    default boolean hasNotification() {
        return StringUtil.isNotBlank(getNotificationTo()) || StringUtil.isNotBlank(getSlackWebhookUrls());
    }

    default void setSlackWebhookUrls(final String value) {
        setSystemProperty(Constants.SLACK_WEBHOOK_URLS_PROPERTY, value);
    }

        // search
        if (StringUtil.isNotBlank(failureUrlPager.url)) {
            cb.query().setUrl_Wildcard(failureUrlPager.url);
        }

        if (StringUtil.isNotBlank(failureUrlPager.errorCountMax)) {
            cb.query().setErrorCount_LessEqual(Integer.parseInt(failureUrlPager.errorCountMax));
        }
        if (StringUtil.isNotBlank(failureUrlPager.errorCountMin)) {

            if (StringUtil.isNotBlank(httpAddress)) {
                cmdList.add("-D" + Constants.FESS_SEARCH_ENGINE_HTTP_ADDRESS + "=" + httpAddress);
            }
        }

        final String systemLastaEnv = System.getProperty("lasta.env");
        if (StringUtil.isNotBlank(systemLastaEnv)) {
            if ("web".equals(systemLastaEnv)) {

            if (StringUtil.isNotBlank(httpAddress)) {
                cmdList.add("-D" + Constants.FESS_SEARCH_ENGINE_HTTP_ADDRESS + "=" + httpAddress);
            }
        }

        final String systemLastaEnv = System.getProperty("lasta.env");
        if (StringUtil.isNotBlank(systemLastaEnv)) {
            if ("web".equals(systemLastaEnv)) {

        putResultDataBody(dataMap, fessConfig.getIndexFieldVirtualHost(),
                stream(crawlingConfig.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
        // TODO date
        // lang
        if (StringUtil.isNotBlank(fessConfig.getCrawlerDocumentFileDefaultLang())) {
            putResultDataBody(dataMap, fessConfig.getIndexFieldLang(), fessConfig.getCrawlerDocumentFileDefaultLang());

        final String proxyHost = fessConfig.getHttpProxyHost();
        final String proxyPort = fessConfig.getHttpProxyPort();
        if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
            paramMap.put(Param.Client.PROXY_HOST, proxyHost);
            paramMap.put(Param.Client.PROXY_PORT, proxyPort);
            final String proxyUsername = fessConfig.getHttpProxyUsername();

        if (StringUtil.isNotBlank(appType)) {
            return appType;
        }
        return StringUtil.EMPTY;
    }

    public static OptionalEntity<String> getOverrideConfPath() {
        if (FESS_APP_DOCKER.equalsIgnoreCase(getAppType())) {
            final String confPath = System.getenv(FESS_OVERRIDE_CONF_PATH);
            if (StringUtil.isNotBlank(confPath)) {

            if (StringUtil.isNotBlank(getIncludedDocUrls())) {
                final List<Pattern> urlPatterList = new ArrayList<>();
                final String[] urls = getIncludedDocUrls().split("[\r\n]");
                for (final String u : urls) {
                    final String v = systemHelper.normalizeConfigPath(u);
                    if (StringUtil.isNotBlank(v)) {

            }

            final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();

            // set urls
            split(urlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
                if (!urlValue.startsWith("#") && protocolHelper.isValidWebProtocol(urlValue)) {
                    final String u = duplicateHostHelper.convert(urlValue);

                stream(crawlingConfig.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
        // id
        putResultDataBody(dataMap, fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
        // parentId
        String parentUrl = responseData.getParentUrl();
        if (StringUtil.isNotBlank(parentUrl)) {
            parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);