}

        // robots txt enabled
        if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
            paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
        }

        final String userAgent = getUserAgent();
        if (!StringUtil.isNotBlank(userAgent) || (userAgent.startsWith(Constants.CRAWLING_USER_AGENT_PREFIX)

     * comment: Whether to ignore robots.txt during crawling.
     * @return The determination, true or false. (if not found, exception but basically no way)
     */
    boolean isCrawlerIgnoreRobotsTxt();

    /**
     * Get the value for the key 'crawler.ignore.robots.tags'. <br>
     * The value is, e.g. false <br>
     * comment: Whether to ignore robots meta tags during crawling.