final String ignore = configMap.get(Config.IGNORE_ROBOTS_TAGS);
        if (ignore == null) {
            if (fessConfig.isCrawlerIgnoreRobotsTags()) {
                return;
            }
        } else if (Boolean.parseBoolean(ignore)) {
            return;
        }

        // meta tag
        try {

     * comment: Whether to ignore robots meta tags during crawling.
     * @return The determination, true or false. (if not found, exception but basically no way)
     */
    boolean isCrawlerIgnoreRobotsTags();

    /**
     * Get the value for the key 'crawler.ignore.content.exception'. <br>
     * The value is, e.g. true <br>
     * comment: Whether to ignore content exceptions during crawling.