L367: /** The key of the configuration. e.g. false */ L368: String CRAWLER_IGNORE_ROBOTS_TXT = "crawler.ignore.robots.txt"; L369: L370: /** The key of the configuration. e.g. false */ L371: String CRAWLER_IGNORE_ROBOTS_TAGS = "crawler.ignore.robots.tags"; L372: L373: /** The key of the configuration. e.g. true */ L374: String CRAWLER_IGNORE_CONTENT_EXCEPTION = "crawler.ignore.content.exception"; L375: L376: /** The key of the configuration. e.g. 404 */ L377: String CRAWLER_F...