Search Options

Display Count
Sort
Preferred Language
Advanced Search

Results 1 - 4 of 4 for isCrawlerIgnoreRobotsTags (0.12 seconds)

  1. src/test/java/org/codelibs/fess/crawler/transformer/FessXpathTransformerTest.java

                }
            };
            transformer.fessConfig = new FessConfig.SimpleImpl() {
                private static final long serialVersionUID = 1L;
    
                @Override
                public boolean isCrawlerIgnoreRobotsTags() {
                    return false;
                };
            };
    
            final ResponseData responseData = new ResponseData();
            responseData.setUrl("http://example.com/");
    
    Created: Tue Mar 31 13:07:34 GMT 2026
    - Last Modified: Fri Mar 13 23:01:26 GMT 2026
    - 62.9K bytes
    - Click Count (0)
  2. src/test/java/org/codelibs/fess/opensearch/config/exentity/WebConfigTest.java

    Shinsuke Sugaya <******@****.***> 1773442886 +0900
    Created: Tue Mar 31 13:07:34 GMT 2026
    - Last Modified: Fri Mar 13 23:01:26 GMT 2026
    - 7.7K bytes
    - Click Count (0)
  3. src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java

            final String ignore = configMap.get(Config.IGNORE_ROBOTS_TAGS);
            if (ignore == null) {
                if (fessConfig.isCrawlerIgnoreRobotsTags()) {
                    return;
                }
            } else if (Boolean.parseBoolean(ignore)) {
                return;
            }
    
            // meta tag
            try {
    Created: Tue Mar 31 13:07:34 GMT 2026
    - Last Modified: Thu Mar 12 01:46:45 GMT 2026
    - 55.3K bytes
    - Click Count (0)
  4. src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

         * comment: Whether to ignore robots meta tags during crawling.
         * @return The determination, true or false. (if not found, exception but basically no way)
         */
        boolean isCrawlerIgnoreRobotsTags();
    
        /**
         * Get the value for the key 'crawler.ignore.content.exception'. <br>
         * The value is, e.g. true <br>
         * comment: Whether to ignore content exceptions during crawling.
    Created: Tue Mar 31 13:07:34 GMT 2026
    - Last Modified: Sat Mar 28 06:59:19 GMT 2026
    - 576.9K bytes
    - Click Count (2)
Back to Top