Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 3 of 3 for isCrawlerIgnoreRobotsTxt (0.16 sec)

  1. src/test/java/org/codelibs/fess/es/config/exentity/WebConfigTest.java

                    return systemPropMap.getOrDefault(key, defaultValue);
                }
    
                @Override
                public boolean isCrawlerIgnoreRobotsTxt() {
                    return false;
                }
    
                @Override
                public String getHttpProxyHost() {
                    return StringUtil.EMPTY;
                }
    
                @Override
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Thu May 09 09:48:04 UTC 2024
    - 7.6K bytes
    - Viewed (0)
  2. src/main/java/org/codelibs/fess/es/config/exentity/WebConfig.java

            }
    
            // robots txt enabled
            if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
                paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
            }
    
            final String userAgent = getUserAgent();
            if (!StringUtil.isNotBlank(userAgent) || (userAgent.startsWith(Constants.CRAWLING_USER_AGENT_PREFIX)
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Fri Oct 11 21:11:58 UTC 2024
    - 9.8K bytes
    - Viewed (0)
  3. src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

         * The value is, e.g. false <br>
         * @return The determination, true or false. (if not found, exception but basically no way)
         */
        boolean isCrawlerIgnoreRobotsTxt();
    
        /**
         * Get the value for the key 'crawler.ignore.robots.tags'. <br>
         * The value is, e.g. false <br>
    Registered: Thu Oct 31 13:40:30 UTC 2024
    - Last Modified: Fri Oct 11 21:11:58 UTC 2024
    - 468.5K bytes
    - Viewed (1)
Back to top