Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 1 of 1 for isCrawlerIgnoreRobotsTxt (0.2 sec)

  2. github.com/codelibs/fess

    src/main/java/org/codelibs/fess/opensearch/config/exentity/WebConfig.java 

            }

        // robots txt enabled
        if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
            paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
        }

        final String userAgent = getUserAgent();
        if (!StringUtil.isNotBlank(userAgent) || (userAgent.startsWith(Constants.CRAWLING_USER_AGENT_PREFIX)
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Sat Mar 15 06:53:53 UTC 2025
    - 9.9K bytes
    - Viewed (0)
Back to top