- Sort Score
- Result 10 results
- Languages All
Results 1 - 2 of 2 for AnyBot (0.21 sec)
-
fess-crawler/src/test/java/org/codelibs/fess/crawler/helper/RobotsTxtHelperTest.java
// Test that orphaned directives (before any User-agent) are ignored // These should not affect any bot assertTrue(robotsTxt.allows("/orphaned1/", "AnyBot")); assertTrue(robotsTxt.allows("/orphaned2/", "AnyBot")); // Test GoodBot - should parse valid directives and ignore invalid ones assertNotNull(robotsTxt.getDirective("goodbot"));Registered: Sat Dec 20 11:21:39 UTC 2025 - Last Modified: Mon Nov 24 03:59:47 UTC 2025 - 20.6K bytes - Viewed (0) -
fess-crawler/src/test/java/org/codelibs/fess/crawler/entity/RobotsTxtTest.java
Directive directive = new Directive("*"); directive.addDisallow("/private/"); robotsTxt.addDirective(directive); Directive matched = robotsTxt.getMatchedDirective("AnyBot"); assertNotNull(matched); assertEquals("*", matched.getUserAgent()); } public void test_getMatchedDirectiveMostSpecific() {Registered: Sat Dec 20 11:21:39 UTC 2025 - Last Modified: Thu Nov 13 13:29:22 UTC 2025 - 14.4K bytes - Viewed (0)