Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 4 of 4 for getDirective (0.07 sec)

  1. fess-crawler/src/test/java/org/codelibs/fess/crawler/entity/RobotsTxtTest.java

            // Test getDirective when directive doesn't exist
            RobotsTxt robotsTxt = new RobotsTxt();
    
            Directive directive = new Directive("MyBot");
            robotsTxt.addDirective(directive);
    
            Directive retrieved = robotsTxt.getDirective("OtherBot");
            assertNull(retrieved);
        }
    
        public void test_getDirectiveWithNull() {
            // Test getDirective with null user agent
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Thu Nov 13 13:29:22 UTC 2025
    - 14.4K bytes
    - Viewed (0)
  2. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/RobotsTxt.java

         * @return The Directive object matching the user agent, or null if no matching directive is found or if userAgent is null
         */
        public Directive getDirective(final String userAgent) {
            if (userAgent == null) {
                return null;
            }
            for (final Directive directive : directiveMap.values()) {
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Mon Nov 24 03:59:47 UTC 2025
    - 18.5K bytes
    - Viewed (0)
  3. fess-crawler/src/main/java/org/codelibs/fess/crawler/helper/RobotsTxtHelper.java

                            // Normalize user-agent to lowercase
                            final String userAgent = value.toLowerCase(Locale.ENGLISH);
                            Directive currentDirective = robotsTxt.getDirective(userAgent);
                            if (currentDirective == null) {
                                currentDirective = new Directive(userAgent);
                                robotsTxt.addDirective(currentDirective);
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Fri Nov 14 12:52:01 UTC 2025
    - 11.4K bytes
    - Viewed (0)
  4. fess-crawler/src/test/java/org/codelibs/fess/crawler/helper/RobotsTxtHelperTest.java

            assertTrue(robotsTxt.allows("/orphaned2/", "AnyBot"));
    
            // Test GoodBot - should parse valid directives and ignore invalid ones
            assertNotNull(robotsTxt.getDirective("goodbot"));
            assertFalse(robotsTxt.allows("/admin/", "GoodBot"));
            assertTrue(robotsTxt.allows("/public/", "GoodBot"));
            // Invalid directives should not cause parsing to fail
    
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Mon Nov 24 03:59:47 UTC 2025
    - 20.6K bytes
    - Viewed (0)
Back to top