Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 2 of 2 for addDirective (0.07 sec)

  1. fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/RobotsTxt.java

         * where '*' is replaced with '.*' for pattern matching, and stored case-insensitively.
         *
         * @param directive The directive to add to the robots.txt rules
         */
        public void addDirective(final Directive directive) {
            directiveMap.put(Pattern.compile(directive.getUserAgent().replace("*", ".*"), Pattern.CASE_INSENSITIVE), directive);
        }
    
        /**
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sun Jul 06 02:13:03 UTC 2025
    - 10K bytes
    - Viewed (0)
  2. fess-crawler/src/main/java/org/codelibs/fess/crawler/helper/RobotsTxtHelper.java

                        if (currentDirective == null) {
                            currentDirective = new Directive(userAgent);
                            robotsTxt.addDirective(currentDirective);
                            currentDirectiveList.add(currentDirective);
                        }
                    } else {
                        isGroupRecordStarted = true;
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sun Jul 06 02:13:03 UTC 2025
    - 7.7K bytes
    - Viewed (0)
Back to top