Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 2 of 2 for joilla (0.22 sec)

  1. fess-crawler/src/test/java/org/codelibs/fess/crawler/helper/RobotsTxtHelperTest.java

            try {
                robotsTxt = robotsTxtHelper.parse(in);
            } finally {
                CloseableUtil.closeQuietly(in);
            }
    
            for (String userAgent : new String[] { "FessCrawler", "FessCrawler/1.0", "Mozilla FessCrawler" }) {
                assertTrue(robotsTxt.allows("/aaa", userAgent));
                assertTrue(robotsTxt.allows("/private/", userAgent));
                assertTrue(robotsTxt.allows("/private/index.html", userAgent));
    Registered: Sat Dec 20 11:21:39 UTC 2025
    - Last Modified: Mon Nov 24 03:59:47 UTC 2025
    - 20.6K bytes
    - Viewed (0)
  2. src/main/java/org/codelibs/fess/Constants.java

        // User Agent Configuration
        // ============================================================
    
        /** Prefix for Fess crawler user agent string. */
        public static final String CRAWLING_USER_AGENT_PREFIX = "Mozilla/5.0 (compatible; Fess/";
    
        /** Suffix for Fess crawler user agent string with bot information URL. */
        public static final String CRAWLING_USER_AGENT_SUFFIX = "; +http://fess.codelibs.org/bot.html)";
    
    Registered: Sat Dec 20 09:19:18 UTC 2025
    - Last Modified: Sat Dec 13 02:21:17 UTC 2025
    - 35.2K bytes
    - Viewed (0)
Back to top