Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 2 of 2 for Sitemap (0.05 sec)

  1. fess-crawler/src/main/java/org/codelibs/fess/crawler/client/http/HcHttpClient.java

                        if (robotsTxt != null) {
                            final String[] sitemaps = robotsTxt.getSitemaps();
                            if (sitemaps.length > 0) {
                                crawlerContext.addSitemaps(sitemaps);
                            }
    
                            final RobotsTxt.Directive directive = robotsTxt.getMatchedDirective(userAgent);
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Thu Aug 07 02:55:08 UTC 2025
    - 52.2K bytes
    - Viewed (0)
  2. fess-crawler-lasta/src/main/resources/crawler/extractor.xml

    				"image/x-freehand",
    				"image/x-jp2-codestream",
    				"image/x-jp2-container",
    				"image/x-niff",
    				"image/x-pcx",
    				"image/x-pict",
    				"image/x-portable-anymap",
    				"image/x-portable-bitmap",
    				"image/x-portable-graymap",
    				"image/x-portable-pixmap",
    				"image/x-raw-adobe",
    				"image/x-raw-hasselblad",
    				"image/x-raw-fuji",
    				"image/x-raw-canon",
    				"image/x-raw-kodak",
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sat Aug 01 21:40:30 UTC 2020
    - 49K bytes
    - Viewed (0)
Back to top