Search Options

Display Count
Sort
Preferred Language
Advanced Search

Results 1 - 2 of 2 for filemap (0.04 seconds)

  2. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/client/http/Hc4HttpClient.java 

            }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {
    Created: Sun Apr 12 03:50:13 GMT 2026
    - Last Modified: Fri Jan 09 23:46:52 GMT 2026
    - 54.4K bytes
    - Click Count (0)
  3. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/client/http/Hc5HttpClient.java 

            }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {
    Created: Sun Apr 12 03:50:13 GMT 2026
    - Last Modified: Sat Jan 31 12:23:29 GMT 2026
    - 62.2K bytes
    - Click Count (0)
Back to Top