Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 2 of 2 for bowker (0.02 sec)

  2. github.com/codelibs/fess-crawler

    .travis.yml 

    dist: xenial

sudo: required
services:
  - docker

language: java

jdk:
  - openjdk11

cache:
  directories:
    - $HOME/.m2/repository

before_script:
  - java -version

script:
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sat Dec 07 14:01:56 UTC 2019
    - 202 bytes
    - Viewed (1)
  3. github.com/codelibs/fess-crawler

    fess-crawler/src/main/java/org/codelibs/fess/crawler/helper/RobotsTxtHelper.java 

     * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sun Jul 06 02:13:03 UTC 2025
    - 7.7K bytes
    - Viewed (0)
Back to top