User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler

     *
     * @return the HTTP method as a String.
     */
    String getMethod();

    /**
     * Sets the HTTP method for the URL queue.
     *
     * @param method the HTTP method to be set (e.g., GET, POST, etc.)
     */
    void setMethod(String method);

    /**
     * Retrieves the URL from the queue.
     *
     * @return the URL as a String.
     */
    String getUrl();

    /**

     *
     *  @param contextNode The node to start searching from.
     *  @param expression A valid XPath string.
     *  @return An XPathEvaluationResult, which can be used to obtain a string, number, nodelist, etc, should never be null.
     *
     * @throws XPathExpressionException if an XPath expression error occurs.
     */

        }
    }

    /**
     * Inserts a single document into the OpenSearch index.
     *
     * @param target The object to insert.
     * @param opType The operation type (CREATE, INDEX, etc.).
     * @return The IndexResponse from OpenSearch.
     * @throws OpenSearchAccessException if the insertion fails.
     * @throws CrawlingAccessException if there's a conflict during insertion.
     */

## Architecture

### Multi-Module Structure

```
fess-crawler-parent/
├── fess-crawler/              # Core crawler framework
│   ├── client/               # Protocol clients (HTTP, FTP, SMB, etc.)
│   ├── extractor/           # Content extractors
│   ├── transformer/         # Data transformers
│   └── service/             # Core services
├── fess-crawler-lasta/       # LastaFlute DI integration

-->
<!--
  Description: This xml file defines the valid mime types used by Tika.
  The mime type data within this file is based on information from various
  sources like Apache Nutch, Apache HTTP Server, the file(1) command, etc.

  Notes:
   * Tika supports a wider range of match types than Freedesktop does
   * Glob patterns must be unique, if there's a clash assign to the most
     popular format