public static final MediaType I_CALENDAR_UTF_8 = createConstantUtf8(TEXT_TYPE, "calendar");

  /**
   * As described in <a href="https://www.rfc-editor.org/rfc/rfc7763.html">RFC 7763</a>, this
   * constant ({@code text/markdown}) is used for Markdown documents.
   *
   * @since 33.3.0
   */
  public static final MediaType MD_UTF_8 = createConstantUtf8(TEXT_TYPE, "markdown");

  public static final MediaType I_CALENDAR_UTF_8 = createConstantUtf8(TEXT_TYPE, "calendar");

  /**
   * As described in <a href="https://www.rfc-editor.org/rfc/rfc7763.html">RFC 7763</a>, this
   * constant ({@code text/markdown}) is used for Markdown documents.
   *
   * @since 33.3.0
   */
  public static final MediaType MD_UTF_8 = createConstantUtf8(TEXT_TYPE, "markdown");

                } else {
                    // Same length - Allow takes precedence (RFC 9309)
                    return true;
                }
            }
        }

        /**
         * Adds an allowed path pattern to this directive.
         * Supports wildcards (*) and end-of-path ($) according to RFC 9309.
         * @param path the path pattern to allow
         */

OkHttp follows modern HTTP specifications such as

* HTTP Semantics - [RFC 9110](https://datatracker.ietf.org/doc/html/rfc9110)
* HTTP Caching- [RFC 9111](https://datatracker.ietf.org/doc/html/rfc9111)
* HTTP/1.1 - [RFC 9112](https://datatracker.ietf.org/doc/html/rfc9112)
* HTTP/2 - [RFC 9113](https://datatracker.ietf.org/doc/html/rfc9113)
* Websockets - [RFC 6455](https://datatracker.ietf.org/doc/html/rfc6455)

  /**
   * The "base64" base encoding specified by <a
   * href="http://tools.ietf.org/html/rfc4648#section-4">RFC 4648 section 4</a>, Base 64 Encoding.
   * (This is the same as the base 64 encoding from <a
   * href="http://tools.ietf.org/html/rfc3548#section-3">RFC 3548</a>.)
   *
   * <p>The character {@code '='} is used for padding, but can be {@linkplain #omitPadding()

# Test robots.txt for wildcard (*) and end-of-path ($) support
# Based on RFC 9309 specification

# Test wildcard patterns
User-agent: WildcardBot
Disallow: /*.pdf$
Disallow: /admin/*.php
Disallow: /*/private/
Allow: /public/*.html

# Test end-of-path ($) patterns
User-agent: EndPathBot
Disallow: /fish$
Disallow: /temp$
Allow: /fishing

# Test complex patterns
User-agent: ComplexBot
Disallow: /
Allow: /$

import org.codelibs.fess.crawler.entity.RobotsTxt;
import org.codelibs.fess.crawler.entity.RobotsTxt.Directive;
import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Parser following RFC 9309 specification.
 *
 * <p>This implementation supports the following features:</p>
 * <ul>
 * <li>User-agent directive with wildcard (*) matching</li>
 * <li>Disallow and Allow directives with pattern matching</li>

      .newBuilder()
      .body(RealResponseBody(contentType, contentLength, cacheWritingSource.buffer()))
      .build()
  }

  companion object {
    /** Combines cached headers with a network headers as defined by RFC 7234, 4.3.4. */
    private fun combine(
      cachedHeaders: Headers,
      networkHeaders: Headers,
    ): Headers {
      val result = Headers.Builder()

      for (index in 0 until cachedHeaders.size) {

import okhttp3.internal.ws.WebSocketProtocol.PAYLOAD_SHORT
import okhttp3.internal.ws.WebSocketProtocol.toggleMask
import okio.Buffer
import okio.BufferedSource
import okio.ByteString

/**
 * An [RFC 6455][rfc_6455]-compatible WebSocket frame reader.
 *
 * This class is not thread safe.
 *
 * [rfc_6455]: http://tools.ietf.org/html/rfc6455
 */
class WebSocketReader(
  private val isClient: Boolean,

**Registration**:
```java
extractorFactory.addExtractor("text/html", htmlExtractor, 2);  // Weight 2
extractorFactory.addExtractor("text/html", tikaExtractor, 1);  // Fallback
```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM