crawler.urlFilter.addInclude(".*\\.pdf$");

// Exclude patterns  
crawler.urlFilter.addExclude(".*\\.js$");
crawler.urlFilter.addExclude(".*login.*");
```

## Supported Protocols and Formats

### Protocols
- **HTTP/HTTPS**: Full web crawling support with cookies, authentication, redirects
- **File System**: Local and network file system access
- **FTP**: FTP server crawling with authentication

| rename              | no        |

MinIO supports following FTP/SFTP based protocols to access and manage data.

- Secure File Transfer Protocol (SFTP) – Defined by the Internet Engineering Task Force (IETF) as an
  extended version of SSH 2.0, allowing file transfer over SSH and for use with Transport Layer
  Security (TLS) and VPN applications.

- File Transfer Protocol over SSL/TLS (FTPS) – Encrypted FTP communication via TLS certificates.

   */
  private fun enableProtocol(protocol: Protocol) {
    enableTls()
    client =
      client
        .newBuilder()
        .protocols(listOf(protocol, Protocol.HTTP_1_1))
        .build()
    server.protocols = client.protocols
  }

  private fun enableTls() {
    client =
      client
        .newBuilder()

}

public final class okhttp3/Protocol : java/lang/Enum {
	public static final field Companion Lokhttp3/Protocol$Companion;
	public static final field H2_PRIOR_KNOWLEDGE Lokhttp3/Protocol;
	public static final field HTTP_1_0 Lokhttp3/Protocol;
	public static final field HTTP_1_1 Lokhttp3/Protocol;
	public static final field HTTP_2 Lokhttp3/Protocol;
	public static final field HTTP_3 Lokhttp3/Protocol;

  }

  fun setUp(protocol: Protocol) {
    this.protocol = protocol
    platform.assumeNotOpenJSSE()
    if (protocol === Protocol.HTTP_2) {
      platform.assumeHttp2Support()
      server.useHttps(handshakeCertificates.sslSocketFactory())
      client =
        clientTestRule
          .newClientBuilder()
          .protocols(listOf(Protocol.HTTP_2, Protocol.HTTP_1_1))
          .sslSocketFactory(

 * If plugin or extension needs anything more complex feature wise (i.e. HTTP range support or alike) it should
 * probably roll its own.
 * <p>
 * This implementation is backed by Maven Resolver API, supported protocols and transport selection depends on it. If
 * resolver preference regarding transport is altered, it will affect this service as well.
 *
 * @since 4.0.0
 */
@Experimental
@Consumer

   *
   * For cipher suites, at least one of the [required cipher suites][cipherSuites] must match the
   * socket's enabled cipher suites. If there are no required cipher suites the socket must have at
   * least one cipher suite enabled.
   *
   * For protocols, at least one of the [required protocols][tlsVersions] must match the socket's
   * enabled protocols.

   * [rfc_2817]: http://www.ietf.org/rfc/rfc2817.txt
   */
  fun requiresTunnel(): Boolean {
    if (proxy.type() != Proxy.Type.HTTP) return false
    return (address.sslSocketFactory != null) ||
      (Protocol.H2_PRIOR_KNOWLEDGE in address.protocols)
  }

  override fun equals(other: Any?): Boolean =
    other is Route &&
      other.address == address &&
      other.proxy == proxy &&
      other.socketAddress == socketAddress

        throw UnknownServiceException(
          "CLEARTEXT communication to $host not permitted by network security policy",
        )
      }
    } else {
      if (Protocol.H2_PRIOR_KNOWLEDGE in route.address.protocols) {
        throw UnknownServiceException("H2_PRIOR_KNOWLEDGE cannot be used with HTTPS")
      }
    }

    val tunnelRequest =
      when {

     *
     * @param protocol
     *            The URL protocol. Must not be {@literal null} or empty.
     * @param factory
     *            The {@link Traverser} factory for the protocol. Must not be {@literal null}.
     */
    public static void addTraverserFactory(final String protocol, final TraverserFactory factory) {
        assertArgumentNotEmpty("protocol", protocol);