proxies.add(proxy)
    return this
  }

  override fun select(uri: URI): List<Proxy> {
    // Don't handle 'socket' schemes, which the RI's Socket class may request (for SOCKS).
    return if (uri.scheme == "http" || uri.scheme == "https") proxies else listOf(Proxy.NO_PROXY)
  }

  override fun connectFailed(
    uri: URI,
    sa: SocketAddress,
    ioe: IOException,
  ) {
  }

 * As policy, implementations of this interface are responsible for selecting which cookies to
 * accept and which to reject. A reasonable policy is to reject all cookies, though that may
 * interfere with session-based authentication schemes that require cookies.
 *
 * As persistence, implementations of this interface must also provide storage of cookies. Simple
 * implementations may store cookies in memory; sophisticated ones may use the file system or

            return entity;
        });
    }

    /**
     * Registers available protocol scheme items for web authentication forms.
     * Includes Basic, Digest, NTLM, and Form authentication schemes.
     *
     * @param data the render data to register the protocol scheme items with
     */
    protected void registerProtocolSchemeItems(final RenderData data) {

     * This method implements incremental crawling by comparing timestamps and checking document
     * expiration. It also handles special cases for different URL schemes (SMB, file, FTP).
     *
     * @param client the crawler client to use for accessing the URL
     * @param urlQueue the URL queue item containing the URL to check

    /**
     * @param scheme either "http" or "https".
     */
    fun scheme(scheme: String) =
      apply {
        when {
          scheme.equals("http", ignoreCase = true) -> this.scheme = "http"
          scheme.equals("https", ignoreCase = true) -> this.scheme = "https"
          else -> throw IllegalArgumentException("unexpected scheme: $scheme")
        }
      }

    /** The underlying index update callback to delegate operations to. */
    protected IndexUpdateCallback indexUpdateCallback;

    /** Factory for creating crawler clients to handle different URL schemes. */
    protected CrawlerClientFactory crawlerClientFactory;

    /** List of URLs to be deleted, cached for batch processing. */
    protected List<String> deleteUrlList = new ArrayList<>(100);

Use `Response.challenges()` to get the schemes and realms of any authentication challenges. When fulfilling a `Basic` challenge, use `Credentials.basic(username, password)` to encode the request header.

=== ":material-language-kotlin: Kotlin"
    ```kotlin

        application/json:
          schema:
            $ref: '#/components/schemas/Error'
    Unauthorized:
      description: Unauthorized request
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
    NotFound:
      description: Page not found
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'

                final String scheme = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".scheme");

                final AuthScheme authScheme = getAuthScheme(paramMap, webAuthName, scheme);
                final AuthScope authScope = getAuthScope(webAuthName, scheme, paramMap);
                final Credentials credentials = getCredentials(webAuthName, scheme, paramMap);

            proxyAddress.port,
            url.scheme,
            challenge.realm,
            challenge.scheme,
            url.toUrl(),
            Authenticator.RequestorType.PROXY,
          )
        } else {
          Authenticator.requestPasswordAuthentication(
            url.host,
            proxy.connectToInetAddress(url, dns),
            url.port,
            url.scheme,
            challenge.realm,