// Compute the next set of routes to attempt.
    val routes = mutableListOf<Route>()
    while (hasNextProxy()) {
      // Postponed routes are always tried last. For example, if we have 2 proxies and all the
      // routes for proxy1 should be postponed, we'll move to proxy2. Only after we've exhausted
      // all the good routes will we attempt the postponed routes.
      val proxy = nextProxy()

                    request.setPath(dunc);
                }

                if (!t.getShare().equals(dr.getShare())) {
                    // this should only happen for standalone roots or if the DC/domain root lookup failed
                    IOException last;
                    final DfsReferralData start = dr;
                    do {
                        if (log.isDebugEnabled()) {

import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *
 * <p>The robots.txt protocol is implemented according to the standard specification,

      "https://www.allizom.org/robots.txt",
      "https://tls13.crypto.mozilla.org/",
      "https://tls.ctf.network/robots.txt",
      "https://rustls.jbp.io/",
      "https://h2o.examp1e.net",
      "https://mew.org/",
      "https://tls13.baishancloud.com/",
      "https://tls13.akamai.io/",
      "https://swifttls.org/",
      "https://www.googleapis.com/robots.txt",
      "https://graph.facebook.com/robots.txt",

    }

    /**
     * Returns the set of robots.txt URLs.
     * @return The set of robots.txt URLs.
     */
    public Set<String> getRobotsTxtUrlSet() {
        return robotsTxtUrlSet;
    }

    /**
     * Sets the set of robots.txt URLs.
     * @param robotsTxtUrlSet The set of robots.txt URLs.
     */
    public void setRobotsTxtUrlSet(final Set<String> robotsTxtUrlSet) {

   */
  internal fun planReusePooledConnection(
    planToReplace: ConnectPlan? = null,
    routes: List<Route>? = null,
  ): ReusePlan? {
    val result =
      connectionPool.callAcquirePooledConnection(
        doExtensiveHealthChecks = doExtensiveHealthChecks,
        address = address,
        call = call,
        routes = routes,
        requireMultiplexed = planToReplace != null && planToReplace.isReady,

 * </ul>
 *
 * <p>References:</p>
 * <ul>
 * <li><a href="https://datatracker.ietf.org/doc/html/rfc9309">RFC 9309 - Robots Exclusion Protocol</a></li>
 * <li><a href="https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt">
 * Google's robots.txt Specification</a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

  val responses = mutableListOf<Response>()
  val routes = mutableListOf<Route>()

  fun onlyResponse() = responses.single()

  fun onlyRoute() = routes.single()

  @Throws(IOException::class)
  override fun authenticate(
    route: Route?,
    response: Response,
  ): Request? {
    if (route == null) throw NullPointerException("route == null")
    responses += response
    routes += route

    dns.assertRequests(uriHost)
    val routes = selection.routes
    assertRoute(routes[0], address, Proxy.NO_PROXY, dns.lookup(uriHost, 0), uriPort)
    assertRoute(routes[1], address, Proxy.NO_PROXY, dns.lookup(uriHost, 1), uriPort)
    assertThat(selection.next()).isSameInstanceAs(routes[0])
    assertThat(selection.next()).isSameInstanceAs(routes[1])
    assertThat(selection.hasNext()).isFalse()

      pingIntervalMillis = pingIntervalMillis,
      retryOnConnectionFailure = retryOnConnectionFailure,
      call = call,
      routePlanner = routePlanner,
      route = route,
      routes = routes,
      attempt = attempt,
      tunnelRequest = tunnelRequest,
      connectionSpecIndex = connectionSpecIndex,
      isTlsFallback = isTlsFallback,
    )

  override fun connectTcp(): ConnectResult {