client =
      OkHttpClient
        .Builder()
        .sslSocketFactory(certificates.sslSocketFactory(), certificates.trustManager)
        .build()
  }

  fun run() {
    showUrl("https://squareup.com/robots.txt")
    showUrl("https://publicobject.com/helloworld.txt")
  }

  private fun showUrl(url: String) {
    val request = Builder().url(url).build()
    client
      .newCall(request)
      .execute()

          handshakeCertificates.trustManager,
        )
    }

    val client = clientBuilder.build()

    val request =
      Request
        .Builder()
        .url("https://valid-isrgrootx1.letsencrypt.org/robots.txt")
        .build()
    client.newCall(request).execute().use { response ->
      assertThat(response.code).isEqualTo(404)
      assertThat(response.protocol).isEqualTo(Protocol.HTTP_2)
    }
  }

     *       to the root of the output directory (see {@link Project#getOutputDirectory(ProjectScope)}).</li>
     *   <li><strong>Relative Path</strong> (e.g., {@code Path.of("META-INF/resources")}) - The path is
     *       intended to be resolved relative to the output directory for this source root's {@link #scope()}.
     *       <ul>

    /** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */

- **Testing**: JUnit 4, UTFlute, Mockito 5.7.0
- **Storage**: In-memory (default), OpenSearch (optional)

### Protocols

- **HTTP/HTTPS**: Full crawling, cookies, auth, robots.txt
- **File**: Local/network file systems
- **FTP**: With authentication
- **SMB/CIFS**: Windows shares (SMB1/SMB2+)
- **Storage**: MinIO/S3-compatible

### Content Formats

    /**
     * Gets the root directory of the project, which is the parent directory
     * containing the {@code .mvn} directory or flagged with {@code root="true"}.
     *
     * @return the root directory of the project
     * @throws IllegalStateException if the root directory could not be found
     * @see Session#getRootDirectory()
     */
    @Nonnull
    Path getRootDirectory();

    /**

import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *
 * <p>The robots.txt protocol is implemented according to the standard specification,

)

// Boot modTimes up to disk count, setting the value to time sentinel.
func bootModtimes(diskCount int) []time.Time {
	modTimes := make([]time.Time, diskCount)
	// Boots up all the modtimes.
	for i := range modTimes {
		modTimes[i] = timeSentinel
	}
	return modTimes
}

func listObjectETags(partsMetadata []FileInfo, errs []error, quorum int) (etags []string) {

 * </ul>
 *
 * <p>References:</p>
 * <ul>
 * <li><a href="https://datatracker.ietf.org/doc/html/rfc9309">RFC 9309 - Robots Exclusion Protocol</a></li>
 * <li><a href="https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt">
 * Google's robots.txt Specification</a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

gc.ca

// cat : https://www.iana.org/domains/root/db/cat.html
cat

// cc : https://www.iana.org/domains/root/db/cc.html
cc

// cd : https://www.iana.org/domains/root/db/cd.html
// see also: https://www.nic.cd/domain/insertDomain_2.jsp?act=1
cd
gov.cd

// cf : https://www.iana.org/domains/root/db/cf.html
cf

// cg : https://www.iana.org/domains/root/db/cg.html
cg