# Release 2.0.0

## Major Features and Improvements

TensorFlow 2.0 focuses on **simplicity** and **ease of use**, featuring updates
like:

*   Easy model building with Keras and eager execution.
*   Robust model deployment in production on any platform.
*   Powerful experimentation for research.
*   API simplification by reducing duplication and removing deprecated
    endpoints.

labels.crawling_info_thread_dump=Vidage de thread
labels.crawling_info_CrawlerStartTime=Heure de début du robot d'exploration
labels.crawling_info_CrawlerEndTime=Heure de fin du robot d'exploration
labels.crawling_info_CrawlerExecTime=Temps d'exécution du robot d'exploration
labels.crawling_info_CrawlerStatus=Statut du robot d'exploration
labels.crawling_info_WebFsCrawlExecTime=Temps d'exécution de l'exploration (Web/Fichier)

    /** Whether robots.txt processing is enabled. */
    protected boolean enabled = true;

    /**
     * Creates a new RobotsTxtHelper instance.
     */
    public RobotsTxtHelper() {
        // Default constructor
    }

    /**
     * Parses a robots.txt file from the given input stream using UTF-8 encoding.

import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *
 * <p>The robots.txt protocol is implemented according to the standard specification,

    }

    /**
     * Returns the set of robots.txt URLs.
     * @return The set of robots.txt URLs.
     */
    public Set<String> getRobotsTxtUrlSet() {
        return robotsTxtUrlSet;
    }

    /**
     * Sets the set of robots.txt URLs.
     * @param robotsTxtUrlSet The set of robots.txt URLs.
     */
    public void setRobotsTxtUrlSet(final Set<String> robotsTxtUrlSet) {

    client.newCall(request).execute().use {
      assertThat(it.protocol).isEqualTo(Protocol.HTTP_2)
      assertThat(it.handshake!!.tlsVersion).isEqualTo(TlsVersion.TLS_1_3)
    }
  }

  @Test
  @Disabled
  fun testGoogle() {
    assumeNetwork()

    val request = Request.Builder().url("https://google.com/robots.txt").build()

  val clientBuilder =
    OkHttpClient.Builder().connectionSpecs(listOf(ConnectionSpec.CLEARTEXT)).dns {
      throw UnknownHostException("shortcircuit")
    }

  val request = Request("http://google.com/robots.txt".toHttpUrl())

  @Test
  fun testHttpLoggingInterceptor() {
    val interceptor =
      HttpLoggingInterceptor().apply {
        level = HttpLoggingInterceptor.Level.BASIC
      }

      "https://www.allizom.org/robots.txt",
      "https://tls13.crypto.mozilla.org/",
      "https://tls.ctf.network/robots.txt",
      "https://rustls.jbp.io/",
      "https://h2o.examp1e.net",
      "https://mew.org/",
      "https://tls13.baishancloud.com/",
      "https://tls13.akamai.io/",
      "https://swifttls.org/",
      "https://www.googleapis.com/robots.txt",
      "https://graph.facebook.com/robots.txt",

            httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

      .Builder()
      .addInterceptor(CompressionInterceptor(Zstd))
      .build()

  sendRequest("https://developers.facebook.com/docs/", client)
  sendRequest("https://www.facebook.com/robots.txt", client)
  sendRequest("https://www.instagram.com/robots.txt", client)
}

private fun sendRequest(
  url: String,
  client: OkHttpClient,
) {
  val req = Request.Builder().url(url).build()

  client.newCall(req).execute().use {