}
                    if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
                        nofollow = true;
                    }
                }
                if (noindex && nofollow) {
                    logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
                    throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
                }

crawler.document.html.canonical.xpath=//LINK[@rel='canonical'][1]/@href
# HTML tags to prune (remove) during document processing.
crawler.document.html.pruned.tags=noscript,script,style,header,footer,aside,nav,a[rel=nofollow]
# Maximum length of digest extracted from HTML documents.
crawler.document.html.max.digest.length=120
# Default language for HTML documents.
crawler.document.html.default.lang=

  integrity sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==

follow-redirects@^1.0.0:
  version "1.15.1"
  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5"
  integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==

   *   Iterator<V> i = values.iterator(); // Must be in synchronized block
   *   while (i.hasNext()) {
   *     foo(i.next());
   *   }
   * }
   * }
   *
   * <p>Failure to follow this advice may result in non-deterministic behavior.
   *
   * <p>Note that the generated multimap's {@link Multimap#removeAll} and {@link
   * Multimap#replaceValues} methods return collections that aren't synchronized.

   *   Iterator<V> i = values.iterator(); // Must be in synchronized block
   *   while (i.hasNext()) {
   *     foo(i.next());
   *   }
   * }
   * }
   *
   * <p>Failure to follow this advice may result in non-deterministic behavior.
   *
   * <p>Note that the generated multimap's {@link Multimap#removeAll} and {@link
   * Multimap#replaceValues} methods return collections that aren't synchronized.

     */
    fun authenticator(authenticator: Authenticator) =
      apply {
        this.authenticator = authenticator
      }

    /** Configure this client to follow redirects. If unset, redirects will be followed. */
    fun followRedirects(followRedirects: Boolean) =
      apply {
        this.followRedirects = followRedirects
      }

    /**

    Once the streams are established no further timeout is enforced.

 *  Fix: Retain the `Route` when a connection is reused on a redirect or other follow-up. This was
    causing some `Authenticator` calls to see a null route when non-null was expected.

 *  Fix: Use the correct key size in the name of `TLS_AES_128_CCM_8_SHA256` which is a TLS 1.3

   *   // Must be in the synchronized block
   *   Iterator<E> it = set2.descendingIterator();
   *   while (it.hasNext()) {
   *     foo(it.next());
   *   }
   * }
   * }
   *
   * <p>Failure to follow this advice may result in non-deterministic behavior.
   *
   * <p>The returned navigable set will be serializable if the specified navigable set is
   * serializable.
   *

      val slashCount = input.slashCount(pos, limit)
      if (slashCount >= 2 || base == null || base.scheme != this.scheme) {
        // Read an authority if either:
        //  * The input starts with 2 or more slashes. These follow the scheme if it exists.
        //  * The input scheme exists and is different from the base URL's scheme.
        //
        // The structure of an authority is:
        //   username:password@host:port
        //