if (clientConfigMap != null) {
            paramMap.putAll(clientConfigMap);
        }

        // robots txt enabled
        if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
            paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
        }

        final String userAgent = getUserAgent();

            public static final String FTP_AUTHENTICATIONS = FtpClient.FTP_AUTHENTICATIONS_PROPERTY;
            public static final String ROBOTS_TXT_ENABLED = HcHttpClient.ROBOTS_TXT_ENABLED_PROPERTY;
            public static final String PROXY_PASSWORD = "proxyPassword";
            public static final String PROXY_USERNAME = "proxyUsername";

    private static final String X_ROBOTS_TAG = "X-Robots-Tag";

    private static final String META_NAME_THUMBNAIL_CONTENT = "//META[@name=\"thumbnail\" or @name=\"THUMBNAIL\"]/@content";

    private static final String META_PROPERTY_OGIMAGE_CONTENT = "//META[@property=\"og:image\"]/@content";

    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    }

    OkHttpClient client = builder.build();

    sendRequest(client, "https://valid-isrgrootx1.letsencrypt.org/robots.txt");

    try {
      sendRequest(client, "https://google.com/robots.txt");
      if (androidMorEarlier) {
        // will pass with default CAs on N or later
        fail();
      }
    } catch (SSLHandshakeException sslhe) {

              .build())
      .build();

  public void run() throws Exception {
    Request request = new Request.Builder()
        .url("https://publicobject.com/robots.txt")
        .build();

    try (Response response = client.newCall(request).execute()) {
      if (!response.isSuccessful()) throw new IOException("Unexpected code " + response);

                return false;
            };
        };

        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/");
        responseData.addMetaData("X-Robots-Tag", "noindex,nofollow");

        try {
            transformer.processXRobotsTag(responseData, new ResultData());
            fail();
        } catch (ChildUrlsException e) {

@RunWith(AndroidJUnit4.class)
public class IssueReproductionTest {
  @Test public void getFailsWithoutAdditionalCert() throws IOException {
    OkHttpClient client = new OkHttpClient();

    sendRequest(client, "https://google.com/robots.txt");
  }

  private void sendRequest(OkHttpClient client, String url) throws IOException {
    Request request = new Request.Builder()
            .url(url)
            .build();

@RunWith(AndroidJUnit4.class)
public class OkHttpClientTest {
  @Test public void get() throws IOException {
    OkHttpClient client = new OkHttpClient();
    Request request = new Request.Builder()
        .url("https://google.com/robots.txt")
        .build();
    try (Response response = client.newCall(request).execute()) {
      assertEquals(200, response.code());
      assertEquals(Protocol.HTTP_2, response.protocol());
    }
  }

    /** The key of the configuration. e.g. false */
    String CRAWLER_IGNORE_ROBOTS_TXT = "crawler.ignore.robots.txt";

    /** The key of the configuration. e.g. false */
    String CRAWLER_IGNORE_ROBOTS_TAGS = "crawler.ignore.robots.tags";

    /** The key of the configuration. e.g. true */
    String CRAWLER_IGNORE_CONTENT_EXCEPTION = "crawler.ignore.content.exception";

                 */
                CacheEntry links = (CacheEntry)roots.get(root);
                if (links != null && now > links.expiration) {
                    roots.remove(root);
                    links = null;
                }

                if (links == null) {
                    if ((trans = getDc(domain, auth)) == null)