client =
      OkHttpClient.Builder()
        .sslSocketFactory(certificates.sslSocketFactory(), certificates.trustManager)
        .build()
  }

  fun run() {
    showUrl("https://squareup.com/robots.txt")
    showUrl("https://publicobject.com/helloworld.txt")
  }

  private fun showUrl(url: String) {
    val request = Builder().url(url).build()
    client.newCall(request)
      .execute()

c107;--danger:#dc3545;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:"Source Sans Pro",-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol";--font-family-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace}*,::after,::before{box-sizing:border-box}html{font-fam...

            httpClient.processRobotsTxt(url);
            assertEquals(1, crawlerContext.getRobotsTxtUrlSet().size());
            assertTrue(crawlerContext.getRobotsTxtUrlSet().contains("http://localhost:7070/robots.txt"));
            assertFalse(urlFilter.match("http://localhost:7070/admin/"));
            assertFalse(urlFilter.match("http://localhost:7070/websvn/"));
        } finally {
            server.stop();
        }
    }

    private static final Logger logger = LoggerFactory.getLogger(EmlExtractor.class);

    protected Properties mailProperties = new Properties();

    /* (non-Javadoc)
     * @see org.codelibs.robot.extractor.Extractor#getText(java.io.InputStream, java.util.Map)
     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {

Do you want to just have a `str`? Or just a `dict`? Or a database class model instance directly? It all works the same way.

You actually don't have users that log in to your application but robots, bots, or other systems, that have just an access token? Again, it all works the same.

        final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
        if (clientConfigMap != null) {
            paramMap.putAll(clientConfigMap);
        }

        // robots txt enabled
        if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
            paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
        }

            return;
        }

        if (logger.isDebugEnabled()) {
            logger.debug("Initializing {}", HcHttpClient.class.getName());
        }

        super.init();

        // robots.txt parser
        final Boolean robotsTxtEnabled = getInitParameter(ROBOTS_TXT_ENABLED_PROPERTY, Boolean.TRUE, Boolean.class);
        if (robotsTxtHelper != null) {
            robotsTxtHelper.setEnabled(robotsTxtEnabled);

  - login: xoflare
    avatarUrl: https://avatars.githubusercontent.com/u/74335107?v=4
    url: https://github.com/xoflare
  - login: marvin-robot
    avatarUrl: https://avatars.githubusercontent.com/u/41086007?u=b9fcab402d0cd0aec738b6574fe60855cb0cd36d&v=4
    url: https://github.com/marvin-robot
  - login: BoostryJP
    avatarUrl: https://avatars.githubusercontent.com/u/57932412?v=4
    url: https://github.com/BoostryJP
  - login: acsone

                logger.debug("Failed to disconnect FTPClient.", e);
            }
        }
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.robot.client.S2RobotClient#doGet(java.lang.String)
     */
    @Override
    public ResponseData doGet(final String uri) {
        return processRequest(uri, true);
    }

        .addHeader("Cookie: s=square")
        .addHeader("Cookie: a=android")
        .addHeader("Cookies: delicious")
    response.setHeader("cookie", "r=robot")
    assertThat(headersToList(response))
      .containsExactly("Cookies: delicious", "cookie: r=robot")
  }

  @Test
  fun mockResponseSetHeaders() {
    val response =
      MockResponse()
        .clearHeaders()