robotsTxtHelper = container.getComponent("robotsTxtHelper");
    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

            httpClient.processRobotsTxt(url);
            assertEquals(1, crawlerContext.getRobotsTxtUrlSet().size());
            assertTrue(crawlerContext.getRobotsTxtUrlSet().contains("http://localhost:7070/robots.txt"));
            assertFalse(urlFilter.match("http://localhost:7070/admin/"));
            assertFalse(urlFilter.match("http://localhost:7070/websvn/"));
        } finally {
            server.stop();
        }
    }

        .addHeader("Cookie: s=square")
        .addHeader("Cookie: a=android")
        .addHeader("Cookies: delicious")
    response.setHeader("cookie", "r=robot")
    assertThat(headersToList(response))
      .containsExactly("Cookies: delicious", "cookie: r=robot")
  }

  @Test
  fun mockResponseSetHeaders() {
    val response =
      MockResponse()
        .clearHeaders()

        assertTrue(directive.allows("/admin/public/page.html"));
        assertFalse(directive.allows("/admin/secret.html"));
    }

    public void test_complexScenario() {
        // Test complex robots.txt scenario
        RobotsTxt robotsTxt = new RobotsTxt();

        // Add directive for all bots
        Directive allBotsDirective = new Directive("*");
        allBotsDirective.addDisallow("/admin/");

    /**
     * Constructs a new EmlExtractor.
     */
    public EmlExtractor() {
        // Default constructor
    }

    /* (non-Javadoc)
     * @see org.codelibs.robot.extractor.Extractor#getText(java.io.InputStream, java.util.Map)
     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {

        .addHeader("Cookie: s=square")
        .addHeader("Cookie: a=android")
        .addHeader("Cookies: delicious")
    builder.setHeader("cookie", "r=robot")
    assertThat(headersToList(builder)).containsExactly("Cookies: delicious", "cookie: r=robot")
  }

  @Test
  fun mockResponseSetHeaders() {
    val builder =
      MockResponse
        .Builder()
        .clearHeaders()

errors.blank_password = Le mot de passe est requis.
errors.invalid_confirm_password = Ne correspond pas à un mot de passe de confirmation.
errors.cannot_delete_doc_because_of_running = Un robot d'exploration est en cours d'exécution. Vous ne pouvez pas supprimer de documents.
errors.failed_to_delete_doc_in_admin = Échec de la suppression d'un document.
errors.failed_to_send_testmail=Échec de l'envoi d'un e-mail de test.

      HandshakeCertificates
        .Builder()
        .heldCertificate(certificate)
        .build()
    server.useHttps(serverHandshakeCertificates.sslSocketFactory())
    url = server.url("/robots.txt")
  }

  /**
   * Test connecting to the main host then an alternative, although only subject alternative names
   * are used if present no special consideration of common name.
   */
  @Test

            CrawlingParameterUtil.setDataService(null);
        }
        log(logHelper, LogType.FINISHED_THREAD, crawlerContext);
    }

    /**
     * Adds sitemaps from robots.txt to the crawling queue.
     * @param urlQueue The URL queue to add sitemaps to.
     */
    protected void addSitemapsFromRobotsTxt(final UrlQueue<?> urlQueue) {

            }
        }
        if (logger.isDebugEnabled()) {
            logger.debug("FTP client closed");
        }
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.robot.client.S2RobotClient#doGet(java.lang.String)
     */
    @Override
    public ResponseData doGet(final String uri) {
        return processRequest(uri, true);
    }

    /**