*/
    protected ExtractData getText(final InputStream inputStream, final Map<String, String> params,
            final BiConsumer<ExtractData, InputStream> postFilter) {
        if (inputStream == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }

        final File tempFile;
        final boolean isByteStream = inputStream instanceof ByteArrayInputStream;

        final String value = "Mon, 01 Jun 2009 21:02:45 GMT";
        final Date date = httpClient.parseLastModifiedDate(value);
        assertNotNull(date);
    }

    public void test_processRobotsTxt() {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/hoge.html";
        try {

            ftpClient.doGet("ftp://localhost:" + FTP_PORT + "/");
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            assertEquals(5, urlSet.size());
            final List<String> urlList = urlSet.stream().map(x -> x.getUrl()).sorted().toList();
            assertEquals("ftp://localhost:10021/dir1", urlList.get(0));

    }

    public void test_execute_web() throws Exception {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final int maxCount = 50;
            final int numOfThread = 10;

            final File file = File.createTempFile("crawler-", "");
            file.delete();

    }

    public void test_execute_web() throws Exception {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final int maxCount = 50;
            final int numOfThread = 10;

            final File file = File.createTempFile("crawler-", "");
            file.delete();

     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        final String resourceName = params == null ? null : params.get(ExtractData.RESOURCE_NAME_KEY);

        String extention;
        String filePrefix;
        if (StringUtil.isNotBlank(resourceName)) {
            final String name = getFileName(resourceName);
            final String[] strings = name.split("\\.");

     */
    public void test_concurrentNameAccess() throws Exception {
        final int threadCount = 100;
        final int operationsPerThread = 100;
        final CountDownLatch startLatch = new CountDownLatch(1);
        final CountDownLatch endLatch = new CountDownLatch(threadCount);
        final AtomicInteger errorCount = new AtomicInteger(0);

        testTransformer.setName("concurrentName");

     */
    protected void storeChildUrls(final Set<RequestData> childUrlList, final String url, final int depth) {
        if (crawlerContext.getMaxDepth() >= 0 && depth > crawlerContext.getMaxDepth()) {
            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()

     */
    protected String getOutputContent(final File outputFile, final String outExt) {
        final Extractor extractor = getExtractor(outExt);
        if (extractor != null) {
            final Map<String, String> params = new HashMap<>();
            params.put(ExtractData.RESOURCE_NAME_KEY, outputFile.getName());
            try (final FileInputStream in = new FileInputStream(outputFile)) {

    public void test_sitemaps_threadLocal() throws Exception {
        final String[] thread1Sitemaps = new String[] { "http://thread1.com/sitemap.xml" };
        final String[] thread2Sitemaps = new String[] { "http://thread2.com/sitemap.xml" };
        final CountDownLatch startLatch = new CountDownLatch(1);
        final CountDownLatch endLatch = new CountDownLatch(2);
        final AtomicInteger successCount = new AtomicInteger(0);