*/
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }

        final MimeTypeHelper mimeTypeHelper = getMimeTypeHelper();
        final ExtractorFactory extractorFactory = getExtractorFactory();
        final StringBuilder buf = new StringBuilder(1000);

        final CrawlerWebServer server2 = new CrawlerWebServer(7071);
        server2.start();

        final String url1 = "http://localhost:7070/";
        final String url2 = "http://localhost:7071/";
        try {
            final int maxCount = 10;
            final int numOfThread = 10;

            final File file = File.createTempFile("crawler-", "");
            file.delete();
            file.mkdirs();

        super();
    }

    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }

        final MimeTypeHelper mimeTypeHelper = getMimeTypeHelper();
        final ExtractorFactory extractorFactory = getExtractorFactory();

 * is deleted when the stream is closed.
 */
public class TemporaryFileInputStream extends InputStream {
    private static final Logger logger = LogManager.getLogger(TemporaryFileInputStream.class);

    private final File tempFile;

    private final FileInputStream fileInputStream;

    /**
     * A class that provides an input stream for reading from a temporary file.

            return tempDir;
        } catch (final Exception e) {
            throw new CrawlerSystemException(e);
        }
    }

    private static void generateContents(final File dir, final int count) throws Exception {
        if (count <= 0) {
            return;
        }

        final String content = getHtmlContent(count);

        final File indexFile = new File(dir, "index.html");

    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

            CopyUtil.copy(is, fos);
        } catch (final Exception e) {
            CloseableUtil.closeQuietly(fos); // for deleting file
            FileUtil.deleteInBackground(tempFile); // clean up
            throw new CrawlingAccessException("Could not read a response body: " + responseData.getUrl(), e);
        } finally {
            CloseableUtil.closeQuietly(fos);
        }
        return tempFile;

/**
 * @author shinsuke
 *
 */
public class CrawlerWebServer {
    private final File docRoot;

    private final Server server;

    private boolean tempDocRoot = false;

    public CrawlerWebServer(final int port) {
        this(port, createDocRoot(3));
        tempDocRoot = true;
    }

    public CrawlerWebServer(final int port, final File docRoot) {
        this.docRoot = docRoot;

                count++;
            }
            final String message = "Failed to access to " + request.getUrl()
                    + exceptionList.stream().map(e -> "; " + e.getMessage()).collect(Collectors.joining());
            throw new MultipleCrawlingAccessException(message, exceptionList.toArray(new Throwable[exceptionList.size()]));
        } finally {
            if (listener != null) {

            return tempDir;
        } catch (final Exception e) {
            throw new CrawlerSystemException(e);
        }
    }

    private static void generateContents(final File dir, final int count) throws Exception {
        if (count <= 0) {
            return;
        }

        final String content = getHtmlContent(count);

        final File indexFile = new File(dir, "index.html");