}

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

            CopyUtil.copy(is, fos);
        } catch (final Exception e) {
            CloseableUtil.closeQuietly(fos); // for deleting file
            FileUtil.deleteInBackground(tempFile); // clean up
            throw new CrawlingAccessException("Could not read a response body: " + responseData.getUrl(), e);
        } finally {
            CloseableUtil.closeQuietly(fos);
        }
        return tempFile;

     */
    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        final String resourceName = params == null ? null : params.get(ExtractData.RESOURCE_NAME_KEY);

        String extention;
        String filePrefix;
        if (StringUtil.isNotBlank(resourceName)) {
            final String name = getFileName(resourceName);
            final String[] strings = name.split("\\.");

     */
    public void test_concurrentNameAccess() throws Exception {
        final int threadCount = 100;
        final int operationsPerThread = 100;
        final CountDownLatch startLatch = new CountDownLatch(1);
        final CountDownLatch endLatch = new CountDownLatch(threadCount);
        final AtomicInteger errorCount = new AtomicInteger(0);

        testTransformer.setName("concurrentName");

     */
    protected void storeChildUrls(final Set<RequestData> childUrlList, final String url, final int depth) {
        if (crawlerContext.getMaxDepth() >= 0 && depth > crawlerContext.getMaxDepth()) {
            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()

     */
    protected String getOutputContent(final File outputFile, final String outExt) {
        final Extractor extractor = getExtractor(outExt);
        if (extractor != null) {
            final Map<String, String> params = new HashMap<>();
            params.put(ExtractData.RESOURCE_NAME_KEY, outputFile.getName());
            try (final FileInputStream in = new FileInputStream(outputFile)) {

    public void test_sitemaps_threadLocal() throws Exception {
        final String[] thread1Sitemaps = new String[] { "http://thread1.com/sitemap.xml" };
        final String[] thread2Sitemaps = new String[] { "http://thread2.com/sitemap.xml" };
        final CountDownLatch startLatch = new CountDownLatch(1);
        final CountDownLatch endLatch = new CountDownLatch(2);
        final AtomicInteger successCount = new AtomicInteger(0);

/**
 * @author shinsuke
 *
 */
public class CrawlerWebServer {
    private final File docRoot;

    private final Server server;

    private boolean tempDocRoot = false;

    public CrawlerWebServer(final int port) {
        this(port, createDocRoot(3));
        tempDocRoot = true;
    }

    public CrawlerWebServer(final int port, final File docRoot) {
        this.docRoot = docRoot;

                count++;
            }
            final String message = "Failed to access to " + request.getUrl()
                    + exceptionList.stream().map(e -> "; " + e.getMessage()).collect(Collectors.joining());
            throw new MultipleCrawlingAccessException(message, exceptionList.toArray(new Throwable[exceptionList.size()]));
        } finally {
            if (listener != null) {

     */
    protected ResponseData getResponseData(final String uri, final boolean includeContent) {
        final ResponseData responseData = new ResponseData();
        responseData.setMethod(Constants.GET_METHOD);
        final String filePath = preprocessUri(uri);
        responseData.setUrl(filePath);

        SmbFile file = null;
        final SmbAuthentication smbAuthentication = smbAuthenticationHolder.get(filePath);