URL_QUEUE_THREAD_LOCAL.set(urlQueue);
        }
    }

    public static CrawlerContext getCrawlerContext() {
        return ROBOT_CONTEXT_THREAD_LOCAL.get();
    }

    public static void setCrawlerContext(final CrawlerContext crawlerContext) {
        if (crawlerContext == null) {
            ROBOT_CONTEXT_THREAD_LOCAL.remove();
        } else {
            ROBOT_CONTEXT_THREAD_LOCAL.set(crawlerContext);
        }

        for (int i = 0; i < crawlerContext.getNumOfThread(); i++) {
            final CrawlerThread crawlerThread = crawlerContainer.getComponent("crawlerThread");
            crawlerThread.setCrawlerContext(crawlerContext);
            crawlerThread.setClientFactory(clientFactory);
            threads[i] =

            final String sessionId = "id1";
            urlFilter.init(sessionId);
            crawlerContext.setUrlFilter(urlFilter);
            CrawlingParameterUtil.setCrawlerContext(crawlerContext);
            httpClient.init();
            httpClient.processRobotsTxt(url);
            assertEquals(1, crawlerContext.getRobotsTxtUrlSet().size());