this.robotsTxtUrlSet = robotsTxtUrlSet;
    }

    /**
     * Returns the lock object for active thread count.
     * @return The lock object.
     */
    public Object getActiveThreadCountLock() {
        return activeThreadCountLock;
    }

    /**
     * Returns the number of threads.
     * @return The number of threads.
     */
    public int getNumOfThread() {

     * Test activeThreadCountLock getter
     */
    public void test_activeThreadCountLock() {
        Object lock = crawlerContext.getActiveThreadCountLock();
        assertNotNull(lock);

        // Should return same instance
        Object sameLock = crawlerContext.getActiveThreadCountLock();
        assertSame(lock, sameLock);
    }

    /**
     * Test numOfThread getter and setter
     */