this.robotsTxtUrlSet = robotsTxtUrlSet;
    }

    /**
     * Returns the lock object for active thread count.
     * @return The lock object.
     */
    public Object getActiveThreadCountLock() {
        return activeThreadCountLock;
    }

    /**
     * Returns the number of threads.
     * @return The number of threads.
     */
    public int getNumOfThread() {