return robotsTxtUrlSet;
    }

    /**
     * Sets the set of robots.txt URLs.
     * @param robotsTxtUrlSet The set of robots.txt URLs.
     */
    public void setRobotsTxtUrlSet(final Set<String> robotsTxtUrlSet) {
        this.robotsTxtUrlSet = robotsTxtUrlSet;
    }

    /**
     * Returns the lock object for active thread count.
     * @return The lock object.
     */

        newSet.add("http://new.com/robots.txt");
        crawlerContext.setRobotsTxtUrlSet(newSet);
        assertSame(newSet, crawlerContext.getRobotsTxtUrlSet());
        assertEquals(1, crawlerContext.getRobotsTxtUrlSet().size());

        // Set null
        crawlerContext.setRobotsTxtUrlSet(null);
        assertNull(crawlerContext.getRobotsTxtUrlSet());
    }

    /**