urlFilter.init(sessionId);
            crawlerContext.setUrlFilter(urlFilter);
            CrawlingParameterUtil.setCrawlerContext(crawlerContext);
            httpClient.init();
            httpClient.processRobotsTxt(url);
            assertEquals(1, crawlerContext.getRobotsTxtUrlSet().size());
            assertTrue(crawlerContext.getRobotsTxtUrlSet().contains("http://localhost:7070/robots.txt"));

     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {
            throw new CrawlerSystemException("url is null or empty.");
        }

        if (robotsTxtHelper == null || !robotsTxtHelper.isEnabled()) {