// Test handling of duplicate URLs in sitemap
        ResponseData responseData = new ResponseData();
        byte[] content = "<sitemap></sitemap>".getBytes();
        responseData.setResponseBody(content);

        SitemapUrl sitemap1 = new SitemapUrl();
        sitemap1.setLoc("https://example.com/duplicate");

        SitemapUrl sitemap2 = new SitemapUrl();
        sitemap2.setLoc("https://example.com/duplicate");

                        + "  </sitemap>\n" + "  <sitemap>\n" + "    <lastmod>2025-01-02</lastmod>\n" + "  </sitemap>\n" + "  <sitemap>\n"
                        + "    <loc>http://www.example.com/sitemap2.xml</loc>\n" + "  </sitemap>\n" + "</sitemapindex>";
        final InputStream in = new ByteArrayInputStream(xml.getBytes());
        final SitemapSet sitemapSet = sitemapsHelper.parse(in);
        final Sitemap[] sitemaps = sitemapSet.getSitemaps();

User-agent: Bot1
User-agent: Bot2
User-agent: Bot3
Disallow: /shared/

# Case 11: Sitemap with various formats
Sitemap: http://example.com/sitemap.xml
sitemap: http://example.com/sitemap2.xml
SITEMAP: http://example.com/sitemap3.xml
Sitemap:    # empty sitemap (should be ignored)
Sitemap: not-a-valid-url

# Case 12: Malformed lines that should be completely ignored
This line is completely invalid
:NoKey

        robotsTxt.addSitemap("https://example.com/sitemap.xml");
        robotsTxt.addSitemap("https://example.com/sitemap2.xml");

        String[] sitemaps = robotsTxt.getSitemaps();
        assertEquals(2, sitemaps.length);
        assertEquals("https://example.com/sitemap.xml", sitemaps[0]);
        assertEquals("https://example.com/sitemap2.xml", sitemaps[1]);
    }

    public void test_addSitemapNoDuplicates() {

/**
 * Represents a URL entry within a sitemap.
 *
 * <p>
 * This class encapsulates the properties of a URL as defined in the sitemap XML format,
 * including its location, last modification date, change frequency, and priority.
 * It also supports sitemap extensions such as images, videos, news, and alternate links.
 * It implements the {@link Sitemap} interface.
 * </p>
 *
 * <p>

/**
 * Helper class for parsing and validating sitemaps.
 * It supports XML sitemaps, XML sitemap indexes, and text sitemaps,
 * and can handle GZIP compressed sitemaps.
 * The class provides methods to check if an input stream is a valid sitemap,
 * and to parse an input stream into a {@link SitemapSet} object.
 * It uses SAX parser for XML sitemaps and XML sitemap indexes,
 * and handles potential exceptions during parsing.

    }

    /**
     * Test sitemaps add and remove operations
     */
    public void test_sitemaps() {
        // Initial state
        assertNull(crawlerContext.removeSitemaps());

        // Add sitemaps
        String[] sitemaps = new String[] { "http://example.com/sitemap.xml", "http://test.com/sitemap.xml" };
        crawlerContext.addSitemaps(sitemaps);

        // Remove and verify

    }

    /**
     * Adds a sitemap URL to the list of sitemaps.
     *
     * @param url The URL of the sitemap to be added
     */
    public void addSitemap(final String url) {
        if (!sitemapList.contains(url)) {
            sitemapList.add(url);
        }
    }

    /**
     * Returns an array of sitemap URLs.
     *
     * @return an array of sitemap URLs
     */

Allow: /page

# Test multiple wildcards
User-agent: MultiWildcardBot
Disallow: /*.cgi*
Disallow: /*?*id=*

# Test literal $ in middle of pattern
User-agent: DollarBot
Disallow: /price$info

 * </p>
 *
 * @see <a href="https://developers.google.com/search/docs/crawling-indexing/sitemaps/news-sitemap">Google News Sitemaps</a>
 */
public class SitemapNews implements Serializable {

    private static final long serialVersionUID = 1L;

    /**
     * The name of the news publication.
     * It must exactly match the name as it appears on your articles.
     */