*
     * @param dataMap the document data map containing URL, roles, and virtual host information
     * @return a unique hashed ID string for the document
     */
    public String generateId(final Map<String, Object> dataMap) {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        final String url = (String) dataMap.get(fessConfig.getIndexFieldUrl());

        ResponseData responseData = new ResponseData();
        byte[] content = "<sitemap></sitemap>".getBytes();
        responseData.setResponseBody(content);

        SitemapUrl sitemap = new SitemapUrl();
        sitemap.setLoc("https://example.com/page1");

        SitemapSet sitemapSet = new SitemapSet();
        sitemapSet.addSitemap(sitemap);

        when(crawlerContainer.getComponent("sitemapsHelper")).thenReturn(sitemapsHelper);

/**
 * Represents a URL entry within a sitemap.
 *
 * <p>
 * This class encapsulates the properties of a URL as defined in the sitemap XML format,
 * including its location, last modification date, change frequency, and priority.
 * It also supports sitemap extensions such as images, videos, news, and alternate links.
 * It implements the {@link Sitemap} interface.
 * </p>
 *
 * <p>

     * @param title the raw title text to process
     * @param dataMap additional data map (not currently used)
     * @return the normalized title, or empty string if title is null
     */
    public String getTitle(final ResponseData responseData, final String title, final Map<String, Object> dataMap) {
        if (title == null) {
            return StringUtil.EMPTY; // empty
        }

        sitemapsRule.addRule("url", Pattern.compile(".*sitemap.*"));
        assertTrue(sitemapsRule.match(responseData));
        InputStream is = responseData.getResponseBody();
        assertTrue(is instanceof InputStream);
        CloseableUtil.closeQuietly(responseData);
    }

    private void assertMatchFalse(ResponseData responseData) {
        sitemapsRule.addRule("url", Pattern.compile(".*sitemap.*"));

        sitemapList.add(sitemap);
    }

    /**
     * Removes a sitemap from this set.
     * @param sitemap the sitemap to remove
     */
    public void removeSitemap(final Sitemap sitemap) {
        sitemapList.remove(sitemap);
    }

    /**
     * Gets all sitemaps in this set as an array.
     * @return an array of sitemaps
     */
    public Sitemap[] getSitemaps() {

     *
     * @param paramMap the data store parameters containing configuration and metadata
     * @param dataMap the document data to be indexed as key-value pairs
     */
    void store(DataStoreParams paramMap, Map<String, Object> dataMap);

    /**
     * Returns the total number of documents processed by this callback.
     *
     * @return the document count
     */

                        + "    <loc>http://www.example.com/sitemap2.xml</loc>\n" + "  </sitemap>\n" + "</sitemapindex>";
        final InputStream in = new ByteArrayInputStream(xml.getBytes());
        final SitemapSet sitemapSet = sitemapsHelper.parse(in);
        final Sitemap[] sitemaps = sitemapSet.getSitemaps();

            final Set<RequestData> requestDataSet = new LinkedHashSet<>();
            for (final Sitemap sitemap : sitemapSet.getSitemaps()) {
                if (sitemap != null) {
                    requestDataSet.add(RequestDataBuilder.newRequestData().get().url(sitemap.getLoc()).build()); // TODO priority
                }
            }

User-agent: Bot1
User-agent: Bot2
User-agent: Bot3
Disallow: /shared/

# Case 11: Sitemap with various formats
Sitemap: http://example.com/sitemap.xml
sitemap: http://example.com/sitemap2.xml
SITEMAP: http://example.com/sitemap3.xml
Sitemap:    # empty sitemap (should be ignored)
Sitemap: not-a-valid-url

# Case 12: Malformed lines that should be completely ignored
This line is completely invalid
:NoKey