final Set<RequestData> requestDataSet = new LinkedHashSet<>();
            for (final Sitemap sitemap : sitemapSet.getSitemaps()) {
                if (sitemap != null) {
                    requestDataSet.add(RequestDataBuilder.newRequestData().get().url(sitemap.getLoc()).build()); // TODO priority
                }
            }

<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <sitemap>
      <loc>http://www.example.com/sitemap1.xml.gz</loc>
      <lastmod>2004-10-01T18:23:17+00:00</lastmod>
   </sitemap>
   <sitemap>
      <loc>http://www.example.com/sitemap2.xml.gz</loc>
      <lastmod>2005-01-01</lastmod>
   </sitemap>

            assertEquals("application/pdf", mimeMap.getMimeType("pdf"));
            assertEquals("application/msword", mimeMap.getMimeType("doc"));
            assertEquals("application/vnd.ms-excel", mimeMap.getMimeType("xls"));
            assertEquals("text/html", mimeMap.getMimeType("html"));
            assertEquals("text/html", mimeMap.getMimeType("htm"));
            assertEquals("image/jpeg", mimeMap.getMimeType("jpg"));

package org.codelibs.fess.crawler.entity;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a Sitemap file entry, conforming to the Sitemap XML format.
 * This class holds information about a single Sitemap, including its location and last modification timestamp.
 * It implements the {@link Sitemap} interface.
 *
 * <p>
 * A Sitemap file provides search engines with a list of URLs available for crawling.

Allow: /page

# Test multiple wildcards
User-agent: MultiWildcardBot
Disallow: /*.cgi*
Disallow: /*?*id=*

# Test literal $ in middle of pattern
User-agent: DollarBot
Disallow: /price$info

 * represents a valid sitemap. It uses a SitemapsHelper to validate the response body as an InputStream.
 * The rule checks if the URL matches the defined regex pattern and then validates the content as a sitemap.
 * If any exception occurs during the sitemap validation, it logs the error and returns false.
 *
 */
public class SitemapsRule extends RegexRule {
    /**

 */
public class SitemapsHelper {
    private static final Logger logger = LogManager.getLogger(SitemapsHelper.class);

    /** The size of the preload buffer for checking file format. */
    protected int preloadSize = 512;

    /** Enable validation of sitemap entries. */
    protected boolean enableValidation = false;

    /** Maximum URL length according to sitemap specification. */

			<component class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>"http[s]?:.*sitemap[^/]*\.xml.*|http[s]?:.*sitemap[^/]*\.gz.*|http[s]?:.*sitemap[^/]*\.txt.*"</arg>
		</postConstruct>
	</component>

	<component name="webHtmlRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule" >
		<property name="ruleId">"webHtmlRule"</property>

        String[] sitemaps = new String[] { "http://example.com/sitemap.xml", "http://test.com/sitemap.xml" };
        crawlerContext.addSitemaps(sitemaps);

        // Remove and verify
        String[] removedSitemaps = crawlerContext.removeSitemaps();
        assertNotNull(removedSitemaps);
        assertEquals(2, removedSitemaps.length);
        assertEquals("http://example.com/sitemap.xml", removedSitemaps[0]);

    }

    /**
     * Adds a sitemap URL to the list of sitemaps.
     *
     * @param url The URL of the sitemap to be added
     */
    public void addSitemap(final String url) {
        if (!sitemapList.contains(url)) {
            sitemapList.add(url);
        }
    }

    /**
     * Returns an array of sitemap URLs.
     *
     * @return an array of sitemap URLs
     */