}

    /**
     * Adds sitemaps to the thread-local storage.
     * @param sitemaps An array of sitemap URLs.
     */
    public void addSitemaps(final String[] sitemaps) {
        sitemapsLocal.set(sitemaps);
    }

    /**
     * Removes sitemaps from the thread-local storage and returns them.
     * @return An array of sitemap URLs, or null if none were present.
     */

     * @param sitemap the sitemap to add
     */
    public void addSitemap(final Sitemap sitemap) {
        sitemapList.add(sitemap);
    }

    /**
     * Removes a sitemap from this set.
     * @param sitemap the sitemap to remove
     */
    public void removeSitemap(final Sitemap sitemap) {
        sitemapList.remove(sitemap);
    }

    /**
     * Gets all sitemaps in this set as an array.

 * and adds them as child URLs to be crawled.
 *
 * <p>
 * This class uses a {@link SitemapsHelper} to parse the sitemap XML or text.
 * It then iterates through the sitemaps in the SitemapSet, extracts the URL
 * from each sitemap, and creates a new {@link RequestData} object for each URL.
 * These RequestData objects are added to a set of child URLs, which are then

/**
 * Helper class for parsing and validating sitemaps.
 * It supports XML sitemaps, XML sitemap indexes, and text sitemaps,
 * and can handle GZIP compressed sitemaps.
 * The class provides methods to check if an input stream is a valid sitemap,
 * and to parse an input stream into a {@link SitemapSet} object.
 * It uses SAX parser for XML sitemaps and XML sitemap indexes,
 * and handles potential exceptions during parsing.

    }

    /**
     * Test sitemaps add and remove operations
     */
    public void test_sitemaps() {
        // Initial state
        assertNull(crawlerContext.removeSitemaps());

        // Add sitemaps
        String[] sitemaps = new String[] { "http://example.com/sitemap.xml", "http://test.com/sitemap.xml" };
        crawlerContext.addSitemaps(sitemaps);

        // Remove and verify

 * and processing of Sitemap data.
 * </p>
 *
 * <p>
 * The {@code loc} attribute specifies the URL of the Sitemap, while the {@code lastmod} attribute
 * indicates the last time the Sitemap file was modified.  The {@code lastmod} attribute is used by crawlers
 * to incrementally fetch sitemaps that have been updated since a certain date.
 * </p>
 *
 * <p>

 * represents a valid sitemap. It uses a SitemapsHelper to validate the response body as an InputStream.
 * The rule checks if the URL matches the defined regex pattern and then validates the content as a sitemap.
 * If any exception occurs during the sitemap validation, it logs the error and returns false.
 *
 */
public class SitemapsRule extends RegexRule {
    /**

    }

    /**
     * Adds sitemaps from robots.txt to the crawling queue.
     * @param urlQueue The URL queue to add sitemaps to.
     */
    protected void addSitemapsFromRobotsTxt(final UrlQueue<?> urlQueue) {
        final String[] sitemaps = crawlerContext.removeSitemaps();
        if (sitemaps != null) {
            for (final String childUrl : sitemaps) {
                try {

    }

    /**
     * Adds a sitemap URL to the list of sitemaps.
     *
     * @param url The URL of the sitemap to be added
     */
    public void addSitemap(final String url) {
        if (!sitemapList.contains(url)) {
            sitemapList.add(url);
        }
    }

    /**
     * Returns an array of sitemap URLs.
     *
     * @return an array of sitemap URLs
     */

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * Exception thrown during sitemaps processing in the crawler.
 * This exception extends {@link org.codelibs.fess.crawler.exception.CrawlerSystemException}
 * and provides constructors for wrapping other exceptions or creating a new exception with a message.
 */