* </p>
 *
 * @see <a href="https://developers.google.com/search/docs/crawling-indexing/sitemaps/news-sitemap">Google News Sitemaps</a>
 */
public class SitemapNews implements Serializable {

    private static final long serialVersionUID = 1L;

    /**
     * The name of the news publication.
     * It must exactly match the name as it appears on your articles.
     */

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * Exception thrown during sitemaps processing in the crawler.
 * This exception extends {@link org.codelibs.fess.crawler.exception.CrawlerSystemException}
 * and provides constructors for wrapping other exceptions or creating a new exception with a message.
 */

	<include path="crawler/extractor.xml"/>
	<include path="crawler/mimetype.xml"/>
	<include path="crawler/encoding.xml"/>
	<include path="crawler/urlconverter.xml"/>
	<include path="crawler/log.xml"/>
	<include path="crawler/sitemaps.xml"/>

	<!-- Crawler -->
	<component name="crawler" class="org.codelibs.fess.crawler.Crawler" instance="prototype" >
	</component>

	<!-- Crawler Thread -->

            file.delete();
            file.mkdirs();
            file.deleteOnExit();
            fileTransformer.setPath(file.getAbsolutePath());
            crawler.addUrl(url + "sitemaps.xml");
            crawler.crawlerContext.setMaxAccessCount(maxCount);
            crawler.crawlerContext.setNumOfThread(numOfThread);
            crawler.urlFilter.addInclude(url + ".*");

    <include path="crawler/mimetype.xml"/>
    <include path="crawler/encoding.xml"/>
    <include path="crawler/urlconverter.xml"/>
    <include path="crawler/log.xml"/>
    <include path="crawler/sitemaps.xml"/>

    <include path="crawler/opensearch.xml"/>

	<!-- Crawler -->
	<component name="crawler" class="org.codelibs.fess.crawler.Crawler" instance="prototype" >
	</component>

	<!-- Crawler Thread -->

    }

    /**
     * Adds a sitemap URL to the list of sitemaps.
     *
     * @param url The URL of the sitemap to be added
     */
    public void addSitemap(final String url) {
        if (!sitemapList.contains(url)) {
            sitemapList.add(url);
        }
    }

    /**
     * Returns an array of sitemap URLs.
     *
     * @return an array of sitemap URLs
     */

            file.delete();
            file.mkdirs();
            file.deleteOnExit();
            fileTransformer.setPath(file.getAbsolutePath());
            crawler.addUrl(url + "sitemaps.xml");
            crawler.crawlerContext.setMaxAccessCount(maxCount);
            crawler.crawlerContext.setNumOfThread(numOfThread);
            crawler.urlFilter.addInclude(url + ".*");

extractorFactory.addExtractor("text/html", tikaExtractor, 1);  // Fallback
```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM
**UrlConvertHelper**: URL normalization

---

## Development Workflow

                    if (robotsTxt != null) {
                        final String[] sitemaps = robotsTxt.getSitemaps();
                        if (sitemaps.length > 0) {
                            crawlerContext.addSitemaps(sitemaps);
                        }

                        final RobotsTxt.Directive directive = robotsTxt.getMatchedDirective(userAgent);

        // Test handling of duplicate URLs in sitemap
        ResponseData responseData = new ResponseData();
        byte[] content = "<sitemap></sitemap>".getBytes();
        responseData.setResponseBody(content);

        SitemapUrl sitemap1 = new SitemapUrl();
        sitemap1.setLoc("https://example.com/duplicate");

        SitemapUrl sitemap2 = new SitemapUrl();
        sitemap2.setLoc("https://example.com/duplicate");