sitemap2.xml http://www.example.com/sitemap1.xml.gz 2004-10-01T18:23:17+00:00 http://www.example.com/sitemap2.xml.gz 2005-01-01...

import java.io.Serializable;

/**
 * The Sitemap interface represents a sitemap entity with location and last modification date.
 * It extends the Serializable interface to allow sitemap objects to be serialized.
 */
public interface Sitemap extends Serializable {

    /**
     * Retrieves the location (URL) of the sitemap.
     *
     * @return the location of the sitemap as a String.
     */
    String getLoc();

User-agent: Bot1
User-agent: Bot2
User-agent: Bot3
Disallow: /shared/

# Case 11: Sitemap with various formats
Sitemap: http://example.com/sitemap.xml
sitemap: http://example.com/sitemap2.xml
SITEMAP: http://example.com/sitemap3.xml
Sitemap:    # empty sitemap (should be ignored)
Sitemap: not-a-valid-url

# Case 12: Malformed lines that should be completely ignored
This line is completely invalid
:NoKey

sitemap1.xml http://www.example.com/ 2005-01-01 monthly 0.8 http://www.example.com/catalog?item=12&desc=vacation_hawaii weekly http://www.example.com/catalog?item=73&desc=vacation_new_zealand 2004-12-23 weekly http://www.example.com/catalog?item=74&desc=vacation_newfoundland 2004-12-23T18:00:15+00:00 0.3 http://www.example.com/catalog?item=83&desc=vacation_usa 2004-11-23...

Shinsuke Sugaya <******@****.***> 1444529815 +0900

/**
 * Represents a URL entry within a sitemap.
 *
 * <p>
 * This class encapsulates the properties of a URL as defined in the sitemap XML format,
 * including its location, last modification date, change frequency, and priority.
 * It also supports sitemap extensions such as images, videos, news, and alternate links.
 * It implements the {@link Sitemap} interface.
 * </p>
 *
 * <p>

Allow: /page

# Test multiple wildcards
User-agent: MultiWildcardBot
Disallow: /*.cgi*
Disallow: /*?*id=*

# Test literal $ in middle of pattern
User-agent: DollarBot
Disallow: /price$info

			<component class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>"http[s]?:.*sitemap[^/]*\.xml.*|http[s]?:.*sitemap[^/]*\.gz.*|http[s]?:.*sitemap[^/]*\.txt.*"</arg>
		</postConstruct>
	</component>

	<component name="webHtmlRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule" >
		<property name="ruleId">"webHtmlRule"</property>

Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler
Disallow: /ddd

sitemap: http://www.example.com/sitmap.xml

			<component
				class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>".*sitemap.*"</arg>
		</postConstruct>
	</component>

	<component name="fileRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule">
		<property name="ruleId">"fileRule"</property>
		<property name="defaultRule">true</property>