User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler
Disallow: /ddd

sitemap: http://www.example.com/sitmap.xml

sitemap1.xml http://www.example.com/ 2005-01-01 monthly 0.8 http://www.example.com/catalog?item=12&desc=vacation_hawaii weekly http://www.example.com/catalog?item=73&desc=vacation_new_zealand 2004-12-23 weekly http://www.example.com/catalog?item=74&desc=vacation_newfoundland 2004-12-23T18:00:15+00:00 0.3 http://www.example.com/catalog?item=83&desc=vacation_usa 2004-11-23...

tableSize; private transient int threshold; private transient int[] sizemap; private transient int seedvalue; public final int _loadFactor(); public final void _loadFactor_$eq(int); public final Object[] table(); public final void table_$eq(Object[]); public final int tableSize(); public final void tableSize_$eq(int); public final int threshold(); public final void threshold_$eq(int); public final int[] sizemap(); public final void sizemap_$eq(int[]); public final int seedvalue(); public final void seedvalue_$eq(int);...

        assertFalse(robotsTxt.allows("/ddd", "Hoge Crawler"));

        String[] sitemaps = robotsTxt.getSitemaps();
        assertEquals(2, sitemaps.length);
        assertEquals("http://www.example.com/sitmap.xml", sitemaps[0]);
        assertEquals("http://www.example.net/sitmap.xml", sitemaps[1]);

    }

    public void testParse_disable() {

            robotTxtFile.deleteOnExit();

            // sitemaps.xml
            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');

            robotTxtFile.deleteOnExit();

            // sitemaps.xml
            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');

			<component class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>"http[s]?:.*sitemap[^/]*\.xml.*|http[s]?:.*sitemap[^/]*\.gz.*|http[s]?:.*sitemap[^/]*\.txt.*"</arg>
		</postConstruct>
	</component>

	<component name="webHtmlRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule" >
		<property name="ruleId">"webHtmlRule"</property>

Shinsuke Sugaya <******@****.***> 1444529815 +0900

    public static final String INDEX = "Index";

    private final List<Sitemap> sitemapList = new ArrayList<>();

    private String type = URLSET;

    public void addSitemap(final Sitemap sitemap) {
        sitemapList.add(sitemap);
    }

    public void removeSitemap(final Sitemap sitemap) {
        sitemapList.remove(sitemap);
    }

    public Sitemap[] getSitemaps() {

            final Set<RequestData> requestDataSet = new LinkedHashSet<>();
            for (final Sitemap sitemap : sitemapSet.getSitemaps()) {
                if (sitemap != null) {
                    requestDataSet.add(RequestDataBuilder.newRequestData().get().url(sitemap.getLoc()).build());
                }
            }