ruleManager = container.getComponent("ruleManager");

        SitemapsRule sitemapsRule = container.getComponent("sitemapsRule");
        sitemapsRule.setRuleId("sitemapsRule");
        sitemapsRule.addRule("url", ".*sitemap.*");
        ruleManager.addRule(sitemapsRule);

        RegexRule fileRule = container.getComponent("fileRule");
        fileRule.setRuleId("fileRule");
        fileRule.setDefaultRule(true);

            }
        }

        final Map<String, Object> dataMap = XmlUtil.getDataMap(accessResultData);
        if (Map.class.equals(dataClass)) {
            return dataMap;
        }

        try {
            final Object obj = dataClass.getDeclaredConstructor().newInstance();
            BeanUtil.copyMapToBean(dataMap, obj);
            return obj;
        } catch (final Exception e) {

     * This is useful for monitoring memory usage during indexing operations.
     *
     * @param dataMap the document data as a map of field names to values
     * @return the estimated memory size in bytes
     */
    public long calculateDocumentSize(final Map<String, Object> dataMap) {
        return MemoryUtil.sizeOf(dataMap);
    }

    /**
     * Sets the maximum number of retry attempts for failed operations.
     *

    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */
    protected static final Pattern SITEMAP_RECORD = Pattern.compile("^sitemap:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Whether robots.txt processing is enabled. */
    protected boolean enabled = true;

    /**

			<component
				class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>".*sitemap.*"</arg>
		</postConstruct>
	</component>

	<component name="fileRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule">
		<property name="ruleId">"fileRule"</property>
		<property name="defaultRule">true</property>

        private final long startTime = System.currentTimeMillis();

        @Override
        public void store(DataStoreParams paramMap, Map<String, Object> dataMap) {
            documentCount.incrementAndGet();
        }

        @Override
        public long getDocumentSize() {
            return documentCount.get();
        }

        @Override

<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <url>
      <loc>http://www.example.com/</loc>
      <lastmod>2005-01-01</lastmod>
      <changefreq>monthly</changefreq>
      <priority>0.8</priority>
   </url>
   <url>
      <loc>http://www.example.com/catalog?item=12&amp;desc=vacation_hawaii</loc>
      <changefreq>weekly</changefreq>
   </url>
   <url>

        String[] sitemaps = robotsTxt.getSitemaps();
        assertEquals(2, sitemaps.length);
        assertEquals("http://www.example.com/sitmap.xml", sitemaps[0]);
        assertEquals("http://www.example.net/sitmap.xml", sitemaps[1]);

    }

    public void testParse_disable() {
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");

image/x-cmu-raster             ras              #
image/x-portable-anymap        pnm              #
image/x-portable-bitmap        pbm              #
image/x-portable-graymap       pgm              #
image/x-portable-pixmap        ppm              #
image/x-rgb                    rgb              # RGB Image File
image/x-xbitmap                xbm              # X-bitmap Image File
image/x-xpixmap                xpm              # X-pixmap Image File

    /**
     * Maximum buffer size the server can handle.
     */
    public int maxBufferSize;
    /**
     * Session key for this connection.
     */
    public int sessKey;
    /**
     * Server capabilities bitmap.
     */
    public int scapabilities;
    /**
     * OEM domain name of the server.
     */
    public String oemDomainName;
    /**
     * Security mode flags.
     */
    public int securityMode;