* governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.entity;

import java.io.Serializable;

/**
 * @author shinsuke
 *
 */
public interface Sitemap extends Serializable {

    String getLoc();

    String getLastmod();

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler
Disallow: /ddd

sitemap: http://www.example.com/sitmap.xml

    default void putResultDataBody(final Map<String, Object> dataMap, final String key, final Object value) {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        if (fessConfig.getIndexFieldUrl().equals(key) || !dataMap.containsKey(key) || !getFessConfig().isCrawlerDocumentAppendData()) {
            dataMap.put(key, value);
        } else {
            final Object oldValue = dataMap.get(key);
            final Object[] oldValues;

            dataMap.put(fessConfig.getIndexFieldLabel(), newLabelSet.toArray(new String[newLabelSet.size()]));
        }

        if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
            dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
        }

        ComponentUtil.getLanguageHelper().updateDocument(dataMap);

        synchronized (docList) {

                crawlingInfoHelper.generateId(dataMap));
    }

    public void test_generateId_virtualHost() {
        final Map<String, Object> dataMap = new HashMap<String, Object>();
        dataMap.put("url", "http://example.com/");
        final List<String> roleTypeList = new ArrayList<String>();
        roleTypeList.add("admin");
        dataMap.put("role", roleTypeList);

        sitemapsRule.addRule("url", Pattern.compile(".*sitemap.*"));
        assertTrue(sitemapsRule.match(responseData));
        InputStream is = responseData.getResponseBody();
        assertTrue(is instanceof InputStream);
        CloseableUtil.closeQuietly(responseData);
    }

    private void assertMatchFalse(ResponseData responseData) {
        sitemapsRule.addRule("url", Pattern.compile(".*sitemap.*"));

    }

    private static class DocHandler extends DefaultHandler {
        private final Map<String, Object> dataMap = new HashMap<>();

        private String fieldName;

        private final StringBuilder buffer = new StringBuilder(1000);

        @Override
        public void startDocument() {
            dataMap.clear();
        }

        @Override

			<component class="org.codelibs.fess.crawler.processor.impl.SitemapsResponseProcessor">
			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>"http[s]?:.*sitemap[^/]*\.xml.*|http[s]?:.*sitemap[^/]*\.gz.*|http[s]?:.*sitemap[^/]*\.txt.*"</arg>
		</postConstruct>
	</component>

	<component name="webHtmlRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule" >
		<property name="ruleId">"webHtmlRule"</property>

    public static final String INDEX = "Index";

    private final List<Sitemap> sitemapList = new ArrayList<>();

    private String type = URLSET;

    public void addSitemap(final Sitemap sitemap) {
        sitemapList.add(sitemap);
    }

    public void removeSitemap(final Sitemap sitemap) {
        sitemapList.remove(sitemap);
    }

    public Sitemap[] getSitemaps() {

            final Set<RequestData> requestDataSet = new LinkedHashSet<>();
            for (final Sitemap sitemap : sitemapSet.getSitemaps()) {
                if (sitemap != null) {
                    requestDataSet.add(RequestDataBuilder.newRequestData().get().url(sitemap.getLoc()).build());
                }
            }