}

            // Add fields
            dataMap.put(fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));

            final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();

            if (fessConfig.getIndexerClickCountEnabledAsBoolean()) {
                addClickCountField(dataMap, url, fessConfig.getIndexFieldClickCount());
            }

import java.io.Serializable;

/**
 * The Sitemap interface represents a sitemap entity with location and last modification date.
 * It extends the Serializable interface to allow sitemap objects to be serialized.
 */
public interface Sitemap extends Serializable {

    /**
     * Retrieves the location (URL) of the sitemap.
     *
     * @return the location of the sitemap as a String.
     */
    String getLoc();

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler
Disallow: /ddd

sitemap: http://www.example.com/sitmap.xml

                    postcard.setDataCrawlStartTime(getValueFromMap(dataMap, "dataCrawlStartTime", StringUtil.EMPTY));
                    postcard.setDataIndexSize(getValueFromMap(dataMap, "dataIndexSize", "0"));
                    postcard.setDataIndexExecTime(getValueFromMap(dataMap, "dataIndexExecTime", "0"));

        assertEquals("", documentHelper.getContent(null, responseData, "", dataMap));
        assertEquals("", documentHelper.getContent(null, responseData, " ", dataMap));
        assertEquals("", documentHelper.getContent(null, responseData, "  ", dataMap));
        assertEquals("", documentHelper.getContent(null, responseData, "\t", dataMap));
        assertEquals("", documentHelper.getContent(null, responseData, "\t\t", dataMap));

            dataMap.put(key, value);
        } else {
            final Object oldValue = dataMap.get(key);
            final Object[] oldValues;
            if (oldValue instanceof Object[]) {
                oldValues = (Object[]) oldValue;
            } else if (oldValue instanceof Collection<?>) {

        Map<String, Object> dataMap = new HashMap<>();
        dataMap.put("a", "A0");
        Map<String, Object> responseDataMap = new HashMap<>();
        responseDataMap.put("b", "B1");

        indexUpdateCallback.mergeResponseData(dataMap, responseDataMap);

        assertEquals(2, dataMap.size());
        assertEquals("A0", dataMap.get("a"));
        assertEquals("B1", dataMap.get("b"));
    }

                // text cache
                putResultDataBody(dataMap, fessConfig.getIndexFieldCache(), cache);
                putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
            }
        }
        // digest
        putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
                documentHelper.getDigest(responseData, bodyBase, dataMap, fessConfig.getCrawlerDocumentFileMaxDigestLengthAsInteger()));

                crawlingInfoHelper.generateId(dataMap));
    }

    public void test_generateId_virtualHost() {
        final Map<String, Object> dataMap = new HashMap<String, Object>();
        dataMap.put("url", "http://example.com/");
        final List<String> roleTypeList = new ArrayList<String>();
        roleTypeList.add("admin");
        dataMap.put("role", roleTypeList);

        IndexUpdateCallback callback = new IndexUpdateCallback() {
            @Override
            public void store(DataStoreParams paramMap, Map<String, Object> dataMap) {
                assertNotNull(paramMap);
                assertNotNull(dataMap);
                assertTrue(dataMap.isEmpty());
                storeCount.incrementAndGet();
            }

            @Override
            public long getDocumentSize() {