}

            // Add fields
            dataMap.put(fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));

            final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();

            if (fessConfig.getIndexerClickCountEnabledAsBoolean()) {
                addClickCountField(dataMap, url, fessConfig.getIndexFieldClickCount());
            }

import java.io.Serializable;

/**
 * The Sitemap interface represents a sitemap entity with location and last modification date.
 * It extends the Serializable interface to allow sitemap objects to be serialized.
 */
public interface Sitemap extends Serializable {

    /**
     * Retrieves the location (URL) of the sitemap.
     *
     * @return the location of the sitemap as a String.
     */
    String getLoc();

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

User-agent: Crawler/2.0
Disallow: /ccc

User-agent: Hoge Crawler
Disallow: /ddd

sitemap: http://www.example.com/sitmap.xml

                    postcard.setDataCrawlStartTime(getValueFromMap(dataMap, "dataCrawlStartTime", StringUtil.EMPTY));
                    postcard.setDataIndexSize(getValueFromMap(dataMap, "dataIndexSize", "0"));
                    postcard.setDataIndexExecTime(getValueFromMap(dataMap, "dataIndexExecTime", "0"));

        assertEquals("", documentHelper.getContent(null, responseData, "", dataMap));
        assertEquals("", documentHelper.getContent(null, responseData, " ", dataMap));
        assertEquals("", documentHelper.getContent(null, responseData, "  ", dataMap));
        assertEquals("", documentHelper.getContent(null, responseData, "\t", dataMap));
        assertEquals("", documentHelper.getContent(null, responseData, "\t\t", dataMap));

            dataMap.put(key, value);
        } else {
            final Object oldValue = dataMap.get(key);
            final Object[] oldValues;
            if (oldValue instanceof Object[]) {
                oldValues = (Object[]) oldValue;
            } else if (oldValue instanceof Collection<?>) {

        Map<String, Object> dataMap = new HashMap<>();
        dataMap.put("a", "A0");
        Map<String, Object> responseDataMap = new HashMap<>();
        responseDataMap.put("b", "B1");

        indexUpdateCallback.mergeResponseData(dataMap, responseDataMap);

        assertEquals(2, dataMap.size());
        assertEquals("A0", dataMap.get("a"));
        assertEquals("B1", dataMap.get("b"));
    }

                // text cache
                putResultDataBody(dataMap, fessConfig.getIndexFieldCache(), cache);
                putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
            }
        }
        // digest
        putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
                documentHelper.getDigest(responseData, bodyBase, dataMap, fessConfig.getCrawlerDocumentFileMaxDigestLengthAsInteger()));

            dataMap.put(fessConfig.getIndexFieldLabel(), newLabelSet.toArray(new String[newLabelSet.size()]));
        }

        if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
            dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
        }

        ComponentUtil.getLanguageHelper().updateDocument(dataMap);

        synchronized (docList) {

        if (!dataMap.containsKey(fessConfig.getIndexFieldUrl()) || dataMap.get(fessConfig.getIndexFieldUrl()) == null) {
            logger.warn("Could not delete a doc. Invalid data: {}", dataMap);
            return false;
        }

        synchronized (indexUpdateCallback) {
            final long maxAccessCount = getMaxAccessCount(paramMap, dataMap);
            final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();