*
     * @param responseData the response data from crawling
     * @param dataMap the data map containing extracted field values
     */
    protected void normalizeData(final ResponseData responseData, final Map<String, Object> dataMap) {
        final Object titleObj = dataMap.get(fessConfig.getIndexFieldTitle());
        if (titleObj != null) {
            dataMap.put(fessConfig.getIndexFieldTitle(),

	} else {
		types = madmin.MetricsAll
	}

	diskMap := make(map[string]struct{})
	for _, disk := range values[peerRESTDisk] {
		diskMap[disk] = struct{}{}
	}

	hostMap := make(map[string]struct{})
	for _, host := range values[peerRESTHost] {
		hostMap[host] = struct{}{}
	}

	info := collectLocalMetrics(types, collectMetricsOpts{
		disks: diskMap,
		hosts: hostMap,
		jobID: values.Get(peerRESTJobID),

        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

				"image/x-freehand",
				"image/x-jp2-codestream",
				"image/x-jp2-container",
				"image/x-niff",
				"image/x-pcx",
				"image/x-pict",
				"image/x-portable-anymap",
				"image/x-portable-bitmap",
				"image/x-portable-graymap",
				"image/x-portable-pixmap",
				"image/x-raw-adobe",
				"image/x-raw-hasselblad",
				"image/x-raw-fuji",
				"image/x-raw-canon",
				"image/x-raw-kodak",