"application/dicom",
				"application/dita+xml",
				"application/dita+xml;format=map",
				"application/dita+xml;format=topic",
				"application/dita+xml;format=task",
				"application/dita+xml;format=concept",
				"application/dita+xml;format=val",
				"application/dns",
				"application/dvcs",
				"application/ecmascript",
				"application/edi-consent",
				"application/edi-x12",
				"application/edifact",

        return childUrlRuleMap.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue()));
    }

    /**
     * Converts child URLs using the URL convert helper.
     *
     * @param requestDataList the list of request data to convert
     * @return the converted list of request data
     */

            throw new CrawlingAccessException("Could not process " + robotTxtUrl + ". ", e);
        } finally {
            EntityUtils.consumeQuietly(httpEntity);
        }
    }

    /**
     * Converts a robots.txt pattern to a regular expression.
     *
     * @param path The robots.txt pattern to convert
     * @return The equivalent regular expression
     */