httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

				"application/vnd.oma.group-usage-list+xml",
				"application/vnd.oma.poc.detailed-progress-report+xml",
				"application/vnd.oma.poc.final-report+xml",
				"application/vnd.oma.poc.groups+xml",
				"application/vnd.oma.poc.invocation-descriptor+xml",
				"application/vnd.oma.poc.optimized-progress-report+xml",
				"application/vnd.oma.xcap-directory+xml",
				"application/vnd.omads-email+xml",

fail",n.Callbacks("once memory"),"rejected"],["notify","progress",n.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return n.Deferred(function(c){n.each(b,function(b,f){var g=n.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&n.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]...