}
    }

    /**
     * Processes robots meta tags in the HTML document.
     * Handles noindex, nofollow, and none directives.
     *
     * @param responseData the response data from crawling
     * @param resultData the result data to store processed information
     * @param document the parsed HTML document
     */

                        }

                        final RobotsTxt.Directive directive = robotsTxt.getMatchedDirective(userAgent);
                        if (directive != null) {
                            if (useRobotsTxtDisallows) {
                                for (String urlPattern : directive.getDisallows()) {
                                    if (StringUtil.isNotBlank(urlPattern)) {

		HTTPStatusCode: http.StatusBadRequest,
	},
	ErrInvalidMetadataDirective: {
		Code:           "InvalidArgument",
		Description:    "Unknown metadata directive.",
		HTTPStatusCode: http.StatusBadRequest,
	},
	ErrInvalidStorageClass: {
		Code:           "InvalidStorageClass",
		Description:    "Invalid storage class.",
		HTTPStatusCode: http.StatusBadRequest,
	},

    characters.
 *  New: Accept a charset in `FormBody.Builder`. Previously form bodies were
    always UTF-8.
 *  New: Support the `immutable` cache-control directive.
 *  Fix: Don't crash when an HTTP/2 call is redirected while the connection is
    being shut down.
 *  Fix: Don't drop headers of healthy streams that raced with `GOAWAY` frames.