is.setEncoding(responseData.getCharSet());
            }
            parser.parse(is);
        } catch (final Exception e) {
            throw new CrawlingAccessException("Could not parse " + responseData.getUrl(), e);
        }

        final Document document = parser.getDocument();

        processMetaRobots(responseData, resultData, document);

        return httpClient.execute(httpRequest, new BasicHttpContext(httpClientContext));
    }

    /**
     * Parses the last modified date from a string value.
     *
     * @param value The date string to parse
     * @return The parsed date, or null if parsing fails
     */
    protected Date parseLastModifiedDate(final String value) {

            this.request = request;
        }

        /**
         * Parses the HTTP request to create a RequestParameter object.
         * @param request The HTTP servlet request containing the parameters
         * @return A new RequestParameter object with parsed values
         */
        protected static RequestParameter parse(final HttpServletRequest request) {
            final String query = request.getParameter("q");

        if (logger.isDebugEnabled()) {
            logger.debug("Parse: {} : {}", url, params);
        }
        try {
            return AuthenticationResponseParser.parse(new URI(url), params);
        } catch (final Exception e) {
            throw new SsoLoginException("Failed to parse an authentication response.", e);
        }
    }

    /**
     * Validates the nonce in the authentication result.

    default PrunedTag[] getCrawlerDocumentHtmlPrunedTagsAsArray() {
        PrunedTag[] tags = (PrunedTag[]) propMap.get("crawlerDocumentHtmlPrunedTags");
        if (tags == null) {
            tags = PrunedTag.parse(getCrawlerDocumentHtmlPrunedTags());
            propMap.put("crawlerDocumentHtmlPrunedTags", tags);
        }
        return tags;
    }

    String getCrawlerDocumentCacheHtmlMimetypes();

				"application/xcon-conference-info-diff+xml",
				"application/xcon-conference-info+xml",
				"application/xenc+xml",
				"application/xhtml-voice+xml",
				"application/xml-dtd",
				"application/xml-external-parsed-entity",
				"application/xmpp+xml",
				"application/xop+xml",
				"application/xslfo+xml",
				"application/xslt+xml",
				"application/xspf+xml",
				"application/xv+xml",
				"application/zip",

                }
                return attrList;
            }
            return Collections.emptyList();
        } catch (final NamingException e) {
            throw new LdapOperationException("Failed to parse attribute values for " + name, e);
        }
    }

    /**
     * Applies LDAP attributes to a user object.
     *
     * @param user the user object to populate with LDAP attributes
     */

crawler.hotthread.timeout=30s
# Type of hot thread monitoring (e.g., cpu).
crawler.hotthread.type=cpu
# Metadata fields to exclude from document content.
crawler.metadata.content.excludes=resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*,X-TIKA.*,X-FESS.*
# Mapping for document metadata names.
crawler.metadata.name.mapping=\
title=title:string\n\
Title=title:string\n\
dc:title=title:string\n\

# html

nction(a,b){return this.off(a,null,b)},delegate:function(a,b,c,d){return this.on(b,a,c,d)},undelegate:function(a,b,c){return 1===arguments.length?this.off(a,"**"):this.off(b,a||"**",c)}});var cc=n.now(),dc=/\?/;n.parseJSON=function(a){return JSON.parse(a+"")},n.parseXML=function(a){var b,c;if(!a||"string"!=typeof a)return null;try{c=new DOMParser,b=c.parseFromString(a,"text/xml")}catch(d){b=void 0}return(!b||b.getElementsByTagName("parsererror").length)&&n.error("Invalid XML: "+a),b};var ec,fc,g...