if (xpath == null) {
            xpath = fessConfig.getCrawlerDocumentHtmlCanonicalXpath();
        }
        if (StringUtil.isBlank(xpath)) {
            return null;
        }
        final String canonicalUrl = getSingleNodeValue(document, xpath, node -> node);
        if (StringUtil.isBlank(canonicalUrl)) {
            return null;
        }
        return normalizeCanonicalUrl(responseData.getUrl(), canonicalUrl);
    }

        includedDocUrls = fessConfig.getCrawlerDocumentHtmlDefaultIncludeSearchPatterns();
        excludedDocUrls = fessConfig.getCrawlerDocumentHtmlDefaultExcludeSearchPatterns();
        boost = 1.0f;
        if (StringUtil.isBlank(userAgent)) {
            userAgent = fessConfig.getUserAgentName();
        }
        numOfThread = Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB;
        intervalTime = Constants.DEFAULT_INTERVAL_TIME_FOR_WEB;

     * @param page The HTML page
     * @return The base path for the virtual host
     */
    protected String getVirtualHostBasePath(final String s, final HtmlNext page) {
        return StringUtil.isBlank(s) ? StringUtil.EMPTY : "/" + s;
    }

    /**
     * Gets the virtual host key from the current request.
     * The key is determined by matching HTTP headers against configured virtual hosts.
     *

     * @throws CrawlerSystemException if data deserialization fails
     */
    public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String crawlingInfoId, final String url) {
        if (StringUtil.isBlank(crawlingInfoId)) {
            throw new CrawlingAccessException("sessionId is null.");
        }

     * @param version the version of the artifact
     * @return the artifact if found, null otherwise
     */
    public Artifact getArtifact(final String name, final String version) {
        if (StringUtil.isBlank(name) || StringUtil.isBlank(version)) {
            return null;
        }
        for (final Artifact artifact : getAvailableArtifacts(ArtifactType.getType(name))) {

            }
            builder.endObject();
            return builder;
        }, XContentType.JSON)) {
            message = ((ByteArrayOutputStream) out).toString(Constants.UTF_8);
            if (StringUtil.isBlank(message)) {
                message = "{}";
            }
        } catch (final IOException e) {
            throw new IORuntimeException(e);
        }
    }

    /**
     * Gets the status.

        String title = DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class);
        if (StringUtil.isBlank(title)) {
            title = DocumentUtil.getValue(document, fessConfig.getIndexFieldFilename(), String.class);
            if (StringUtil.isBlank(title)) {
                title = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
            }
        }

,[0-9]{1,3}%,[0-9]{1,3}%,[0,1]{1}.?[0-9]*\)/i;if(e.match(f)){var g=e.replace(/\(/g,"").replace(/\)/g,""),h=g.split(","),i=!0;return h.forEach(function(a,c){var e=b(a);Number.isInteger(e)?0===c?(d=0<=e&&e<=360,d||(i=!1)):(d=0<=e&&e<=100,d||(i=!1)):isNaN(e)?(e=parseInt(a),d=0<=e&&e<=100,d||(i=!1)):(d=0<=e&&e<=1,d||(i=!1))}),i}return!1},errorMessage:"",errorMessageKey:"badHsla"})}(a)});...

     */
    private static int process(final Options options) {
        final Crawler crawler = ComponentUtil.getComponent(Crawler.class);

        if (StringUtil.isBlank(options.sessionId)) {
            // use a default session id
            final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
            options.sessionId = sdf.format(new Date());
        } else {

     * @return The script type.
     */
    protected String getScriptType(final DataStoreParams paramMap) {
        final String value = paramMap.getAsString(SCRIPT_TYPE);
        if (StringUtil.isBlank(value)) {
            return Constants.DEFAULT_SCRIPT;
        }
        return value;
    }

    /**
     * Convert the value.
     * @param scriptType The script type.