if (xpath == null) {
            xpath = fessConfig.getCrawlerDocumentHtmlCanonicalXpath();
        }
        if (StringUtil.isBlank(xpath)) {
            return null;
        }
        final String canonicalUrl = getSingleNodeValue(document, xpath, node -> node);
        if (StringUtil.isBlank(canonicalUrl)) {
            return null;
        }
        return normalizeCanonicalUrl(responseData.getUrl(), canonicalUrl);
    }

            }
        }
        return LaResponseUtil.getResponse().encodeURL(sb.toString());
    }

    public static String sdh(final String input) {
        if (StringUtil.isBlank(input)) {
            return input;
        }
        return ComponentUtil.getDocumentHelper().encodeSimilarDocHash(input);
    }

    public static String join(final Object input) {

    private static final long serialVersionUID = 1L;

    @Override
    public String getScriptType() {
        final String st = super.getScriptType();
        if (StringUtil.isBlank(st)) {
            return "groovy";
        }
        return st;
    }

    public boolean isLoggingEnabled() {
        return Constants.T.equals(getJobLogging());
    }

            throw new PluginException("Failed to delete the artifact " + fileName, e);
        }
    }

    public Artifact getArtifact(final String name, final String version) {
        if (StringUtil.isBlank(name) || StringUtil.isBlank(version)) {
            return null;
        }
        for (final Artifact artifact : getAvailableArtifacts(ArtifactType.getType(name))) {

            dfos = out;
            CopyUtil.copy(in, out);
            out.flush();

            Extractor extractor = StringUtil.isBlank(mimeType) ? null : extractorFactory.getExtractor(mimeType);
            if (extractor == null) {
                final String detectedMimeType = getMimeType(out);

            return new HtmlNext(basePath + page.getRoutingPath());
        }, page);
    }

    protected String getVirtualHostBasePath(final String s, final HtmlNext page) {
        return StringUtil.isBlank(s) ? StringUtil.EMPTY : "/" + s;
    }

    public String getVirtualHostKey() {
        return LaRequestUtil.getOptionalRequest().map(req -> (String) req.getAttribute(FessConfig.VIRTUAL_HOST_VALUE)).orElseGet(() -> {

        includedDocUrls = fessConfig.getCrawlerDocumentHtmlDefaultIncludeSearchPatterns();
        excludedDocUrls = fessConfig.getCrawlerDocumentHtmlDefaultExcludeSearchPatterns();
        boost = 1.0f;
        if (StringUtil.isBlank(userAgent)) {
            userAgent = fessConfig.getUserAgentName();
        }
        numOfThread = Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB;
        intervalTime = Constants.DEFAULT_INTERVAL_TIME_FOR_WEB;

        }
    }

    private static int process(final Options options) {
        final Crawler crawler = ComponentUtil.getComponent(Crawler.class);

        if (StringUtil.isBlank(options.sessionId)) {
            // use a default session id
            final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
            options.sessionId = sdf.format(new Date());
        } else {

            return "Unknown error";
        }

        final Supplier<String> stacktraceString = () -> {
            final StringBuilder sb = new StringBuilder();
            if (StringUtil.isBlank(cause.getMessage())) {
                sb.append(cause.getClass().getName());
            } else {
                sb.append(cause.getMessage());
            }

            return LaRequestUtil.getOptionalRequest().map(request -> {
                final String userAgent = request.getHeader("user-agent");
                if (StringUtil.isBlank(userAgent)) {
                    return false;
                }

                return pathMapping.getUAMatcher(userAgent).find();
            }).orElse(false);
        }
        return false;
    }