if (list == null) {
            final String value = getSystemProperty(Constants.DEFAULT_SORT_VALUE_PROPERTY);
            if (StringUtil.isBlank(value)) {
                list = Collections.emptyList();
            } else {
                final Set<String> keySet = new HashSet<>();

                        final String userCode = userInfoHelper.getUserCode();

                        if (StringUtil.isBlank(userCode)) {
                            throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, "No user session.");
                        }
                        if (StringUtil.isBlank(favoriteUrl)) {
                            throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, "URL is null.");

        if (xpath == null) {
            xpath = fessConfig.getCrawlerDocumentHtmlCanonicalXpath();
        }
        if (StringUtil.isBlank(xpath)) {
            return null;
        }
        final String canonicalUrl = getSingleNodeValue(document, xpath, node -> node);
        if (StringUtil.isBlank(canonicalUrl)) {
            return null;
        }
        return normalizeCanonicalUrl(responseData.getUrl(), canonicalUrl);
    }

        String title = DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class);
        if (StringUtil.isBlank(title)) {
            title = DocumentUtil.getValue(document, fessConfig.getIndexFieldFilename(), String.class);
            if (StringUtil.isBlank(title)) {
                title = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
            }
        }

     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {
            throw new CrawlerSystemException("url is null or empty.");
        }

        if (robotsTxtHelper == null || !robotsTxtHelper.isEnabled()) {
            // not support robots.txt

     * @return an optional containing the authenticated user if successful, empty otherwise
     */
    public OptionalEntity<FessUser> login(final String username, final String password) {
        if (StringUtil.isBlank(fessConfig.getLdapProviderUrl()) || !validate()) {
            return OptionalEntity.empty();
        }

        final Hashtable<String, String> env = createSearchEnv(username, password);