* @param roleList The list to add role names to.
     * @param url The Microsoft Graph API URL.
     */
    protected void processMemberOf(final EntraIdUser user, final List<String> groupList, final List<String> roleList, final String url) {
        if (logger.isDebugEnabled()) {
            logger.debug("url={}", url);
        }
        try (CurlResponse response = Curl.get(url)

				"text/sgml",
				"text/t140",
				"text/tab-separated-values",
				"text/troff",
				"text/ulpfec",
				"text/uri-list",
				"text/vnd.abc",
				"text/vnd.curl",
				"text/vnd.curl.dcurl",
				"text/vnd.curl.scurl",
				"text/vnd.curl.mcurl",
				"text/vnd.dmclientscript",
				"text/vnd.esmertec.theme-descriptor",
				"text/vnd.fly",
				"text/vnd.fmi.flexstor",
				"text/vnd.graphviz",

    }

    public void setUrl_Equal(String url) {
        setUrl_Term(url, null);
    }

    public void setUrl_Equal(String url, ConditionOptionCall<TermQueryBuilder> opLambda) {
        setUrl_Term(url, opLambda);
    }

    public void setUrl_Term(String url) {
        setUrl_Term(url, null);
    }

    public void setUrl_Term(String url, ConditionOptionCall<TermQueryBuilder> opLambda) {

    }

    /**
     * Validates if the canonical URL is valid relative to the original URL.
     * Specifically checks for HTTPS to HTTP downgrades.
     *
     * @param url the original URL
     * @param canonicalUrl the canonical URL to validate
     * @return true if the canonical URL is valid, false otherwise
     */
    protected boolean isValidCanonicalUrl(final String url, final String canonicalUrl) {

        final boolean isHttpUrl = url.startsWith("http:") || url.startsWith("https:");

        if (isSmbUrl) {
            url = url.replace("smb:", "file:");
            url = url.replace("smb1:", "file:");
        }

        if (isHttpUrl && isSmbOrFtpUrl) {
            //  smb/ftp->http
            // encode
            final StringBuilder buf = new StringBuilder(url.length() + 100);

    }

    public void setUrl_Equal(String url) {
        setUrl_Term(url, null);
    }

    public void setUrl_Equal(String url, ConditionOptionCall<TermQueryBuilder> opLambda) {
        setUrl_Term(url, opLambda);
    }

    public void setUrl_Term(String url) {
        setUrl_Term(url, null);
    }

    public void setUrl_Term(String url, ConditionOptionCall<TermQueryBuilder> opLambda) {

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

labels.excludedUrls=Исключенные URL для сканирования
labels.excludedDocPaths=Исключенные пути для поиска
labels.excludedDocUrls=Исключенные URL для поиска
labels.hostname=Имя хоста
labels.id=ID
labels.includedPaths=Включенные пути для сканирования
labels.includedUrls=Включенные URL для сканирования
labels.includedDocPaths=Включенные пути для поиска
labels.includedDocUrls=Включенные URL для поиска

labels.web_crawling_configuration=Exploration Web
labels.web_crawling_title_details=Configuration de l'exploration Web
labels.included_urls=URL incluses
labels.excluded_urls=URL exclues
labels.included_doc_urls=URL de documents incluses
labels.excluded_doc_urls=URL de documents exclues
labels.user_agent=Agent utilisateur
labels.web_crawling_button_create=Créer
labels.web_crawling_button_create_job=Créer une nouvelle tâche

<td align="left" width="15" style="width:15px;padding:0;margin:0;line-height:1px;font-size:1px;"></td>