}
                return;
            }
            logger.warn("Failed to store failure url: url={}", urlQueue.getUrl(), e);
        }

        if (cae.isDebugEnabled()) {
            logger.debug("[{}] Crawling Access Exception: url={}", failureUrlId, urlQueue.getUrl(), cae);
        } else if (cae.isInfoEnabled()) {
            logger.info("[{}] {}", failureUrlId, cae.getMessage());

        return this.max_xmit;
    }

    /**
     * Get a handle to a service
     *
     * @param url
     *            the DCE/RPC service URL
     * @param tc
     *            context to use
     * @return a DCERPC handle for the given url
     * @throws MalformedURLException
     *            if the URL is malformed
     * @throws DcerpcException
     *            if there is an error establishing the connection

    }

    /**
     * Loads the specified {@link URL} into the {@link Properties} (wraps exception handling).
     *
     * @param props
     *            Property set. Must not be {@literal null}.
     * @param url
     *            URL. Must not be {@literal null}.
     */
    public static void load(final Properties props, final URL url) {
        assertArgumentNotNull("props", props);

                }
            }
        }
        return data;
    }

    /**
     * Sets the URL of the API endpoint.
     * @param url The URL to set.
     */
    public void setUrl(final String url) {
        this.url = url;
    }

    /**
     * Sets the connection timeout.
     * @param connectionTimeout The connection timeout in milliseconds.
     */

     * @throws CrawlerSystemException if an error occurs while processing the URL
     */
    @Override
    protected void delayBeforeProcessing() {
        final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
        if (urlQueue == null) {
            return;
        }

        final String url = urlQueue.getUrl();
        if (StringUtil.isBlank(url) || url.startsWith("file:")) {
            // not target
            return;

import org.opensearch.search.sort.SortBuilders;
import org.opensearch.search.sort.SortOrder;

/**
 * Fess-specific URL queue service that extends OpenSearch URL queue functionality.
 * This service provides customized URL fetching strategies including sequential and random ordering
 * based on crawling configuration parameters.
 */
public class FessUrlQueueService extends OpenSearchUrlQueueService {

 * <pre>
 * # All SP URLs will be derived from this base URL
 * saml.sp.base.url=https://your-fess-server.example.com
 * </pre>
 *
 * <h3>Option 2: Set individual SP URLs</h3>
 * <pre>
 * # SP Entity ID (Audience URI in IdP)
 * saml.sp.entityid=https://your-fess-server.example.com/sso/metadata
 *
 * # Assertion Consumer Service URL

        final String url = params != null ? params.get(ExtractData.URL) : null;
        if (!passwordMap.isEmpty()) {
            final String resourceName = params != null ? params.get(ExtractData.RESOURCE_NAME_KEY) : null;

            String value = null;
            if (StringUtil.isNotEmpty(url)) {
                value = url;
            } else if (StringUtil.isNotEmpty(resourceName)) {

    }

    public void test_doGet() {
        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final ResponseData responseData = httpClient.doGet(url);
            assertEquals(200, responseData.getHttpStatusCode());
        } finally {
            server.stop();
        }
    }

            return method(Method.POST);
        }

        /**
         * Sets the URL for this request data.
         *
         * @param url the URL string to be set
         * @return the current RequestDataContext instance for method chaining
         */
        public RequestDataContext url(final String url) {
            data.setUrl(url);
            return this;
        }

        /**