}

    /**
     * Validates if the canonical URL is valid relative to the original URL.
     * Specifically checks for HTTPS to HTTP downgrades.
     *
     * @param url the original URL
     * @param canonicalUrl the canonical URL to validate
     * @return true if the canonical URL is valid, false otherwise
     */
    protected boolean isValidCanonicalUrl(final String url, final String canonicalUrl) {

/**
 * UrlFilter checks if a given url is a target one.
 *
 * @author shinsuke
 *
 */
public interface UrlFilter {

    /**
     * Initialize a url filter by sessionId.
     *
     * @param sessionId Session ID
     */
    void init(String sessionId);

    /**
     * Check if a given url is a target.
     *
     * @param url URL
     * @return true if url is matched
     */

        // This test ensures that for a standard HTTP URL, openConnection wraps the connection
        // in an NtlmHttpURLConnection, using mock handlers to avoid dependency on system handlers.
        setupMockProtocolHandlers();
        URL url = new URL("http://example.com/resource");
        URLConnection connection = handler.openConnection(url);

        assertNotNull(connection, "Connection should not be null.");

        final boolean isHttpUrl = url.startsWith("http:") || url.startsWith("https:");

        if (isSmbUrl) {
            url = url.replace("smb:", "file:");
            url = url.replace("smb1:", "file:");
        }

        if (isHttpUrl && isSmbOrFtpUrl) {
            //  smb/ftp->http
            // encode
            final StringBuilder buf = new StringBuilder(url.length() + 100);

        super(message, cause);
        this.url = url;
        this.abort = abort;
    }

    /**
     * Gets the URL where the crawling error occurred.
     *
     * @return the URL associated with this exception
     */
    public String getUrl() {
        return url;
    }

    /**
     * Checks whether the crawling process should be aborted due to this exception.
     *

        final String url = "http://test.com/";
        final ResponseData response = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
        assertEquals(1, testListener.startCount);
        assertEquals(1, testListener.requestCount);
        assertEquals(0, testListener.exceptionCount);
        assertEquals(1, testListener.endCount);
        assertEquals(url, testListener.requestUrl);

    /** Delay in milliseconds after processing a URL */
    protected long delayMillisAfterProcessing = 0L;

    /** Delay in milliseconds when no URL is in the queue */
    protected long delayMillisAtNoUrlInQueue = 500L;

    /** Delay in milliseconds before processing a URL */
    protected long delayMillisBeforeProcessing = 0L;

    }

    /**
     * Returns the URL.
     * @return The URL.
     */
    public String getUrl() {
        return url;
    }

    /**
     * Sets the URL.
     * @param url The URL.
     */
    public void setUrl(final String url) {
        this.url = url;
    }

    /**
     * Converts this URL filter to XContent format for OpenSearch indexing.
     *

     *
     * @param url
     *            The URL. Must not be {@literal null}.
     * @return A {@link URLConnection} object to the URL.
     */
    public static URLConnection openConnection(final URL url) {
        assertArgumentNotNull("url", url);

        try {
            final URLConnection connection = url.openConnection();
            connection.setUseCaches(false);

                    }
                });
    }

    /**
     * Adds an include URL filter for the specified session.
     *
     * @param sessionId The session ID.
     * @param url The URL pattern to include.
     */
    @Override
    public void addIncludeUrlFilter(final String sessionId, final String url) {
        final OpenSearchUrlFilter esUrlFilter = new OpenSearchUrlFilter();