* Constructs a new S3URLConnection for the specified URL.
         * This constructor parses the URL to extract bucket and object names.
         *
         * @param url The S3 URL to connect to
         */
        protected S3URLConnection(final URL url) {
            super(url);
            // Extract bucket name from host
            bucketName = url.getHost() != null ? url.getHost() : StringUtil.EMPTY;

     * This method applies duplicate host conversion before storing the URL.
     *
     * @param childUrl the child URL to store
     * @param parentUrl the parent URL that referenced this child URL
     * @param weight the weight/priority of the child URL
     * @param depth the crawling depth of the child URL
     */
    @Override

/**
 * Service interface for managing URL filters.
 * Provides methods to add and remove include/exclude URL filters,
 * as well as retrieve the patterns of these filters.
 */
public interface UrlFilterService {

    /**
     * Adds a URL to the include filter list for the specified session.
     *
     * @param sessionId the ID of the session for which the URL filter is being added

         *
         * @param url the URL to crawl
         * @param depth the depth of this URL in the crawling hierarchy
         */
        CrawlRequest(final String url, final int depth) {
            this.url = url;
            this.depth = depth;
        }

        /**
         * Gets the URL of this crawl request.
         *
         * @return the URL to be crawled
         */
        public String getUrl() {

     */
    public void test_protocol() throws Exception {
        URL url = new URL("storage://bucket/object");
        assertEquals("storage", url.getProtocol());
    }

    /**
     * Test URL parsing with bucket and object path.
     */
    public void test_urlParsing_bucketAndObject() throws Exception {
        URL url = new URL("storage://mybucket/path/to/object.txt");
        Handler handler = new Handler();

         * Constructs a new GcsURLConnection for the specified URL.
         * This constructor parses the URL to extract bucket and object names.
         *
         * @param url The GCS URL to connect to
         */
        protected GcsURLConnection(final URL url) {
            super(url);
            // Extract bucket name from host
            bucketName = url.getHost() != null ? url.getHost() : StringUtil.EMPTY;

                        + "  <url>\n" + "    <loc>http://www.example.com/page1.html</loc>\n" + "    <lastmod>2025-01-01</lastmod>\n"
                        + "    <priority>0.8</priority>\n" + "  </url>\n" + "  <url>\n" + "    <loc></loc>\n" + "  </url>\n" + "  <url>\n"
                        + "    <loc>http://www.example.com/page2.html</loc>\n" + "    <changefreq>daily</changefreq>\n" + "  </url>\n"

        return url.startsWith("file:") || url.startsWith("smb:") || url.startsWith("smb1:") || url.startsWith("ftp:")
                || url.startsWith("storage:") || url.startsWith("s3:") || url.startsWith("gcs:");
    }

    /**
     * Checks if the given URL should skip URL decoding when extracting file names.
     * Some protocols (like SMB, FTP, S3, GCS) should preserve the original URL encoding.
     *

        this.data = data;
    }

    /**
     * Add a child URL.
     * @param url the request data to add to the child URL set
     */
    public void addUrl(final RequestData url) {
        childUrlSet.add(url);
    }

    /**
     * Add child URLs.
     * @param c the collection of request data to add to the child URL set
     */
    public void addAllUrl(final Collection<RequestData> c) {

     *
     * @param url the URL that requires authentication
     * @param sae the authentication exception that was thrown
     * @return credentials returned by prompt
     */
    public static NtlmPasswordAuthenticator requestNtlmPasswordAuthentication(final String url, final SmbAuthException sae) {
        return requestNtlmPasswordAuthentication(auth, url, sae);
    }

    /**