url = url.substring(0, idx);
        }

        url = url.replace("/./", "/");

        idx = url.indexOf(";jsessionid");
        if (idx >= 0) {
            url = url.replaceFirst(";jsessionid=[a-zA-Z0-9\\.]*", "");
        }

        if (url.indexOf(' ') >= 0) {
            url = url.replace(" ", "%20");
        }

        String oldUrl = null;

     * Checks if a URL exists in the queue by searching OpenSearch and local caches.
     *
     * @param sessionId The session ID.
     * @param url The URL to check.
     * @return true if the URL exists in the queue, false otherwise.
     */
    @Override
    protected boolean exists(final String sessionId, final String url) {
        final boolean ret = super.exists(sessionId, url);
        if (!ret) {

         * Constructs a new S3URLConnection for the specified URL.
         * This constructor parses the URL to extract bucket and object names.
         *
         * @param url The S3 URL to connect to
         */
        protected S3URLConnection(final URL url) {
            super(url);
            // Extract bucket name from host
            bucketName = url.getHost() != null ? url.getHost() : StringUtil.EMPTY;

         * Constructs a new GcsURLConnection for the specified URL.
         * This constructor parses the URL to extract bucket and object names.
         *
         * @param url The GCS URL to connect to
         */
        protected GcsURLConnection(final URL url) {
            super(url);
            // Extract bucket name from host
            bucketName = url.getHost() != null ? url.getHost() : StringUtil.EMPTY;

                        + "  <url>\n" + "    <loc>http://www.example.com/page1.html</loc>\n" + "    <lastmod>2025-01-01</lastmod>\n"
                        + "    <priority>0.8</priority>\n" + "  </url>\n" + "  <url>\n" + "    <loc></loc>\n" + "  </url>\n" + "  <url>\n"
                        + "    <loc>http://www.example.com/page2.html</loc>\n" + "    <changefreq>daily</changefreq>\n" + "  </url>\n"

         * This constructor parses the URL to extract bucket and object names.
         *
         * @param url The storage URL to connect to
         */
        protected StorageURLConnection(final URL url) {
            super(url);
            // Extract bucket name from host
            bucketName = url.getHost() != null ? url.getHost() : StringUtil.EMPTY;

    /**
     * Get the host on file.
     * @param url The URL.
     * @return The host on file.
     */
    protected String getHostOnFile(final String url) {
        if (StringUtil.isBlank(url)) {
            return StringUtil.EMPTY; // empty
        }

        if (url.startsWith("file:////")) {
            final String value = decodeUrlAsName(url.substring(9), true);
            final int pos = value.indexOf('/');

    /** Delay in milliseconds after processing a URL */
    protected long delayMillisAfterProcessing = 0L;

    /** Delay in milliseconds when no URL is in the queue */
    protected long delayMillisAtNoUrlInQueue = 500L;

    /** Delay in milliseconds before processing a URL */
    protected long delayMillisBeforeProcessing = 0L;

        return url.startsWith("file:") || url.startsWith("smb:") || url.startsWith("smb1:") || url.startsWith("ftp:")
                || url.startsWith("storage:") || url.startsWith("s3:") || url.startsWith("gcs:");
    }

    /**
     * Checks if the given URL should skip URL decoding when extracting file names.
     * Some protocols (like SMB, FTP, S3, GCS) should preserve the original URL encoding.
     *

     * This method applies duplicate host conversion before storing the URL.
     *
     * @param childUrl the child URL to store
     * @param parentUrl the parent URL that referenced this child URL
     * @param weight the weight/priority of the child URL
     * @param depth the crawling depth of the child URL
     */
    @Override