*/
    public void setFlag(final int flag) {
        this.flag = flag;
    }

    /**
     * Sets the encoding for URL decoding.
     *
     * @param encoding the encoding to set
     */
    public void setEncoding(final String encoding) {
        this.encoding = encoding;
    }

        return url;
    }

    /**
     * Extracts and processes the site path from a URL with proper encoding handling.
     * Removes protocol, query parameters, and applies URL decoding based on encoding settings.
     *
     * @param u the URL string to process
     * @param encoding the character encoding to use for URL decoding
     * @return the processed site path, abbreviated if necessary
     */

 */
package org.codelibs.core.misc;

import java.util.Base64;

import org.codelibs.core.collection.ArrayUtil;
import org.codelibs.core.lang.StringUtil;

/**
 * Utility class for handling Base64 encoding and decoding.
 * <p>
 * This class now uses the standard {@link java.util.Base64} implementation
 * instead of a custom implementation, providing better security and performance.

 * This class provides utilities for processing document content, titles, and digests,
 * handling text normalization, content extraction, and similar document hash encoding/decoding.
 * It also manages document processing requests and integrates with the crawler system.
 *
 */
public class DocumentHelper {
    private static final Logger logger = LogManager.getLogger(DocumentHelper.class);

    }

    /**
     * Checks if the given URL should skip URL decoding when extracting file names.
     * Some protocols (like SMB, FTP, S3, GCS) should preserve the original URL encoding.
     *
     * @param url the URL to check
     * @return true if URL decoding should be skipped for this protocol
     */
    public boolean shouldSkipUrlDecode(final String url) {

        URLConnection conn = handler.openConnection(url);

        Handler.GcsURLConnection gcsConn = (Handler.GcsURLConnection) conn;

        assertEquals("mybucket", getField(gcsConn, "bucketName"));
        // URL decoding is handled by URL class
        assertEquals("path/file with spaces.txt", getField(gcsConn, "objectName"));
    }

    /**
     * Test URL parsing with hyphenated bucket name.
     */

            // Extract bucket name from host
            bucketName = url.getHost() != null ? url.getHost() : StringUtil.EMPTY;
            // Extract object name from path, removing leading slash if present and decoding URL encoding
            final String path = url.getPath();
            if (path != null && !path.isEmpty()) {
                final String pathWithoutLeadingSlash = path.startsWith("/") ? path.substring(1) : path;

            // Extract bucket name from host
            bucketName = url.getHost() != null ? url.getHost() : StringUtil.EMPTY;
            // Extract object name from path, removing leading slash if present and decoding URL encoding
            final String path = url.getPath();
            if (path != null && !path.isEmpty()) {
                final String pathWithoutLeadingSlash = path.startsWith("/") ? path.substring(1) : path;

            // Extract bucket name from host
            bucketName = url.getHost() != null ? url.getHost() : StringUtil.EMPTY;
            // Extract object name from path, removing leading slash if present and decoding URL encoding
            final String path = url.getPath();
            if (path != null && !path.isEmpty()) {
                final String pathWithoutLeadingSlash = path.startsWith("/") ? path.substring(1) : path;

    protected String transformation = BLOWFISH;

    /**
     * The key to use for encryption/decryption.
     */
    protected String key;

    /**
     * The character set name to use for encoding/decoding strings.
     */
    protected String charsetName = CoreLibConstants.UTF_8;

    /**
     * The queue of ciphers for encryption.
     */