*/
package org.codelibs.fess.crawler.util;

/**
 * Utility class for character-related operations.
 */
public final class CharUtil {
    /**
     * Private constructor to prevent instantiation of this utility class.
     */
    private CharUtil() {
    }

    /**
     * Checks if the given character is a valid URL character.
     *
     * Valid URL characters include:
     * - Lowercase letters (a-z)

        }
        return c;
    }

    /**
     * Merges two characters, handling voiced and semi-voiced sound marks.
     * @param c1 The first character.
     * @param c2 The second character.
     * @return The merged character.
     */
    public static char mergeChar(final char c1, final char c2) {
        if (c2 == 'ﾞ') {
            if ("ｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃﾄﾊﾋﾌﾍﾎ".indexOf(c1) >= 0) {

     * according to the XML 1.0 specification. Valid characters include:
     * - Tab (0x9)
     * - Line feed (0xA)
     * - Carriage return (0xD)
     * - Any character between 0x20 and 0xD7FF
     * - Any character between 0xE000 and 0xFFFD
     * - Any character between 0x10000 and 0x10FFFF
     *
     * @param in the input string to be processed
     * @return a new string with invalid XML characters removed, or the original

        return parse(stream, Constants.UTF_8);
    }

    /**
     * Parses a robots.txt file from the given input stream using the specified character encoding.
     * @param stream the input stream to parse
     * @param charsetName the character encoding to use
     * @return the parsed RobotsTxt object, or null if disabled
     */
    public RobotsTxt parse(final InputStream stream, final String charsetName) {

import java.util.HashMap;
import java.util.Locale;
import java.util.Map;

import org.codelibs.core.lang.StringUtil;

/**
 * EncodingHelper provides utility methods for managing and normalizing character encodings.
 * It allows setting a default encoding, mapping various encoding names to preferred ones,
 * and normalizing an encoding string to its preferred form or the default if no mapping is found.
 */

import java.nio.charset.StandardCharsets;

/**
 * Constants used in the fess-crawler.
 * This class provides a collection of constant values for HTTP methods, status codes,
 * transformer names, boolean values, character encodings, date/time formats, and XML features.
 * It is designed to avoid the instantiation.
 */
public final class Constants {
    /**
     * The GET method.
     */

            new LookupTranslator(EntityArrays.HTML40_EXTENDED_UNESCAPE), new NumericEntityUnescaper());

    /**
     * Default character encoding for content extraction.
     */
    protected String encoding = Constants.UTF_8;

    /**
     * The preload size for charset detection.
     */
    protected int preloadSizeForCharset = 2048;

 * It also provides a method to retrieve the extracted data as a String from an AccessResultData object.
 *
 * <p>
 * The class handles character encoding issues by attempting to use the specified charset.
 * If the specified charset is invalid, it falls back to UTF-8.
 * </p>
 *
 * <p>