whitespaceChars(ctype2, 0, ' ');
    }

    /**
     * Sets a range of characters as word characters.
     *
     * @param ctype2
     *            The array of character types.
     * @param low
     *            The lower bound of the character range (inclusive).
     * @param hi
     *            The upper bound of the character range (inclusive).
     */

        /**
         * Compiles a robots.txt path pattern into a regular expression pattern.
         * According to RFC 9309:
         * - '*' matches any sequence of characters (including the empty sequence)
         * - '$' matches the end of the URL path
         * - All other characters are matched literally
         *
         * @param pattern the robots.txt path pattern
         * @return the compiled Pattern object
         */

   *     separator
   */
  public abstract BaseEncoding withPadChar(char padChar);

  /**
   * Returns an encoding that behaves equivalently to this encoding, but adds a separator string
   * after every {@code n} characters. Any occurrences of any characters that occur in the separator
   * are skipped over in decoding.
   *

        // Test all special characters
        String input = "Test & < > \"quote\" 'apos'";
        String expected = "Test &amp; &lt; &gt; &quot;quote&quot; &apos;apos&apos;";
        assertEquals(expected, XmlUtil.escapeXml(input));
    }

    public void test_escapeXml_noSpecialChars() {
        // Test string without special characters
        String input = "Simple text without special characters";

            assertTrue("Error message should contain 'text content'", message.contains("text content"));
        }
    }

    /**
     * Test extraction with special characters and control characters.
     */
    public void test_getText_withSpecialCharacters_handlesCorrectly() {
        final String specialContent = "Tab\there\nNewline\rCarriage return\0Null byte";

      verifyHostname(hostname, it)
    }
  }

  /**
   * This is like [toLowerCase] except that it does nothing if this contains any non-ASCII
   * characters. We want to avoid lower casing special chars like U+212A (Kelvin symbol) because
   * they can return ASCII characters that match real hostnames.
   */
  private fun String.asciiToLowercase(): String =
    when {
      isAscii() -> lowercase(Locale.US) // This is an ASCII string.

        if (chars.length >= 2 && Character.isUpperCase(chars[0]) && Character.isUpperCase(chars[1])) {
            return name;
        }
        chars[0] = Character.toLowerCase(chars[0]);
        return new String(chars);
    }

    /**
     * Capitalizes a string according to JavaBeans conventions.
     * Note: If the first two characters are uppercase, the string will not be capitalized.
     * <p>

package org.codelibs.fess.crawler.util;

/**
 * Utility class for character-related operations.
 */
public final class CharUtil {
    /**
     * Private constructor to prevent instantiation of this utility class.
     */
    private CharUtil() {
    }

    /**
     * Checks if the given character is a valid URL character.
     *
     * Valid URL characters include:
     * - Lowercase letters (a-z)

	file *os.File // If non-nil, file descriptor to close.
}

func NewTokenizer(name string, r io.Reader, file *os.File) *Tokenizer {
	var s scanner.Scanner
	s.Init(r)
	// Newline is like a semicolon; other space characters are fine.
	s.Whitespace = 1<<'\t' | 1<<'\r' | 1<<' '
	// Don't skip comments: we need to count newlines.
	s.Mode = scanner.ScanChars |
		scanner.ScanFloats |
		scanner.ScanIdents |
		scanner.ScanInts |

    return asByteSource(url).read();
  }

  /**
   * Reads all characters from a URL into a {@link String}, using the given character set.
   *
   * @param url the URL to read from
   * @param charset the charset used to decode the input stream; see {@link StandardCharsets} for
   *     helpful predefined constants
   * @return a string containing all the characters from the URL
   * @throws IOException if an I/O error occurs.
   */