*
         * This method reads characters from the reader and processes them according to the following rules:
         * - ISO control characters and space characters are treated as spaces.
         * - Alphanumeric characters (0-9, A-Z, a-z) are appended to the buffer.
         * - Symbol characters (!-/, :-@, [-`, {-~) are appended to the buffer.

 * It also provides a method to retrieve the extracted data as a String from an AccessResultData object.
 *
 * <p>
 * The class handles character encoding issues by attempting to use the specified charset.
 * If the specified charset is invalid, it falls back to UTF-8.
 * </p>
 *
 * <p>

      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all

 * FormScheme formScheme = new FormScheme(params);
 * formScheme.authenticate(credentials, executor);
 * }
 * </pre>
 *
 * <p>Parameters:
 * <ul>
 * <li>ENCODING - The character encoding to use for request parameters.</li>
 * <li>TOKEN_URL - The URL to request the token from.</li>
 * <li>TOKEN_PATTERN - The regex pattern to extract the token from the response.</li>

        final StringBuilder buf = new StringBuilder(searchWord.length());
        char prev = 0;
        for (final char c : searchWord.toCharArray()) {
            if (!Character.isWhitespace(c)) {
                buf.append(c);
            } else if (!Character.isWhitespace(prev)) {
                buf.append(' ');
            }
            prev = c;
        }
        final String[] words = buf.toString().trim().split(" ");

      negligent acts) or agreed to in writing, shall any Contributor be
      liable to You for damages, including any direct, indirect, special,
      incidental, or consequential damages of any character arising as a
      result of this License or out of the use or inability to use the
      Work (including but not limited to damages for loss of goodwill,
      work stoppage, computer failure or malfunction, or any and all

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.suggest.request.suggest;

import java.io.IOException;
import java.lang.Character.UnicodeBlock;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.codelibs.fess.suggest.concurrent.Deferred;
import org.codelibs.fess.suggest.constants.FieldNames;

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.suggest.normalizer;

/**
 * Normalizes full-width alphanumeric characters to half-width alphanumeric characters.
 */
public class FullWidthToHalfWidthAlphabetNormalizer implements Normalizer {
    /**
     * Constructs a new {@link FullWidthToHalfWidthAlphabetNormalizer}.
     */

    protected int initialBufferSize = 10000;

    /**
     * If true, duplicated terms are replaced.
     */
    protected boolean replaceDuplication = false;

    /**
     * Space characters. Default includes common space characters.
     */
    protected int[] spaceChars = { '\u0020', '\u00a0', '\u3000', '\ufffd' };

    /**
     * Memory size.
     */
    protected int memorySize = 1024 * 1024; //1mb

 *
 * <p>
 * This class provides methods to:
 * </p>
 * <ul>
 *   <li>Detect the MIME type based on the input stream and filename.</li>
 *   <li>Normalize the filename to handle special characters.</li>
 *   <li>Configure whether to use the filename for MIME type detection.</li>
 *   <li>Configure whether to use the filename for MIME type detection when the stream is octet-stream.</li>
 * </ul>
 *
 * <p>