"number_of_shards": 1,
      "number_of_replicas": 0,
      "auto_expand_replicas": "0-1"
    },
    "analysis": {
      "analyzer": {
        "standard_analyzer": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "cjk_width",
            "asciifolding",
            "lowercase",
            "stop",
            "stemmer"
          ]
        }
      }
    }
  }

 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.opensearch.extension.analysis;

import org.apache.lucene.analysis.Tokenizer;
import org.opensearch.common.settings.Settings;
import org.opensearch.env.Environment;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.analysis.AbstractTokenizerFactory;

/**

          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "truncate20_filter",
            "lowercase",
            "arabic_stop",
            "arabic_normalization",
            "arabic_keywords",
            "arabic_override",
            "arabic_stemmer"
          ]
        },
        "armenian_analyzer": {
          "tokenizer":  "standard",
          "filter": [

          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "truncate20_filter",
            "lowercase",
            "arabic_stop",
            "arabic_normalization",
            "arabic_keywords",
            "arabic_override",
            "arabic_stemmer"
          ]
        },
        "armenian_analyzer": {
          "tokenizer":  "standard",
          "filter": [

String escaped = JsonUtil.escape("Hello \"World\" with special chars");
String unescaped = JsonUtil.unescape(escaped);

// Text tokenization
Tokenizer tokenizer = new Tokenizer("field1,field2,field3", ",");
while (tokenizer.hasMoreTokens()) {
    String token = tokenizer.nextToken();
    // Process each token
}

// Decimal formatting
DecimalFormat format = DecimalFormatUtil.getDecimalFormat("###,###.00");
```

        List<AnalyzeToken> tokens = analyzer.analyzeAndReading(text, field, lang);

        assertNotNull(tokens);
        assertEquals(2, tokens.size()); // "Test123" and "ABC-456" (hyphen is not a split char in our tokenizer)
        assertEquals("Test123", tokens.get(0).getTerm());
        assertEquals("ABC-456", tokens.get(1).getTerm());
        assertEquals("TEST123", ((TestAnalyzeToken) tokens.get(0)).getReading());

      "number_of_shards": 1,
      "number_of_replicas": 0,
      "auto_expand_replicas": "0-1"
    },
    "analysis": {
      "analyzer": {
        "standard_analyzer": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "cjk_width",
            "asciifolding",
            "lowercase",
            "stop",
            "stemmer"
          ]
        }
      }
    }
  }

      "number_of_shards": 1,
      "number_of_replicas": 0,
      "auto_expand_replicas": "0-1"
    },
    "analysis": {
      "analyzer": {
        "standard_analyzer": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "cjk_width",
            "asciifolding",
            "lowercase",
            "stop",
            "stemmer"
          ]
        }
      }
    }
  }

                }
            }
            if (idx < tok.length) {
                qualifier = tok[idx];
                fallback = isDigits(qualifier);
            }

            // string tokenizer won't detect these and ignores them
            if (part1.contains("..") || part1.startsWith(".") || part1.endsWith(".")) {
                fallback = true;
            }

            if (fallback) {

 */

import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.synonym.SynonymMap;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;