* either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.opensearch.extension.analysis;

import org.apache.lucene.analysis.Tokenizer;
import org.opensearch.common.settings.Settings;
import org.opensearch.env.Environment;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.analysis.AbstractTokenizerFactory;

/**

        List<AnalyzeToken> tokens = analyzer.analyzeAndReading(text, field, lang);

        assertNotNull(tokens);
        assertEquals(2, tokens.size()); // "Test123" and "ABC-456" (hyphen is not a split char in our tokenizer)
        assertEquals("Test123", tokens.get(0).getTerm());
        assertEquals("ABC-456", tokens.get(1).getTerm());
        assertEquals("TEST123", ((TestAnalyzeToken) tokens.get(0)).getReading());

├── nio/         # NIO file utilities
├── security/    # Security & MessageDigest utilities
├── sql/         # SQL type utilities
├── stream/      # Stream utilities
├── text/        # Text processing (JSON, Tokenizer)
├── timer/       # Timer utilities
├── xml/         # XML processing
├── zip/         # ZIP file utilities
└── CoreLibConstants.java
src/test/java/   # Test classes (mirrors main structure)
```

            if (handler == null) {
                final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                while (tokenizer.hasMoreTokens()) {
                    final String provider = tokenizer.nextToken().trim();
                    if (provider.equals("jcifs")) {
                        continue;
                    }

String escaped = JsonUtil.escape("Hello \"World\" with special chars");
String unescaped = JsonUtil.unescape(escaped);

// Text tokenization
Tokenizer tokenizer = new Tokenizer("field1,field2,field3", ",");
while (tokenizer.hasMoreTokens()) {
    String token = tokenizer.nextToken();
    // Process each token
}

// Decimal formatting
DecimalFormat format = DecimalFormatUtil.getDecimalFormat("###,###.00");
```

t n=this.lexer.state.inRawBlock;return{type:"text",raw:t[0],text:t[0],escaped:n}}}};var x=class l{tokens;options;state;inlineQueue;tokenizer;constructor(e){this.tokens=[],this.tokens.links=Object.create(null),this.options=e||R,this.options.tokenizer=this.options.tokenizer||new w,this.tokenizer=this.options.tokenizer,this.tokenizer.options=this.options,this.tokenizer.lexer=this,this.inlineQueue=[],this.state={inLink:!1,inRawBlock:!1,top:!0};let t={other:m,block:B.normal,inline:z.normal};this.opti...

                }
            }
            if (idx < tok.length) {
                qualifier = tok[idx];
                fallback = isDigits(qualifier);
            }

            // string tokenizer won't detect these and ignores them
            if (part1.contains("..") || part1.startsWith(".") || part1.endsWith(".")) {
                fallback = true;
            }

            if (fallback) {

 */

import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.synonym.SynonymMap;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;

      },
      "swedish_stemmer": {
        "type": "stemmer",
        "language": "swedish"
      }
    },
    "analyzer": {
      "sv_analyzer": {
        "type": "custom",
        "tokenizer": "standard",
        "filter": ["lowercase", "swedish_stop", "swedish_stemmer"]
      }
    }
  }
}
```

### 2. Add Custom Dictionaries (Optional)

		if i > 0 {
			name, value = name[:i], name[i+1:]
		}
		tokens := Tokenize(name)
		if len(tokens) != 1 || tokens[0].ScanToken != scanner.Ident {
			fmt.Fprintf(os.Stderr, "asm: parsing -D: %q is not a valid identifier name\n", tokens[0])
			flags.Usage()
		}
		macros[name] = &Macro{
			name:   name,
			args:   nil,
			tokens: Tokenize(value),
		}
	}
	return macros
}

var panicOnError bool // For testing.