return i > 0 && unicode.IsDigit(ch)
}

func (t *Tokenizer) Text() string {
	switch t.tok {
	case LSH:
		return "<<"
	case RSH:
		return ">>"
	case ARR:
		return "->"
	case ROT:
		return "@>"
	}
	return t.s.TokenText()
}

func (t *Tokenizer) File() string {
	return t.base.Filename()
}

func (t *Tokenizer) Base() *src.PosBase {
	return t.base
}

    static {
        setup(defaultCtype);
    }

    /**
     * Creates a {@link Tokenizer}.
     *
     * @param str
     *            The string. Must not be {@literal null}.
     */
    public Tokenizer(final String str) {
        this(str, defaultCtype);
    }

    /**
     * Creates a {@link Tokenizer}.
     *
     * @param str
     *            The string. Must not be {@literal null}.

        "tokenizer" : "standard"
      },
      "reading_term_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard"
      },
      "normalize_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "keyword",
        "char_filter" : ["mapping_char"],
        "filter" : ["lowercase"]
      },
      "contents_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard",

        "tokenizer" : "standard"
      },
      "reading_term_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard"
      },
      "normalize_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "keyword",
        "char_filter" : ["mapping_char"],
        "filter" : ["lowercase"]
      },
      "contents_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard",

            if (handler == null) {
                final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                while (tokenizer.hasMoreTokens()) {
                    final String provider = tokenizer.nextToken().trim();
                    if (provider.equals("jcifs")) {
                        continue;
                    }

          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "truncate20_filter",
            "lowercase",
            "arabic_stop",
            "arabic_normalization",
            "arabic_keywords",
            "arabic_override",
            "arabic_stemmer"
          ]
        },
        "armenian_analyzer": {
          "tokenizer":  "standard",
          "filter": [

            if (handler == null) {
                final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                while (tokenizer.hasMoreTokens()) {
                    final String provider = tokenizer.nextToken().trim();
                    if (provider.equals("jcifs.smb1")) {
                        continue;
                    }

        "tokenizer" : "standard"
      },
      "reading_term_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard"
      },
      "normalize_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "keyword",
        "char_filter" : ["mapping_char"],
        "filter" : ["lowercase"]
      },
      "contents_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard",

        "tokenizer" : "standard"
      },
      "reading_term_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard"
      },
      "normalize_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "keyword",
        "char_filter" : ["mapping_char"],
        "filter" : ["lowercase"]
      },
      "contents_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard",

    public void testEOF() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("");
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
    }

    /**
     * @throws Exception
     */
    @Test
    public void testWhitespace() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("\t       \n");