Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 10 of 19 for tokenizer (0.03 sec)

  2. github.com/codelibs/corelib

    src/main/java/org/codelibs/core/text/Tokenizer.java 

        static {
        setup(defaultCtype);
    }

    /**
     * Creates a {@link Tokenizer}.
     *
     * @param str
     *            The string. Must not be {@literal null}.
     */
    public Tokenizer(final String str) {
        this(str, defaultCtype);
    }

    /**
     * Creates a {@link Tokenizer}.
     *
     * @param str
     *            The string. Must not be {@literal null}.
    Registered: Fri Sep 05 20:58:11 UTC 2025
    - Last Modified: Sat Jul 05 00:11:05 UTC 2025
    - 8.8K bytes
    - Viewed (0)
  3. github.com/codelibs/fess

    src/main/resources/suggest_indices/_cloud/suggest_analyzer.json 

            "tokenizer" : "standard"
      },
      "reading_term_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard"
      },
      "normalize_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "keyword",
        "char_filter" : ["mapping_char"],
        "filter" : ["lowercase"]
      },
      "contents_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard",
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Sat Feb 27 09:26:16 UTC 2021
    - 57.4K bytes
    - Viewed (1)
  4. github.com/codelibs/fess

    src/main/resources/fess_indices/fess.json 

              "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "truncate20_filter",
            "lowercase",
            "arabic_stop",
            "arabic_normalization",
            "arabic_keywords",
            "arabic_override",
            "arabic_stemmer"
          ]
        },
        "armenian_analyzer": {
          "tokenizer":  "standard",
          "filter": [
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Fri Jun 06 14:17:42 UTC 2025
    - 39.8K bytes
    - Viewed (0)
  5. github.com/google/guava

    android/guava/src/com/google/common/net/MediaType.java 

        checkNotNull(input);
    Tokenizer tokenizer = new Tokenizer(input);
    try {
      String type = tokenizer.consumeToken(TOKEN_MATCHER);
      consumeSeparator(tokenizer, '/');
      String subtype = tokenizer.consumeToken(TOKEN_MATCHER);
      ImmutableListMultimap.Builder<String, String> parameters = ImmutableListMultimap.builder();
      while (tokenizer.hasMore()) {
        consumeSeparator(tokenizer, ';');
    Registered: Fri Sep 05 12:43:10 UTC 2025
    - Last Modified: Tue May 13 17:27:14 UTC 2025
    - 48K bytes
    - Viewed (0)
  6. github.com/google/guava

    guava/src/com/google/common/net/MediaType.java 

        checkNotNull(input);
    Tokenizer tokenizer = new Tokenizer(input);
    try {
      String type = tokenizer.consumeToken(TOKEN_MATCHER);
      consumeSeparator(tokenizer, '/');
      String subtype = tokenizer.consumeToken(TOKEN_MATCHER);
      ImmutableListMultimap.Builder<String, String> parameters = ImmutableListMultimap.builder();
      while (tokenizer.hasMore()) {
        consumeSeparator(tokenizer, ';');
    Registered: Fri Sep 05 12:43:10 UTC 2025
    - Last Modified: Tue May 13 17:27:14 UTC 2025
    - 48K bytes
    - Viewed (0)
  7. github.com/codelibs/fess

    src/main/resources/suggest_indices/_aws/suggest_analyzer.json 

            "tokenizer" : "standard"
      },
      "reading_term_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard"
      },
      "normalize_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "keyword",
        "char_filter" : ["mapping_char"],
        "filter" : ["lowercase"]
      },
      "contents_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard",
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Wed Mar 24 12:55:37 UTC 2021
    - 57.4K bytes
    - Viewed (0)
  8. github.com/codelibs/fess

    src/main/resources/suggest_indices/suggest_analyzer.json 

            "tokenizer" : "standard"
      },
      "reading_term_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard"
      },
      "normalize_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "keyword",
        "char_filter" : ["mapping_char"],
        "filter" : ["lowercase"]
      },
      "contents_analyzer_mk" : {
        "type" : "custom",
        "tokenizer" : "standard",
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Thu May 23 05:09:51 UTC 2019
    - 57.7K bytes
    - Viewed (0)
  9. github.com/codelibs/corelib

    src/test/java/org/codelibs/core/text/TokenizerTest.java 

        public void testEOF() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("");
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
    }

    /**
     * @throws Exception
     */
    @Test
    public void testWhitespace() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("\t       \n");
    Registered: Fri Sep 05 20:58:11 UTC 2025
    - Last Modified: Sat May 10 01:32:17 UTC 2025
    - 2K bytes
    - Viewed (0)
  10. github.com/codelibs/fess

    src/main/resources/fess_indices/fess_config.data_config.json 

          "number_of_shards": 1,
      "number_of_replicas": 0,
      "auto_expand_replicas": "0-1"
    },
    "analysis": {
      "analyzer": {
        "standard_analyzer": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "cjk_width",
            "asciifolding",
            "lowercase",
            "stop",
            "stemmer"
          ]
        }
      }
    }
  }
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Thu Dec 02 13:14:56 UTC 2021
    - 484 bytes
    - Viewed (0)
  11. github.com/codelibs/fess

    src/main/resources/fess_indices/fess_config.file_config.json 

          "number_of_shards": 1,
      "number_of_replicas": 0,
      "auto_expand_replicas": "0-1"
    },
    "analysis": {
      "analyzer": {
        "standard_analyzer": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": [
            "cjk_width",
            "asciifolding",
            "lowercase",
            "stop",
            "stemmer"
          ]
        }
      }
    }
  }
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Thu Dec 02 13:14:56 UTC 2021
    - 484 bytes
    - Viewed (0)
Back to top