Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 10 of 38 for Tokenizer (0.12 sec)

  1. src/cmd/asm/internal/lex/tokenizer.go

    	return i > 0 && unicode.IsDigit(ch)
    }
    
    func (t *Tokenizer) Text() string {
    	switch t.tok {
    	case LSH:
    		return "<<"
    	case RSH:
    		return ">>"
    	case ARR:
    		return "->"
    	case ROT:
    		return "@>"
    	}
    	return t.s.TokenText()
    }
    
    func (t *Tokenizer) File() string {
    	return t.base.Filename()
    }
    
    func (t *Tokenizer) Base() *src.PosBase {
    	return t.base
    }
    
    Registered: Tue Sep 09 11:13:09 UTC 2025
    - Last Modified: Thu Aug 04 20:35:21 UTC 2022
    - 3K bytes
    - Viewed (0)
  2. src/main/java/org/codelibs/core/text/Tokenizer.java

        static {
            setup(defaultCtype);
        }
    
        /**
         * Creates a {@link Tokenizer}.
         *
         * @param str
         *            The string. Must not be {@literal null}.
         */
        public Tokenizer(final String str) {
            this(str, defaultCtype);
        }
    
        /**
         * Creates a {@link Tokenizer}.
         *
         * @param str
         *            The string. Must not be {@literal null}.
    Registered: Fri Sep 05 20:58:11 UTC 2025
    - Last Modified: Sat Jul 05 00:11:05 UTC 2025
    - 8.8K bytes
    - Viewed (0)
  3. src/main/resources/suggest_indices/suggest_analyzer.json

            "tokenizer" : "standard"
          },
          "reading_term_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard"
          },
          "normalize_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "keyword",
            "char_filter" : ["mapping_char"],
            "filter" : ["lowercase"]
          },
          "contents_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard",
    Registered: Fri Sep 19 09:08:11 UTC 2025
    - Last Modified: Thu Oct 03 02:51:59 UTC 2019
    - 57.3K bytes
    - Viewed (0)
  4. src/main/resources/suggest_indices/_cloud/suggest_analyzer.json

            "tokenizer" : "standard"
          },
          "reading_term_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard"
          },
          "normalize_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "keyword",
            "char_filter" : ["mapping_char"],
            "filter" : ["lowercase"]
          },
          "contents_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard",
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Sat Feb 27 09:26:16 UTC 2021
    - 57.4K bytes
    - Viewed (1)
  5. src/main/java/jcifs/http/Handler.java

                if (handler == null) {
                    final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                    final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                    while (tokenizer.hasMoreTokens()) {
                        final String provider = tokenizer.nextToken().trim();
                        if (provider.equals("jcifs")) {
                            continue;
                        }
    Registered: Sun Sep 07 00:10:21 UTC 2025
    - Last Modified: Sat Aug 16 01:32:48 UTC 2025
    - 6.9K bytes
    - Viewed (0)
  6. src/main/resources/fess_indices/fess.json

    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Fri Jun 06 14:17:42 UTC 2025
    - 39.8K bytes
    - Viewed (0)
  7. src/main/java/jcifs/smb1/http/Handler.java

                if (handler == null) {
                    final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                    final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                    while (tokenizer.hasMoreTokens()) {
                        final String provider = tokenizer.nextToken().trim();
                        if (provider.equals("jcifs.smb1")) {
                            continue;
                        }
    Registered: Sun Sep 07 00:10:21 UTC 2025
    - Last Modified: Sat Aug 16 01:32:48 UTC 2025
    - 6.1K bytes
    - Viewed (0)
  8. src/main/resources/suggest_indices/_aws/suggest_analyzer.json

            "tokenizer" : "standard"
          },
          "reading_term_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard"
          },
          "normalize_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "keyword",
            "char_filter" : ["mapping_char"],
            "filter" : ["lowercase"]
          },
          "contents_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard",
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Wed Mar 24 12:55:37 UTC 2021
    - 57.4K bytes
    - Viewed (0)
  9. src/main/resources/suggest_indices/suggest_analyzer.json

            "tokenizer" : "standard"
          },
          "reading_term_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard"
          },
          "normalize_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "keyword",
            "char_filter" : ["mapping_char"],
            "filter" : ["lowercase"]
          },
          "contents_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard",
    Registered: Thu Sep 04 12:52:25 UTC 2025
    - Last Modified: Thu May 23 05:09:51 UTC 2019
    - 57.7K bytes
    - Viewed (0)
  10. android/guava/src/com/google/common/net/MediaType.java

        checkNotNull(input);
        Tokenizer tokenizer = new Tokenizer(input);
        try {
          String type = tokenizer.consumeToken(TOKEN_MATCHER);
          consumeSeparator(tokenizer, '/');
          String subtype = tokenizer.consumeToken(TOKEN_MATCHER);
          ImmutableListMultimap.Builder<String, String> parameters = ImmutableListMultimap.builder();
          while (tokenizer.hasMore()) {
            consumeSeparator(tokenizer, ';');
    Registered: Fri Sep 05 12:43:10 UTC 2025
    - Last Modified: Tue May 13 17:27:14 UTC 2025
    - 48K bytes
    - Viewed (0)
Back to top