Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 10 of 10 for Tokenize (0.64 sec)

  1. src/cmd/asm/internal/lex/input.go

    		if i > 0 {
    			name, value = name[:i], name[i+1:]
    		}
    		tokens := Tokenize(name)
    		if len(tokens) != 1 || tokens[0].ScanToken != scanner.Ident {
    			fmt.Fprintf(os.Stderr, "asm: parsing -D: %q is not a valid identifier name\n", tokens[0])
    			flags.Usage()
    		}
    		macros[name] = &Macro{
    			name:   name,
    			args:   nil,
    			tokens: Tokenize(value),
    		}
    	}
    	return macros
    }
    
    var panicOnError bool // For testing.
    Registered: Tue Dec 30 11:13:12 UTC 2025
    - Last Modified: Wed Nov 12 03:59:40 UTC 2025
    - 12.4K bytes
    - Viewed (0)
  2. src/main/java/jcifs/smb1/http/Handler.java

                if (handler == null) {
                    final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                    final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                    while (tokenizer.hasMoreTokens()) {
                        final String provider = tokenizer.nextToken().trim();
                        if (provider.equals("jcifs.smb1")) {
                            continue;
                        }
    Registered: Sat Dec 20 13:44:44 UTC 2025
    - Last Modified: Sat Aug 16 01:32:48 UTC 2025
    - 6.1K bytes
    - Viewed (0)
  3. src/test/java/jcifs/internal/smb2/session/Smb2SessionSetupRequestTest.java

            // Verify token length
            assertEquals(tokenSize, SMBUtil.readInt2(buffer, bodyOffset + 14));
    
            if (tokenSize > 0) {
                // Verify token content
                int tokenOffset = bodyOffset + 24;
                while ((tokenOffset % 8) != 0) {
                    tokenOffset++; // Account for pad8
                }
                byte[] actualToken = new byte[tokenSize];
    Registered: Sat Dec 20 13:44:44 UTC 2025
    - Last Modified: Thu Aug 14 05:31:44 UTC 2025
    - 21.2K bytes
    - Viewed (0)
  4. android/guava/src/com/google/common/net/MediaType.java

        checkNotNull(input);
        Tokenizer tokenizer = new Tokenizer(input);
        try {
          String type = tokenizer.consumeToken(TOKEN_MATCHER);
          consumeSeparator(tokenizer, '/');
          String subtype = tokenizer.consumeToken(TOKEN_MATCHER);
          ImmutableListMultimap.Builder<String, String> parameters = ImmutableListMultimap.builder();
          while (tokenizer.hasMore()) {
            consumeSeparator(tokenizer, ';');
    Registered: Fri Dec 26 12:43:10 UTC 2025
    - Last Modified: Thu Oct 02 01:46:40 UTC 2025
    - 48K bytes
    - Viewed (0)
  5. guava/src/com/google/common/net/MediaType.java

        checkNotNull(input);
        Tokenizer tokenizer = new Tokenizer(input);
        try {
          String type = tokenizer.consumeToken(TOKEN_MATCHER);
          consumeSeparator(tokenizer, '/');
          String subtype = tokenizer.consumeToken(TOKEN_MATCHER);
          ImmutableListMultimap.Builder<String, String> parameters = ImmutableListMultimap.builder();
          while (tokenizer.hasMore()) {
            consumeSeparator(tokenizer, ';');
    Registered: Fri Dec 26 12:43:10 UTC 2025
    - Last Modified: Thu Oct 02 01:46:40 UTC 2025
    - 48K bytes
    - Viewed (0)
  6. src/main/java/jcifs/http/Handler.java

                if (handler == null) {
                    final String path = System.getProperty(HANDLER_PKGS_PROPERTY);
                    final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                    while (tokenizer.hasMoreTokens()) {
                        final String provider = tokenizer.nextToken().trim();
                        if (provider.equals("jcifs")) {
                            continue;
                        }
    Registered: Sat Dec 20 13:44:44 UTC 2025
    - Last Modified: Sat Aug 16 01:32:48 UTC 2025
    - 6.9K bytes
    - Viewed (0)
  7. src/test/java/org/codelibs/fess/suggest/converter/AnalyzerConverterTest.java

                    .put("analysis.analyzer.test_reading_analyzer.tokenizer", "standard")
                    .put("analysis.analyzer.test_reading_term_analyzer.type", "custom")
                    .put("analysis.analyzer.test_reading_term_analyzer.tokenizer", "standard")
                    .put("analysis.analyzer.test_reading_analyzer_ja.type", "custom")
                    .put("analysis.analyzer.test_reading_analyzer_ja.tokenizer", "keyword")
    Registered: Sat Dec 20 13:04:59 UTC 2025
    - Last Modified: Mon Sep 01 13:33:03 UTC 2025
    - 12.5K bytes
    - Viewed (0)
  8. README.md

    String escaped = JsonUtil.escape("Hello \"World\" with special chars");
    String unescaped = JsonUtil.unescape(escaped);
    
    // Text tokenization
    Tokenizer tokenizer = new Tokenizer("field1,field2,field3", ",");
    while (tokenizer.hasMoreTokens()) {
        String token = tokenizer.nextToken();
        // Process each token
    }
    
    // Decimal formatting
    DecimalFormat format = DecimalFormatUtil.getDecimalFormat("###,###.00");
    ```
    
    Registered: Sat Dec 20 08:55:33 UTC 2025
    - Last Modified: Sun Aug 31 02:56:02 UTC 2025
    - 12.7K bytes
    - Viewed (0)
  9. src/test/java/org/codelibs/fess/suggest/analysis/SuggestAnalyzerTest.java

            List<AnalyzeToken> tokens = analyzer.analyzeAndReading(text, field, lang);
    
            assertNotNull(tokens);
            assertEquals(2, tokens.size()); // "Test123" and "ABC-456" (hyphen is not a split char in our tokenizer)
            assertEquals("Test123", tokens.get(0).getTerm());
            assertEquals("ABC-456", tokens.get(1).getTerm());
            assertEquals("TEST123", ((TestAnalyzeToken) tokens.get(0)).getReading());
    Registered: Sat Dec 20 13:04:59 UTC 2025
    - Last Modified: Mon Sep 01 13:33:03 UTC 2025
    - 15.7K bytes
    - Viewed (0)
  10. ADDING_NEW_LANGUAGE.md

          },
          "swedish_stemmer": {
            "type": "stemmer",
            "language": "swedish"
          }
        },
        "analyzer": {
          "sv_analyzer": {
            "type": "custom",
            "tokenizer": "standard",
            "filter": ["lowercase", "swedish_stop", "swedish_stemmer"]
          }
        }
      }
    }
    ```
    
    ### 2. Add Custom Dictionaries (Optional)
    
    Registered: Sat Dec 20 09:19:18 UTC 2025
    - Last Modified: Thu Nov 06 11:36:30 UTC 2025
    - 10.4K bytes
    - Viewed (1)
Back to top