- Sort Score
- Result 10 results
- Languages All
Results 1 - 10 of 19 for tokenize (0.03 sec)
-
src/cmd/asm/internal/asm/pseudo_test.go
package asm import ( "strings" "testing" "cmd/asm/internal/lex" ) func tokenize(s string) [][]lex.Token { res := [][]lex.Token{} if len(s) == 0 { return res } for _, o := range strings.Split(s, ",") { res = append(res, lex.Tokenize(o)) } return res } func TestErroneous(t *testing.T) { type errtest struct { pseudo string operands string
Registered: Tue Sep 09 11:13:09 UTC 2025 - Last Modified: Tue Aug 29 07:48:38 UTC 2023 - 3.1K bytes - Viewed (0) -
src/cmd/asm/internal/asm/expr_test.go
} func TestExpr(t *testing.T) { p := NewParser(nil, nil, nil) // Expression evaluation uses none of these fields of the parser. for i, test := range exprTests { p.start(lex.Tokenize(test.input)) result := int64(p.expr()) if result != test.output { t.Errorf("%d: %q evaluated to %d; expected %d", i, test.input, result, test.output) } tok := p.next()
Registered: Tue Sep 09 11:13:09 UTC 2025 - Last Modified: Tue Aug 29 07:48:38 UTC 2023 - 3.2K bytes - Viewed (0) -
src/cmd/asm/internal/lex/lex.go
type Macro struct { name string // The #define name. args []string // Formal arguments. tokens []Token // Body of macro. } // Tokenize turns a string into a list of Tokens; used to parse the -D flag and in tests. func Tokenize(str string) []Token { t := NewTokenizer("command line", strings.NewReader(str), nil) var tokens []Token for { tok := t.Next() if tok == scanner.EOF { breakRegistered: Tue Sep 09 11:13:09 UTC 2025 - Last Modified: Tue Aug 29 18:31:05 UTC 2023 - 4.1K bytes - Viewed (0) -
src/cmd/asm/internal/lex/input.go
if i > 0 { name, value = name[:i], name[i+1:] } tokens := Tokenize(name) if len(tokens) != 1 || tokens[0].ScanToken != scanner.Ident { fmt.Fprintf(os.Stderr, "asm: parsing -D: %q is not a valid identifier name\n", tokens[0]) flags.Usage() } macros[name] = &Macro{ name: name, args: nil, tokens: Tokenize(value), } } return macros } var panicOnError bool // For testing.Registered: Tue Sep 09 11:13:09 UTC 2025 - Last Modified: Fri Sep 06 13:17:27 UTC 2024 - 12.5K bytes - Viewed (0) -
src/cmd/asm/internal/asm/operand_test.go
} }() parse() return nil } func testBadOperandParser(t *testing.T, parser *Parser, tests []badOperandTest) { for _, test := range tests { err := tryParse(t, func() { parser.start(lex.Tokenize(test.input)) addr := obj.Addr{} parser.operand(&addr) }) switch { case err == nil: t.Errorf("fail at %s: got no errors; expected %s\n", test.input, test.error)
Registered: Tue Sep 09 11:13:09 UTC 2025 - Last Modified: Tue Aug 29 18:31:05 UTC 2023 - 23.9K bytes - Viewed (0) -
src/bufio/scan.go
scanCalled bool // Scan has been called; buffer is in use. done bool // Scan has finished. } // SplitFunc is the signature of the split function used to tokenize the // input. The arguments are an initial substring of the remaining unprocessed // data and a flag, atEOF, that reports whether the [Reader] has no more data
Registered: Tue Sep 09 11:13:09 UTC 2025 - Last Modified: Wed May 21 18:05:26 UTC 2025 - 14.2K bytes - Viewed (0) -
src/cmd/asm/internal/lex/tokenizer.go
return i > 0 && unicode.IsDigit(ch) } func (t *Tokenizer) Text() string { switch t.tok { case LSH: return "<<" case RSH: return ">>" case ARR: return "->" case ROT: return "@>" } return t.s.TokenText() } func (t *Tokenizer) File() string { return t.base.Filename() } func (t *Tokenizer) Base() *src.PosBase { return t.base }
Registered: Tue Sep 09 11:13:09 UTC 2025 - Last Modified: Thu Aug 04 20:35:21 UTC 2022 - 3K bytes - Viewed (0) -
fess-crawler/src/test/resources/ajax/js/jquery-2.1.1.min.js
sh(d);return a})}},d.pseudos.nth=d.pseudos.eq;for(b in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})d.pseudos[b]=lb(b);for(b in{submit:!0,reset:!0})d.pseudos[b]=mb(b);function pb(){}pb.prototype=d.filters=d.pseudos,d.setFilters=new pb,g=fb.tokenize=function(a,b){var c,e,f,g,h,i,j,k=z[a+" "];if(k)return b?0:k.slice(0);h=a,i=[],j=d.preFilter;while(h){(!c||(e=S.exec(h)))&&(e&&(h=h.slice(e[0].length)||h),i.push(f=[])),c=!1,(e=T.exec(h))&&(c=e.shift(),f.push({value:c,type:e[0].replace(R," ")}),...
Registered: Sun Sep 21 03:50:09 UTC 2025 - Last Modified: Sun Oct 11 02:16:55 UTC 2015 - 82.3K bytes - Viewed (0) -
src/main/resources/suggest_indices/suggest_analyzer.json
"tokenizer" : "standard" }, "reading_term_analyzer_mk" : { "type" : "custom", "tokenizer" : "standard" }, "normalize_analyzer_mk" : { "type" : "custom", "tokenizer" : "keyword", "char_filter" : ["mapping_char"], "filter" : ["lowercase"] }, "contents_analyzer_mk" : { "type" : "custom", "tokenizer" : "standard",
Registered: Fri Sep 19 09:08:11 UTC 2025 - Last Modified: Thu Oct 03 02:51:59 UTC 2019 - 57.3K bytes - Viewed (0) -
src/main/java/org/codelibs/fess/suggest/converter/KatakanaConverter.java
* use a tokenizer to process the input. * * <p> * The class provides methods to initialize the converter, convert strings, and * check if a tokenizer is enabled. It also includes methods to create a token * stream and extract reading information from the stream's attributes, although * the tokenizer-related functionality is currently commented out. * </p> */
Registered: Fri Sep 19 09:08:11 UTC 2025 - Last Modified: Fri Jul 04 14:00:23 UTC 2025 - 6.1K bytes - Viewed (0)