Search Options

Results per page
Sort
Preferred Languages
Advance

Results 1 - 10 of 19 for tokenize (0.03 sec)

  1. src/cmd/asm/internal/asm/pseudo_test.go

    package asm
    
    import (
    	"strings"
    	"testing"
    
    	"cmd/asm/internal/lex"
    )
    
    func tokenize(s string) [][]lex.Token {
    	res := [][]lex.Token{}
    	if len(s) == 0 {
    		return res
    	}
    	for _, o := range strings.Split(s, ",") {
    		res = append(res, lex.Tokenize(o))
    	}
    	return res
    }
    
    func TestErroneous(t *testing.T) {
    
    	type errtest struct {
    		pseudo   string
    		operands string
    Registered: Tue Sep 09 11:13:09 UTC 2025
    - Last Modified: Tue Aug 29 07:48:38 UTC 2023
    - 3.1K bytes
    - Viewed (0)
  2. src/cmd/asm/internal/asm/expr_test.go

    }
    
    func TestExpr(t *testing.T) {
    	p := NewParser(nil, nil, nil) // Expression evaluation uses none of these fields of the parser.
    	for i, test := range exprTests {
    		p.start(lex.Tokenize(test.input))
    		result := int64(p.expr())
    		if result != test.output {
    			t.Errorf("%d: %q evaluated to %d; expected %d", i, test.input, result, test.output)
    		}
    		tok := p.next()
    Registered: Tue Sep 09 11:13:09 UTC 2025
    - Last Modified: Tue Aug 29 07:48:38 UTC 2023
    - 3.2K bytes
    - Viewed (0)
  3. src/cmd/asm/internal/lex/lex.go

    type Macro struct {
    	name   string   // The #define name.
    	args   []string // Formal arguments.
    	tokens []Token  // Body of macro.
    }
    
    // Tokenize turns a string into a list of Tokens; used to parse the -D flag and in tests.
    func Tokenize(str string) []Token {
    	t := NewTokenizer("command line", strings.NewReader(str), nil)
    	var tokens []Token
    	for {
    		tok := t.Next()
    		if tok == scanner.EOF {
    			break
    Registered: Tue Sep 09 11:13:09 UTC 2025
    - Last Modified: Tue Aug 29 18:31:05 UTC 2023
    - 4.1K bytes
    - Viewed (0)
  4. src/cmd/asm/internal/lex/input.go

    		if i > 0 {
    			name, value = name[:i], name[i+1:]
    		}
    		tokens := Tokenize(name)
    		if len(tokens) != 1 || tokens[0].ScanToken != scanner.Ident {
    			fmt.Fprintf(os.Stderr, "asm: parsing -D: %q is not a valid identifier name\n", tokens[0])
    			flags.Usage()
    		}
    		macros[name] = &Macro{
    			name:   name,
    			args:   nil,
    			tokens: Tokenize(value),
    		}
    	}
    	return macros
    }
    
    var panicOnError bool // For testing.
    Registered: Tue Sep 09 11:13:09 UTC 2025
    - Last Modified: Fri Sep 06 13:17:27 UTC 2024
    - 12.5K bytes
    - Viewed (0)
  5. src/cmd/asm/internal/asm/operand_test.go

    		}
    	}()
    
    	parse()
    
    	return nil
    }
    
    func testBadOperandParser(t *testing.T, parser *Parser, tests []badOperandTest) {
    	for _, test := range tests {
    		err := tryParse(t, func() {
    			parser.start(lex.Tokenize(test.input))
    			addr := obj.Addr{}
    			parser.operand(&addr)
    		})
    
    		switch {
    		case err == nil:
    			t.Errorf("fail at %s: got no errors; expected %s\n", test.input, test.error)
    Registered: Tue Sep 09 11:13:09 UTC 2025
    - Last Modified: Tue Aug 29 18:31:05 UTC 2023
    - 23.9K bytes
    - Viewed (0)
  6. src/bufio/scan.go

    	scanCalled   bool      // Scan has been called; buffer is in use.
    	done         bool      // Scan has finished.
    }
    
    // SplitFunc is the signature of the split function used to tokenize the
    // input. The arguments are an initial substring of the remaining unprocessed
    // data and a flag, atEOF, that reports whether the [Reader] has no more data
    Registered: Tue Sep 09 11:13:09 UTC 2025
    - Last Modified: Wed May 21 18:05:26 UTC 2025
    - 14.2K bytes
    - Viewed (0)
  7. src/cmd/asm/internal/lex/tokenizer.go

    	return i > 0 && unicode.IsDigit(ch)
    }
    
    func (t *Tokenizer) Text() string {
    	switch t.tok {
    	case LSH:
    		return "<<"
    	case RSH:
    		return ">>"
    	case ARR:
    		return "->"
    	case ROT:
    		return "@>"
    	}
    	return t.s.TokenText()
    }
    
    func (t *Tokenizer) File() string {
    	return t.base.Filename()
    }
    
    func (t *Tokenizer) Base() *src.PosBase {
    	return t.base
    }
    
    Registered: Tue Sep 09 11:13:09 UTC 2025
    - Last Modified: Thu Aug 04 20:35:21 UTC 2022
    - 3K bytes
    - Viewed (0)
  8. fess-crawler/src/test/resources/ajax/js/jquery-2.1.1.min.js

    sh(d);return a})}},d.pseudos.nth=d.pseudos.eq;for(b in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})d.pseudos[b]=lb(b);for(b in{submit:!0,reset:!0})d.pseudos[b]=mb(b);function pb(){}pb.prototype=d.filters=d.pseudos,d.setFilters=new pb,g=fb.tokenize=function(a,b){var c,e,f,g,h,i,j,k=z[a+" "];if(k)return b?0:k.slice(0);h=a,i=[],j=d.preFilter;while(h){(!c||(e=S.exec(h)))&&(e&&(h=h.slice(e[0].length)||h),i.push(f=[])),c=!1,(e=T.exec(h))&&(c=e.shift(),f.push({value:c,type:e[0].replace(R," ")}),...
    Registered: Sun Sep 21 03:50:09 UTC 2025
    - Last Modified: Sun Oct 11 02:16:55 UTC 2015
    - 82.3K bytes
    - Viewed (0)
  9. src/main/resources/suggest_indices/suggest_analyzer.json

            "tokenizer" : "standard"
          },
          "reading_term_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard"
          },
          "normalize_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "keyword",
            "char_filter" : ["mapping_char"],
            "filter" : ["lowercase"]
          },
          "contents_analyzer_mk" : {
            "type" : "custom",
            "tokenizer" : "standard",
    Registered: Fri Sep 19 09:08:11 UTC 2025
    - Last Modified: Thu Oct 03 02:51:59 UTC 2019
    - 57.3K bytes
    - Viewed (0)
  10. src/main/java/org/codelibs/fess/suggest/converter/KatakanaConverter.java

     * use a tokenizer to process the input.
     *
     * <p>
     * The class provides methods to initialize the converter, convert strings, and
     * check if a tokenizer is enabled. It also includes methods to create a token
     * stream and extract reading information from the stream's attributes, although
     * the tokenizer-related functionality is currently commented out.
     * </p>
     */
    Registered: Fri Sep 19 09:08:11 UTC 2025
    - Last Modified: Fri Jul 04 14:00:23 UTC 2025
    - 6.1K bytes
    - Viewed (0)
Back to top