// Package token defines the recognized tokens of the language. It also // provides types, functions and values to record position of those tokens. package token import "go/token" type ( File = token.File FileSet = token.FileSet Position = token.Position Pos = token.Pos ) const NoPos = token.NoPos var NewFileSet = token.NewFileSet // the set of lexical tokens of the programming language. type Token int // The list of tokens. const ( // Special tokens Illegal Token = iota EOF Comment litStart // Identifiers and basic type literals // (these tokens stand for classes of literals) Ident Int Float String litEnd opStart // Operators and delimiters Add // + Sub // - Mul // * Div // / Mod // % Assign // = Lparen // ( Lbrace // { Comma // , Rparen // ) Rbrace // } Colon // : Rarrow // -> At // @ Semicolon // ; Or // || And // && Eq // == NotEq // != Lt // < Lte // <= Gt // > Gte // >= Not // ! Dot // . Dollar // $ Lbrack // [ Rbrack // ] opEnd kwStart // Keywords Let Var Fn Return If Guard Else Struct Ref Interface kwEnd ) var tokens = [...]string{ Illegal: "illegal", EOF: "eof", Comment: "comment", Ident: "ident", Int: "int", Float: "float", String: "string", Add: "+", Sub: "-", Mul: "*", Div: "/", Mod: "%", Assign: "=", Lparen: "(", Lbrace: "{", Comma: ",", Rparen: ")", Rbrace: "}", Colon: ":", Rarrow: "->", At: "@", Semicolon: ";", Or: "||", And: "&&", Eq: "==", NotEq: "!=", Lt: "<", Lte: "<=", Gt: ">", Gte: ">=", Not: "!", Dot: ".", Dollar: "$", Lbrack: "[", Rbrack: "]", Let: "let", Var: "var", Fn: "fn", Return: "return", If: "if", Guard: "guard", Else: "else", Struct: "struct", Ref: "ref", Interface: "interface", } func (tok Token) String() string { return tokens[tok] } var ( keywords = func() map[string]Token { kw := make(map[string]Token) for i := kwStart + 1; i < kwEnd; i++ { kw[tokens[i]] = i } return kw }() operators = func() map[string]Token { ops := make(map[string]Token) for i := opStart + 1; i < opEnd; i++ { ops[tokens[i]] = i } return ops }() ) // LookupKw maps an identifier to its keyword token or Ident (if not a // keyword). func LookupKw(ident string) Token { if tok, ok := keywords[ident]; ok { return tok } return Ident } // LookupOp maps an operator to its operator token or Illegal (if not an // operator). func LookupOp(op string) Token { if tok, ok := operators[op]; ok { return tok } return Illegal } // Literal returns true for tokens corresponding to identifiers // and basic type literals, false otherwise. func (tok Token) Literal() bool { return litStart < tok && tok < litEnd } // Operator returns true for tokens corresponding to operators and // delimiters, false otherwise. func (tok Token) Operator() bool { return opStart < tok && tok < opEnd } // Keyword returns true for tokens corresponding to keywords, // false otherwise. func (tok Token) Keyword() bool { return kwStart < tok && tok < kwEnd }