~mna/snow

ref: 51726a5ef2959dd97add58914fe45d829880e258 snow/pkg/token/token.go -rw-r--r-- 3.3 KiB
51726a5eMartin Angers Merge branch 'wip-static-func-return' 1 year, 11 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
// Package token defines the recognized tokens of the language. It also
// provides types, functions and values to record position of those tokens.
package token

import "go/token"

type (
	File     = token.File
	FileSet  = token.FileSet
	Position = token.Position
	Pos      = token.Pos
)

const NoPos = token.NoPos

var NewFileSet = token.NewFileSet

// the set of lexical tokens of the programming language.
type Token int

// The list of tokens.
const (
	// Special tokens
	Illegal Token = iota
	EOF
	Comment

	litStart
	// Identifiers and basic type literals
	// (these tokens stand for classes of literals)
	Ident
	Int
	Float
	String
	litEnd

	opStart
	// Operators and delimiters
	Add       // +
	Sub       // -
	Mul       // *
	Div       // /
	Mod       // %
	Assign    // =
	Lparen    // (
	Lbrace    // {
	Comma     // ,
	Rparen    // )
	Rbrace    // }
	Colon     // :
	Rarrow    // ->
	At        // @
	Semicolon // ;
	Or        // ||
	And       // &&
	Eq        // ==
	NotEq     // !=
	Lt        // <
	Lte       // <=
	Gt        // >
	Gte       // >=
	Not       // !
	Dot       // .
	Dollar    // $
	Lbrack    // [
	Rbrack    // ]
	opEnd

	kwStart
	// Keywords
	Let
	Var
	Fn
	Return
	If
	Guard
	Else
	Struct
	Ref
	Interface
	kwEnd
)

var tokens = [...]string{
	Illegal: "illegal",
	EOF:     "eof",
	Comment: "comment",

	Ident:  "ident",
	Int:    "int",
	Float:  "float",
	String: "string",

	Add:       "+",
	Sub:       "-",
	Mul:       "*",
	Div:       "/",
	Mod:       "%",
	Assign:    "=",
	Lparen:    "(",
	Lbrace:    "{",
	Comma:     ",",
	Rparen:    ")",
	Rbrace:    "}",
	Colon:     ":",
	Rarrow:    "->",
	At:        "@",
	Semicolon: ";",
	Or:        "||",
	And:       "&&",
	Eq:        "==",
	NotEq:     "!=",
	Lt:        "<",
	Lte:       "<=",
	Gt:        ">",
	Gte:       ">=",
	Not:       "!",
	Dot:       ".",
	Dollar:    "$",
	Lbrack:    "[",
	Rbrack:    "]",

	Let:       "let",
	Var:       "var",
	Fn:        "fn",
	Return:    "return",
	If:        "if",
	Guard:     "guard",
	Else:      "else",
	Struct:    "struct",
	Ref:       "ref",
	Interface: "interface",
}

func (tok Token) String() string {
	return tokens[tok]
}

var (
	keywords = func() map[string]Token {
		kw := make(map[string]Token)
		for i := kwStart + 1; i < kwEnd; i++ {
			kw[tokens[i]] = i
		}
		return kw
	}()
	operators = func() map[string]Token {
		ops := make(map[string]Token)
		for i := opStart + 1; i < opEnd; i++ {
			ops[tokens[i]] = i
		}
		return ops
	}()
)

// LookupKw maps an identifier to its keyword token or Ident (if not a
// keyword).
func LookupKw(ident string) Token {
	if tok, ok := keywords[ident]; ok {
		return tok
	}
	return Ident
}

// LookupOp maps an operator to its operator token or Illegal (if not an
// operator).
func LookupOp(op string) Token {
	if tok, ok := operators[op]; ok {
		return tok
	}
	return Illegal
}

// Literal returns true for tokens corresponding to identifiers
// and basic type literals, false otherwise.
func (tok Token) Literal() bool { return litStart < tok && tok < litEnd }

// Operator returns true for tokens corresponding to operators and
// delimiters, false otherwise.
func (tok Token) Operator() bool { return opStart < tok && tok < opEnd }

// Keyword returns true for tokens corresponding to keywords,
// false otherwise.
func (tok Token) Keyword() bool { return kwStart < tok && tok < kwEnd }