blob: edc9050090c6afa57b486981e2d3213e0deb6de3 [file] [log] [blame]
Marcel van Lohuizen8a7fc452018-12-10 15:19:41 +01001// Copyright 2018 The CUE Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// Package token defines constants representing the lexical tokens of the Go
16// programming language and basic operations on tokens (printing, predicates).
17package token // import "cuelang.org/go/cue/token"
18
19import "strconv"
20
21// Token is the set of lexical tokens of the CUE configuration language.
22type Token int
23
24// The list of tokens.
25const (
26 // Special tokens
27 ILLEGAL Token = iota
28 EOF
29 COMMENT
30
31 literalBeg
32 // Identifiers and basic type literals
33 // (these tokens stand for classes of literals)
34 IDENT // main, _tmp
35 INT // 12_345Mi, 0700, 0xdeadbeef, 1.2M
36 FLOAT // 123.45,
37 // DURATION // 3m4s TODO
38 STRING // "abc"
39 INTERPOLATION // a part of a template string, e.g. `"age: \(`
40 BOTTOM // _|_
41
42 literalEnd
43
44 operatorBeg
45 // Operators and delimiters
46 ADD // +
47 SUB // -
48 MUL // *
49 POW // ^
50 QUO // /
51 REM // %
52
53 IQUO // quo
54 IREM // rem
55 IDIV // div
56 IMOD // mod
57
58 UNIFY // &
59 DISJUNCTION // |
60
61 LAND // &&
62 LOR // ||
63
Marcel van Lohuizen2bf066f2018-12-16 11:43:49 +010064 BIND // =
65 EQL // ==
66 LSS // <
67 GTR // >
68 NOT // !
69 ARROW // <-
Marcel van Lohuizen8a7fc452018-12-10 15:19:41 +010070
71 NEQ // !=
72 LEQ // <=
73 GEQ // >=
74
75 LPAREN // (
76 LBRACK // [
77 LBRACE // {
78 COMMA // ,
79 PERIOD // .
80 RANGE // ..
81 ELLIPSIS // ...
82
83 RPAREN // )
84 RBRACK // ]
85 RBRACE // }
86 SEMICOLON // :
87 COLON // :
88 operatorEnd
89
90 keywordBeg
91
92 IF
93 FOR
94 IN
95 LET
96
97 TRUE
98 FALSE
99 NULL
100
101 keywordEnd
102)
103
104var tokens = [...]string{
105 ILLEGAL: "ILLEGAL",
106
107 EOF: "EOF",
108 COMMENT: "COMMENT",
109
110 IDENT: "IDENT",
111 INT: "INT",
112 FLOAT: "FLOAT",
113 // DURATION: "DURATION", // TODO
114 STRING: "STRING",
115 INTERPOLATION: "INTERPOLATION",
116
117 ADD: "+",
118 SUB: "-",
119 MUL: "*",
120 POW: "^",
121 QUO: "/",
122 REM: "%",
123
124 IQUO: "quo",
125 IREM: "rem",
126 IDIV: "div",
127 IMOD: "mod",
128
129 UNIFY: "&",
130 DISJUNCTION: "|",
131
132 LAND: "&&",
133 LOR: "||",
134
Marcel van Lohuizen2bf066f2018-12-16 11:43:49 +0100135 BIND: "=",
136 EQL: "==",
137 LSS: "<",
138 GTR: ">",
139 NOT: "!",
140 ARROW: "<-",
Marcel van Lohuizen8a7fc452018-12-10 15:19:41 +0100141
142 NEQ: "!=",
143 LEQ: "<=",
144 GEQ: ">=",
145
146 LPAREN: "(",
147 LBRACK: "[",
148 LBRACE: "{",
149 COMMA: ",",
150 PERIOD: ".",
151 RANGE: "..",
152 ELLIPSIS: "...",
153
154 RPAREN: ")",
155 RBRACK: "]",
156 RBRACE: "}",
157 SEMICOLON: ";",
158 COLON: ":",
159
160 BOTTOM: "_|_",
161
162 FALSE: "false",
163 TRUE: "true",
164 NULL: "null",
165
166 FOR: "for",
167 IF: "if",
168 IN: "in",
169 LET: "let",
170}
171
172// String returns the string corresponding to the token tok.
173// For operators, delimiters, and keywords the string is the actual
174// token character sequence (e.g., for the token ADD, the string is
175// "+"). For all other tokens the string corresponds to the token
176// constant name (e.g. for the token IDENT, the string is "IDENT").
177func (tok Token) String() string {
178 s := ""
179 if 0 <= tok && tok < Token(len(tokens)) {
180 s = tokens[tok]
181 }
182 if s == "" {
183 s = "token(" + strconv.Itoa(int(tok)) + ")"
184 }
185 return s
186}
187
188// A set of constants for precedence-based expression parsing.
189// Non-operators have lowest precedence, followed by operators
190// starting with precedence 1 up to unary operators. The highest
191// precedence serves as "catch-all" precedence for selector,
192// indexing, and other operator and delimiter tokens.
193const (
194 LowestPrec = lowestPrec
195 UnaryPrec = unaryPrec
196 HighestPrec = highestPrec
197)
198
199const (
200 lowestPrec = 0 // non-operators
201 unaryPrec = 8
202 highestPrec = 9
203)
204
205// Precedence returns the operator precedence of the binary
206// operator op. If op is not a binary operator, the result
207// is LowestPrecedence.
208//
209func (tok Token) Precedence() int {
210 switch tok {
211 case DISJUNCTION:
212 return 1
213 case UNIFY:
214 return 2
215 case LOR:
216 return 3
217 case LAND:
218 return 4
219 case EQL, NEQ, LSS, LEQ, GTR, GEQ:
220 return 5
221 case ADD, SUB:
222 return 6
223 case MUL, QUO, REM, IDIV, IMOD, IQUO, IREM:
224 return 7
225 case RANGE:
226 return 8
227 }
228 return lowestPrec
229}
230
231var keywords map[string]Token
232
233func init() {
234 keywords = make(map[string]Token)
235 for i := keywordBeg + 1; i < keywordEnd; i++ {
236 keywords[tokens[i]] = i
237 }
238}
239
240// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
241//
242func Lookup(ident string) Token {
243 if tok, isKeyword := keywords[ident]; isKeyword {
244 return tok
245 }
246 return IDENT
247}
248
249// Predicates
250
251// IsLiteral returns true for tokens corresponding to identifiers
252// and basic type literals; it returns false otherwise.
253func (tok Token) IsLiteral() bool { return literalBeg < tok && tok < literalEnd }
254
255// IsOperator returns true for tokens corresponding to operators and
256// delimiters; it returns false otherwise.
257func (tok Token) IsOperator() bool { return operatorBeg < tok && tok < operatorEnd }
258
259// IsKeyword returns true for tokens corresponding to keywords;
260// it returns false otherwise.
261func (tok Token) IsKeyword() bool { return keywordBeg < tok && tok < keywordEnd }