1
2 /* Compiler implementation of the D programming language
3 * Copyright (C) 1999-2023 by The D Language Foundation, All Rights Reserved
4 * written by Walter Bright
5 * https://www.digitalmars.com
6 * Distributed under the Boost Software License, Version 1.0.
7 * https://www.boost.org/LICENSE_1_0.txt
8 * https://github.com/dlang/dmd/blob/master/src/dmd/tokens.h
9 */
10
11 #pragma once
12
13 #include "root/dcompat.h"
14 #include "root/port.h"
15 #include "globals.h"
16
17 class Identifier;
18
19 /* Tokens:
20 ( )
21 [ ]
22 { }
23 < > <= >= == != === !==
24 << >> <<= >>= >>> >>>=
25 + - += -=
26 * / % *= /= %=
27 & | ^ &= |= ^=
28 = ! ~ @
29 ^^ ^^=
30 ++ --
31 . -> : , =>
32 ? && ||
33 */
34
35 enum class TOK : unsigned char
36 {
37 reserved,
38
39 // Other
40 leftParenthesis,
41 rightParenthesis,
42 leftBracket,
43 rightBracket,
44 leftCurly,
45 rightCurly,
46 colon,
47 semicolon,
48 dotDotDot,
49 endOfFile,
50 cast_,
51 null_,
52 assert_,
53 true_,
54 false_,
55 throw_,
56 new_,
57 delete_,
58 variable,
59 slice,
60 version_,
61 module_,
62 dollar,
63 template_,
64 typeof_,
65 pragma_,
66 typeid_,
67 comment,
68
69 // Operators
70 lessThan,
71 greaterThan,
72 lessOrEqual,
73 greaterOrEqual,
74 equal,
75 notEqual,
76 identity,
77 notIdentity,
78 is_,
79
80 leftShift,
81 rightShift,
82 leftShiftAssign,
83 rightShiftAssign,
84 unsignedRightShift,
85 unsignedRightShiftAssign,
86 concatenateAssign, // ~=
87 add,
88 min,
89 addAssign,
90 minAssign,
91 mul,
92 div,
93 mod,
94 mulAssign,
95 divAssign,
96 modAssign,
97 and_,
98 or_,
99 xor_,
100 andAssign,
101 orAssign,
102 xorAssign,
103 assign,
104 not_,
105 tilde,
106 plusPlus,
107 minusMinus,
108 dot,
109 comma,
110 question,
111 andAnd,
112 orOr,
113
114 // Numeric literals
115 int32Literal,
116 uns32Literal,
117 int64Literal,
118 uns64Literal,
119 int128Literal,
120 uns128Literal,
121 float32Literal,
122 float64Literal,
123 float80Literal,
124 imaginary32Literal,
125 imaginary64Literal,
126 imaginary80Literal,
127
128 // Char constants
129 charLiteral,
130 wcharLiteral,
131 dcharLiteral,
132
133 // Leaf operators
134 identifier,
135 string_,
136 this_,
137 super_,
138 error,
139
140 // Basic types
141 void_,
142 int8,
143 uns8,
144 int16,
145 uns16,
146 int32,
147 uns32,
148 int64,
149 uns64,
150 int128,
151 uns128,
152 float32,
153 float64,
154 float80,
155 imaginary32,
156 imaginary64,
157 imaginary80,
158 complex32,
159 complex64,
160 complex80,
161 char_,
162 wchar_,
163 dchar_,
164 bool_,
165
166 // Aggregates
167 struct_,
168 class_,
169 interface_,
170 union_,
171 enum_,
172 import_,
173 alias_,
174 override_,
175 delegate_,
176 function_,
177 mixin_,
178 align_,
179 extern_,
180 private_,
181 protected_,
182 public_,
183 export_,
184 static_,
185 final_,
186 const_,
187 abstract_,
188 debug_,
189 deprecated_,
190 in_,
191 out_,
192 inout_,
193 lazy_,
194 auto_,
195 package_,
196 immutable_,
197
198 // Statements
199 if_,
200 else_,
201 while_,
202 for_,
203 do_,
204 switch_,
205 case_,
206 default_,
207 break_,
208 continue_,
209 with_,
210 synchronized_,
211 return_,
212 goto_,
213 try_,
214 catch_,
215 finally_,
216 asm_,
217 foreach_,
218 foreach_reverse_,
219 scope_,
220 onScopeExit,
221 onScopeFailure,
222 onScopeSuccess,
223
224 // Contracts
225 invariant_,
226
227 // Testing
228 unittest_,
229
230 // Added after 1.0
231 argumentTypes,
232 ref_,
233 macro_,
234
235 parameters,
236 traits,
237 pure_,
238 nothrow_,
239 gshared,
240 line,
241 file,
242 fileFullPath,
243 moduleString, // __MODULE__
244 functionString, // __FUNCTION__
245 prettyFunction, // __PRETTY_FUNCTION__
246 shared_,
247 at,
248 pow,
249 powAssign,
250 goesTo,
251 vector,
252 pound,
253
254 arrow, // ->
255 colonColon, // ::
256 wchar_tLiteral,
257 endOfLine, // \n, \r, \u2028, \u2029
258 whitespace,
259
260 // C only keywords
261 inline_,
262 register_,
263 restrict_,
264 signed_,
265 sizeof_,
266 typedef_,
267 unsigned_,
268 volatile_,
269 _Alignas_,
270 _Alignof_,
271 _Atomic_,
272 _Bool_,
273 _Complex_,
274 _Generic_,
275 _Imaginary_,
276 _Noreturn_,
277 _Static_assert_,
278 _Thread_local_,
279
280 // C only extended keywords
281 _assert,
282 _import,
283 cdecl_,
284 declspec,
285 stdcall,
286 pragma,
287 int128_,
288 attribute__,
289
290 MAX,
291 };
292
293 enum class EXP : unsigned char
294 {
295 reserved,
296
297 // Other
298 negate,
299 cast_,
300 null_,
301 assert_,
302 true_,
303 false_,
304 array,
305 call,
306 address,
307 type,
308 throw_,
309 new_,
310 delete_,
311 star,
312 symbolOffset,
313 variable,
314 dotVariable,
315 dotIdentifier,
316 dotTemplateInstance,
317 dotType,
318 slice,
319 arrayLength,
320 version_,
321 dollar,
322 template_,
323 dotTemplateDeclaration,
324 declaration,
325 typeof_,
326 pragma_,
327 dSymbol,
328 typeid_,
329 uadd,
330 remove,
331 newAnonymousClass,
332 arrayLiteral,
333 assocArrayLiteral,
334 structLiteral,
335 classReference,
336 thrownException,
337 delegatePointer,
338 delegateFunctionPointer,
339
340 // Operators
341 lessThan,
342 greaterThan,
343 lessOrEqual,
344 greaterOrEqual,
345 equal,
346 notEqual,
347 identity,
348 notIdentity,
349 index,
350 is_,
351
352 leftShift,
353 rightShift,
354 leftShiftAssign,
355 rightShiftAssign,
356 unsignedRightShift,
357 unsignedRightShiftAssign,
358 concatenate,
359 concatenateAssign, // ~=
360 concatenateElemAssign,
361 concatenateDcharAssign,
362 add,
363 min,
364 addAssign,
365 minAssign,
366 mul,
367 div,
368 mod,
369 mulAssign,
370 divAssign,
371 modAssign,
372 and_,
373 or_,
374 xor_,
375 andAssign,
376 orAssign,
377 xorAssign,
378 assign,
379 not_,
380 tilde,
381 plusPlus,
382 minusMinus,
383 construct,
384 blit,
385 dot,
386 comma,
387 question,
388 andAnd,
389 orOr,
390 prePlusPlus,
391 preMinusMinus,
392
393 // Leaf operators
394 identifier,
395 string_,
396 this_,
397 super_,
398 halt,
399 tuple,
400 error,
401
402 // Basic types
403 void_,
404 int64,
405 float64,
406 complex80,
407 char_,
408 import_,
409 delegate_,
410 function_,
411 mixin_,
412 in_,
413 default_,
414 break_,
415 continue_,
416 goto_,
417 scope_,
418
419 traits,
420 overloadSet,
421 line,
422 file,
423 fileFullPath,
424 moduleString, // __MODULE__
425 functionString, // __FUNCTION__
426 prettyFunction, // __PRETTY_FUNCTION__
427 shared_,
428 pow,
429 powAssign,
430 vector,
431
432 voidExpression,
433 cantExpression,
434 showCtfeContext,
435 objcClassReference,
436 vectorArray,
437 arrow, // ->
438 compoundLiteral, // ( type-name ) { initializer-list }
439 _Generic_,
440 interval,
441
442 MAX
443 };
444
445 #define TOKwild TOKinout
446
447 // Token has an anonymous struct, which is not strict ISO C++.
448 #if defined(__GNUC__)
449 #pragma GCC diagnostic push
450 #pragma GCC diagnostic ignored "-Wpedantic"
451 #endif
452
453 struct Token
454 {
455 Token *next;
456 Loc loc;
457 const utf8_t *ptr; // pointer to first character of this token within buffer
458 TOK value;
459 DString blockComment; // doc comment string prior to this token
460 DString lineComment; // doc comment for previous token
461 union
462 {
463 // Integers
464 sinteger_t intvalue;
465 uinteger_t unsvalue;
466
467 // Floats
468 real_t floatvalue;
469
470 struct
471 { utf8_t *ustring; // UTF8 string
472 unsigned len;
473 unsigned char postfix; // 'c', 'w', 'd'
474 };
475
476 Identifier *ident;
477 };
478
479 void free();
480
481 Token() : next(NULL) {}
482 int isKeyword();
483 const char *toChars() const;
484
485 static const char *toChars(TOK value);
486 };
487
488 #if defined(__GNUC__)
489 #pragma GCC diagnostic pop
490 #endif