1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
|
/##
Copyright 2021 Kyle Gunger
This file is licensed under the CDDL 1.0 (the License)
and may only be used in accordance with the License.
You should have received a copy of the License with this
software/source code. If you did not, a copy can be found
at the following URL:
https://opensource.org/licenses/CDDL-1.0
THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
EXPRESS OR IMPLIED
#/
/; export module parse
:include "parse/token.tnsl"
:include "parse/tokenizer.tnsl"
;/
/; create_panic ({}charp err)
;tnsl.io.println("ABOUT TO INDUCE PANIC... STAND BY")
;tnsl.io.print("Error code given: ")
;tnsl.io.println(err)
;{}int i = {0}
;i{2}
;/
/# The various types of tokens #/
; enum TOKEN_TYPE [int] {
LINESEP = 0,
INLNSEP = 1,
DELIMIT = 2,
AUGMENT = 3,
LITERAL = 4,
KEYTYPE = 5,
PREWORD = 6,
KEYWORD = 7,
DEFWORD = 8
}
/# Token struct definition #/
; struct Token {
int
token_type,
line,
col,
~{}charp
data
}
/; method Token
/; print
;tnsl.io.print("{ ")
;tnsl.io.print(self.token_type)
;tnsl.io.print(" ")
;tnsl.io.print(self.data`)
;tnsl.io.print(" ")
;tnsl.io.print(self.line)
;tnsl.io.print(" ")
;tnsl.io.print(self.col)
;tnsl.io.print(" } ")
;/
/; operator delete
;delete self.data
;/
/; add_char (~{}charp part)
# ;uint l = len self.data`
# ;realloc self.data, l + len part
/; loop (int i = 0; i < len part`) [i++]
# ;self.data`{l + i} = part{i}
;self.data`.append(part`{i})
;/
;/
;/
/; print_tokens(~{}Token dat)
/; loop (int i = 0; i < len dat`) [i++]
;dat`{i}.print()
;/
;tnsl.io.print("\n")
;/
|
