summaryrefslogtreecommitdiff
path: root/tnslc/parse
diff options
context:
space:
mode:
Diffstat (limited to 'tnslc/parse')
-rw-r--r--tnslc/parse/token.tnsl26
-rw-r--r--tnslc/parse/tokenizer.tnsl8
2 files changed, 20 insertions, 14 deletions
diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl
index 92748d8..e7d64e7 100644
--- a/tnslc/parse/token.tnsl
+++ b/tnslc/parse/token.tnsl
@@ -32,16 +32,13 @@
"abi"
}
-;const {}{}charp KEYTYPES = {
- "bool",
- "char",
- "charp",
-
+;{}{}charp KEYTYPES = {
"int8",
"int16",
"int32",
"int64",
"int",
+
"uint8",
"uint16",
"uint32",
@@ -52,6 +49,7 @@
"float64",
"float",
+ "bool",
"void",
"type"
}
@@ -60,7 +58,6 @@
"struct",
"interface",
"enum",
- "is",
"extends",
"loop",
@@ -83,8 +80,6 @@
"method",
"override",
- "self",
- "super",
"operator",
"raw",
@@ -93,6 +88,9 @@
"virtual",
"delete",
+ "alloc",
+ "salloc",
+ "realloc",
"module",
"export"
@@ -101,7 +99,11 @@
;{}{}charp LITERALS = {
"true",
"false",
- "null"
+
+ "null",
+
+ "self",
+ "super"
}
;{}charp RESERVED = "`~!#%^&*()-=+[]{}|;:,.<>/"
@@ -169,7 +171,11 @@
# Increment and De-increment
"++",
- "--"
+ "--",
+
+ "is",
+ "len",
+ "size"
}
;int MAX_MRESERVED = 3
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 3a66e24..7a1f085 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -14,11 +14,11 @@
EXPRESS OR IMPLIED
#/
-/; is_float (~{}charp dat) [bool]
+/; is_float (~{}uint8 dat) [bool]
;return is_numeric_literal(dat) && is_in_string(dat, '.')
;/
-/; break_token ({}charp dat, charp c) [bool]
+/; break_token ({}uint8 dat, uint8 c) [bool]
/; if (len dat == 0)
;return false
@@ -74,7 +74,7 @@
/; tokenize (tnsl.io.File fstr) [~{}Token]
;{}Token out = {}
- ;{}charp tdat = {}
+ ;{}uint8 tdat = {}
;bool comment = false
;int line = 1, col = 1
@@ -84,7 +84,7 @@
;tdat = {}
;comment = true
;; else if (len tdat > 0)
- ;{}charp tmp = tdat
+ ;{}uint8 tmp = tdat
;Token ttk = {get_token_type(~tmp), line, col, ~tmp}
;out.append(ttk)
;tdat = {}