diff options
Diffstat (limited to 'tnslc/parse')
| -rw-r--r-- | tnslc/parse/token.tnsl | 26 | ||||
| -rw-r--r-- | tnslc/parse/tokenizer.tnsl | 8 |
2 files changed, 20 insertions, 14 deletions
diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl index 92748d8..e7d64e7 100644 --- a/tnslc/parse/token.tnsl +++ b/tnslc/parse/token.tnsl @@ -32,16 +32,13 @@ "abi" } -;const {}{}charp KEYTYPES = { - "bool", - "char", - "charp", - +;{}{}charp KEYTYPES = { "int8", "int16", "int32", "int64", "int", + "uint8", "uint16", "uint32", @@ -52,6 +49,7 @@ "float64", "float", + "bool", "void", "type" } @@ -60,7 +58,6 @@ "struct", "interface", "enum", - "is", "extends", "loop", @@ -83,8 +80,6 @@ "method", "override", - "self", - "super", "operator", "raw", @@ -93,6 +88,9 @@ "virtual", "delete", + "alloc", + "salloc", + "realloc", "module", "export" @@ -101,7 +99,11 @@ ;{}{}charp LITERALS = { "true", "false", - "null" + + "null", + + "self", + "super" } ;{}charp RESERVED = "`~!#%^&*()-=+[]{}|;:,.<>/" @@ -169,7 +171,11 @@ # Increment and De-increment "++", - "--" + "--", + + "is", + "len", + "size" } ;int MAX_MRESERVED = 3 diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index 3a66e24..7a1f085 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -14,11 +14,11 @@ EXPRESS OR IMPLIED #/ -/; is_float (~{}charp dat) [bool] +/; is_float (~{}uint8 dat) [bool] ;return is_numeric_literal(dat) && is_in_string(dat, '.') ;/ -/; break_token ({}charp dat, charp c) [bool] +/; break_token ({}uint8 dat, uint8 c) [bool] /; if (len dat == 0) ;return false @@ -74,7 +74,7 @@ /; tokenize (tnsl.io.File fstr) [~{}Token] ;{}Token out = {} - ;{}charp tdat = {} + ;{}uint8 tdat = {} ;bool comment = false ;int line = 1, col = 1 @@ -84,7 +84,7 @@ ;tdat = {} ;comment = true ;; else if (len tdat > 0) - ;{}charp tmp = tdat + ;{}uint8 tmp = tdat ;Token ttk = {get_token_type(~tmp), line, col, ~tmp} ;out.append(ttk) ;tdat = {} |