summaryrefslogtreecommitdiff
path: root/tnslc/compile
diff options
context:
space:
mode:
authorKyle Gunger <kgunger12@gmail.com>2024-03-31 10:18:13 -0400
committerKyle Gunger <kgunger12@gmail.com>2024-03-31 10:18:13 -0400
commit4a15d64a65cca37a708af4f954415b5c205a9655 (patch)
treebef833da928309e489325389bf7d485500cf49b5 /tnslc/compile
parentacc33ffeb8f5eae5e6bb805f1cb409841f0aad75 (diff)
better tokens
Diffstat (limited to 'tnslc/compile')
-rw-r--r--tnslc/compile/error.tnsl1
-rw-r--r--tnslc/compile/tokenizer.tnsl34
2 files changed, 20 insertions, 15 deletions
diff --git a/tnslc/compile/error.tnsl b/tnslc/compile/error.tnsl
index 7857075..255aec1 100644
--- a/tnslc/compile/error.tnsl
+++ b/tnslc/compile/error.tnsl
@@ -5,6 +5,7 @@
/; report_error (utils.File file, Token token, ~uint8 message)
~uint s = file.path.to_cstr('/')
_printf(s)
+ _delete(s)
_print_num(ERR_NUM, token.line)
_print_num(ERR_NUM, token.col)
_print_num(TOK_PRNT, token.data)
diff --git a/tnslc/compile/tokenizer.tnsl b/tnslc/compile/tokenizer.tnsl
index e528e34..30fc8e8 100644
--- a/tnslc/compile/tokenizer.tnsl
+++ b/tnslc/compile/tokenizer.tnsl
@@ -96,10 +96,10 @@ struct Token {
~uint8 KEYTYPES = "uint8,uint16,uint32,uint64,uint,int8,int16,int32,int64,int,float32,float64,float,bool,vect,void\0"
~uint8 LITERALS = "false,true\0"
-~uint8 RESERVED = "~`!@#$%^&*()[]{}+=\"\'\\|;:/?.>,<\0"
+~uint8 RESERVED = "~`!@#$%^&*()[]{}-+=\"\'\\|;:/?.>,<\0"
-~uint8 OPS = "`~!%^&*-=+./><\0"
-~uint8 MULTI_OPS = "==,&&,||,^^,!=,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>=,<=,len\0"
+~uint8 OPS = "`~!%^&|*-=+./><\0"
+~uint8 MULTI_OPS = "==,&&,||,^^,!==,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>==,<==,len,!=\0"
~uint8 DELIMS = "()[]{}\0"
~uint8 MULTI_DELIMS = ";:#\0"
@@ -317,6 +317,8 @@ int TT_INVALID = 7
return out
;/
+~uint8 ERROR_RESERVED = "unexpected reserved token in file\0"
+
/; parse_reserved_tokens (~utils.File fin, ~uint8 char, ~int line, col, ~utils.Vector out)
Token tmp
tmp.line = line`
@@ -333,9 +335,20 @@ int TT_INVALID = 7
int after = token_type(res.as_cstr())
/; if (after == TT_DEFWORD)
- res.pop()
+ bool res_unexpected = true
+ /; if (res.count > 1)
+ res.pop()
+ res_unexpected = false
+ ;/
+
tmp.data = res.as_cstr()
tmp._type = token_type(tmp.data)
+
+ /; if (res_unexpected == true)
+ HAD_ERROR = true
+ report_error(fin`, tmp, ERROR_RESERVED)
+ ;/
+
out`.push(~tmp)
res.init(1)
@@ -368,13 +381,12 @@ int TT_INVALID = 7
Token tok
tok._type = TT_INVALID
- utils.Vector out, delims, str
+ utils.Vector out, delims
# init vectors
out.init(len tok)
delims.init(8) # A stack of delimiters
- str.init(1)
# open file for reading
fin`.open()
@@ -404,7 +416,7 @@ int TT_INVALID = 7
;; else if (is_reserved(char) == true)
parse_reserved_tokens(fin, ~char, ~line, ~col, ~out)
- ;; else if (char != '\n')
+ ;; else if (char !== '\n')
# word tokens
tok = parse_word_token(fin, ~char, ~line, ~col)
;/
@@ -422,14 +434,6 @@ int TT_INVALID = 7
;/
;/
- /; if (str.count > 0)
- tok.data = str.as_cstr()
- tok._type = token_type(tok.data)
- out.push(~tok)
- ;; else
- str.end()
- ;/
-
delims.end()
# done with file