From 4a15d64a65cca37a708af4f954415b5c205a9655 Mon Sep 17 00:00:00 2001
From: Kyle Gunger <kgunger12@gmail.com>
Date: Sun, 31 Mar 2024 10:18:13 -0400
Subject: better tokens

---
 tnslc/compile/error.tnsl     |  1 +
 tnslc/compile/tokenizer.tnsl | 34 +++++++++++++++++++---------------
 tnslc/test.tnsl              |  4 +++-
 3 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/tnslc/compile/error.tnsl b/tnslc/compile/error.tnsl
index 7857075..255aec1 100644
--- a/tnslc/compile/error.tnsl
+++ b/tnslc/compile/error.tnsl
@@ -5,6 +5,7 @@
 /; report_error (utils.File file, Token token, ~uint8 message)
 	~uint s = file.path.to_cstr('/')
 	_printf(s)
+	_delete(s)
 	_print_num(ERR_NUM, token.line)
 	_print_num(ERR_NUM, token.col)
 	_print_num(TOK_PRNT, token.data)
diff --git a/tnslc/compile/tokenizer.tnsl b/tnslc/compile/tokenizer.tnsl
index e528e34..30fc8e8 100644
--- a/tnslc/compile/tokenizer.tnsl
+++ b/tnslc/compile/tokenizer.tnsl
@@ -96,10 +96,10 @@ struct Token {
 ~uint8 KEYTYPES = "uint8,uint16,uint32,uint64,uint,int8,int16,int32,int64,int,float32,float64,float,bool,vect,void\0"
 ~uint8 LITERALS = "false,true\0"
 
-~uint8 RESERVED = "~`!@#$%^&*()[]{}+=\"\'\\|;:/?.>,<\0"
+~uint8 RESERVED = "~`!@#$%^&*()[]{}-+=\"\'\\|;:/?.>,<\0"
 
-~uint8 OPS = "`~!%^&*-=+./><\0"
-~uint8 MULTI_OPS = "==,&&,||,^^,!=,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>=,<=,len\0"
+~uint8 OPS = "`~!%^&|*-=+./><\0"
+~uint8 MULTI_OPS = "==,&&,||,^^,!==,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>==,<==,len,!=\0"
 
 ~uint8 DELIMS = "()[]{}\0"
 ~uint8 MULTI_DELIMS = ";:#\0"
@@ -317,6 +317,8 @@ int TT_INVALID = 7
 	return out
 ;/
 
+~uint8 ERROR_RESERVED = "unexpected reserved token in file\0"
+
 /; parse_reserved_tokens (~utils.File fin, ~uint8 char, ~int line, col, ~utils.Vector out)
 	Token tmp
 	tmp.line = line`
@@ -333,9 +335,20 @@ int TT_INVALID = 7
 		int after = token_type(res.as_cstr())
 
 		/; if (after == TT_DEFWORD)
-			res.pop()
+			bool res_unexpected = true
+			/; if (res.count > 1)
+				res.pop()
+				res_unexpected = false
+			;/
+
 			tmp.data = res.as_cstr()
 			tmp._type = token_type(tmp.data)
+			
+			/; if (res_unexpected == true)
+				HAD_ERROR = true
+				report_error(fin`, tmp, ERROR_RESERVED)
+			;/
+			
 			out`.push(~tmp)
 
 			res.init(1)
@@ -368,13 +381,12 @@ int TT_INVALID = 7
 	Token tok
 	tok._type = TT_INVALID
 
-	utils.Vector out, delims, str
+	utils.Vector out, delims
 
 	# init vectors
 
 	out.init(len tok)
 	delims.init(8) # A stack of delimiters
-	str.init(1)
 	
 	# open file for reading
 	fin`.open()
@@ -404,7 +416,7 @@ int TT_INVALID = 7
 		;; else if (is_reserved(char) == true)
 			parse_reserved_tokens(fin, ~char, ~line, ~col, ~out)
 
-		;; else if (char != '\n')
+		;; else if (char !== '\n')
 			# word tokens
 			tok = parse_word_token(fin, ~char, ~line, ~col)
 		;/
@@ -422,14 +434,6 @@ int TT_INVALID = 7
 		;/
 	;/
 
-	/; if (str.count > 0)
-		tok.data = str.as_cstr()
-		tok._type = token_type(tok.data)
-		out.push(~tok)
-	;; else
-		str.end()
-	;/
-
 	delims.end()
 	
 	# done with file
diff --git a/tnslc/test.tnsl b/tnslc/test.tnsl
index 7a51951..7572f1f 100644
--- a/tnslc/test.tnsl
+++ b/tnslc/test.tnsl
@@ -1,3 +1,5 @@
+# should not be included
 /; main [int]
+	
+	return 0
 ;/
-
-- 
cgit v1.2.3