From 8fea53ab8fa095ff7909b5bfbb0c6e94f2206b52 Mon Sep 17 00:00:00 2001
From: Kyle Gunger <kgunger12@gmail.com>
Date: Tue, 12 Apr 2022 17:18:05 -0400
Subject: Update tnslc

---
 tnslc/parse/tokenizer.tnsl | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

(limited to 'tnslc/parse/tokenizer.tnsl')

diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 54671fe..0b80a15 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -18,10 +18,22 @@
 	;return c == '\t' || c == '\n' || c == ' '
 ;/
 
+/; is_digit (charp c) [bool]
+	;return c !< '0' && c !> '9'
+;/
+
 /; break_token ({}charp dat, charp c) [bool]
 	/; if (len dat == 0)
 		;return false
 	
+	;; else if (is_in_string(~RESERVED, dat{len dat - 1}))
+		/; if (is_in_string(~RESERVED, c))
+			;dat.append(c)
+			;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD
+		;; else if (len dat == 1 && dat{0} == '.')
+			;return !is_digit(c)
+		;/
+		;return true
 	;/
 	;return is_space(c)
 ;/
@@ -38,15 +50,19 @@
 		/; if (break_token(tdat, i))
 			/; if (len tdat > 0)
 				;{}charp tmp = tdat
-				;Token ttk = {0, 0, 0, ~tmp}
+				;Token ttk = {get_token_type(~tmp), 0, 0, ~tmp}
 				;out.append(ttk)
 				;tdat = {}
 			;/
-		;; else if ( !is_space(i) )
+		;/
+		/; if ( !is_space(i) )
 			;tdat.append(i)
 		;/
 	;/
 
+	/; if (len tdat > 0)
+	;/
+
 	;tnsl.io.println(tdat)
 
 	;return ~out
-- 
cgit v1.2.3