From 44355ceb4db780b370d148008c5048a5075b3978 Mon Sep 17 00:00:00 2001
From: Kyle Gunger <kgunger12@gmail.com>
Date: Wed, 13 Apr 2022 00:55:46 -0400
Subject: Numeric literals

---
 tnslc/parse/tokenizer.tnsl | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

(limited to 'tnslc/parse/tokenizer.tnsl')

diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 0b80a15..b57c766 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -14,12 +14,8 @@
 	EXPRESS OR IMPLIED
 #/
 
-/; is_space (charp c) [bool]
-	;return c == '\t' || c == '\n' || c == ' '
-;/
-
-/; is_digit (charp c) [bool]
-	;return c !< '0' && c !> '9'
+/; is_float (~{}charp dat) [bool]
+	;return is_numeric_literal(dat) && is_in_string(dat, '.')
 ;/
 
 /; break_token ({}charp dat, charp c) [bool]
@@ -34,8 +30,13 @@
 			;return !is_digit(c)
 		;/
 		;return true
+	;; else if (is_in_string(~RESERVED, c))
+		/; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
+			;return false
+		;/
+		;return true
 	;/
-	;return is_space(c)
+	;return is_whitespace(c)
 ;/
 
 /; parse_reserved ({}charp dat) [{}Token]
@@ -55,7 +56,7 @@
 				;tdat = {}
 			;/
 		;/
-		/; if ( !is_space(i) )
+		/; if ( !is_whitespace(i) )
 			;tdat.append(i)
 		;/
 	;/
-- 
cgit v1.2.3