Numeric literals

2 files changed, 45 insertions, 8 deletions

diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl
index 668312b..6c402fd 100644
--- a/tnslc/parse/token.tnsl
+++ b/tnslc/parse/token.tnsl
@@ -250,6 +250,38 @@
 	;return false
 ;/
 
+/; is_numeric_literal(~{}charp dat) [bool]
+	/; if (len dat` == 0)
+		;return false
+	;/
+
+	;bool dec = true, flt = false
+	
+	;int i = 0
+
+	/; if (len dat` > 1)
+		/; if (dat`{0} == '0' && !is_digit(dat`{1}) && dat`{1} !== '.')
+			;dec = false
+			;i = 2
+		;/
+	;/
+
+	/; loop (i < len dat`) [i++]
+		/; if (!is_digit(dat`{i}) && dec)
+			/; if (dat`{i} == '.')
+				/; if (flt || !dec)
+					;return false
+				;/
+				;flt = true
+			;; else if (dec)
+				;return false
+			;/
+		;/
+	;/
+	
+	;return true
+;/
+
 /#
 	Get the token_type value for a given string of character points
 
@@ -257,6 +289,10 @@
 
 	/; if (len s` > 1)
 
+		/; if (is_numeric_literal(s))
+			;return TOKEN_TYPE.LITERAL
+		;/
+
 		/; if (is_in_string_list(~PREWORDS, s))
 			;return TOKEN_TYPE.PREWORD
 		;; else if (is_in_string_list(~KEYTYPES, s))
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 0b80a15..b57c766 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -14,12 +14,8 @@
 	EXPRESS OR IMPLIED
 #/
 
-/; is_space (charp c) [bool]
-	;return c == '\t' || c == '\n' || c == ' '
-;/
-
-/; is_digit (charp c) [bool]
-	;return c !< '0' && c !> '9'
+/; is_float (~{}charp dat) [bool]
+	;return is_numeric_literal(dat) && is_in_string(dat, '.')
 ;/
 
 /; break_token ({}charp dat, charp c) [bool]
@@ -34,8 +30,13 @@
 			;return !is_digit(c)
 		;/
 		;return true
+	;; else if (is_in_string(~RESERVED, c))
+		/; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
+			;return false
+		;/
+		;return true
 	;/
-	;return is_space(c)
+	;return is_whitespace(c)
 ;/
 
 /; parse_reserved ({}charp dat) [{}Token]
@@ -55,7 +56,7 @@
 				;tdat = {}
 			;/
 		;/
-		/; if ( !is_space(i) )
+		/; if ( !is_whitespace(i) )
 			;tdat.append(i)
 		;/
 	;/