[TNSLC] AST Updates

- Change how string_equate works
- Call AST Node generator from main
- Flush out ast.tnsl
- Add a few initial methods to AST
- Move some code from token.tnsl to parse.tnsl

1 files changed, 13 insertions, 4 deletions

diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 139877c..3a66e24 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -21,41 +21,50 @@
 /; break_token ({}charp dat, charp c) [bool]
 	/; if (len dat == 0)
 		;return false
+
 	;; else if (dat{0} == '"' || dat{0} == '\'')
 		;return string_closed(dat, c)
+
 	;; else if (is_in_string(~RESERVED, dat{len dat - 1}))
+
 		/; if (is_in_string(~RESERVED, c))
 			;dat.append(c)
 			;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD
+
 		;; else if (len dat == 1 && dat{0} == '.')
 			;return !is_digit(c)
+
 		;/
+
 		;return true
+
 	;; else if (is_in_string(~RESERVED, c))
+
 		/; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
 			;return false
+
 		;/
+
 		;return true
 	;/
+	
 	;return is_whitespace(c)
 ;/
 
 /; strip_and_expand (~{}Token dat) [{}Token]
 	;{}Token out = {}
 
-	;{}charp cbst = "/#", cben = "#/"
-
 	;bool cblk = false
 
 	/; loop (int i = 0; i < len dat`) [i++]
 		/; if (!cblk)
-			/; if (string_equate(dat`{i}.data, ~cbst))
+			/; if (string_equate(dat`{i}.data`, "/#"))
 				;cblk = true
 			;; else
 				;out.append(dat`{i})
 			;/
 
-		;; else if (string_equate(dat`{i}.data, ~cben))
+		;; else if (string_equate(dat`{i}.data`, "#/"))
 			;cblk = false
 		;/
 	;/