From 46aa6b65376ea62deb1d5ea1611b59dc222a5141 Mon Sep 17 00:00:00 2001
From: Kyle Gunger <kgunger12@gmail.com>
Date: Mon, 2 May 2022 02:57:49 -0400
Subject: [TNSLC] AST Updates - Change how string_equate works - Call AST Node
 generator from main - Flush out ast.tnsl - Add a few initial methods to AST -
 Move some code from token.tnsl to parse.tnsl

---
 tnslc/parse/parse.tnsl     | 56 ++++++++++++++++++++++++++++++++++++++++++++--
 tnslc/parse/token.tnsl     | 56 +++-------------------------------------------
 tnslc/parse/tokenizer.tnsl | 17 ++++++++++----
 3 files changed, 70 insertions(+), 59 deletions(-)

(limited to 'tnslc/parse')

diff --git a/tnslc/parse/parse.tnsl b/tnslc/parse/parse.tnsl
index 5d3bfa5..9927d56 100644
--- a/tnslc/parse/parse.tnsl
+++ b/tnslc/parse/parse.tnsl
@@ -14,12 +14,64 @@
 	EXPRESS OR IMPLIED
 #/
 
-/; module parse
+/; export module parse
 	:include "parse/token.tnsl"
 	:include "parse/tokenizer.tnsl"
 ;/
 
-/; print_tokens(~{}parse.Token dat)
+/# The various types of tokens #/
+; enum TOKEN_TYPE [int] {
+	LINESEP = 0,
+	INLNSEP = 1,
+	DELIMIT = 2,
+	AUGMENT = 3,
+	LITERAL = 4,
+	KEYTYPE = 5,
+	PREWORD = 6,
+	KEYWORD = 7,
+	DEFWORD = 8
+}
+
+/# Token struct definition #/
+; struct Token {
+	int
+		token_type,
+		line,
+		col,
+
+	~{}charp
+		data
+}
+
+/; method Token
+
+	/; print
+		;tnsl.io.print("{ ")
+		;tnsl.io.print(self.token_type)
+		;tnsl.io.print(" ")
+		;tnsl.io.print(self.data`)
+		;tnsl.io.print(" ")
+		;tnsl.io.print(self.line)
+		;tnsl.io.print(" ")
+		;tnsl.io.print(self.col)
+		;tnsl.io.print(" } ")
+	;/
+
+	/; operator delete
+		;delete self.data
+	;/
+
+	/; add_char (~{}charp part)
+		# ;uint l = len self.data`
+		# ;realloc self.data, l + len part
+		/;loop (int i = 0; i < len part`) [i++]
+			# ;self.data`{l + i} = part{i}
+			;self.data`.append(part`{i})
+		;/
+	;/
+;/
+
+/; print_tokens(~{}Token dat)
 	/;loop (int i = 0; i < len dat`) [i++]
 		;dat`{i}.print()
 	;/
diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl
index ce2bcdb..92748d8 100644
--- a/tnslc/parse/token.tnsl
+++ b/tnslc/parse/token.tnsl
@@ -14,58 +14,6 @@
 	EXPRESS OR IMPLIED
 #/
 
-/# The various types of tokens #/
-; enum TOKEN_TYPE [int] {
-	LINESEP = 0,
-	INLNSEP = 1,
-	DELIMIT = 2,
-	AUGMENT = 3,
-	LITERAL = 4,
-	KEYTYPE = 5,
-	PREWORD = 6,
-	KEYWORD = 7,
-	DEFWORD = 8
-}
-
-/# Token struct definition #/
-; struct Token {
-	int
-		token_type,
-		line,
-		col,
-
-	~{}charp
-		data
-}
-
-/; method Token
-
-	/; print
-		;tnsl.io.print("{ ")
-		;tnsl.io.print(self.token_type)
-		;tnsl.io.print(" ")
-		;tnsl.io.print(self.data`)
-		;tnsl.io.print(" ")
-		;tnsl.io.print(self.line)
-		;tnsl.io.print(" ")
-		;tnsl.io.print(self.col)
-		;tnsl.io.print(" } ")
-	;/
-
-	/; operator delete
-		;delete self.data
-	;/
-
-	/; add_char (~{}charp part)
-		# ;uint l = len self.data`
-		# ;realloc self.data, l + len part
-		/;loop (int i = 0; i < len part`) [i++]
-			# ;self.data`{l + i} = part{i}
-			;self.data`.append(part`{i})
-		;/
-	;/
-;/
-
 /#
 	Reserved words and characters, as well as
 	helper funcs for checking their token types.
@@ -142,6 +90,7 @@
 	"raw",
 	"asm",
 	"inline",
+	"virtual",
 
 	"delete",
 	
@@ -151,7 +100,8 @@
 
 ;{}{}charp LITERALS = {
 	"true",
-	"false"
+	"false",
+	"null"
 }
 
 ;{}charp RESERVED = "`~!#%^&*()-=+[]{}|;:,.<>/"
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 139877c..3a66e24 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -21,41 +21,50 @@
 /; break_token ({}charp dat, charp c) [bool]
 	/; if (len dat == 0)
 		;return false
+
 	;; else if (dat{0} == '"' || dat{0} == '\'')
 		;return string_closed(dat, c)
+
 	;; else if (is_in_string(~RESERVED, dat{len dat - 1}))
+
 		/; if (is_in_string(~RESERVED, c))
 			;dat.append(c)
 			;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD
+
 		;; else if (len dat == 1 && dat{0} == '.')
 			;return !is_digit(c)
+
 		;/
+
 		;return true
+
 	;; else if (is_in_string(~RESERVED, c))
+
 		/; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
 			;return false
+
 		;/
+
 		;return true
 	;/
+	
 	;return is_whitespace(c)
 ;/
 
 /; strip_and_expand (~{}Token dat) [{}Token]
 	;{}Token out = {}
 
-	;{}charp cbst = "/#", cben = "#/"
-
 	;bool cblk = false
 
 	/; loop (int i = 0; i < len dat`) [i++]
 		/; if (!cblk)
-			/; if (string_equate(dat`{i}.data, ~cbst))
+			/; if (string_equate(dat`{i}.data`, "/#"))
 				;cblk = true
 			;; else
 				;out.append(dat`{i})
 			;/
 
-		;; else if (string_equate(dat`{i}.data, ~cben))
+		;; else if (string_equate(dat`{i}.data`, "#/"))
 			;cblk = false
 		;/
 	;/
-- 
cgit v1.2.3