[TNSLC] AST Updates

- Change how string_equate works
- Call AST Node generator from main
- Flush out ast.tnsl
- Add a few initial methods to AST
- Move some code from token.tnsl to parse.tnsl

12 files changed, 271 insertions, 82 deletions

diff --git a/tnslc/ast/ast.tnsl b/tnslc/ast/ast.tnsl
index 466fe08..bece84e 100644
--- a/tnslc/ast/ast.tnsl
+++ b/tnslc/ast/ast.tnsl
@@ -15,5 +15,33 @@
 #/
 
 /; export module ast
-	:include "ast/node.tnsl"
+	:include "ast/block.tnsl"
+	:include "ast/list.tnsl"
+	:include "ast/statement.tnsl"
+	:include "ast/tree.tnsl"
+	:include "ast/value.tnsl"
 ;/
+
+;{}charp CNULL = ""
+
+# AST node (non-block)
+;struct Node {
+	tnslc.Token
+		# associated token to the node
+		tok,
+
+	~{}Node
+		# sub-nodes
+		sub
+}
+
+;enum NODE_TYPE [int] {
+	FILE  = 9,
+	TYPE  = 10,
+	VALUE = 11,
+	DEFN  = 12,
+
+	#...
+
+	PREP = 1000
+}
\ No newline at end of file
diff --git a/tnslc/ast/block.tnsl b/tnslc/ast/block.tnsl
new file mode 100644
index 0000000..17a9e22
--- /dev/null
+++ b/tnslc/ast/block.tnsl
@@ -0,0 +1,15 @@
+/##
+	Copyright 2021 Kyle Gunger
+
+	This file is licensed under the CDDL 1.0 (the License)
+	and may only be used in accordance with the License.
+	You should have received a copy of the License with this
+	software/source code. If you did not, a copy can be found
+	at the following URL:
+
+	https://opensource.org/licenses/CDDL-1.0
+
+	THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
+	WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
+	EXPRESS OR IMPLIED
+#/
\ No newline at end of file
diff --git a/tnslc/ast/node.tnsl b/tnslc/ast/list.tnsl
index d1db325..ee5d0cf 100644
--- a/tnslc/ast/node.tnsl
+++ b/tnslc/ast/list.tnsl
@@ -14,13 +14,8 @@
 	EXPRESS OR IMPLIED
 #/
 
-# AST node (non-block)
-;struct Node {
-	tnslc.parse.Token
-		# associated token to the node
-		tok,
+/; tree_list_value [{}Node]
+	;{}Node out = {}
 
-	~{}Node
-		# sub-nodes
-		sub
-}
+	;return out
+;/
diff --git a/tnslc/ast/statement.tnsl b/tnslc/ast/statement.tnsl
new file mode 100644
index 0000000..d8b4877
--- /dev/null
+++ b/tnslc/ast/statement.tnsl
@@ -0,0 +1,48 @@
+/##
+	Copyright 2021 Kyle Gunger
+
+	This file is licensed under the CDDL 1.0 (the License)
+	and may only be used in accordance with the License.
+	You should have received a copy of the License with this
+	software/source code. If you did not, a copy can be found
+	at the following URL:
+
+	https://opensource.org/licenses/CDDL-1.0
+
+	THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
+	WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
+	EXPRESS OR IMPLIED
+#/
+
+/; is_keyword_statement ({}charp word) [bool]
+	;return string_equate(word, "struct") || string_equate(word, "interface") || string_equate(word, "enum") ||
+			string_equate(word, "continue") || string_equate(word, "break") || string_equate(word, "label") ||
+			string_equate(word, "goto") || string_equate(word, "virtual") || string_equate(word, "asm") ||
+			string_equate(word, "delete")
+;/
+
+/; is_definition (~{}Token tokens, ~int i) [bool] ;/
+
+/; tree_keyword_statement (~{}Token tokens, ~int i) [Node]
+
+;/
+
+/; tree_statement (~{}Token tokens, ~int i) [Node]
+	;i`++
+	/; if (i` !< len tokens`)
+		;return make_null_node()
+	
+	;; else if (is_keyword_statement(tokens`{i`}.data`))
+		;return tree_keyword_statement(tokens, i)
+
+	;; else if (is_definition(tokens, i))
+		;return tree_definition(tokens, i)
+
+	;/
+	
+	;return tree_value(tokens, i)
+;/
+
+/; tree_preproc [Node]
+
+;/
\ No newline at end of file
diff --git a/tnslc/ast/tree.tnsl b/tnslc/ast/tree.tnsl
new file mode 100644
index 0000000..fe6baab
--- /dev/null
+++ b/tnslc/ast/tree.tnsl
@@ -0,0 +1,52 @@
+/##
+	Copyright 2021 Kyle Gunger
+
+	This file is licensed under the CDDL 1.0 (the License)
+	and may only be used in accordance with the License.
+	You should have received a copy of the License with this
+	software/source code. If you did not, a copy can be found
+	at the following URL:
+
+	https://opensource.org/licenses/CDDL-1.0
+
+	THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
+	WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
+	EXPRESS OR IMPLIED
+#/
+
+/; make_null_node [Node]
+	;{}charp null_str = ""
+	;Token null_tok = {0, 0, 0, ~null_str}
+	;{}Node sub = {}
+	;return {null_tok, ~sub}
+;/
+
+/#
+ # This should be the main entry point into the AST module
+ # 
+#; make_tree (~{}Token tokens, {}charp name) [Node]
+	;Token root = {0, 0, NODE_TYPE.FILE, ~name}
+	;{}Node sub = {}
+
+	/; loop (int i = 0; i < len tokens`) [i++]
+		;{}charp to_check = tokens`{i}.data`
+
+		/; if (string_equate(to_check, ";"))
+			;sub.append(tree_statement(tokens, ~i))
+
+		;; else if (string_equate(to_check, ":"))
+			;sub.append(tree_preproc(tokens, ~i))
+
+		;; else if (string_equate(to_check, "/;"))
+			;sub.append(tree_block(tokens, ~i))
+
+		;; else if (string_equate(to_check, "/:"))
+			;sub.append(tree_preblock(tokens, ~i))
+
+		;; else
+			;break
+		;/
+	;/
+
+	;return {root, ~sub}
+;/
diff --git a/tnslc/ast/value.tnsl b/tnslc/ast/value.tnsl
new file mode 100644
index 0000000..2cefb62
--- /dev/null
+++ b/tnslc/ast/value.tnsl
@@ -0,0 +1,40 @@
+/##
+	Copyright 2021 Kyle Gunger
+
+	This file is licensed under the CDDL 1.0 (the License)
+	and may only be used in accordance with the License.
+	You should have received a copy of the License with this
+	software/source code. If you did not, a copy can be found
+	at the following URL:
+
+	https://opensource.org/licenses/CDDL-1.0
+
+	THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
+	WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
+	EXPRESS OR IMPLIED
+#/
+
+/; tree_type [Node]
+	;Token type_tok = {NODE_TYPE.TYPE, 0, 0, ~CNULL}
+	;{}Node type_sub = {}
+	;return {type_tok, ~type_sub}
+;/
+
+/; tree_value [Node]
+
+;/
+
+/; tree_definition [Node]
+	;Token def_tok = {NODE_TYPE.DEFN, 0, 0, ~CNULL}
+	;{}Node def_sub = {}
+
+	;def_sub.append(tree_type())
+	
+	;{}Node vals = tree_list_value()
+
+	/; loop (int i = 0; i < len vals) [i++]
+		;def_sub.append(vals{i})
+	;/
+
+	;return {def_tok, ~def_sub}
+;/
\ No newline at end of file
diff --git a/tnslc/dummy.tnsl b/tnslc/dummy.tnsl
index a2a8156..ebc9c3f 100644
--- a/tnslc/dummy.tnsl
+++ b/tnslc/dummy.tnsl
@@ -1,5 +1 @@
-/; main [float]
-	;int i = 0
-	;{}charp c = "abc"
-	;return a.b
-;/
+;int a = 0
\ No newline at end of file
diff --git a/tnslc/parse/parse.tnsl b/tnslc/parse/parse.tnsl
index 5d3bfa5..9927d56 100644
--- a/tnslc/parse/parse.tnsl
+++ b/tnslc/parse/parse.tnsl
@@ -14,12 +14,64 @@
 	EXPRESS OR IMPLIED
 #/
 
-/; module parse
+/; export module parse
 	:include "parse/token.tnsl"
 	:include "parse/tokenizer.tnsl"
 ;/
 
-/; print_tokens(~{}parse.Token dat)
+/# The various types of tokens #/
+; enum TOKEN_TYPE [int] {
+	LINESEP = 0,
+	INLNSEP = 1,
+	DELIMIT = 2,
+	AUGMENT = 3,
+	LITERAL = 4,
+	KEYTYPE = 5,
+	PREWORD = 6,
+	KEYWORD = 7,
+	DEFWORD = 8
+}
+
+/# Token struct definition #/
+; struct Token {
+	int
+		token_type,
+		line,
+		col,
+
+	~{}charp
+		data
+}
+
+/; method Token
+
+	/; print
+		;tnsl.io.print("{ ")
+		;tnsl.io.print(self.token_type)
+		;tnsl.io.print(" ")
+		;tnsl.io.print(self.data`)
+		;tnsl.io.print(" ")
+		;tnsl.io.print(self.line)
+		;tnsl.io.print(" ")
+		;tnsl.io.print(self.col)
+		;tnsl.io.print(" } ")
+	;/
+
+	/; operator delete
+		;delete self.data
+	;/
+
+	/; add_char (~{}charp part)
+		# ;uint l = len self.data`
+		# ;realloc self.data, l + len part
+		/;loop (int i = 0; i < len part`) [i++]
+			# ;self.data`{l + i} = part{i}
+			;self.data`.append(part`{i})
+		;/
+	;/
+;/
+
+/; print_tokens(~{}Token dat)
 	/;loop (int i = 0; i < len dat`) [i++]
 		;dat`{i}.print()
 	;/
diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl
index ce2bcdb..92748d8 100644
--- a/tnslc/parse/token.tnsl
+++ b/tnslc/parse/token.tnsl
@@ -14,58 +14,6 @@
 	EXPRESS OR IMPLIED
 #/
 
-/# The various types of tokens #/
-; enum TOKEN_TYPE [int] {
-	LINESEP = 0,
-	INLNSEP = 1,
-	DELIMIT = 2,
-	AUGMENT = 3,
-	LITERAL = 4,
-	KEYTYPE = 5,
-	PREWORD = 6,
-	KEYWORD = 7,
-	DEFWORD = 8
-}
-
-/# Token struct definition #/
-; struct Token {
-	int
-		token_type,
-		line,
-		col,
-
-	~{}charp
-		data
-}
-
-/; method Token
-
-	/; print
-		;tnsl.io.print("{ ")
-		;tnsl.io.print(self.token_type)
-		;tnsl.io.print(" ")
-		;tnsl.io.print(self.data`)
-		;tnsl.io.print(" ")
-		;tnsl.io.print(self.line)
-		;tnsl.io.print(" ")
-		;tnsl.io.print(self.col)
-		;tnsl.io.print(" } ")
-	;/
-
-	/; operator delete
-		;delete self.data
-	;/
-
-	/; add_char (~{}charp part)
-		# ;uint l = len self.data`
-		# ;realloc self.data, l + len part
-		/;loop (int i = 0; i < len part`) [i++]
-			# ;self.data`{l + i} = part{i}
-			;self.data`.append(part`{i})
-		;/
-	;/
-;/
-
 /#
 	Reserved words and characters, as well as
 	helper funcs for checking their token types.
@@ -142,6 +90,7 @@
 	"raw",
 	"asm",
 	"inline",
+	"virtual",
 
 	"delete",
 	
@@ -151,7 +100,8 @@
 
 ;{}{}charp LITERALS = {
 	"true",
-	"false"
+	"false",
+	"null"
 }
 
 ;{}charp RESERVED = "`~!#%^&*()-=+[]{}|;:,.<>/"
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 139877c..3a66e24 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -21,41 +21,50 @@
 /; break_token ({}charp dat, charp c) [bool]
 	/; if (len dat == 0)
 		;return false
+
 	;; else if (dat{0} == '"' || dat{0} == '\'')
 		;return string_closed(dat, c)
+
 	;; else if (is_in_string(~RESERVED, dat{len dat - 1}))
+
 		/; if (is_in_string(~RESERVED, c))
 			;dat.append(c)
 			;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD
+
 		;; else if (len dat == 1 && dat{0} == '.')
 			;return !is_digit(c)
+
 		;/
+
 		;return true
+
 	;; else if (is_in_string(~RESERVED, c))
+
 		/; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
 			;return false
+
 		;/
+
 		;return true
 	;/
+	
 	;return is_whitespace(c)
 ;/
 
 /; strip_and_expand (~{}Token dat) [{}Token]
 	;{}Token out = {}
 
-	;{}charp cbst = "/#", cben = "#/"
-
 	;bool cblk = false
 
 	/; loop (int i = 0; i < len dat`) [i++]
 		/; if (!cblk)
-			/; if (string_equate(dat`{i}.data, ~cbst))
+			/; if (string_equate(dat`{i}.data`, "/#"))
 				;cblk = true
 			;; else
 				;out.append(dat`{i})
 			;/
 
-		;; else if (string_equate(dat`{i}.data, ~cben))
+		;; else if (string_equate(dat`{i}.data`, "#/"))
 			;cblk = false
 		;/
 	;/
diff --git a/tnslc/tnslc.tnsl b/tnslc/tnslc.tnsl
index d8adc24..194464e 100644
--- a/tnslc/tnslc.tnsl
+++ b/tnslc/tnslc.tnsl
@@ -16,7 +16,7 @@
 
 :include "util.tnsl"
 
-/; module tnslc
+/; export module tnslc
 	:include "parse/parse.tnsl"
 	:include "ast/ast.tnsl"
 ;/
@@ -34,11 +34,15 @@
 
 	;tnsl.io.File src = tnsl.io.readFile(args{0})
 
-	;~{}tnslc.parse.Token psrc = tnslc.parse.tokenize(src)
-	
-	;tnslc.print_tokens(psrc)
+	;~{}tnslc.Token psrc = tnslc.parse.tokenize(src)
 
 	;src.close()
 
+	;tnslc.print_tokens(psrc)
+	
+	;tnslc.Node tree_node = tnslc.ast.make_tree(psrc, args{0})
+
+	
+
 	;return 0
 ;/
diff --git a/tnslc/util.tnsl b/tnslc/util.tnsl
index a29d544..f1644e7 100644
--- a/tnslc/util.tnsl
+++ b/tnslc/util.tnsl
@@ -18,13 +18,13 @@
     Utility functions that may be useful in many places.
 #/
 
-/; string_equate(~{}charp s1, s2) [bool]
-    /; if (len s1` !== len s2`)
+/; string_equate({}charp s1, s2) [bool]
+    /; if (len s1 !== len s2)
         ;return false
     ;/
 
-    /; loop (int i = 0; i < len s1`) [i++]
-        /; if (s1`{i} !== s2`{i})
+    /; loop (int i = 0; i < len s1) [i++]
+        /; if (s1{i} !== s2{i})
             ;return false
         ;/
     ;/