summaryrefslogtreecommitdiff
path: root/tnslc
diff options
context:
space:
mode:
authorKyle Gunger <kgunger12@gmail.com>2022-05-02 02:57:49 -0400
committerKyle Gunger <kgunger12@gmail.com>2022-05-02 02:57:49 -0400
commit46aa6b65376ea62deb1d5ea1611b59dc222a5141 (patch)
treee92b26bc2d2653b9009a230ee1d6b7ed1185e3a0 /tnslc
parent9478e157ec2cfe4de704b3bd78b07aee8824774f (diff)
[TNSLC] AST Updates
- Change how string_equate works - Call AST Node generator from main - Flush out ast.tnsl - Add a few initial methods to AST - Move some code from token.tnsl to parse.tnsl
Diffstat (limited to 'tnslc')
-rw-r--r--tnslc/ast/ast.tnsl30
-rw-r--r--tnslc/ast/block.tnsl15
-rw-r--r--tnslc/ast/list.tnsl (renamed from tnslc/ast/node.tnsl)13
-rw-r--r--tnslc/ast/statement.tnsl48
-rw-r--r--tnslc/ast/tree.tnsl52
-rw-r--r--tnslc/ast/value.tnsl40
-rw-r--r--tnslc/dummy.tnsl6
-rw-r--r--tnslc/parse/parse.tnsl56
-rw-r--r--tnslc/parse/token.tnsl56
-rw-r--r--tnslc/parse/tokenizer.tnsl17
-rw-r--r--tnslc/tnslc.tnsl12
-rw-r--r--tnslc/util.tnsl8
12 files changed, 271 insertions, 82 deletions
diff --git a/tnslc/ast/ast.tnsl b/tnslc/ast/ast.tnsl
index 466fe08..bece84e 100644
--- a/tnslc/ast/ast.tnsl
+++ b/tnslc/ast/ast.tnsl
@@ -15,5 +15,33 @@
#/
/; export module ast
- :include "ast/node.tnsl"
+ :include "ast/block.tnsl"
+ :include "ast/list.tnsl"
+ :include "ast/statement.tnsl"
+ :include "ast/tree.tnsl"
+ :include "ast/value.tnsl"
;/
+
+;{}charp CNULL = ""
+
+# AST node (non-block)
+;struct Node {
+ tnslc.Token
+ # associated token to the node
+ tok,
+
+ ~{}Node
+ # sub-nodes
+ sub
+}
+
+;enum NODE_TYPE [int] {
+ FILE = 9,
+ TYPE = 10,
+ VALUE = 11,
+ DEFN = 12,
+
+ #...
+
+ PREP = 1000
+} \ No newline at end of file
diff --git a/tnslc/ast/block.tnsl b/tnslc/ast/block.tnsl
new file mode 100644
index 0000000..17a9e22
--- /dev/null
+++ b/tnslc/ast/block.tnsl
@@ -0,0 +1,15 @@
+/##
+ Copyright 2021 Kyle Gunger
+
+ This file is licensed under the CDDL 1.0 (the License)
+ and may only be used in accordance with the License.
+ You should have received a copy of the License with this
+ software/source code. If you did not, a copy can be found
+ at the following URL:
+
+ https://opensource.org/licenses/CDDL-1.0
+
+ THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
+ WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
+ EXPRESS OR IMPLIED
+#/ \ No newline at end of file
diff --git a/tnslc/ast/node.tnsl b/tnslc/ast/list.tnsl
index d1db325..ee5d0cf 100644
--- a/tnslc/ast/node.tnsl
+++ b/tnslc/ast/list.tnsl
@@ -14,13 +14,8 @@
EXPRESS OR IMPLIED
#/
-# AST node (non-block)
-;struct Node {
- tnslc.parse.Token
- # associated token to the node
- tok,
+/; tree_list_value [{}Node]
+ ;{}Node out = {}
- ~{}Node
- # sub-nodes
- sub
-}
+ ;return out
+;/
diff --git a/tnslc/ast/statement.tnsl b/tnslc/ast/statement.tnsl
new file mode 100644
index 0000000..d8b4877
--- /dev/null
+++ b/tnslc/ast/statement.tnsl
@@ -0,0 +1,48 @@
+/##
+ Copyright 2021 Kyle Gunger
+
+ This file is licensed under the CDDL 1.0 (the License)
+ and may only be used in accordance with the License.
+ You should have received a copy of the License with this
+ software/source code. If you did not, a copy can be found
+ at the following URL:
+
+ https://opensource.org/licenses/CDDL-1.0
+
+ THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
+ WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
+ EXPRESS OR IMPLIED
+#/
+
+/; is_keyword_statement ({}charp word) [bool]
+ ;return string_equate(word, "struct") || string_equate(word, "interface") || string_equate(word, "enum") ||
+ string_equate(word, "continue") || string_equate(word, "break") || string_equate(word, "label") ||
+ string_equate(word, "goto") || string_equate(word, "virtual") || string_equate(word, "asm") ||
+ string_equate(word, "delete")
+;/
+
+/; is_definition (~{}Token tokens, ~int i) [bool] ;/
+
+/; tree_keyword_statement (~{}Token tokens, ~int i) [Node]
+
+;/
+
+/; tree_statement (~{}Token tokens, ~int i) [Node]
+ ;i`++
+ /; if (i` !< len tokens`)
+ ;return make_null_node()
+
+ ;; else if (is_keyword_statement(tokens`{i`}.data`))
+ ;return tree_keyword_statement(tokens, i)
+
+ ;; else if (is_definition(tokens, i))
+ ;return tree_definition(tokens, i)
+
+ ;/
+
+ ;return tree_value(tokens, i)
+;/
+
+/; tree_preproc [Node]
+
+;/ \ No newline at end of file
diff --git a/tnslc/ast/tree.tnsl b/tnslc/ast/tree.tnsl
new file mode 100644
index 0000000..fe6baab
--- /dev/null
+++ b/tnslc/ast/tree.tnsl
@@ -0,0 +1,52 @@
+/##
+ Copyright 2021 Kyle Gunger
+
+ This file is licensed under the CDDL 1.0 (the License)
+ and may only be used in accordance with the License.
+ You should have received a copy of the License with this
+ software/source code. If you did not, a copy can be found
+ at the following URL:
+
+ https://opensource.org/licenses/CDDL-1.0
+
+ THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
+ WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
+ EXPRESS OR IMPLIED
+#/
+
+/; make_null_node [Node]
+ ;{}charp null_str = ""
+ ;Token null_tok = {0, 0, 0, ~null_str}
+ ;{}Node sub = {}
+ ;return {null_tok, ~sub}
+;/
+
+/#
+ # This should be the main entry point into the AST module
+ #
+#; make_tree (~{}Token tokens, {}charp name) [Node]
+ ;Token root = {0, 0, NODE_TYPE.FILE, ~name}
+ ;{}Node sub = {}
+
+ /; loop (int i = 0; i < len tokens`) [i++]
+ ;{}charp to_check = tokens`{i}.data`
+
+ /; if (string_equate(to_check, ";"))
+ ;sub.append(tree_statement(tokens, ~i))
+
+ ;; else if (string_equate(to_check, ":"))
+ ;sub.append(tree_preproc(tokens, ~i))
+
+ ;; else if (string_equate(to_check, "/;"))
+ ;sub.append(tree_block(tokens, ~i))
+
+ ;; else if (string_equate(to_check, "/:"))
+ ;sub.append(tree_preblock(tokens, ~i))
+
+ ;; else
+ ;break
+ ;/
+ ;/
+
+ ;return {root, ~sub}
+;/
diff --git a/tnslc/ast/value.tnsl b/tnslc/ast/value.tnsl
new file mode 100644
index 0000000..2cefb62
--- /dev/null
+++ b/tnslc/ast/value.tnsl
@@ -0,0 +1,40 @@
+/##
+ Copyright 2021 Kyle Gunger
+
+ This file is licensed under the CDDL 1.0 (the License)
+ and may only be used in accordance with the License.
+ You should have received a copy of the License with this
+ software/source code. If you did not, a copy can be found
+ at the following URL:
+
+ https://opensource.org/licenses/CDDL-1.0
+
+ THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
+ WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
+ EXPRESS OR IMPLIED
+#/
+
+/; tree_type [Node]
+ ;Token type_tok = {NODE_TYPE.TYPE, 0, 0, ~CNULL}
+ ;{}Node type_sub = {}
+ ;return {type_tok, ~type_sub}
+;/
+
+/; tree_value [Node]
+
+;/
+
+/; tree_definition [Node]
+ ;Token def_tok = {NODE_TYPE.DEFN, 0, 0, ~CNULL}
+ ;{}Node def_sub = {}
+
+ ;def_sub.append(tree_type())
+
+ ;{}Node vals = tree_list_value()
+
+ /; loop (int i = 0; i < len vals) [i++]
+ ;def_sub.append(vals{i})
+ ;/
+
+ ;return {def_tok, ~def_sub}
+;/ \ No newline at end of file
diff --git a/tnslc/dummy.tnsl b/tnslc/dummy.tnsl
index a2a8156..ebc9c3f 100644
--- a/tnslc/dummy.tnsl
+++ b/tnslc/dummy.tnsl
@@ -1,5 +1 @@
-/; main [float]
- ;int i = 0
- ;{}charp c = "abc"
- ;return a.b
-;/
+;int a = 0 \ No newline at end of file
diff --git a/tnslc/parse/parse.tnsl b/tnslc/parse/parse.tnsl
index 5d3bfa5..9927d56 100644
--- a/tnslc/parse/parse.tnsl
+++ b/tnslc/parse/parse.tnsl
@@ -14,12 +14,64 @@
EXPRESS OR IMPLIED
#/
-/; module parse
+/; export module parse
:include "parse/token.tnsl"
:include "parse/tokenizer.tnsl"
;/
-/; print_tokens(~{}parse.Token dat)
+/# The various types of tokens #/
+; enum TOKEN_TYPE [int] {
+ LINESEP = 0,
+ INLNSEP = 1,
+ DELIMIT = 2,
+ AUGMENT = 3,
+ LITERAL = 4,
+ KEYTYPE = 5,
+ PREWORD = 6,
+ KEYWORD = 7,
+ DEFWORD = 8
+}
+
+/# Token struct definition #/
+; struct Token {
+ int
+ token_type,
+ line,
+ col,
+
+ ~{}charp
+ data
+}
+
+/; method Token
+
+ /; print
+ ;tnsl.io.print("{ ")
+ ;tnsl.io.print(self.token_type)
+ ;tnsl.io.print(" ")
+ ;tnsl.io.print(self.data`)
+ ;tnsl.io.print(" ")
+ ;tnsl.io.print(self.line)
+ ;tnsl.io.print(" ")
+ ;tnsl.io.print(self.col)
+ ;tnsl.io.print(" } ")
+ ;/
+
+ /; operator delete
+ ;delete self.data
+ ;/
+
+ /; add_char (~{}charp part)
+ # ;uint l = len self.data`
+ # ;realloc self.data, l + len part
+ /;loop (int i = 0; i < len part`) [i++]
+ # ;self.data`{l + i} = part{i}
+ ;self.data`.append(part`{i})
+ ;/
+ ;/
+;/
+
+/; print_tokens(~{}Token dat)
/;loop (int i = 0; i < len dat`) [i++]
;dat`{i}.print()
;/
diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl
index ce2bcdb..92748d8 100644
--- a/tnslc/parse/token.tnsl
+++ b/tnslc/parse/token.tnsl
@@ -14,58 +14,6 @@
EXPRESS OR IMPLIED
#/
-/# The various types of tokens #/
-; enum TOKEN_TYPE [int] {
- LINESEP = 0,
- INLNSEP = 1,
- DELIMIT = 2,
- AUGMENT = 3,
- LITERAL = 4,
- KEYTYPE = 5,
- PREWORD = 6,
- KEYWORD = 7,
- DEFWORD = 8
-}
-
-/# Token struct definition #/
-; struct Token {
- int
- token_type,
- line,
- col,
-
- ~{}charp
- data
-}
-
-/; method Token
-
- /; print
- ;tnsl.io.print("{ ")
- ;tnsl.io.print(self.token_type)
- ;tnsl.io.print(" ")
- ;tnsl.io.print(self.data`)
- ;tnsl.io.print(" ")
- ;tnsl.io.print(self.line)
- ;tnsl.io.print(" ")
- ;tnsl.io.print(self.col)
- ;tnsl.io.print(" } ")
- ;/
-
- /; operator delete
- ;delete self.data
- ;/
-
- /; add_char (~{}charp part)
- # ;uint l = len self.data`
- # ;realloc self.data, l + len part
- /;loop (int i = 0; i < len part`) [i++]
- # ;self.data`{l + i} = part{i}
- ;self.data`.append(part`{i})
- ;/
- ;/
-;/
-
/#
Reserved words and characters, as well as
helper funcs for checking their token types.
@@ -142,6 +90,7 @@
"raw",
"asm",
"inline",
+ "virtual",
"delete",
@@ -151,7 +100,8 @@
;{}{}charp LITERALS = {
"true",
- "false"
+ "false",
+ "null"
}
;{}charp RESERVED = "`~!#%^&*()-=+[]{}|;:,.<>/"
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 139877c..3a66e24 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -21,41 +21,50 @@
/; break_token ({}charp dat, charp c) [bool]
/; if (len dat == 0)
;return false
+
;; else if (dat{0} == '"' || dat{0} == '\'')
;return string_closed(dat, c)
+
;; else if (is_in_string(~RESERVED, dat{len dat - 1}))
+
/; if (is_in_string(~RESERVED, c))
;dat.append(c)
;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD
+
;; else if (len dat == 1 && dat{0} == '.')
;return !is_digit(c)
+
;/
+
;return true
+
;; else if (is_in_string(~RESERVED, c))
+
/; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
;return false
+
;/
+
;return true
;/
+
;return is_whitespace(c)
;/
/; strip_and_expand (~{}Token dat) [{}Token]
;{}Token out = {}
- ;{}charp cbst = "/#", cben = "#/"
-
;bool cblk = false
/; loop (int i = 0; i < len dat`) [i++]
/; if (!cblk)
- /; if (string_equate(dat`{i}.data, ~cbst))
+ /; if (string_equate(dat`{i}.data`, "/#"))
;cblk = true
;; else
;out.append(dat`{i})
;/
- ;; else if (string_equate(dat`{i}.data, ~cben))
+ ;; else if (string_equate(dat`{i}.data`, "#/"))
;cblk = false
;/
;/
diff --git a/tnslc/tnslc.tnsl b/tnslc/tnslc.tnsl
index d8adc24..194464e 100644
--- a/tnslc/tnslc.tnsl
+++ b/tnslc/tnslc.tnsl
@@ -16,7 +16,7 @@
:include "util.tnsl"
-/; module tnslc
+/; export module tnslc
:include "parse/parse.tnsl"
:include "ast/ast.tnsl"
;/
@@ -34,11 +34,15 @@
;tnsl.io.File src = tnsl.io.readFile(args{0})
- ;~{}tnslc.parse.Token psrc = tnslc.parse.tokenize(src)
-
- ;tnslc.print_tokens(psrc)
+ ;~{}tnslc.Token psrc = tnslc.parse.tokenize(src)
;src.close()
+ ;tnslc.print_tokens(psrc)
+
+ ;tnslc.Node tree_node = tnslc.ast.make_tree(psrc, args{0})
+
+
+
;return 0
;/
diff --git a/tnslc/util.tnsl b/tnslc/util.tnsl
index a29d544..f1644e7 100644
--- a/tnslc/util.tnsl
+++ b/tnslc/util.tnsl
@@ -18,13 +18,13 @@
Utility functions that may be useful in many places.
#/
-/; string_equate(~{}charp s1, s2) [bool]
- /; if (len s1` !== len s2`)
+/; string_equate({}charp s1, s2) [bool]
+ /; if (len s1 !== len s2)
;return false
;/
- /; loop (int i = 0; i < len s1`) [i++]
- /; if (s1`{i} !== s2`{i})
+ /; loop (int i = 0; i < len s1) [i++]
+ /; if (s1{i} !== s2{i})
;return false
;/
;/