From 46aa6b65376ea62deb1d5ea1611b59dc222a5141 Mon Sep 17 00:00:00 2001 From: Kyle Gunger Date: Mon, 2 May 2022 02:57:49 -0400 Subject: [TNSLC] AST Updates - Change how string_equate works - Call AST Node generator from main - Flush out ast.tnsl - Add a few initial methods to AST - Move some code from token.tnsl to parse.tnsl --- tnslc/ast/ast.tnsl | 30 ++++++++++++++++++++++++- tnslc/ast/block.tnsl | 15 +++++++++++++ tnslc/ast/list.tnsl | 21 +++++++++++++++++ tnslc/ast/node.tnsl | 26 --------------------- tnslc/ast/statement.tnsl | 48 +++++++++++++++++++++++++++++++++++++++ tnslc/ast/tree.tnsl | 52 ++++++++++++++++++++++++++++++++++++++++++ tnslc/ast/value.tnsl | 40 +++++++++++++++++++++++++++++++++ tnslc/dummy.tnsl | 6 +---- tnslc/parse/parse.tnsl | 56 ++++++++++++++++++++++++++++++++++++++++++++-- tnslc/parse/token.tnsl | 56 +++------------------------------------------- tnslc/parse/tokenizer.tnsl | 17 ++++++++++---- tnslc/tnslc.tnsl | 12 ++++++---- tnslc/util.tnsl | 8 +++---- 13 files changed, 288 insertions(+), 99 deletions(-) create mode 100644 tnslc/ast/block.tnsl create mode 100644 tnslc/ast/list.tnsl delete mode 100644 tnslc/ast/node.tnsl create mode 100644 tnslc/ast/statement.tnsl create mode 100644 tnslc/ast/tree.tnsl create mode 100644 tnslc/ast/value.tnsl (limited to 'tnslc') diff --git a/tnslc/ast/ast.tnsl b/tnslc/ast/ast.tnsl index 466fe08..bece84e 100644 --- a/tnslc/ast/ast.tnsl +++ b/tnslc/ast/ast.tnsl @@ -15,5 +15,33 @@ #/ /; export module ast - :include "ast/node.tnsl" + :include "ast/block.tnsl" + :include "ast/list.tnsl" + :include "ast/statement.tnsl" + :include "ast/tree.tnsl" + :include "ast/value.tnsl" ;/ + +;{}charp CNULL = "" + +# AST node (non-block) +;struct Node { + tnslc.Token + # associated token to the node + tok, + + ~{}Node + # sub-nodes + sub +} + +;enum NODE_TYPE [int] { + FILE = 9, + TYPE = 10, + VALUE = 11, + DEFN = 12, + + #... + + PREP = 1000 +} \ No newline at end of file diff --git a/tnslc/ast/block.tnsl b/tnslc/ast/block.tnsl new file mode 100644 index 0000000..17a9e22 --- /dev/null +++ b/tnslc/ast/block.tnsl @@ -0,0 +1,15 @@ +/## + Copyright 2021 Kyle Gunger + + This file is licensed under the CDDL 1.0 (the License) + and may only be used in accordance with the License. + You should have received a copy of the License with this + software/source code. If you did not, a copy can be found + at the following URL: + + https://opensource.org/licenses/CDDL-1.0 + + THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO + WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE + EXPRESS OR IMPLIED +#/ \ No newline at end of file diff --git a/tnslc/ast/list.tnsl b/tnslc/ast/list.tnsl new file mode 100644 index 0000000..ee5d0cf --- /dev/null +++ b/tnslc/ast/list.tnsl @@ -0,0 +1,21 @@ +/## + Copyright 2021 Kyle Gunger + + This file is licensed under the CDDL 1.0 (the License) + and may only be used in accordance with the License. + You should have received a copy of the License with this + software/source code. If you did not, a copy can be found + at the following URL: + + https://opensource.org/licenses/CDDL-1.0 + + THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO + WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE + EXPRESS OR IMPLIED +#/ + +/; tree_list_value [{}Node] + ;{}Node out = {} + + ;return out +;/ diff --git a/tnslc/ast/node.tnsl b/tnslc/ast/node.tnsl deleted file mode 100644 index d1db325..0000000 --- a/tnslc/ast/node.tnsl +++ /dev/null @@ -1,26 +0,0 @@ -/## - Copyright 2021 Kyle Gunger - - This file is licensed under the CDDL 1.0 (the License) - and may only be used in accordance with the License. - You should have received a copy of the License with this - software/source code. If you did not, a copy can be found - at the following URL: - - https://opensource.org/licenses/CDDL-1.0 - - THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO - WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE - EXPRESS OR IMPLIED -#/ - -# AST node (non-block) -;struct Node { - tnslc.parse.Token - # associated token to the node - tok, - - ~{}Node - # sub-nodes - sub -} diff --git a/tnslc/ast/statement.tnsl b/tnslc/ast/statement.tnsl new file mode 100644 index 0000000..d8b4877 --- /dev/null +++ b/tnslc/ast/statement.tnsl @@ -0,0 +1,48 @@ +/## + Copyright 2021 Kyle Gunger + + This file is licensed under the CDDL 1.0 (the License) + and may only be used in accordance with the License. + You should have received a copy of the License with this + software/source code. If you did not, a copy can be found + at the following URL: + + https://opensource.org/licenses/CDDL-1.0 + + THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO + WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE + EXPRESS OR IMPLIED +#/ + +/; is_keyword_statement ({}charp word) [bool] + ;return string_equate(word, "struct") || string_equate(word, "interface") || string_equate(word, "enum") || + string_equate(word, "continue") || string_equate(word, "break") || string_equate(word, "label") || + string_equate(word, "goto") || string_equate(word, "virtual") || string_equate(word, "asm") || + string_equate(word, "delete") +;/ + +/; is_definition (~{}Token tokens, ~int i) [bool] ;/ + +/; tree_keyword_statement (~{}Token tokens, ~int i) [Node] + +;/ + +/; tree_statement (~{}Token tokens, ~int i) [Node] + ;i`++ + /; if (i` !< len tokens`) + ;return make_null_node() + + ;; else if (is_keyword_statement(tokens`{i`}.data`)) + ;return tree_keyword_statement(tokens, i) + + ;; else if (is_definition(tokens, i)) + ;return tree_definition(tokens, i) + + ;/ + + ;return tree_value(tokens, i) +;/ + +/; tree_preproc [Node] + +;/ \ No newline at end of file diff --git a/tnslc/ast/tree.tnsl b/tnslc/ast/tree.tnsl new file mode 100644 index 0000000..fe6baab --- /dev/null +++ b/tnslc/ast/tree.tnsl @@ -0,0 +1,52 @@ +/## + Copyright 2021 Kyle Gunger + + This file is licensed under the CDDL 1.0 (the License) + and may only be used in accordance with the License. + You should have received a copy of the License with this + software/source code. If you did not, a copy can be found + at the following URL: + + https://opensource.org/licenses/CDDL-1.0 + + THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO + WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE + EXPRESS OR IMPLIED +#/ + +/; make_null_node [Node] + ;{}charp null_str = "" + ;Token null_tok = {0, 0, 0, ~null_str} + ;{}Node sub = {} + ;return {null_tok, ~sub} +;/ + +/# + # This should be the main entry point into the AST module + # +#; make_tree (~{}Token tokens, {}charp name) [Node] + ;Token root = {0, 0, NODE_TYPE.FILE, ~name} + ;{}Node sub = {} + + /; loop (int i = 0; i < len tokens`) [i++] + ;{}charp to_check = tokens`{i}.data` + + /; if (string_equate(to_check, ";")) + ;sub.append(tree_statement(tokens, ~i)) + + ;; else if (string_equate(to_check, ":")) + ;sub.append(tree_preproc(tokens, ~i)) + + ;; else if (string_equate(to_check, "/;")) + ;sub.append(tree_block(tokens, ~i)) + + ;; else if (string_equate(to_check, "/:")) + ;sub.append(tree_preblock(tokens, ~i)) + + ;; else + ;break + ;/ + ;/ + + ;return {root, ~sub} +;/ diff --git a/tnslc/ast/value.tnsl b/tnslc/ast/value.tnsl new file mode 100644 index 0000000..2cefb62 --- /dev/null +++ b/tnslc/ast/value.tnsl @@ -0,0 +1,40 @@ +/## + Copyright 2021 Kyle Gunger + + This file is licensed under the CDDL 1.0 (the License) + and may only be used in accordance with the License. + You should have received a copy of the License with this + software/source code. If you did not, a copy can be found + at the following URL: + + https://opensource.org/licenses/CDDL-1.0 + + THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO + WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE + EXPRESS OR IMPLIED +#/ + +/; tree_type [Node] + ;Token type_tok = {NODE_TYPE.TYPE, 0, 0, ~CNULL} + ;{}Node type_sub = {} + ;return {type_tok, ~type_sub} +;/ + +/; tree_value [Node] + +;/ + +/; tree_definition [Node] + ;Token def_tok = {NODE_TYPE.DEFN, 0, 0, ~CNULL} + ;{}Node def_sub = {} + + ;def_sub.append(tree_type()) + + ;{}Node vals = tree_list_value() + + /; loop (int i = 0; i < len vals) [i++] + ;def_sub.append(vals{i}) + ;/ + + ;return {def_tok, ~def_sub} +;/ \ No newline at end of file diff --git a/tnslc/dummy.tnsl b/tnslc/dummy.tnsl index a2a8156..ebc9c3f 100644 --- a/tnslc/dummy.tnsl +++ b/tnslc/dummy.tnsl @@ -1,5 +1 @@ -/; main [float] - ;int i = 0 - ;{}charp c = "abc" - ;return a.b -;/ +;int a = 0 \ No newline at end of file diff --git a/tnslc/parse/parse.tnsl b/tnslc/parse/parse.tnsl index 5d3bfa5..9927d56 100644 --- a/tnslc/parse/parse.tnsl +++ b/tnslc/parse/parse.tnsl @@ -14,12 +14,64 @@ EXPRESS OR IMPLIED #/ -/; module parse +/; export module parse :include "parse/token.tnsl" :include "parse/tokenizer.tnsl" ;/ -/; print_tokens(~{}parse.Token dat) +/# The various types of tokens #/ +; enum TOKEN_TYPE [int] { + LINESEP = 0, + INLNSEP = 1, + DELIMIT = 2, + AUGMENT = 3, + LITERAL = 4, + KEYTYPE = 5, + PREWORD = 6, + KEYWORD = 7, + DEFWORD = 8 +} + +/# Token struct definition #/ +; struct Token { + int + token_type, + line, + col, + + ~{}charp + data +} + +/; method Token + + /; print + ;tnsl.io.print("{ ") + ;tnsl.io.print(self.token_type) + ;tnsl.io.print(" ") + ;tnsl.io.print(self.data`) + ;tnsl.io.print(" ") + ;tnsl.io.print(self.line) + ;tnsl.io.print(" ") + ;tnsl.io.print(self.col) + ;tnsl.io.print(" } ") + ;/ + + /; operator delete + ;delete self.data + ;/ + + /; add_char (~{}charp part) + # ;uint l = len self.data` + # ;realloc self.data, l + len part + /;loop (int i = 0; i < len part`) [i++] + # ;self.data`{l + i} = part{i} + ;self.data`.append(part`{i}) + ;/ + ;/ +;/ + +/; print_tokens(~{}Token dat) /;loop (int i = 0; i < len dat`) [i++] ;dat`{i}.print() ;/ diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl index ce2bcdb..92748d8 100644 --- a/tnslc/parse/token.tnsl +++ b/tnslc/parse/token.tnsl @@ -14,58 +14,6 @@ EXPRESS OR IMPLIED #/ -/# The various types of tokens #/ -; enum TOKEN_TYPE [int] { - LINESEP = 0, - INLNSEP = 1, - DELIMIT = 2, - AUGMENT = 3, - LITERAL = 4, - KEYTYPE = 5, - PREWORD = 6, - KEYWORD = 7, - DEFWORD = 8 -} - -/# Token struct definition #/ -; struct Token { - int - token_type, - line, - col, - - ~{}charp - data -} - -/; method Token - - /; print - ;tnsl.io.print("{ ") - ;tnsl.io.print(self.token_type) - ;tnsl.io.print(" ") - ;tnsl.io.print(self.data`) - ;tnsl.io.print(" ") - ;tnsl.io.print(self.line) - ;tnsl.io.print(" ") - ;tnsl.io.print(self.col) - ;tnsl.io.print(" } ") - ;/ - - /; operator delete - ;delete self.data - ;/ - - /; add_char (~{}charp part) - # ;uint l = len self.data` - # ;realloc self.data, l + len part - /;loop (int i = 0; i < len part`) [i++] - # ;self.data`{l + i} = part{i} - ;self.data`.append(part`{i}) - ;/ - ;/ -;/ - /# Reserved words and characters, as well as helper funcs for checking their token types. @@ -142,6 +90,7 @@ "raw", "asm", "inline", + "virtual", "delete", @@ -151,7 +100,8 @@ ;{}{}charp LITERALS = { "true", - "false" + "false", + "null" } ;{}charp RESERVED = "`~!#%^&*()-=+[]{}|;:,.<>/" diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index 139877c..3a66e24 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -21,41 +21,50 @@ /; break_token ({}charp dat, charp c) [bool] /; if (len dat == 0) ;return false + ;; else if (dat{0} == '"' || dat{0} == '\'') ;return string_closed(dat, c) + ;; else if (is_in_string(~RESERVED, dat{len dat - 1})) + /; if (is_in_string(~RESERVED, c)) ;dat.append(c) ;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD + ;; else if (len dat == 1 && dat{0} == '.') ;return !is_digit(c) + ;/ + ;return true + ;; else if (is_in_string(~RESERVED, c)) + /; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.') ;return false + ;/ + ;return true ;/ + ;return is_whitespace(c) ;/ /; strip_and_expand (~{}Token dat) [{}Token] ;{}Token out = {} - ;{}charp cbst = "/#", cben = "#/" - ;bool cblk = false /; loop (int i = 0; i < len dat`) [i++] /; if (!cblk) - /; if (string_equate(dat`{i}.data, ~cbst)) + /; if (string_equate(dat`{i}.data`, "/#")) ;cblk = true ;; else ;out.append(dat`{i}) ;/ - ;; else if (string_equate(dat`{i}.data, ~cben)) + ;; else if (string_equate(dat`{i}.data`, "#/")) ;cblk = false ;/ ;/ diff --git a/tnslc/tnslc.tnsl b/tnslc/tnslc.tnsl index d8adc24..194464e 100644 --- a/tnslc/tnslc.tnsl +++ b/tnslc/tnslc.tnsl @@ -16,7 +16,7 @@ :include "util.tnsl" -/; module tnslc +/; export module tnslc :include "parse/parse.tnsl" :include "ast/ast.tnsl" ;/ @@ -34,11 +34,15 @@ ;tnsl.io.File src = tnsl.io.readFile(args{0}) - ;~{}tnslc.parse.Token psrc = tnslc.parse.tokenize(src) - - ;tnslc.print_tokens(psrc) + ;~{}tnslc.Token psrc = tnslc.parse.tokenize(src) ;src.close() + ;tnslc.print_tokens(psrc) + + ;tnslc.Node tree_node = tnslc.ast.make_tree(psrc, args{0}) + + + ;return 0 ;/ diff --git a/tnslc/util.tnsl b/tnslc/util.tnsl index a29d544..f1644e7 100644 --- a/tnslc/util.tnsl +++ b/tnslc/util.tnsl @@ -18,13 +18,13 @@ Utility functions that may be useful in many places. #/ -/; string_equate(~{}charp s1, s2) [bool] - /; if (len s1` !== len s2`) +/; string_equate({}charp s1, s2) [bool] + /; if (len s1 !== len s2) ;return false ;/ - /; loop (int i = 0; i < len s1`) [i++] - /; if (s1`{i} !== s2`{i}) + /; loop (int i = 0; i < len s1) [i++] + /; if (s1{i} !== s2{i}) ;return false ;/ ;/ -- cgit v1.2.3