From 46aa6b65376ea62deb1d5ea1611b59dc222a5141 Mon Sep 17 00:00:00 2001
From: Kyle Gunger <kgunger12@gmail.com>
Date: Mon, 2 May 2022 02:57:49 -0400
Subject: [TNSLC] AST Updates - Change how string_equate works - Call AST Node
generator from main - Flush out ast.tnsl - Add a few initial methods to AST -
Move some code from token.tnsl to parse.tnsl
---
tnslc/parse/parse.tnsl | 56 ++++++++++++++++++++++++++++++++++++++++++++--
tnslc/parse/token.tnsl | 56 +++-------------------------------------------
tnslc/parse/tokenizer.tnsl | 17 ++++++++++----
3 files changed, 70 insertions(+), 59 deletions(-)
(limited to 'tnslc/parse')
diff --git a/tnslc/parse/parse.tnsl b/tnslc/parse/parse.tnsl
index 5d3bfa5..9927d56 100644
--- a/tnslc/parse/parse.tnsl
+++ b/tnslc/parse/parse.tnsl
@@ -14,12 +14,64 @@
EXPRESS OR IMPLIED
#/
-/; module parse
+/; export module parse
:include "parse/token.tnsl"
:include "parse/tokenizer.tnsl"
;/
-/; print_tokens(~{}parse.Token dat)
+/# The various types of tokens #/
+; enum TOKEN_TYPE [int] {
+ LINESEP = 0,
+ INLNSEP = 1,
+ DELIMIT = 2,
+ AUGMENT = 3,
+ LITERAL = 4,
+ KEYTYPE = 5,
+ PREWORD = 6,
+ KEYWORD = 7,
+ DEFWORD = 8
+}
+
+/# Token struct definition #/
+; struct Token {
+ int
+ token_type,
+ line,
+ col,
+
+ ~{}charp
+ data
+}
+
+/; method Token
+
+ /; print
+ ;tnsl.io.print("{ ")
+ ;tnsl.io.print(self.token_type)
+ ;tnsl.io.print(" ")
+ ;tnsl.io.print(self.data`)
+ ;tnsl.io.print(" ")
+ ;tnsl.io.print(self.line)
+ ;tnsl.io.print(" ")
+ ;tnsl.io.print(self.col)
+ ;tnsl.io.print(" } ")
+ ;/
+
+ /; operator delete
+ ;delete self.data
+ ;/
+
+ /; add_char (~{}charp part)
+ # ;uint l = len self.data`
+ # ;realloc self.data, l + len part
+ /;loop (int i = 0; i < len part`) [i++]
+ # ;self.data`{l + i} = part{i}
+ ;self.data`.append(part`{i})
+ ;/
+ ;/
+;/
+
+/; print_tokens(~{}Token dat)
/;loop (int i = 0; i < len dat`) [i++]
;dat`{i}.print()
;/
diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl
index ce2bcdb..92748d8 100644
--- a/tnslc/parse/token.tnsl
+++ b/tnslc/parse/token.tnsl
@@ -14,58 +14,6 @@
EXPRESS OR IMPLIED
#/
-/# The various types of tokens #/
-; enum TOKEN_TYPE [int] {
- LINESEP = 0,
- INLNSEP = 1,
- DELIMIT = 2,
- AUGMENT = 3,
- LITERAL = 4,
- KEYTYPE = 5,
- PREWORD = 6,
- KEYWORD = 7,
- DEFWORD = 8
-}
-
-/# Token struct definition #/
-; struct Token {
- int
- token_type,
- line,
- col,
-
- ~{}charp
- data
-}
-
-/; method Token
-
- /; print
- ;tnsl.io.print("{ ")
- ;tnsl.io.print(self.token_type)
- ;tnsl.io.print(" ")
- ;tnsl.io.print(self.data`)
- ;tnsl.io.print(" ")
- ;tnsl.io.print(self.line)
- ;tnsl.io.print(" ")
- ;tnsl.io.print(self.col)
- ;tnsl.io.print(" } ")
- ;/
-
- /; operator delete
- ;delete self.data
- ;/
-
- /; add_char (~{}charp part)
- # ;uint l = len self.data`
- # ;realloc self.data, l + len part
- /;loop (int i = 0; i < len part`) [i++]
- # ;self.data`{l + i} = part{i}
- ;self.data`.append(part`{i})
- ;/
- ;/
-;/
-
/#
Reserved words and characters, as well as
helper funcs for checking their token types.
@@ -142,6 +90,7 @@
"raw",
"asm",
"inline",
+ "virtual",
"delete",
@@ -151,7 +100,8 @@
;{}{}charp LITERALS = {
"true",
- "false"
+ "false",
+ "null"
}
;{}charp RESERVED = "`~!#%^&*()-=+[]{}|;:,.<>/"
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 139877c..3a66e24 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -21,41 +21,50 @@
/; break_token ({}charp dat, charp c) [bool]
/; if (len dat == 0)
;return false
+
;; else if (dat{0} == '"' || dat{0} == '\'')
;return string_closed(dat, c)
+
;; else if (is_in_string(~RESERVED, dat{len dat - 1}))
+
/; if (is_in_string(~RESERVED, c))
;dat.append(c)
;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD
+
;; else if (len dat == 1 && dat{0} == '.')
;return !is_digit(c)
+
;/
+
;return true
+
;; else if (is_in_string(~RESERVED, c))
+
/; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
;return false
+
;/
+
;return true
;/
+
;return is_whitespace(c)
;/
/; strip_and_expand (~{}Token dat) [{}Token]
;{}Token out = {}
- ;{}charp cbst = "/#", cben = "#/"
-
;bool cblk = false
/; loop (int i = 0; i < len dat`) [i++]
/; if (!cblk)
- /; if (string_equate(dat`{i}.data, ~cbst))
+ /; if (string_equate(dat`{i}.data`, "/#"))
;cblk = true
;; else
;out.append(dat`{i})
;/
- ;; else if (string_equate(dat`{i}.data, ~cben))
+ ;; else if (string_equate(dat`{i}.data`, "#/"))
;cblk = false
;/
;/
--
cgit v1.2.3