summaryrefslogtreecommitdiff
path: root/tnslc
diff options
context:
space:
mode:
authorKyle Gunger <kgunger12@gmail.com>2024-08-04 13:31:25 -0400
committerKyle Gunger <kgunger12@gmail.com>2024-08-04 13:31:25 -0400
commit6eaced4b97a87fed40c10cb85f0c0f8e96ad909d (patch)
treeb68723ea21804e220bcaf208647655f01f78a166 /tnslc
parent0d8ed3e169dba42fc43a4544a9c28fc63cf3053f (diff)
[tnslc] Resolve issue with tokenizer
Diffstat (limited to 'tnslc')
-rw-r--r--tnslc/parse/ast.tnsl5
-rw-r--r--tnslc/parse/tokenizer.tnsl6
2 files changed, 8 insertions, 3 deletions
diff --git a/tnslc/parse/ast.tnsl b/tnslc/parse/ast.tnsl
index ba8408b..3f92675 100644
--- a/tnslc/parse/ast.tnsl
+++ b/tnslc/parse/ast.tnsl
@@ -202,8 +202,9 @@ struct Node {
Node num
num.init(NTYPE_LITERAL, first`.data)
arr.sub.push(~num)
-
+
first` = produce_next_token(fin, first`)
+
/; if (first`.eq("}\0") == true)
Token tmp = produce_next_token(fin, first`)
first`.end()
@@ -325,7 +326,7 @@ struct Node {
first` = produce_next_token(fin, first`)
/; loop (first`._type !== TTYPE_ERR && first`.data` !== end)
- _ast_type(fin, mod, first)
+ _ast_value(fin, ~list, first)
/; if (_advance_check(fin, first, ",\0") == false && (first`._type !== TTYPE_DELIM || first`.data` !== end))
_ast_print_err(first, "Expected ',' to continue the value list or a closing delimiter\0")
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index f17ea34..6066f9d 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -265,7 +265,11 @@ uint MAX_MULTI = 3
/; if (ch == 0)
out.data = tmp.as_cstr()
return out
- ;; else if (is_reserved(ch) == false && is_whitespace(ch) == false && is_numeric(ch) == false)
+ ;; else if (is_reserved(ch) == true || is_whitespace(ch) == true)
+ fin`.unread()
+ out.data = tmp.as_cstr()
+ return out
+ ;; else if (is_numeric(ch) == false)
base = true
tmp.push(~ch)
;/