diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2024-08-04 13:31:25 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2024-08-04 13:31:25 -0400 |
| commit | 6eaced4b97a87fed40c10cb85f0c0f8e96ad909d (patch) | |
| tree | b68723ea21804e220bcaf208647655f01f78a166 | |
| parent | 0d8ed3e169dba42fc43a4544a9c28fc63cf3053f (diff) | |
[tnslc] Resolve issue with tokenizer
| -rw-r--r-- | tnslc/parse/ast.tnsl | 5 | ||||
| -rw-r--r-- | tnslc/parse/tokenizer.tnsl | 6 |
2 files changed, 8 insertions, 3 deletions
diff --git a/tnslc/parse/ast.tnsl b/tnslc/parse/ast.tnsl index ba8408b..3f92675 100644 --- a/tnslc/parse/ast.tnsl +++ b/tnslc/parse/ast.tnsl @@ -202,8 +202,9 @@ struct Node { Node num num.init(NTYPE_LITERAL, first`.data) arr.sub.push(~num) - + first` = produce_next_token(fin, first`) + /; if (first`.eq("}\0") == true) Token tmp = produce_next_token(fin, first`) first`.end() @@ -325,7 +326,7 @@ struct Node { first` = produce_next_token(fin, first`) /; loop (first`._type !== TTYPE_ERR && first`.data` !== end) - _ast_type(fin, mod, first) + _ast_value(fin, ~list, first) /; if (_advance_check(fin, first, ",\0") == false && (first`._type !== TTYPE_DELIM || first`.data` !== end)) _ast_print_err(first, "Expected ',' to continue the value list or a closing delimiter\0") diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index f17ea34..6066f9d 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -265,7 +265,11 @@ uint MAX_MULTI = 3 /; if (ch == 0) out.data = tmp.as_cstr() return out - ;; else if (is_reserved(ch) == false && is_whitespace(ch) == false && is_numeric(ch) == false) + ;; else if (is_reserved(ch) == true || is_whitespace(ch) == true) + fin`.unread() + out.data = tmp.as_cstr() + return out + ;; else if (is_numeric(ch) == false) base = true tmp.push(~ch) ;/ |