diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2024-08-04 13:31:25 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2024-08-04 13:31:25 -0400 |
| commit | 6eaced4b97a87fed40c10cb85f0c0f8e96ad909d (patch) | |
| tree | b68723ea21804e220bcaf208647655f01f78a166 /tnslc/parse/tokenizer.tnsl | |
| parent | 0d8ed3e169dba42fc43a4544a9c28fc63cf3053f (diff) | |
[tnslc] Resolve issue with tokenizer
Diffstat (limited to 'tnslc/parse/tokenizer.tnsl')
| -rw-r--r-- | tnslc/parse/tokenizer.tnsl | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index f17ea34..6066f9d 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -265,7 +265,11 @@ uint MAX_MULTI = 3 /; if (ch == 0) out.data = tmp.as_cstr() return out - ;; else if (is_reserved(ch) == false && is_whitespace(ch) == false && is_numeric(ch) == false) + ;; else if (is_reserved(ch) == true || is_whitespace(ch) == true) + fin`.unread() + out.data = tmp.as_cstr() + return out + ;; else if (is_numeric(ch) == false) base = true tmp.push(~ch) ;/ |