diff options
Diffstat (limited to 'tnslc/parse/tokenizer.tnsl')
| -rw-r--r-- | tnslc/parse/tokenizer.tnsl | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index f17ea34..6066f9d 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -265,7 +265,11 @@ uint MAX_MULTI = 3 /; if (ch == 0) out.data = tmp.as_cstr() return out - ;; else if (is_reserved(ch) == false && is_whitespace(ch) == false && is_numeric(ch) == false) + ;; else if (is_reserved(ch) == true || is_whitespace(ch) == true) + fin`.unread() + out.data = tmp.as_cstr() + return out + ;; else if (is_numeric(ch) == false) base = true tmp.push(~ch) ;/ |