diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2024-07-21 02:53:02 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2024-07-21 02:53:02 -0400 |
| commit | 7d6b6dfc10c549238ceb11b0a01b3e440508cec4 (patch) | |
| tree | 9f5b4b5ae7e94afe8406bc59814d6a6cbcd9bd01 /tnslc/parse/tokenizer.tnsl | |
| parent | 0c04a61ef15f02eadc24d46b9527e0afb74040a4 (diff) | |
semi-broken file imports
Diffstat (limited to 'tnslc/parse/tokenizer.tnsl')
| -rw-r--r-- | tnslc/parse/tokenizer.tnsl | 13 |
1 files changed, 9 insertions, 4 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index 0df0ef8..c2ef091 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -84,12 +84,12 @@ uint MAX_MULTI = 3 uint8 ch = fin`.read() - /; loop (fin`.at_end == false && is_reserved(ch) == false && is_whitespace(ch) == false) + /; loop (ch !== 0 && is_reserved(ch) == false && is_whitespace(ch) == false) tmp.push(~ch) ch = fin`.read() ;/ - /; if (fin`.at_end == false) + /; if (ch !== 0) fin`.unread() ;/ @@ -262,7 +262,10 @@ uint MAX_MULTI = 3 bool base = false /; if (ch == '0') ch = fin`.read() - /; if (is_reserved(ch) == false && is_whitespace(ch) == false && is_numeric(ch) == false) + /; if (ch == 0) + out.data = tmp.as_cstr() + return out + ;; else if (is_reserved(ch) == false && is_whitespace(ch) == false && is_numeric(ch) == false) base = true tmp.push(~ch) ;/ @@ -271,7 +274,9 @@ uint MAX_MULTI = 3 bool decimal = false /; loop (bool run = true; run == true && fin`.at_end == false) ch = fin`.read() - /; if (decimal == false && ch == '.') + /; if (ch == 0) + run = false + ;; else if (decimal == false && ch == '.') decimal = true tmp.push(~ch) ;; else if (is_reserved(ch) == true || is_whitespace(ch) == true) |