From 7d6b6dfc10c549238ceb11b0a01b3e440508cec4 Mon Sep 17 00:00:00 2001 From: Kyle Gunger Date: Sun, 21 Jul 2024 02:53:02 -0400 Subject: semi-broken file imports --- tnslc/parse/tokenizer.tnsl | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'tnslc/parse/tokenizer.tnsl') diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index 0df0ef8..c2ef091 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -84,12 +84,12 @@ uint MAX_MULTI = 3 uint8 ch = fin`.read() - /; loop (fin`.at_end == false && is_reserved(ch) == false && is_whitespace(ch) == false) + /; loop (ch !== 0 && is_reserved(ch) == false && is_whitespace(ch) == false) tmp.push(~ch) ch = fin`.read() ;/ - /; if (fin`.at_end == false) + /; if (ch !== 0) fin`.unread() ;/ @@ -262,7 +262,10 @@ uint MAX_MULTI = 3 bool base = false /; if (ch == '0') ch = fin`.read() - /; if (is_reserved(ch) == false && is_whitespace(ch) == false && is_numeric(ch) == false) + /; if (ch == 0) + out.data = tmp.as_cstr() + return out + ;; else if (is_reserved(ch) == false && is_whitespace(ch) == false && is_numeric(ch) == false) base = true tmp.push(~ch) ;/ @@ -271,7 +274,9 @@ uint MAX_MULTI = 3 bool decimal = false /; loop (bool run = true; run == true && fin`.at_end == false) ch = fin`.read() - /; if (decimal == false && ch == '.') + /; if (ch == 0) + run = false + ;; else if (decimal == false && ch == '.') decimal = true tmp.push(~ch) ;; else if (is_reserved(ch) == true || is_whitespace(ch) == true) -- cgit v1.2.3