From 7793ee1739ccb239819c6664c89727697be4bf97 Mon Sep 17 00:00:00 2001 From: Kyle Gunger Date: Fri, 2 Aug 2024 11:52:50 -0400 Subject: type_helper_pre --- tnslc/parse/tokenizer.tnsl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'tnslc/parse/tokenizer.tnsl') diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index 745100c..e44e648 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -333,7 +333,13 @@ uint MAX_MULTI = 3 /; if (first == '\'' || first == '\"') return produce_string_token(fin, prev) ;; else if (is_reserved(first) == true) - return produce_reserved_token(fin, prev) + Token out = produce_reserved_token(fin, prev) + /; loop (out._type == TTYPE_COMNT) + Token tmp = produce_next_token(fin, out) + out.end() + out = tmp + ;/ + return out ;; else if (is_numeric(first) == true) return produce_numeric_token(fin, prev) ;; else if (first !== 0) -- cgit v1.2.3