summaryrefslogtreecommitdiff
path: root/tnslc/parse/tokenizer.tnsl
diff options
context:
space:
mode:
Diffstat (limited to 'tnslc/parse/tokenizer.tnsl')
-rw-r--r--tnslc/parse/tokenizer.tnsl8
1 files changed, 7 insertions, 1 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 745100c..e44e648 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -333,7 +333,13 @@ uint MAX_MULTI = 3
/; if (first == '\'' || first == '\"')
return produce_string_token(fin, prev)
;; else if (is_reserved(first) == true)
- return produce_reserved_token(fin, prev)
+ Token out = produce_reserved_token(fin, prev)
+ /; loop (out._type == TTYPE_COMNT)
+ Token tmp = produce_next_token(fin, out)
+ out.end()
+ out = tmp
+ ;/
+ return out
;; else if (is_numeric(first) == true)
return produce_numeric_token(fin, prev)
;; else if (first !== 0)