summaryrefslogtreecommitdiff
path: root/tnslc/tokenizer.tnsl
diff options
context:
space:
mode:
authorKyle Gunger <kgunger12@gmail.com>2023-08-05 20:56:31 -0400
committerKyle Gunger <kgunger12@gmail.com>2023-08-05 20:56:31 -0400
commite070620bf7c66b4bdfeaa8e6bf96ca98fa9a1024 (patch)
treee796892fda818ddb7f207a8c60359a628d7aaaa2 /tnslc/tokenizer.tnsl
parentc02a409236d407b39e87fa1e476e35959b358262 (diff)
Fix weird augments like 'len' and 'is'
Diffstat (limited to 'tnslc/tokenizer.tnsl')
-rw-r--r--tnslc/tokenizer.tnsl11
1 files changed, 9 insertions, 2 deletions
diff --git a/tnslc/tokenizer.tnsl b/tnslc/tokenizer.tnsl
index 882dcef..3220313 100644
--- a/tnslc/tokenizer.tnsl
+++ b/tnslc/tokenizer.tnsl
@@ -113,7 +113,7 @@ struct Token {
;/
;/
- return is_reserved(tok.data{0}) && in_csv(~CSV_AUGMENTS, tok.data)
+ return in_csv(~CSV_AUGMENTS, tok.data)
;/
# True if the token is a valid number (integer or float)
@@ -176,7 +176,14 @@ struct Token {
tok`.pop()
bool a = is_whitespace(c) && type_after !== TOKEN_TYPE.LITERAL
- bool b = is_reserved(c) && type_after == TOKEN_TYPE.DEFWORD
+
+ bool b = false
+ /; if (is_reserved(c) == true)
+ b = type_after == TOKEN_TYPE.DEFWORD
+ ;; else if (tok`._len() > 0)
+ b = is_reserved(tok`.data{0})
+ ;/
+
bool c = type_before == TOKEN_TYPE.LITERAL && type_after == TOKEN_TYPE.DEFWORD
return a || b || c