summaryrefslogtreecommitdiff
path: root/tnslc/tokenizer.tnsl
diff options
context:
space:
mode:
authorKyle Gunger <kgunger12@gmail.com>2023-07-26 00:48:43 -0400
committerKyle Gunger <kgunger12@gmail.com>2023-07-26 00:48:43 -0400
commit7f0e6ea654037eda11c2e84946f99bad3f1ab6e4 (patch)
tree88aae0b23c8cdcaaa4a80daf44f87e1969b71de4 /tnslc/tokenizer.tnsl
parentceaeb8df4d9bf9b518239ea623d813add5a71072 (diff)
Division fixes
Diffstat (limited to 'tnslc/tokenizer.tnsl')
-rw-r--r--tnslc/tokenizer.tnsl72
1 files changed, 47 insertions, 25 deletions
diff --git a/tnslc/tokenizer.tnsl b/tnslc/tokenizer.tnsl
index ada1b8e..ae52468 100644
--- a/tnslc/tokenizer.tnsl
+++ b/tnslc/tokenizer.tnsl
@@ -141,6 +141,10 @@ struct Token {
;; else if (contains_char(~AUGMENTS, tok.data{0}))
return TOKEN_TYPE.AUGMENT
;/
+ ;; else if (in_csv(~CSV_KEYWORDS, tok.data))
+ return TOKEN_TYPE.KEYWORD
+ ;; else if (in_csv(~CSV_KEYTYPES, tok.data))
+ return TOKEN_TYPE.KEYTYPE
;; else if (tok_literal(tok) == true)
return TOKEN_TYPE.LITERAL
;/
@@ -158,9 +162,17 @@ struct Token {
tok.append(c)
uint type_after = get_tok_type(tok)
tok.pop()
- log_one_nl('g')
- bool a = type_before !== TOKEN_TYPE.DEFWORD && type_after == TOKEN_TYPE.DEFWORD
+ log_one('g')
+ bool a = type_before !== TOKEN_TYPE.DEFWORD && type_before != TOKEN_TYPE.KEYTYPE && type_before != TOKEN_TYPE.KEYWORD && type_after == TOKEN_TYPE.DEFWORD
+ log_one(' ')
+ log_one('[')
+ log_one(a)
+ log_one(']')
bool b = type_after !== TOKEN_TYPE.LITERAL && is_whitespace(c) == true
+ log_one(' ')
+ log_one('[')
+ log_one(b)
+ log_one_nl(']')
return a || b
;/
@@ -178,31 +190,41 @@ struct Token {
/; if (read_count == 0)
break
;/
+
log_one_nl('b')
- # /; if (break_token(tmp, buf) == true)
- # log_one_nl('c')
- # /; if (tmp._len() > 0)
- # log_one_nl('d')
- # print_token(tmp, file_out)
- # ;/
- #
- # tmp._del()
- # tmp.start()
-#
- # /; if (is_whitespace(buf) == false)
- # log_one_nl('e')
- # tmp.append(buf)
- # ;; else if (buf == WHITESPACE{2})
- # log_one_nl('f')
- # tmp.append(WHITESPACE{2})
- # print_token(tmp, file_out)
- # tmp._del()
- # tmp.start()
- # ;/
- # log_one_nl('c')
- # ;; else
+
+ /; if (buf == '#')
+ /; loop (_read_byte(file_in, ~buf, ~read_count))
+ /; if (buf == '\n' || read_count == 0)
+ break
+ ;/
+ ;/
+
+ ;; else if (break_token(tmp, buf) == true)
+ log_one_nl('c')
+ /; if (tmp._len() > 0)
+ log_one_nl('d')
+ print_token(tmp, file_out)
+ ;/
+ tmp._del()
+ tmp.start()
+ /; if (is_whitespace(buf) == false)
+ log_one_nl('e')
+ tmp.append(buf)
+ ;; else if (buf == WHITESPACE{2})
+ log_one_nl('f')
+ tmp.append(WHITESPACE{2})
+ print_token(tmp, file_out)
+ tmp._del()
+ tmp.start()
+ ;/
+ log_one_nl('c')
+
+ ;; else
tmp.append(buf)
- # ;/
+
+ ;/
+
log_one_nl('b')
read_count = 0
;/