diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2023-08-05 19:44:18 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2023-08-05 19:44:18 -0400 |
| commit | c02a409236d407b39e87fa1e476e35959b358262 (patch) | |
| tree | 786ca022ca0aa1c628253edfbcc46c98d3a4b520 | |
| parent | adc97807daa20efd16fcc9ef82f773bdd23f63bd (diff) | |
Done with tokenizer for native compiler
| -rw-r--r-- | tnslc/tokenizer.tnsl | 27 |
1 files changed, 15 insertions, 12 deletions
diff --git a/tnslc/tokenizer.tnsl b/tnslc/tokenizer.tnsl index 0ae71a1..882dcef 100644 --- a/tnslc/tokenizer.tnsl +++ b/tnslc/tokenizer.tnsl @@ -108,10 +108,12 @@ struct Token { /; if (tok._len() == 1) return is_reserved(tok.data{0}) ;; else if (tok._len() == 2) - return contains_char(~MULTI_PARENS, tok.data{0}) && contains_char(~MULTI_PARENS, tok.data{1}) - ;/ + /; if (contains_char(~MULTI_PARENS, tok.data{0}) && contains_char(~MULTI_PARENS, tok.data{1})) + return true + ;/ + ;/ - return in_csv(~CSV_AUGMENTS, tok.data) + return is_reserved(tok.data{0}) && in_csv(~CSV_AUGMENTS, tok.data) ;/ # True if the token is a valid number (integer or float) @@ -174,8 +176,8 @@ struct Token { tok`.pop() bool a = is_whitespace(c) && type_after !== TOKEN_TYPE.LITERAL - bool b = is_reserved(c) && type_before == TOKEN_TYPE.DEFWORD - bool c = type_before !== TOKEN_TYPE.DEFWORD && type_after == TOKEN_TYPE.DEFWORD + bool b = is_reserved(c) && type_after == TOKEN_TYPE.DEFWORD + bool c = type_before == TOKEN_TYPE.LITERAL && type_after == TOKEN_TYPE.DEFWORD return a || b || c ;/ @@ -198,31 +200,24 @@ struct Token { tmp._type = TOKEN_TYPE.LITERAL tmp.append(buf) read_count = 0 - log_one_nl(first) /; loop _read_byte(file_in, ~buf, ~read_count) - log_one_nl(buf) - log_num_nl(read_count) /; if (read_count == 0) - log_one_nl('b') break ;/ /; if (buf == '\\') - log_one_nl('c') tmp.append(buf) read_count = 0 _read_byte(file_in, ~buf, ~read_count) column`++ tmp.append(buf) ;; else if (buf == first) - log_one_nl('d') tmp.append(buf) break ;; else - log_one_nl('e') tmp.append(buf) ;/ @@ -266,6 +261,14 @@ struct Token { ;; else if (buf == '\'' || buf == '"') + # Don't rope the last token into this + /; if (tmp._len() > 0) + tmp._type = get_tok_type(tmp) + print_token(tmp, file_out) + tmp._del() + tmp.start() + ;/ + # Handle char/string literal tmp = handle_str(file_in, tmp, ~line, ~column, buf) |