diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2024-03-31 10:18:13 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2024-03-31 10:18:13 -0400 |
| commit | 4a15d64a65cca37a708af4f954415b5c205a9655 (patch) | |
| tree | bef833da928309e489325389bf7d485500cf49b5 | |
| parent | acc33ffeb8f5eae5e6bb805f1cb409841f0aad75 (diff) | |
better tokens
| -rw-r--r-- | tnslc/compile/error.tnsl | 1 | ||||
| -rw-r--r-- | tnslc/compile/tokenizer.tnsl | 34 | ||||
| -rw-r--r-- | tnslc/test.tnsl | 4 |
3 files changed, 23 insertions, 16 deletions
diff --git a/tnslc/compile/error.tnsl b/tnslc/compile/error.tnsl index 7857075..255aec1 100644 --- a/tnslc/compile/error.tnsl +++ b/tnslc/compile/error.tnsl @@ -5,6 +5,7 @@ /; report_error (utils.File file, Token token, ~uint8 message) ~uint s = file.path.to_cstr('/') _printf(s) + _delete(s) _print_num(ERR_NUM, token.line) _print_num(ERR_NUM, token.col) _print_num(TOK_PRNT, token.data) diff --git a/tnslc/compile/tokenizer.tnsl b/tnslc/compile/tokenizer.tnsl index e528e34..30fc8e8 100644 --- a/tnslc/compile/tokenizer.tnsl +++ b/tnslc/compile/tokenizer.tnsl @@ -96,10 +96,10 @@ struct Token { ~uint8 KEYTYPES = "uint8,uint16,uint32,uint64,uint,int8,int16,int32,int64,int,float32,float64,float,bool,vect,void\0" ~uint8 LITERALS = "false,true\0" -~uint8 RESERVED = "~`!@#$%^&*()[]{}+=\"\'\\|;:/?.>,<\0" +~uint8 RESERVED = "~`!@#$%^&*()[]{}-+=\"\'\\|;:/?.>,<\0" -~uint8 OPS = "`~!%^&*-=+./><\0" -~uint8 MULTI_OPS = "==,&&,||,^^,!=,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>=,<=,len\0" +~uint8 OPS = "`~!%^&|*-=+./><\0" +~uint8 MULTI_OPS = "==,&&,||,^^,!==,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>==,<==,len,!=\0" ~uint8 DELIMS = "()[]{}\0" ~uint8 MULTI_DELIMS = ";:#\0" @@ -317,6 +317,8 @@ int TT_INVALID = 7 return out ;/ +~uint8 ERROR_RESERVED = "unexpected reserved token in file\0" + /; parse_reserved_tokens (~utils.File fin, ~uint8 char, ~int line, col, ~utils.Vector out) Token tmp tmp.line = line` @@ -333,9 +335,20 @@ int TT_INVALID = 7 int after = token_type(res.as_cstr()) /; if (after == TT_DEFWORD) - res.pop() + bool res_unexpected = true + /; if (res.count > 1) + res.pop() + res_unexpected = false + ;/ + tmp.data = res.as_cstr() tmp._type = token_type(tmp.data) + + /; if (res_unexpected == true) + HAD_ERROR = true + report_error(fin`, tmp, ERROR_RESERVED) + ;/ + out`.push(~tmp) res.init(1) @@ -368,13 +381,12 @@ int TT_INVALID = 7 Token tok tok._type = TT_INVALID - utils.Vector out, delims, str + utils.Vector out, delims # init vectors out.init(len tok) delims.init(8) # A stack of delimiters - str.init(1) # open file for reading fin`.open() @@ -404,7 +416,7 @@ int TT_INVALID = 7 ;; else if (is_reserved(char) == true) parse_reserved_tokens(fin, ~char, ~line, ~col, ~out) - ;; else if (char != '\n') + ;; else if (char !== '\n') # word tokens tok = parse_word_token(fin, ~char, ~line, ~col) ;/ @@ -422,14 +434,6 @@ int TT_INVALID = 7 ;/ ;/ - /; if (str.count > 0) - tok.data = str.as_cstr() - tok._type = token_type(tok.data) - out.push(~tok) - ;; else - str.end() - ;/ - delims.end() # done with file diff --git a/tnslc/test.tnsl b/tnslc/test.tnsl index 7a51951..7572f1f 100644 --- a/tnslc/test.tnsl +++ b/tnslc/test.tnsl @@ -1,3 +1,5 @@ +# should not be included /; main [int] + + return 0 ;/ - |