diff options
Diffstat (limited to 'tnslc/compile/tokenizer.tnsl')
| -rw-r--r-- | tnslc/compile/tokenizer.tnsl | 34 |
1 files changed, 19 insertions, 15 deletions
diff --git a/tnslc/compile/tokenizer.tnsl b/tnslc/compile/tokenizer.tnsl index e528e34..30fc8e8 100644 --- a/tnslc/compile/tokenizer.tnsl +++ b/tnslc/compile/tokenizer.tnsl @@ -96,10 +96,10 @@ struct Token { ~uint8 KEYTYPES = "uint8,uint16,uint32,uint64,uint,int8,int16,int32,int64,int,float32,float64,float,bool,vect,void\0" ~uint8 LITERALS = "false,true\0" -~uint8 RESERVED = "~`!@#$%^&*()[]{}+=\"\'\\|;:/?.>,<\0" +~uint8 RESERVED = "~`!@#$%^&*()[]{}-+=\"\'\\|;:/?.>,<\0" -~uint8 OPS = "`~!%^&*-=+./><\0" -~uint8 MULTI_OPS = "==,&&,||,^^,!=,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>=,<=,len\0" +~uint8 OPS = "`~!%^&|*-=+./><\0" +~uint8 MULTI_OPS = "==,&&,||,^^,!==,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>==,<==,len,!=\0" ~uint8 DELIMS = "()[]{}\0" ~uint8 MULTI_DELIMS = ";:#\0" @@ -317,6 +317,8 @@ int TT_INVALID = 7 return out ;/ +~uint8 ERROR_RESERVED = "unexpected reserved token in file\0" + /; parse_reserved_tokens (~utils.File fin, ~uint8 char, ~int line, col, ~utils.Vector out) Token tmp tmp.line = line` @@ -333,9 +335,20 @@ int TT_INVALID = 7 int after = token_type(res.as_cstr()) /; if (after == TT_DEFWORD) - res.pop() + bool res_unexpected = true + /; if (res.count > 1) + res.pop() + res_unexpected = false + ;/ + tmp.data = res.as_cstr() tmp._type = token_type(tmp.data) + + /; if (res_unexpected == true) + HAD_ERROR = true + report_error(fin`, tmp, ERROR_RESERVED) + ;/ + out`.push(~tmp) res.init(1) @@ -368,13 +381,12 @@ int TT_INVALID = 7 Token tok tok._type = TT_INVALID - utils.Vector out, delims, str + utils.Vector out, delims # init vectors out.init(len tok) delims.init(8) # A stack of delimiters - str.init(1) # open file for reading fin`.open() @@ -404,7 +416,7 @@ int TT_INVALID = 7 ;; else if (is_reserved(char) == true) parse_reserved_tokens(fin, ~char, ~line, ~col, ~out) - ;; else if (char != '\n') + ;; else if (char !== '\n') # word tokens tok = parse_word_token(fin, ~char, ~line, ~col) ;/ @@ -422,14 +434,6 @@ int TT_INVALID = 7 ;/ ;/ - /; if (str.count > 0) - tok.data = str.as_cstr() - tok._type = token_type(tok.data) - out.push(~tok) - ;; else - str.end() - ;/ - delims.end() # done with file |