diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2024-03-30 04:24:29 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2024-03-30 04:24:29 -0400 |
| commit | d90f254a65e7443e7d6ee7d4a3e89df21bdd4e8c (patch) | |
| tree | 7c27b62ed2be94b7817a090966c0af9d1917cf51 /tnslc/compile | |
| parent | bb7bbcc03386089dce1353e98c071c15c45aa82f (diff) | |
port strings from c based compiler
Diffstat (limited to 'tnslc/compile')
| -rw-r--r-- | tnslc/compile/tokenizer.tnsl | 22 |
1 files changed, 22 insertions, 0 deletions
diff --git a/tnslc/compile/tokenizer.tnsl b/tnslc/compile/tokenizer.tnsl index 428815f..722a5a0 100644 --- a/tnslc/compile/tokenizer.tnsl +++ b/tnslc/compile/tokenizer.tnsl @@ -43,6 +43,28 @@ struct Token { return along !< 0 && str{along} == 0 ;/ +/; _str_contains (~uint8 str, uint8 ch) [bool] + /; loop (str` !== 0) [str++] + /; if (str` == ch) + return true + ;/ + ;/ + return false +;/ + +~uint8 KEYWORDS = "module,export,asm,if,else,loop,label,goto,continue,break,return,import,as,using,struct,method,interface,enum,implements,operator,is\0" +~uint8 KEYTYPES = "uint8,uint16,uint32,uint64,uint,int8,int16,int32,int64,int,float32,float64,float,bool,vect,void\0" +~uint8 LITERALS = "false,true\0" + +~uint8 RESERVED = "~`!@#$%^&*()[]{}+_=\"\'\\|;:/?.>,<\0" + +~uint8 OPS = "`~!%^&*-=+./><\0" +~uint8 MULTI_OPS = "==,&&,||,^^,!==,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>==,<==,len\0" + +~uint8 DELIMS = "()[]{}\0" +~uint8 MULTI_DELIMS = ";:#\0" + + /; tokenize(utils.File fin) [utils.Vector] Token tok |