summaryrefslogtreecommitdiff
path: root/tnslc/compile
diff options
context:
space:
mode:
Diffstat (limited to 'tnslc/compile')
-rw-r--r--tnslc/compile/tokenizer.tnsl22
1 files changed, 22 insertions, 0 deletions
diff --git a/tnslc/compile/tokenizer.tnsl b/tnslc/compile/tokenizer.tnsl
index 428815f..722a5a0 100644
--- a/tnslc/compile/tokenizer.tnsl
+++ b/tnslc/compile/tokenizer.tnsl
@@ -43,6 +43,28 @@ struct Token {
return along !< 0 && str{along} == 0
;/
+/; _str_contains (~uint8 str, uint8 ch) [bool]
+ /; loop (str` !== 0) [str++]
+ /; if (str` == ch)
+ return true
+ ;/
+ ;/
+ return false
+;/
+
+~uint8 KEYWORDS = "module,export,asm,if,else,loop,label,goto,continue,break,return,import,as,using,struct,method,interface,enum,implements,operator,is\0"
+~uint8 KEYTYPES = "uint8,uint16,uint32,uint64,uint,int8,int16,int32,int64,int,float32,float64,float,bool,vect,void\0"
+~uint8 LITERALS = "false,true\0"
+
+~uint8 RESERVED = "~`!@#$%^&*()[]{}+_=\"\'\\|;:/?.>,<\0"
+
+~uint8 OPS = "`~!%^&*-=+./><\0"
+~uint8 MULTI_OPS = "==,&&,||,^^,!==,!&&,!||,!^^,!<,!>,<<,>>,!&,!|,!^,++,--,>==,<==,len\0"
+
+~uint8 DELIMS = "()[]{}\0"
+~uint8 MULTI_DELIMS = ";:#\0"
+
+
/; tokenize(utils.File fin) [utils.Vector]
Token tok