summaryrefslogtreecommitdiff
path: root/tnslc/compile/tokenizer.tnsl
diff options
context:
space:
mode:
Diffstat (limited to 'tnslc/compile/tokenizer.tnsl')
-rw-r--r--tnslc/compile/tokenizer.tnsl53
1 files changed, 53 insertions, 0 deletions
diff --git a/tnslc/compile/tokenizer.tnsl b/tnslc/compile/tokenizer.tnsl
index e69de29..428815f 100644
--- a/tnslc/compile/tokenizer.tnsl
+++ b/tnslc/compile/tokenizer.tnsl
@@ -0,0 +1,53 @@
+struct Token {
+ ~uint8 data,
+ int
+ _type,
+ line,
+ col
+}
+
+/; method Token
+ /; eq (Token tok) [bool]
+ return utils.strcmp(self.data, tok.data)
+ ;/
+
+ /; eq_str(~uint8 str) [bool]
+ return utils.strcmp(self.data, str)
+ ;/
+;/
+
+/; _is_space(uint8 char) [bool]
+ /; if (char == '\t' || char == '\n' || char == '\r' || char == ' ')
+ return true
+ ;/
+ return false
+;/
+
+/; _in_csv (~uint8 csv, ~uint8 str) [bool]
+ int along = 0
+
+ /; loop (csv` !== 0) [csv++]
+ /; if (csv` == ',')
+ /; if (along !< 0 && str{along} == 0)
+ return true
+ ;/
+ along = 0
+ ;; else if (along !< 0 && str{along} == csv`)
+ along++
+ ;; else
+ along = 0
+ along--
+ ;/
+ ;/
+
+ return along !< 0 && str{along} == 0
+;/
+
+/; tokenize(utils.File fin) [utils.Vector]
+ Token tok
+
+ utils.Vector out
+ out.init(len tok)
+
+ return out
+;/