summaryrefslogtreecommitdiff
path: root/tnslc/parse/tokenizer.tnsl
diff options
context:
space:
mode:
Diffstat (limited to 'tnslc/parse/tokenizer.tnsl')
-rw-r--r--tnslc/parse/tokenizer.tnsl53
1 files changed, 53 insertions, 0 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
new file mode 100644
index 0000000..54671fe
--- /dev/null
+++ b/tnslc/parse/tokenizer.tnsl
@@ -0,0 +1,53 @@
+/#
+ Copyright 2021 Kyle Gunger
+
+ This file is licensed under the CDDL 1.0 (the License)
+ and may only be used in accordance with the License.
+ You should have received a copy of the License with this
+ software/source code. If you did not, a copy can be found
+ at the following URL:
+
+ https://opensource.org/licenses/CDDL-1.0
+
+ THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
+ WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
+ EXPRESS OR IMPLIED
+#/
+
+/; is_space (charp c) [bool]
+ ;return c == '\t' || c == '\n' || c == ' '
+;/
+
+/; break_token ({}charp dat, charp c) [bool]
+ /; if (len dat == 0)
+ ;return false
+
+ ;/
+ ;return is_space(c)
+;/
+
+/; parse_reserved ({}charp dat) [{}Token]
+
+;/
+
+/; tokenize (tnsl.io.File fstr) [~{}Token]
+ ;{}Token out = {}
+ ;{}charp tdat = {}
+
+ /; loop (int i = fstr.read(); i !== -1) [i = fstr.read()]
+ /; if (break_token(tdat, i))
+ /; if (len tdat > 0)
+ ;{}charp tmp = tdat
+ ;Token ttk = {0, 0, 0, ~tmp}
+ ;out.append(ttk)
+ ;tdat = {}
+ ;/
+ ;; else if ( !is_space(i) )
+ ;tdat.append(i)
+ ;/
+ ;/
+
+ ;tnsl.io.println(tdat)
+
+ ;return ~out
+;/