summaryrefslogtreecommitdiff
path: root/tnslc/parse/tokenizer.tnsl
diff options
context:
space:
mode:
Diffstat (limited to 'tnslc/parse/tokenizer.tnsl')
-rw-r--r--tnslc/parse/tokenizer.tnsl114
1 files changed, 0 insertions, 114 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
deleted file mode 100644
index 30ccd4b..0000000
--- a/tnslc/parse/tokenizer.tnsl
+++ /dev/null
@@ -1,114 +0,0 @@
-/#
- Copyright 2021-2022 Kyle Gunger
-
- This file is licensed under the CDDL 1.0 (the License)
- and may only be used in accordance with the License.
- You should have received a copy of the License with this
- software/source code. If you did not, a copy can be found
- at the following URL:
-
- https://opensource.org/licenses/CDDL-1.0
-
- THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
- WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
- EXPRESS OR IMPLIED
-#/
-
-/; is_float (~{}uint8 dat) [bool]
- ;return is_numeric_literal(dat) && is_in_string(dat, '.')
-;/
-
-/; break_token ({}uint8 dat, uint8 c) [bool]
- /; if (len dat == 0)
- ;return false
-
- ;; else if (dat{0} == '"' || dat{0} == '\'')
- ;return string_closed(dat, c)
-
- ;; else if (is_in_string(~RESERVED, dat{len dat - 1}))
-
- /; if (is_in_string(~RESERVED, c))
- ;dat.append(c)
- ;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD
-
- ;; else if (len dat == 1 && dat{0} == '.')
- ;return !is_digit(c)
-
- ;/
-
- ;return true
-
- ;; else if (is_in_string(~RESERVED, c))
-
- /; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
- ;return false
-
- ;/
-
- ;return true
- ;/
-
- ;return is_whitespace(c)
-;/
-
-/; strip_and_expand (~{}Token dat) [{}Token]
- ;{}Token out = {}
-
- ;bool cblk = false
-
- /; loop (int i = 0; i < len dat`) [i++]
- /; if (!cblk)
- /; if (string_equate(dat`{i}.data`, "/#"))
- ;cblk = true
- ;; else
- ;out.append(dat`{i})
- ;/
-
- ;; else if (string_equate(dat`{i}.data`, "#/"))
- ;cblk = false
- ;/
- ;/
-
- ;return out
-;/
-
-/; tokenize (tnsl.io.File fstr) [~{}Token]
- ;{}Token out = {}
- ;{}uint8 tdat = {}
- ;bool comment = false
- ;int line = 1, col = 1
-
- /; loop (int i = fstr.read(); i !== -1) [i = fstr.read()]
- /; if (break_token(tdat, i) && !comment)
- /; if (len tdat == 1 && tdat{0} == '#')
- ;tdat = {}
- ;comment = true
- ;; else if (len tdat > 0)
- ;{}uint8 tmp = tdat
- ;Token ttk = {get_token_type(~tmp), line, col, ~tmp}
- ;out.append(ttk)
- ;tdat = {}
- ;/
- ;/
-
- /; if ( (!is_whitespace(i) || is_text_literal(~tdat)) && !comment )
- ;tdat.append(i)
- ;; else if (i == '\n')
- ;line++
- ;col = 0
- /; if (comment)
- ;comment = false
- ;/
- ;/
-
- ;col++
- ;/
-
- /; if (len tdat > 0)
- ;Token ttk = {get_token_type(~tdat), line, col, ~tdat}
- ;out.append(ttk)
- ;/
-
- ;out = strip_and_expand(~out)
- ;return ~out
-;/