diff options
author | Kyle Gunger <kgunger12@gmail.com> | 2022-04-11 18:50:08 -0400 |
---|---|---|
committer | Kyle Gunger <kgunger12@gmail.com> | 2022-04-11 18:50:08 -0400 |
commit | d0213b4202ac12d7e92f7125e87c31d9ecb637dd (patch) | |
tree | 50164dfa2295c6806f3866c24a897789856337a7 /tnslc/parse/tokenizer.tnsl | |
parent | 41cb9e107b20153449526cedec101e33f21d8630 (diff) |
Remove src folders
Diffstat (limited to 'tnslc/parse/tokenizer.tnsl')
-rw-r--r-- | tnslc/parse/tokenizer.tnsl | 53 |
1 files changed, 53 insertions, 0 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl new file mode 100644 index 0000000..54671fe --- /dev/null +++ b/tnslc/parse/tokenizer.tnsl @@ -0,0 +1,53 @@ +/# + Copyright 2021 Kyle Gunger + + This file is licensed under the CDDL 1.0 (the License) + and may only be used in accordance with the License. + You should have received a copy of the License with this + software/source code. If you did not, a copy can be found + at the following URL: + + https://opensource.org/licenses/CDDL-1.0 + + THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO + WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE + EXPRESS OR IMPLIED +#/ + +/; is_space (charp c) [bool] + ;return c == '\t' || c == '\n' || c == ' ' +;/ + +/; break_token ({}charp dat, charp c) [bool] + /; if (len dat == 0) + ;return false + + ;/ + ;return is_space(c) +;/ + +/; parse_reserved ({}charp dat) [{}Token] + +;/ + +/; tokenize (tnsl.io.File fstr) [~{}Token] + ;{}Token out = {} + ;{}charp tdat = {} + + /; loop (int i = fstr.read(); i !== -1) [i = fstr.read()] + /; if (break_token(tdat, i)) + /; if (len tdat > 0) + ;{}charp tmp = tdat + ;Token ttk = {0, 0, 0, ~tmp} + ;out.append(ttk) + ;tdat = {} + ;/ + ;; else if ( !is_space(i) ) + ;tdat.append(i) + ;/ + ;/ + + ;tnsl.io.println(tdat) + + ;return ~out +;/ |