diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2022-04-12 17:18:05 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2022-04-12 17:18:05 -0400 |
| commit | 8fea53ab8fa095ff7909b5bfbb0c6e94f2206b52 (patch) | |
| tree | 3a06f26b7a9fd5edfcb9c84b8f66bc1455f7357e /tnslc/parse | |
| parent | d0213b4202ac12d7e92f7125e87c31d9ecb637dd (diff) | |
Update tnslc
Diffstat (limited to 'tnslc/parse')
| -rw-r--r-- | tnslc/parse/parse.tnsl | 17 | ||||
| -rw-r--r-- | tnslc/parse/token.tnsl | 6 | ||||
| -rw-r--r-- | tnslc/parse/tokenizer.tnsl | 20 |
3 files changed, 35 insertions, 8 deletions
diff --git a/tnslc/parse/parse.tnsl b/tnslc/parse/parse.tnsl index ec65f7b..4983e13 100644 --- a/tnslc/parse/parse.tnsl +++ b/tnslc/parse/parse.tnsl @@ -15,6 +15,17 @@ #/ /; module parse - :include "tnslc/parse/token.tnsl" - :include "tnslc/parse/tokenizer.tnsl" -;/
\ No newline at end of file + :include "parse/token.tnsl" + :include "parse/tokenizer.tnsl" +;/ + +/; print_tokens(~{}parse.Token dat) + /;loop (int i = 0; i < len dat`) [i++] + ;tnsl.io.print("{ ") + ;tnsl.io.print(dat`{i}.token_type) + ;tnsl.io.print(" ") + ;tnsl.io.print(dat`{i}.data`) + ;tnsl.io.print(" } ") + ;/ + ;tnsl.io.print("\n") +;/ diff --git a/tnslc/parse/token.tnsl b/tnslc/parse/token.tnsl index 0134483..668312b 100644 --- a/tnslc/parse/token.tnsl +++ b/tnslc/parse/token.tnsl @@ -218,7 +218,7 @@ #; is_in_string (~{}charp cmp, charp p) [bool] /; loop (int i = 0; i < len cmp`) [i++] - /; if (s == cmp`{i}) + /; if (p == cmp`{i}) ;return true ;/ ;/ @@ -233,7 +233,7 @@ /; loop (int i = 0; i < len cmp`) [i++] - /; if (len s == len cmp`{i}) + /; if (len s` == len cmp`{i}) /; loop (int j = 0; j < len s`) [j++] @@ -255,7 +255,7 @@ #; get_token_type (~{}charp s) [int] - /; if (len s > 1) + /; if (len s` > 1) /; if (is_in_string_list(~PREWORDS, s)) ;return TOKEN_TYPE.PREWORD diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index 54671fe..0b80a15 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -18,10 +18,22 @@ ;return c == '\t' || c == '\n' || c == ' ' ;/ +/; is_digit (charp c) [bool] + ;return c !< '0' && c !> '9' +;/ + /; break_token ({}charp dat, charp c) [bool] /; if (len dat == 0) ;return false + ;; else if (is_in_string(~RESERVED, dat{len dat - 1})) + /; if (is_in_string(~RESERVED, c)) + ;dat.append(c) + ;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD + ;; else if (len dat == 1 && dat{0} == '.') + ;return !is_digit(c) + ;/ + ;return true ;/ ;return is_space(c) ;/ @@ -38,15 +50,19 @@ /; if (break_token(tdat, i)) /; if (len tdat > 0) ;{}charp tmp = tdat - ;Token ttk = {0, 0, 0, ~tmp} + ;Token ttk = {get_token_type(~tmp), 0, 0, ~tmp} ;out.append(ttk) ;tdat = {} ;/ - ;; else if ( !is_space(i) ) + ;/ + /; if ( !is_space(i) ) ;tdat.append(i) ;/ ;/ + /; if (len tdat > 0) + ;/ + ;tnsl.io.println(tdat) ;return ~out |