diff options
Diffstat (limited to 'tnslc/src/parse')
| -rw-r--r-- | tnslc/src/parse/token.tnsl | 18 | ||||
| -rw-r--r-- | tnslc/src/parse/tokenizer.tnsl | 17 |
2 files changed, 28 insertions, 7 deletions
diff --git a/tnslc/src/parse/token.tnsl b/tnslc/src/parse/token.tnsl index a841f58..cbbb31f 100644 --- a/tnslc/src/parse/token.tnsl +++ b/tnslc/src/parse/token.tnsl @@ -30,9 +30,9 @@ /# Token struct definition #/ ;raw struct Token { uint - type, + token_type, line, - char, + col, ~{}charp data @@ -43,6 +43,14 @@ /; operator delete ;delete this.data ;/ + + /; add_char (`{}charp part) + ;uint l = len `this.data + ;realloc this.data, l + len part + /;loop (uint i = 0; i < len part) [i++] + `this.data{l + i} = part{i} + ;/ + ;/ ;/ /# @@ -206,7 +214,7 @@ #; is_in_string (`const {}charp cmp, charp p) [bool] - /; for (int i = 0; i < len cmp) [i++] + /; loop (int i = 0; i < len cmp) [i++] /; if (s == cmp{i}) ;return true @@ -222,11 +230,11 @@ #; is_in_string_list (`const {}{}charp cmp, `{}charp s) [bool] - /; for (int i = 0; i < len cmp) [i++] + /; loop (int i = 0; i < len cmp) [i++] /; if (len s == len cmp{i}) - /; for (int j = 0; j < len s) [j++] + /; loop (int j = 0; j < len s) [j++] /; if (s{j} !== cmp{i}{j}) ;goto cont_outer diff --git a/tnslc/src/parse/tokenizer.tnsl b/tnslc/src/parse/tokenizer.tnsl index ec34d83..4e5eeb7 100644 --- a/tnslc/src/parse/tokenizer.tnsl +++ b/tnslc/src/parse/tokenizer.tnsl @@ -14,12 +14,25 @@ EXPRESS OR IMPLIED #/ +:using 'tnsl' /## parse.numeric_literal tokenizes the next numeric literal value in a file. Returns a token with the proper data as well as the number of characters read -#; numeric_literal () [Token, uint] +#; numeric_literal (io.text_stream fstream) [Token, uint] + ;Token out = {token_type: TOKEN_TYPE.LITERAL} + ;uint counter = 0 - + ;bool l, d, run = false, false, true + ;~{}charp num + + ;num, run = fstream.read_number() + + /; loop (run) [num, run = fstream.read_number()] + ;out.add_data(num) + ;delete num + ;/ + + ;return out, counter ;/
\ No newline at end of file |