diff options
Diffstat (limited to 'tnslc/src/parse/tokenizer.tnsl')
| -rw-r--r-- | tnslc/src/parse/tokenizer.tnsl | 17 |
1 files changed, 15 insertions, 2 deletions
diff --git a/tnslc/src/parse/tokenizer.tnsl b/tnslc/src/parse/tokenizer.tnsl index ec34d83..4e5eeb7 100644 --- a/tnslc/src/parse/tokenizer.tnsl +++ b/tnslc/src/parse/tokenizer.tnsl @@ -14,12 +14,25 @@ EXPRESS OR IMPLIED #/ +:using 'tnsl' /## parse.numeric_literal tokenizes the next numeric literal value in a file. Returns a token with the proper data as well as the number of characters read -#; numeric_literal () [Token, uint] +#; numeric_literal (io.text_stream fstream) [Token, uint] + ;Token out = {token_type: TOKEN_TYPE.LITERAL} + ;uint counter = 0 - + ;bool l, d, run = false, false, true + ;~{}charp num + + ;num, run = fstream.read_number() + + /; loop (run) [num, run = fstream.read_number()] + ;out.add_data(num) + ;delete num + ;/ + + ;return out, counter ;/
\ No newline at end of file |