diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2021-10-05 14:04:29 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2021-10-05 14:04:29 -0400 |
| commit | d2772d77ce3b0908495bb298a0597a02bf300d1d (patch) | |
| tree | 24524d91b375a02313ff0c2a25bedbd5434895d9 /tnslc/src/parse/tokenizer.tnsl | |
| parent | 3add402da9fc5b574f34e37e951779212ce28ed1 (diff) | |
[Docs] FIP additions
Diffstat (limited to 'tnslc/src/parse/tokenizer.tnsl')
| -rw-r--r-- | tnslc/src/parse/tokenizer.tnsl | 17 |
1 files changed, 15 insertions, 2 deletions
diff --git a/tnslc/src/parse/tokenizer.tnsl b/tnslc/src/parse/tokenizer.tnsl index ec34d83..4e5eeb7 100644 --- a/tnslc/src/parse/tokenizer.tnsl +++ b/tnslc/src/parse/tokenizer.tnsl @@ -14,12 +14,25 @@ EXPRESS OR IMPLIED #/ +:using 'tnsl' /## parse.numeric_literal tokenizes the next numeric literal value in a file. Returns a token with the proper data as well as the number of characters read -#; numeric_literal () [Token, uint] +#; numeric_literal (io.text_stream fstream) [Token, uint] + ;Token out = {token_type: TOKEN_TYPE.LITERAL} + ;uint counter = 0 - + ;bool l, d, run = false, false, true + ;~{}charp num + + ;num, run = fstream.read_number() + + /; loop (run) [num, run = fstream.read_number()] + ;out.add_data(num) + ;delete num + ;/ + + ;return out, counter ;/
\ No newline at end of file |