summaryrefslogtreecommitdiff
path: root/tnslc/src/parse/tokenizer.tnsl
blob: 4e5eeb7f047368e9421a484ed0824c9f228f4fd4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
/#
	Copyright 2021 Kyle Gunger

	This file is licensed under the CDDL 1.0 (the License)
	and may only be used in accordance with the License.
	You should have received a copy of the License with this
	software/source code. If you did not, a copy can be found
	at the following URL:

	https://opensource.org/licenses/CDDL-1.0

	THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
	WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
	EXPRESS OR IMPLIED
#/

:using 'tnsl'

/## 
	parse.numeric_literal tokenizes the next numeric literal value in a file.
	Returns a token with the proper data as well as the number of characters read

#; numeric_literal (io.text_stream fstream) [Token, uint]
	;Token out = {token_type: TOKEN_TYPE.LITERAL}
	;uint counter = 0

	;bool l, d, run = false, false, true
	;~{}charp num

	;num, run = fstream.read_number()

	/; loop (run) [num, run = fstream.read_number()]
		;out.add_data(num)
		;delete num
	;/

	;return out, counter
;/