From 46aa6b65376ea62deb1d5ea1611b59dc222a5141 Mon Sep 17 00:00:00 2001 From: Kyle Gunger Date: Mon, 2 May 2022 02:57:49 -0400 Subject: [TNSLC] AST Updates - Change how string_equate works - Call AST Node generator from main - Flush out ast.tnsl - Add a few initial methods to AST - Move some code from token.tnsl to parse.tnsl --- tnslc/parse/tokenizer.tnsl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'tnslc/parse/tokenizer.tnsl') diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index 139877c..3a66e24 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -21,41 +21,50 @@ /; break_token ({}charp dat, charp c) [bool] /; if (len dat == 0) ;return false + ;; else if (dat{0} == '"' || dat{0} == '\'') ;return string_closed(dat, c) + ;; else if (is_in_string(~RESERVED, dat{len dat - 1})) + /; if (is_in_string(~RESERVED, c)) ;dat.append(c) ;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD + ;; else if (len dat == 1 && dat{0} == '.') ;return !is_digit(c) + ;/ + ;return true + ;; else if (is_in_string(~RESERVED, c)) + /; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.') ;return false + ;/ + ;return true ;/ + ;return is_whitespace(c) ;/ /; strip_and_expand (~{}Token dat) [{}Token] ;{}Token out = {} - ;{}charp cbst = "/#", cben = "#/" - ;bool cblk = false /; loop (int i = 0; i < len dat`) [i++] /; if (!cblk) - /; if (string_equate(dat`{i}.data, ~cbst)) + /; if (string_equate(dat`{i}.data`, "/#")) ;cblk = true ;; else ;out.append(dat`{i}) ;/ - ;; else if (string_equate(dat`{i}.data, ~cben)) + ;; else if (string_equate(dat`{i}.data`, "#/")) ;cblk = false ;/ ;/ -- cgit v1.2.3