From 9478e157ec2cfe4de704b3bd78b07aee8824774f Mon Sep 17 00:00:00 2001 From: Kyle Gunger Date: Mon, 18 Apr 2022 01:19:02 -0400 Subject: Add tokenizer support for strings --- tnslc/parse/tokenizer.tnsl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'tnslc/parse/tokenizer.tnsl') diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index c6e1ab6..139877c 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -21,7 +21,8 @@ /; break_token ({}charp dat, charp c) [bool] /; if (len dat == 0) ;return false - + ;; else if (dat{0} == '"' || dat{0} == '\'') + ;return string_closed(dat, c) ;; else if (is_in_string(~RESERVED, dat{len dat - 1})) /; if (is_in_string(~RESERVED, c)) ;dat.append(c) @@ -99,8 +100,6 @@ ;out.append(ttk) ;/ - ;tnsl.io.println(tdat) - ;out = strip_and_expand(~out) ;return ~out ;/ -- cgit v1.2.3