diff options
Diffstat (limited to 'tnslc/parse/tokenizer.tnsl')
| -rw-r--r-- | tnslc/parse/tokenizer.tnsl | 17 |
1 files changed, 13 insertions, 4 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl index 139877c..3a66e24 100644 --- a/tnslc/parse/tokenizer.tnsl +++ b/tnslc/parse/tokenizer.tnsl @@ -21,41 +21,50 @@ /; break_token ({}charp dat, charp c) [bool] /; if (len dat == 0) ;return false + ;; else if (dat{0} == '"' || dat{0} == '\'') ;return string_closed(dat, c) + ;; else if (is_in_string(~RESERVED, dat{len dat - 1})) + /; if (is_in_string(~RESERVED, c)) ;dat.append(c) ;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD + ;; else if (len dat == 1 && dat{0} == '.') ;return !is_digit(c) + ;/ + ;return true + ;; else if (is_in_string(~RESERVED, c)) + /; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.') ;return false + ;/ + ;return true ;/ + ;return is_whitespace(c) ;/ /; strip_and_expand (~{}Token dat) [{}Token] ;{}Token out = {} - ;{}charp cbst = "/#", cben = "#/" - ;bool cblk = false /; loop (int i = 0; i < len dat`) [i++] /; if (!cblk) - /; if (string_equate(dat`{i}.data, ~cbst)) + /; if (string_equate(dat`{i}.data`, "/#")) ;cblk = true ;; else ;out.append(dat`{i}) ;/ - ;; else if (string_equate(dat`{i}.data, ~cben)) + ;; else if (string_equate(dat`{i}.data`, "#/")) ;cblk = false ;/ ;/ |