summaryrefslogtreecommitdiff
path: root/tnslc/parse/tokenizer.tnsl
diff options
context:
space:
mode:
Diffstat (limited to 'tnslc/parse/tokenizer.tnsl')
-rw-r--r--tnslc/parse/tokenizer.tnsl17
1 files changed, 13 insertions, 4 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index 139877c..3a66e24 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -21,41 +21,50 @@
/; break_token ({}charp dat, charp c) [bool]
/; if (len dat == 0)
;return false
+
;; else if (dat{0} == '"' || dat{0} == '\'')
;return string_closed(dat, c)
+
;; else if (is_in_string(~RESERVED, dat{len dat - 1}))
+
/; if (is_in_string(~RESERVED, c))
;dat.append(c)
;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD
+
;; else if (len dat == 1 && dat{0} == '.')
;return !is_digit(c)
+
;/
+
;return true
+
;; else if (is_in_string(~RESERVED, c))
+
/; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
;return false
+
;/
+
;return true
;/
+
;return is_whitespace(c)
;/
/; strip_and_expand (~{}Token dat) [{}Token]
;{}Token out = {}
- ;{}charp cbst = "/#", cben = "#/"
-
;bool cblk = false
/; loop (int i = 0; i < len dat`) [i++]
/; if (!cblk)
- /; if (string_equate(dat`{i}.data, ~cbst))
+ /; if (string_equate(dat`{i}.data`, "/#"))
;cblk = true
;; else
;out.append(dat`{i})
;/
- ;; else if (string_equate(dat`{i}.data, ~cben))
+ ;; else if (string_equate(dat`{i}.data`, "#/"))
;cblk = false
;/
;/