summaryrefslogtreecommitdiff
path: root/tnslc
diff options
context:
space:
mode:
authorKyle Gunger <kgunger12@gmail.com>2022-04-13 14:33:48 -0400
committerKyle Gunger <kgunger12@gmail.com>2022-04-13 14:33:48 -0400
commit6674b44eb75218e1f00a1cc63a82aab82c641806 (patch)
treefda8d28a8098e99a6d37191012956b7d89514ac6 /tnslc
parent44355ceb4db780b370d148008c5048a5075b3978 (diff)
Fix comments and add to token printer.
Diffstat (limited to 'tnslc')
-rw-r--r--tnslc/parse/parse.tnsl4
-rw-r--r--tnslc/parse/tokenizer.tnsl46
2 files changed, 45 insertions, 5 deletions
diff --git a/tnslc/parse/parse.tnsl b/tnslc/parse/parse.tnsl
index 4983e13..52fd0c7 100644
--- a/tnslc/parse/parse.tnsl
+++ b/tnslc/parse/parse.tnsl
@@ -25,6 +25,10 @@
;tnsl.io.print(dat`{i}.token_type)
;tnsl.io.print(" ")
;tnsl.io.print(dat`{i}.data`)
+ ;tnsl.io.print(" ")
+ ;tnsl.io.print(dat`{i}.line)
+ ;tnsl.io.print(" ")
+ ;tnsl.io.print(dat`{i}.col)
;tnsl.io.print(" } ")
;/
;tnsl.io.print("\n")
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
index b57c766..c6e1ab6 100644
--- a/tnslc/parse/tokenizer.tnsl
+++ b/tnslc/parse/tokenizer.tnsl
@@ -39,32 +39,68 @@
;return is_whitespace(c)
;/
-/; parse_reserved ({}charp dat) [{}Token]
+/; strip_and_expand (~{}Token dat) [{}Token]
+ ;{}Token out = {}
+
+ ;{}charp cbst = "/#", cben = "#/"
+
+ ;bool cblk = false
+ /; loop (int i = 0; i < len dat`) [i++]
+ /; if (!cblk)
+ /; if (string_equate(dat`{i}.data, ~cbst))
+ ;cblk = true
+ ;; else
+ ;out.append(dat`{i})
+ ;/
+
+ ;; else if (string_equate(dat`{i}.data, ~cben))
+ ;cblk = false
+ ;/
+ ;/
+
+ ;return out
;/
/; tokenize (tnsl.io.File fstr) [~{}Token]
;{}Token out = {}
;{}charp tdat = {}
+ ;bool comment = false
+ ;int line = 1, col = 1
/; loop (int i = fstr.read(); i !== -1) [i = fstr.read()]
- /; if (break_token(tdat, i))
- /; if (len tdat > 0)
+ /; if (break_token(tdat, i) && !comment)
+ /; if (len tdat == 1 && tdat{0} == '#')
+ ;tdat = {}
+ ;comment = true
+ ;; else if (len tdat > 0)
;{}charp tmp = tdat
- ;Token ttk = {get_token_type(~tmp), 0, 0, ~tmp}
+ ;Token ttk = {get_token_type(~tmp), line, col, ~tmp}
;out.append(ttk)
;tdat = {}
;/
;/
- /; if ( !is_whitespace(i) )
+
+ /; if ( !is_whitespace(i) && !comment )
;tdat.append(i)
+ ;; else if (i == '\n')
+ ;line++
+ ;col = 0
+ /; if (comment)
+ ;comment = false
+ ;/
;/
+
+ ;col++
;/
/; if (len tdat > 0)
+ ;Token ttk = {get_token_type(~tdat), line, col, ~tdat}
+ ;out.append(ttk)
;/
;tnsl.io.println(tdat)
+ ;out = strip_and_expand(~out)
;return ~out
;/