From acc33ffeb8f5eae5e6bb805f1cb409841f0aad75 Mon Sep 17 00:00:00 2001
From: Kyle Gunger <kgunger12@gmail.com>
Date: Sun, 31 Mar 2024 03:09:06 -0400
Subject: Tenative tokenizer

---
 tnslc/compile/generator.tnsl | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

(limited to 'tnslc/compile/generator.tnsl')

diff --git a/tnslc/compile/generator.tnsl b/tnslc/compile/generator.tnsl
index 28a834d..eedc552 100644
--- a/tnslc/compile/generator.tnsl
+++ b/tnslc/compile/generator.tnsl
@@ -1,16 +1,25 @@
-/; generate (utils.File fin, fout)
 
-	fin.open()
-	fout.create()
+~uint8 TOKEN_COUNT = "Token count: %d\n\0"
 
-	uint8 buf = fin.read()
-	/; loop (fin.at_end == false && fout.at_end == false)
-		fout.write(buf)
-		buf = fin.read()
+/; generate (~utils.File fin, fout)
+
+	utils.Vector tokens = tokenize(fin)
+
+	_print_num(TOKEN_COUNT, tokens.count)
+
+	fout`.create()
+
+	/; loop (int i = 0; i < tokens.count) [i++]
+		~Token tok = tokens.get(i)
+		~uint8 buf = tok`.sprint()
+		fout`.write_cstr(buf)
+		fout`.write('\n')
+		_delete(buf)
 	;/
 
-	fin.close()
-	fout.close()
+	fout`.close()
+
+	free_token_list(~tokens)
 
 ;/
 
-- 
cgit v1.2.3