[tnslc] Tokenizer bootstrap

6 files changed, 141 insertions, 118 deletions

diff --git a/tnslc/compile/codegen.tnsl b/tnslc/compile/codegen.tnsl
index dae3764..d30ef37 100644
--- a/tnslc/compile/codegen.tnsl
+++ b/tnslc/compile/codegen.tnsl
@@ -65,9 +65,9 @@
 	# var_tests(~buffer)
 
 	# Write assembly to output file
-	fout.create()
+	fout`.create()
 	buffer.write_to(fout)
-	fout.close()
+	fout`.close()
 	
 	# Free all structs
 	mod.end()
diff --git a/tnslc/compile/function.tnsl b/tnslc/compile/function.tnsl
index 0de6451..d87c207 100644
--- a/tnslc/compile/function.tnsl
+++ b/tnslc/compile/function.tnsl
@@ -233,8 +233,8 @@ struct Function {
 		~Var in
 		/; loop (int i = 0; i < self.inputs.count) [i++]
 			in = self.inputs.get(i)
-			/; if (in.is_ref())
-				~int32 ptc = in.top_ptrc()
+			/; if (in`.is_ref())
+				~int32 ptc = in`.top_ptrc()
 				int32 set = 0
 				set = set - 1
 				ptc` = set
@@ -871,7 +871,10 @@ struct Function {
 		;/
 
 		# Save caller saved tmp variables
-		Var handle = s`.save_caller_tmp()
+		int saved = s`.save_caller_tmp()
+		# Generate handle to this point in the temp stack so we
+		# can properly restore saved registers (and clean up after ourselves)
+		int handle = s`.get_tmp_handle()
 
 		# Check that parameter count matches
 		int param_count = params`.sub.count
@@ -933,6 +936,9 @@ struct Function {
 			;/
 		;/
 
+		# Store where we are in the stack after the stack variables
+		int param_handle = s`.get_tmp_handle()
+
 		# Create all register based tmps
 		Var last_reg
 		last_reg.loc = 0
@@ -951,6 +957,7 @@ struct Function {
 				;/
 
 				tmps.replace(i, ~to_set)
+				self._set_var_ptr(~last_reg, ~to_set)
 				last_reg = to_set
 			;/
 		;/
@@ -980,7 +987,7 @@ struct Function {
 			param.end()
 		;/
 		
-		# Move all register parameters into registers and free the stack space
+		# Move all register parameters into registers
 		~Var ptmp
 		/; loop (int i = 0; i < f`.inputs.count) [i++]
 			inp = f`.inputs.get(i)
@@ -996,9 +1003,8 @@ struct Function {
 			;/
 		;/
 		
-		/; if (last_stack.loc !== 0)
-			s`.free_after(~last_stack, true)
-		;/
+		# Free stack space
+		s`.free_after_handle(param_handle)
 
 		# Pad out the stack for return if required
 		~CompBuf buf = s`.cb
@@ -1021,44 +1027,24 @@ struct Function {
 		/; if (f`.outputs.count > 0)
 			/; if (out`.loc < 0)
 				Var out_pos = out`.copy()
-
-				/; if (out_pos.is_ref() == true)
-					int32 pp = 0
-					pp = pp - 1
-					out_pos.ptr_pop()
-					out_pos.ptr_push(pp)
-				;/
-
-				int sloc = 0
-				out_pos.loc = sloc - 1
-				out_pos.offset = last_stack.offset
-
-				/; if (f`.call_padding > 0)
-					int offset = out_pos.offset
-					offset = offset - f`.call_padding
-					out_pos.offset = offset
-				;/
-				
-				bool is_ref = false
-				/; if (result.is_ref() == true)
-					is_ref = true
-					int32 pp = 0
-					pp = pp - 1
-					result.ptr_pop()
-					result.ptr_push(pp)
-				;/
+				out_pos.loc = 7
+				out_pos.offset = 0
 
 				~CompBuf buf = s`.cb
 				result.set(s`.cb, ~out_pos)
 				out_pos.end()
-				
-				/; if (is_ref == true)
-					result.ptr_pop()
-					result.ptr_push(0)
-				;/
 			;/
 		;/
 
+		# Un-pad the stack
+		/; if (f`.call_padding > 0)
+			~uint8 pad_num = utils.int_to_str(f`.call_padding)
+			buf`.add_c("    add rsp, \0")
+			buf`.add_c(pad_num)
+			buf`.add_c("\n\0")
+			_delete(pad_num)
+		;/
+
 		# Free all in tmp vector
 		/; loop (int i = 0; i < tmps.count) [i++]
 			inp = tmps.get(i)
@@ -1067,19 +1053,8 @@ struct Function {
 		tmps.end()
 
 		# Restore tmps in regs
-		s`.restore_caller_tmp(~handle)
-
-		/; if (result.loc < 0)
-			/; if (last_stack.loc !== 0)
-				s`.free_after(~result, true)
-			;; else if (f`.call_padding > 0)
-				~uint8 pad_num = utils.int_to_str(f`.call_padding)
-				buf`.add_c("    add rsp, \0")
-				buf`.add_c(pad_num)
-				buf`.add_c("\n\0")
-				_delete(pad_num)
-			;/
-		;/
+		s`.free_after_handle(handle)
+		s`.restore_caller_tmp(saved)
 
 		return result
 	;/
@@ -1132,9 +1107,13 @@ struct Function {
 
 			# Compute index
 			Var idx = self._compile_value(s, post)
-			/; if (idx.loc == 6 || idx.loc == 5)
+			/; if (idx.loc == 6)
+				~CompBuf buf = s`.cb
+				buf`.add_c("    mov rax, rdi ; idx\n\0")
+				idx.loc = 1
+			;; else if (idx.loc == 5)
 				~CompBuf buf = s`.cb
-				buf`.add_c("    mov rax, rdi\n\0")
+				buf`.add_c("    mov rax, rsi ; idx\n\0")
 				idx.loc = 1
 			;/
 			
diff --git a/tnslc/compile/module.tnsl b/tnslc/compile/module.tnsl
index 86ede96..b5eda8a 100644
--- a/tnslc/compile/module.tnsl
+++ b/tnslc/compile/module.tnsl
@@ -233,7 +233,7 @@ struct Module {
 				;; else
 					_printf("Failed to find module for method\n\0")
 				;/
-				_method_id_end(~id)
+				self._method_id_end(~id)
 			;; else if (sub`._type == parse.NTYPE_MODULE)
 				~Module m = self._find_sub(sub`.data)
 				/; if (m !== NULL)
@@ -354,7 +354,7 @@ struct Module {
 			/; loop (int i = 0; i < self.subs.count) [i++]
 				m = self.subs.get(i)
 				/; if (utils.strcmp(str`, m`.name) == true)
-					~void v = m._find(stype, key, lvl + 1)
+					~void v = m`._find(stype, key, lvl + 1)
 					/; if (v !== NULL)
 						return v
 					;/
diff --git a/tnslc/compile/scope.tnsl b/tnslc/compile/scope.tnsl
index 8a6747e..96d30fd 100644
--- a/tnslc/compile/scope.tnsl
+++ b/tnslc/compile/scope.tnsl
@@ -429,7 +429,7 @@ struct Scope {
 		;/
 	;/
 
-	/; save_caller_tmp [Var]
+	/; save_caller_tmp [int]
 		Var tmp
 		utils.Vector tmps
 		tmps.init(len tmp)
@@ -438,7 +438,7 @@ struct Scope {
 
 		/; if (tmps.count == 0)
 			tmps.end()
-			return tmp
+			return 0
 		;/
 
 		~Var to_save
@@ -461,53 +461,47 @@ struct Scope {
 				tmp.end()
 			;/
 		;/
+		int out = tmps.count
 		tmps.end()
 
-		return tmp
+		return out
 	;/
 
-	/; restore_caller_tmp (~Var handle)
-		Var tmp
-		utils.Vector tmps
-		tmps.init(len tmp)
-
-		self._get_reg_tmp(~tmps)
-
-		/; if (tmps.count == 0)
+	/; restore_caller_tmp (int count)
+		/; if (count == 0)
 			return
 		;/
 
-		int end_idx = 0
-		~Var tmp
-		/; loop (int i = 0; i < self.tmps.count) [i++]
-			tmp = self.tmps.get(i)
-			/; if (tmp`.offset == handle`.offset)
-				/; if (tmp`.loc == handle`.loc)
-					end_idx = i + 1
-					i = self.tmps.count
-				;/
-			;/
-		;/
+		Var t
+		utils.Vector tmps
+		tmps.init(len t)
+
+		self._get_reg_tmp(~tmps)
 
+		~Var to_load
 		~Var to_set
-		/; loop (int i = 0; i < tmps.count) [i++]
-			int idx = i + end_idx - tmps.count
-			to_set = tmps.get(i)
-			tmp = self.tmps.get(idx)
+		/; loop (int i = 1; i !> count) [i++]
+			int idx_tmp = count - i
+			int idx_saved = self.tmps.count
+			idx_saved = idx_saved - i
+
+			to_set = tmps.get(idx_tmp)
+			to_load = self.tmps.get(idx_saved)
 			
 			/; if (to_set`.is_ref() == true)
 				# Handle ref
-				Var ref_val = tmp`.take_ptr(self.cb, 1)
+				Var ref_val = to_load`.take_ptr(self.cb, 1)
 				Var ref_set = to_set`.take_ptr(self.cb, 2)
 				ref_set.set(self.cb, ~ref_val)
 				ref_val.end()
 				ref_set.end()
 			;; else
-				to_set`.set(self.cb, tmp)
+				to_set`.set(self.cb, to_load)
 			;/
 		;/
 
-		handle`.end()
+		self.free_tmp(count, true)
+		tmps.end()
 	;/
 
 	# Free n tmp variables (starts from most recently created)
@@ -518,28 +512,28 @@ struct Scope {
 			tmps = self.tmps.count
 		;/
 
-		int stk_mv = 0
-		int tmp_count = self.tmps.count
+		bool saw_stack = false
 
 		~Var tmp
 		/; loop (int i = 1; i !> tmps) [i++]
-			tmp = self.tmps.get(tmp_count - i)
-
+			int tmp_count = self.tmps.count
+			tmp = self.tmps.get(tmp_count - 1)
+			
 			/; if (tmp`.loc < 0)
-				stk_mv = stk_mv + tmp`.actual_size()
+				saw_stack = true
 			;/
-			tmp`.end()
-		;/
 
-		/; loop (int i = 0; i < tmps) [i++]
+			tmp`.end()
 			self.tmps.pop()
 		;/
 
-		/; if (stk_mv > 0)
-			~uint8 stk_mv_str = utils.int_to_str(stk_mv)
-			self.cb`.add_c("    add rsp, \0")
+		/; if (saw_stack == true)
+			int stk = self._next_stack_slot()
+			stk = 0 - stk
+			~uint8 stk_mv_str = utils.int_to_str(stk)
+			self.cb`.add_c("    lea rsp, [rbp - \0")
 			self.cb`.add_c(stk_mv_str)
-			self.cb`.add_c(" ; Free stack space (freeing tmp variables)\n\0")
+			self.cb`.add_c("] ; Free stack space (freeing tmp variables)\n\0")
 			_delete(stk_mv_str)
 		;/
 	;/
@@ -566,9 +560,11 @@ struct Scope {
 		~Var v
 		/; loop (int i = 1; i !> self.tmps.count) [i++]
 			v = self.tmps.get(self.tmps.count - i)
-			/; if (v`.offset == tmp`.offset)
-				tmps = i
-				i = self.tmps.count + 1
+			/; if (v`.loc == tmp`.loc)
+				/; if (v`.offset == tmp`.offset)
+					tmps = i
+					i = self.tmps.count + 1
+				;/
 			;/
 		;/
 
@@ -580,7 +576,7 @@ struct Scope {
 			_printf("\"\n\0")
 			_print_num("                Was looking for tmp with offset %d but could not find one.\n\0", off)
 		;; else if (tmps > 0)
-			self.free_tmp(tmps, code)
+			self.free_tmp(tmps, false)
 		;/
 	;/
 
@@ -592,9 +588,11 @@ struct Scope {
 		~Var v
 		/; loop (int i = 1; i !> self.tmps.count) [i++]
 			v = self.tmps.get(self.tmps.count - i)
-			/; if (v`.offset == tmp`.offset)
-				tmps = i - 1
-				i = self.tmps.count + 1
+			/; if (v`.loc == tmp`.loc)
+				/; if (v`.offset == tmp`.offset)
+					tmps = i - 1
+					i = self.tmps.count + 1
+				;/
 			;/
 		;/
 
@@ -610,6 +608,33 @@ struct Scope {
 		;/
 	;/
 
+	/; get_tmp_handle [int]
+		return self.tmps.count
+	;/
+
+	/; free_after_handle (int handle)
+		/; if (handle !< self.tmps.count)
+			return
+		;/
+
+		int tmps = self.tmps.count
+		tmps = tmps - handle
+		self.free_tmp(tmps, true)
+	;/
+
+	/; free_to_handle (int handle)
+		/; if (handle > self.tmps.count)
+			return
+		;; else if (self.tmps.count == 0)
+			return
+		;/
+
+		int tmps = self.tmps.count
+		tmps = tmps - handle
+		tmps++
+		self.free_tmp(tmps, true)
+	;/
+
 	#
 	# Sub scope
 	#
diff --git a/tnslc/test.tnsl b/tnslc/test.tnsl
index a130386..7570215 100644
--- a/tnslc/test.tnsl
+++ b/tnslc/test.tnsl
@@ -1,20 +1,39 @@
+import "utils/utils.tnsl"
+import "parse/parse.tnsl"
+# :import "compile/compile.tnsl"
 
-struct TTT {
-	int a, b, c
-}
+~uint8 USAGE = "
+TNSLP v0.1.0 (C) 2026 CircleShift (MPL 2.0)
 
-/; asdf [TTT]
-	TTT out
-	out.a = 0
-	out.b = 0
-	return out
-;/
+usage:
+	tnslp (file in)
+		generates a parse tree of the given file
+
+\0"
+
+~uint8 FOPEN_ERR = "Error opening file\n\0"
+
+~uint8 char_str = "%c\0"
+~uint8 newline = "\n\0"
 
 /; main (int argc, ~~uint8 argv) [int]
+	asm "mov r10, rdi"
+	asm "mov r11, rsi"
+
+	/; if (argc < 2)
+		_printf(USAGE)
+		return 1
+	;/
+
+	utils.File fin
+	fin.init(argv{1})
+	
+	utils.Vector vec = parse.gen_token_list(~fin)
+	parse.print_token_list(~vec)
+	parse.end_token_list(~vec)
 
-	TTT out
-	out = asdf()
+	fin.end()
 
-	return out.a
+	return 0
 ;/
 
diff --git a/tnslc/vec_test.tnsl b/tnslc/vec_test.tnsl
index 95544e3..c3d3948 100644
--- a/tnslc/vec_test.tnsl
+++ b/tnslc/vec_test.tnsl
@@ -1,4 +1,4 @@
-:import "utils/utils.tnsl"
+import "utils/utils.tnsl"
 
 /; print_vec(~utils.Vector v)
 	_printf("vec: [ \0")