Returns inside of complex operations

13 files changed, 490 insertions, 135 deletions

diff --git a/tnslc/basic.tnsl b/tnslc/basic.tnsl
index 5d4f040..a0cf385 100644
--- a/tnslc/basic.tnsl
+++ b/tnslc/basic.tnsl
@@ -1,4 +1,8 @@
+:include "c_wrap.tnsl"
+
+{}uint8 str = "ASDF\0"
 
 /; main [int]
+    _printf(~str{0})
     return 0
 ;/
\ No newline at end of file
diff --git a/tnslc/c_wrap.tnsl b/tnslc/c_wrap.tnsl
index fc60a52..c0fa91c 100644
--- a/tnslc/c_wrap.tnsl
+++ b/tnslc/c_wrap.tnsl
@@ -15,11 +15,14 @@ asm "extern CloseHandle"
 
 /; _alloc (uint size) [~void]
     ~void out
+
     # Mov size into proper register, and set all extras to zero
     asm "mov rcx, r8"
     asm "mov rdx, 0"
     asm "mov r8, 0"
     asm "mov r9, 0"
+
+    asm "sub rsp, 32"
     asm "call malloc"
     # Set out to the returned value
     # (The compiler assignes spaces sequentially, and we have a uint in r8)
@@ -29,12 +32,14 @@ asm "extern CloseHandle"
 
 /; _realloc (~void ptr, uint new_size) [~void]
     ~void out
+
     # Mov ptr and new size into proper registers, and set all extras to zero
     asm "mov rcx, r8"
     asm "mov rdx, r9"
     asm "mov r8, 0"
     asm "mov r9, 0"
     # Do call
+    asm "sub rsp, 32"
     asm "call realloc"
     # Set out to the returned value
     # (The compiler assignes spaces sequentially. We have a ptr in r8, and a uint in r9)
@@ -43,36 +48,45 @@ asm "extern CloseHandle"
 ;/
 
 /; _delete (~void ptr)
+
     # setup call by clearing most values
-    asm "mov rcx, rax"
+    asm "mov rcx, r8"
     asm "mov rdx, 0"
     asm "mov r8, 0"
     asm "mov r9, 0"
     # do call
+    asm "sub rsp, 32"
     asm "call free"
     # there's no more to do 'cause free returns nothing 
 ;/
 
 /; _printf (~void str)
+
     # setup call by clearing most values
-    asm "mov rcx, rax"
+    asm "mov rcx, r8"
     asm "mov rdx, 0"
     asm "mov r8, 0"
     asm "mov r9, 0"
     # do call
+    asm "push qword 0"
+    asm "push qword 0"
+    asm "push qword 0"
+    asm "push qword 0"
     asm "call printf"
     # there's no more to do 'cause printf returns nothing 
 ;/
 
 /; _print_num (~void str, int num)
+
     # setup call by clearing most values
-    asm "mov rcx, rax"
-    asm "mov rdx, rbx"
+    asm "mov rcx, r8"
+    asm "mov rdx, r9"
     asm "mov rdi, 0"
     asm "mov rsi, 0"
     asm "mov r8, 0"
     asm "mov r9, 0"
     # do call
+    asm "sub rsp, 32"
     asm "call printf"
     # there's no more to do 'cause printf returns nothing 
 ;/
@@ -139,6 +153,7 @@ asm "extern CloseHandle"
 ;/
 
 /; _close_file (~void handle)
+
     asm "mov rcx, r8" # handle
     asm "mov rdx, 0"
     asm "mov r8, 0"
diff --git a/tnslc/copy.tnsl b/tnslc/copy.tnsl
new file mode 100644
index 0000000..90e1126
--- /dev/null
+++ b/tnslc/copy.tnsl
@@ -0,0 +1,41 @@
+:include "c_wrap.tnsl"
+:include "vector.tnsl"
+:include "utils.tnsl"
+
+
+{}uint8 wrong_args = "Usage: copy [from] [to]"
+{}uint8 write_one = "\0\n\0"
+
+# Proof of concept copy program
+/; main (int argc, ~~uint8 argv) [int]
+    asm "mov r8, rcx"
+    asm "mov r9, rdx"
+
+    /; if (argc < 3)
+        _printf(~wrong_args{0})
+        return 1
+    ;/
+
+    ~void read_handle = _open_file(argv{1})
+    ~void write_handle = _create_file(argv{2})
+
+    _print_num(~_dec{0}, read_handle)
+    _print_num(~_dec{0}, write_handle)
+
+    uint8 buf = 0
+    int read_count = 0
+    int tries = 0
+    _read_byte(read_handle, ~buf, ~read_count)
+    /; loop (_read_byte(read_handle, ~buf, ~read_count))
+        /; if (read_count == 0)
+            break
+        ;/
+        _write_byte(write_handle, ~buf)
+        read_count = 0
+    ;/
+
+    _close_file(read_handle)
+    _close_file(write_handle)
+
+    return 0
+;/
\ No newline at end of file
diff --git a/tnslc/else.tnsl b/tnslc/else.tnsl
index a87aad3..d2a7c94 100644
--- a/tnslc/else.tnsl
+++ b/tnslc/else.tnsl
@@ -1,13 +1,19 @@
-/; main [int]
+{}uint8 a = "asdf", b = "asdf\0"
 
-    int i = 1
+{}uint8 csv_pr = ",\0"
 
+/; cstr_len (~void cstr, int add) [int]
+    int i = 0
+    /; loop (cstr{i} !== 0) [i++] ;/
+    return i + add
+;/
 
-    /; if (i == 3)
-        i = 1
-    ;; else if (i == 1)
-        i = 10
-    ;/
 
-    return i
+
+/; main (int argc, ~~uint8 argv) [int]
+
+    uint8 bub = 0
+    cstr_len(~b{0}, bub)
+
+    return 0
 ;/
\ No newline at end of file
diff --git a/tnslc/enum.tnsl b/tnslc/enum.tnsl
index e8e9c03..5dd241b 100644
--- a/tnslc/enum.tnsl
+++ b/tnslc/enum.tnsl
@@ -1,11 +1,11 @@
 # to get working
 
 enum TEST_ENUM [int] {
-    i = 0
+    i = 1
 }
 
 /; module m
-    int i = 0
+    int i = 1
 ;/
 
 /; main [int]
diff --git a/tnslc/logging.tnsl b/tnslc/logging.tnsl
new file mode 100644
index 0000000..7a60364
--- /dev/null
+++ b/tnslc/logging.tnsl
@@ -0,0 +1,65 @@
+##
+## LOG UTILITIES
+##
+
+# Log levels:
+# 0 - Visual queues and errors only
+# 1 - Info (default)
+# 2 - Debugging information (useful for me, probably less for you)
+# 3 - Also logs the state changes of the log itself
+int log_level = 2
+int log_mode = 1
+
+{}uint8 _log_print_one = "\0\0"
+{}uint8 _log_nl = "\n\0"
+{}uint8 _log_prefix = "[TNSLC] [%d] \0"
+
+/; log_state(int new_state)
+    /; if (new_state !== log_mode)
+        _printf(~_log_nl{0})
+    ;/
+    
+    log_mode = new_state
+;/
+
+/; log_err (~uint8 msg)
+    log_state(0)
+    _print_num(~_log_prefix{0}, log_mode)
+    _printf(msg)
+    _printf(~_log_nl{0})
+;/
+
+/; log_info (~uint8 msg)
+    /; if (log_level > 0)
+        log_state(1)
+        _print_num(~_log_prefix{0}, log_mode)
+        _printf(msg)
+        _printf(~_log_nl{0})
+    ;/
+;/
+
+/; log_vis (~uint8 msg)
+    log_state(0)
+    _printf(msg)
+;/
+
+/; log_debug (~uint8 msg)
+    /; if (log_level > 1)
+        log_state(2)
+        _print_num(~_log_prefix{0}, log_mode)
+        _printf(msg)
+        _printf(~_log_nl{0})
+    ;/
+;/
+
+# bypass logging framework (mostly for in-place debugging)
+
+/; log_one (uint8 c)
+    _log_print_one{0} = c
+    _printf(~_log_print_one{0})
+;/
+
+/; log_one_nl (uint8 c)
+    log_one(c)
+    _printf(~_log_nl{0})
+;/
diff --git a/tnslc/path.tnsl b/tnslc/path.tnsl
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tnslc/path.tnsl
diff --git a/tnslc/simple.tnsl b/tnslc/simple.tnsl
index 1c54577..1541df2 100644
--- a/tnslc/simple.tnsl
+++ b/tnslc/simple.tnsl
@@ -1,19 +1,7 @@
+:include "c_wrap.tnsl"
+:include "logging.tnsl"
 
-{}uint8 str1 = "abcd"
-
-struct Stress {
-    int i, j, k
-}
-
-struct Test {
-    ~Stress s
-}
-
-/; s_call (Test t)
-    t.s`.i = 1
-    t.s`.j = 1 + t.s`.i
-    t.s`.k = t.s`.j + 1
-;/
+{}uint8 eee = "eee\0"
 
 /; main (int argc, ~~uint argv) [int]
     # On windows, the first two arguments are passed in RCX and RDX, so we need to
@@ -21,14 +9,10 @@ struct Test {
     asm "mov r8, rcx"
     asm "mov r9, rdx"
 
-    Stress test
-    Test stress
-
-    stress.s = ~test
-
-    s_call(stress)
-
-    # return 3
-    return stress.s`.k
+    log_err(~eee{0})
+    log_info(~eee{0})
+    log_debug(~eee{0})
+    
+    return 0
 ;/
 
diff --git a/tnslc/str_test.tnsl b/tnslc/str_test.tnsl
new file mode 100644
index 0000000..7875636
--- /dev/null
+++ b/tnslc/str_test.tnsl
@@ -0,0 +1,9 @@
+{}uint8 a = "\n\0\""
+
+/; str_call({}uint str) [int]
+    return len str
+;/
+
+/; main [int]
+    return str_call(~a)
+;/
\ No newline at end of file
diff --git a/tnslc/tnslc.tnsl b/tnslc/tnslc.tnsl
index d5e1f7d..5c1e6f9 100644
--- a/tnslc/tnslc.tnsl
+++ b/tnslc/tnslc.tnsl
@@ -7,7 +7,7 @@
 # 1 - Info (default)
 # 2 - Debugging information (useful for me, probably less for you)
 # 3 - Also logs the state changes of the log itself
-;{}uint8 log_level = "2"
+;{}uint8 log_level = "1"
 ;{}uint8 log_mode = "single"
 
 /; log_state({}uint8 new_state)
@@ -151,6 +151,8 @@
         ;return 0
     ;; else if (cmp == '\'')
         ;return '\''
+    ;; else if (cmp == '"')
+        ;return '"'
     ;/
 ;/
 
@@ -523,12 +525,15 @@
 /; is_num_literal ({}uint8 str) [bool]
     /; if (len str < 1)
         ;return false
-    ;; if (len str == 1 && str{0} == '.')
-        ;return false
+    ;/
+
+    ;int i = 0
+    /; if ((str{i} == '-' || str{i} == '.') && len str > 1)
+        ;i++
     ;/
 
     ;bool dec = false
-    /; loop (int i = 0; i < len str) [i++]
+    /; loop (i < len str) [i++]
         /; if (str{i} == '.')
             /; if (!dec)
                 ;dec = true
@@ -567,9 +572,11 @@
 ;/
 
 /; break_token (Token current, uint8 to_append) [bool]
-    /; if (is_literal(current.data))
-        ;current.data.append(to_append)
-        ;return !(is_literal(current.data))
+    ;{}uint8 data = current.data
+    ;data.append(to_append)
+
+    /; if (is_literal(current.data) || is_literal(data))
+        ;return !(is_literal(data))
     ;/
 
     /; if (is_whitespace(to_append) || current.cmp("\n"))
@@ -578,7 +585,7 @@
 
     /; if (is_reserved(current.data))
         /; if (is_reserved({to_append}))
-            ;current.data.append(to_append)
+            ;current.data = data
             ;return gen_type(current) == TOKEN.DEFWORD
         ;/
         ;return true
@@ -1021,6 +1028,10 @@
         ;; if (!(to_match.is_prim()) || (len(to_match.data_type.ptr_chain) > 0 && !(to_match.is_ref())))
             ;return to_match
         ;/
+
+        /; loop (out.is_ref())
+            ;out.data_type.ptr_chain = strip_int(out.data_type.ptr_chain)
+        ;/
         
         ;{}uint8 mov = "mov"
         
@@ -1048,7 +1059,8 @@
             ;v.norm_op(op, sz, self, data)
         ;; else
             ;v = self.match_types(v, data)
-	    /; if ((self.is_ref() || self.loc_type == LOCATION.STACK) && (v.is_ref() || v.loc_type == LOCATION.STACK))
+
+            /; if ((self.is_ref() || self.loc_type == LOCATION.STACK || self.loc_type == LOCATION.LABEL) && (v.is_ref() || v.loc_type == LOCATION.STACK || v.loc_type == LOCATION.LABEL))
                 ;data`.csec = string_join( {
                     data`.csec,
                     "\tmov ", get_reg(4, sz), ", ", v.norm_loc(sz), "\n",
@@ -1071,6 +1083,14 @@
         ;return false
     ;/
 
+    /; is_arr [bool]
+        ;int s = len (self.data_type.ptr_chain)
+        /; if (s > 0)
+            ;return self.data_type.ptr_chain{s - 1} == PTYPE.ARRAY
+        ;/
+        ;return false
+    ;/
+
     /; is_prim [bool]
         /; if (len(self.data_type.ptr_chain) == 0)
             ;return is_primitive(self.data_type.name) !< 0
@@ -1271,16 +1291,16 @@
 
     # Todo: Make sure this works well.
     /; set (Variable v, ~CompData data)	
-	;int sz = self.norm_size()
+	    ;int sz = self.norm_size()
         ;v = self.match_types(v, data)
         /; if (self.loc_type == LOCATION.LITERAL && v.loc_type == LOCATION.LITERAL)
             ;self.location = v.location
         ;; else if (self.is_prim())
-	    ;self.norm_op("mov", self.norm_size(), v, data)
+	        ;self.norm_op("mov", self.norm_size(), v, data)
         ;; else
-	    ;log_debug("Full struct set")
-	    ;log_debug(self.sprint())
-	    ;log_debug(v.sprint())
+	        ;log_debug("Full struct set")
+	        ;log_debug(self.sprint())
+	        ;log_debug(v.sprint())
             /; if (self.is_ref() && self.loc_type == LOCATION.STACK)
                 ;data`.csec = string_join( {
                     data`.csec,
@@ -1433,7 +1453,7 @@
 	    ;ind.set(i, data)
         ;ind.mul({"#mul", {8, "uint", {}, {}, 0}, sz, LOCATION.LITERAL}, data)
 
-        /; if (out.is_ref())
+        /; if (out.is_ref() || out.is_arr())
             ;data`.csec = string_join( {
                 data`.csec,
                 "\tadd rsi, 8\n"
@@ -2938,6 +2958,8 @@
             ;Scope call_sc = scope`.new_sub_cf("call")
             ;Function to_call = _setup_call(tok, start`, wk, out, mov, current, ~call_sc)
             ;wk = _perform_call(to_call, ~call_sc, out)
+            ;log_debug(to_call.sprint())
+            ;log_debug(wk.sprint())
             ;call_sc.clear_tmp(out)
 
             /; if (layer` > 2)
@@ -2970,8 +2992,13 @@
             ;tt = scope`.new_tmp_var(wk.data_type, out)
         ;/
         
-        ;tt.set_raw(wk, out)
-        ;tt.data_type = wk.data_type
+        /; if (len (wk.data_type.ptr_chain) > 0)
+            ;tt.set_raw(wk, out)
+            ;tt.data_type = wk.data_type
+        ;; else
+            ;tt.set(wk, out)
+        ;/
+        
         ;wk = tt
     ;; else if (layer` == 1)
         ;wk.move_register(2, out)
@@ -3441,7 +3468,7 @@
             ;/
         ;; else if (tok`{cur`}.cmp(";;") || (tok`{cur`}.cmp(";/") && tok`{next_non_nl(tok, cur` + 1)}.cmp("/;")))
             /; if (tok`{cur`}.cmp(";/"))
-                ;cur`++
+                ;cur` = next_non_nl(tok, cur` + 1)
             ;/
 
             /; if ((cf.cf_type("if") || cf.cf_type("elif")) && tok`{next_non_nl(tok, cur` + 1)}.cmp("else"))
diff --git a/tnslc/tnslc_wrapped.tnsl b/tnslc/tnslc_wrapped.tnsl
index 77d7c13..d58c045 100644
--- a/tnslc/tnslc_wrapped.tnsl
+++ b/tnslc/tnslc_wrapped.tnsl
@@ -1,90 +1,20 @@
 :include "c_wrap.tnsl"
+:include "logging.tnsl"
 :include "vector.tnsl"
 :include "utils.tnsl"
+:include "tokenizer.tnsl"
 
-enum TOKEN_TYPE [int] {
-    DEFWORD = 0
-}
-
-struct Token {
-    int line, ch, _type,
-    ~uint8 data
-}
-
-/; method Token
-    /; init
-        self.data = _alloc(1)
-        self.data{0} = 0
-    ;/
-
-    /; data_len [int]
-        return cstr_len(self.data)
-    ;/
-
-    /; append (uint8 ch)
-        int l = self.data_len()
-        self.data = _realloc(self.data, self.data_len() + 1)
-        self.data{l + 1} = 0
-        self.data{l} = ch
-    ;/
-
-    /; pop
-        int l = self.data_len()
-        self.data = _realloc(self.data, l - 1)
-        self.data{l - 1} = 0
-    ;/
-
-    /; clear
-        _delete(self.data)
-    ;/
-
-    /; eq (~Token t) [bool]
-        return cstr_eq(self.data, t`.data)
-    ;/
-;/
-
-/; tokenize (~void file) [Vector]
-    Vector out
-    out.start(32)
-
-
-
-    return out
-;/
-
-{}uint8 wrong_args = "Usage: copy [from] [to]"
-{}uint8 write_one = "\0\n\0"
-
-# Proof of concept copy program
 /; main (int argc, ~~uint8 argv) [int]
     asm "mov r8, rcx"
     asm "mov r9, rdx"
 
-    /; if (argc < 3)
-        _printf(~wrong_args{0})
-        return 1
-    ;/
-
-    ~void read_handle = _open_file(argv{1})
+    ~void open_handle = _open_file(argv{1})
     ~void write_handle = _create_file(argv{2})
-
-    _print_num(~_dec{0}, read_handle)
-    _print_num(~_dec{0}, write_handle)
-
-    uint8 buf = 0
-    int read_count = 0
-    int tries = 0
-    _read_byte(read_handle, ~buf, ~read_count)
-    /; loop (_read_byte(read_handle, ~buf, ~read_count))
-        /; if (read_count == 0)
-            break
-        ;/
-        _write_byte(write_handle, ~buf)
-        read_count = 0
-    ;/
-
-    _close_file(read_handle)
+    log_one_nl('a')
+    tokenize_file(open_handle, write_handle)
+    log_one_nl('a')
+    _close_file(open_handle)
     _close_file(write_handle)
 
     return 0
-;/
\ No newline at end of file
+;/
diff --git a/tnslc/tokenizer.tnsl b/tnslc/tokenizer.tnsl
new file mode 100644
index 0000000..ada1b8e
--- /dev/null
+++ b/tnslc/tokenizer.tnsl
@@ -0,0 +1,215 @@
+# All single reserved characters
+{}uint8 MULTI_PARENS = "/;:#"
+{}uint8 PARENS = "()[]{}"
+{}uint8 SEPS = "\n;:,"
+{}uint8 RESERVED = "`~!%^&*()-+=[]{}|;:/?<>.,"
+{}uint8 AUGMENTS = "=~!<>&|^+-*/%`."
+
+{}uint8 WHITESPACE = " \r\n\t"
+
+# All lists of keywords are comma delim because the compiler does not yet support arrays of strings
+{}uint8 CSV_AUGMENTS = "++,--,==,!==,&&,||,^^,<==,>==,!>,!<,~=,`=,%=,^=,&=,*=,!=,|=,/=,<<,>>,!&,!|,!^,len,is"
+
+{}uint8 CSV_KEYWORDS = "if,else,loop,continue,break,return,method,struct,enum,interface,export,module,const,static,volatile,raw,extends,override,asm"
+
+{}uint8 CSV_KEYTYPES = "uint8,uint16,uint32,uint64,uint,int8,int16,int32,int64,int,float32,float64,float,comp32,comp64,comp,vect,bool,type,void"
+
+# Types of tokens
+enum TOKEN_TYPE [uint] {
+    SEPARATOR = 0,
+    DELIMITER = 1,
+    AUGMENT = 2,
+    KEYTYPE = 3,
+    KEYWORD = 4,
+    LITERAL = 5,
+    DEFWORD = 6
+}
+
+# Token structure represents a single token in the program
+struct Token {
+    uint 
+        _type,
+        line,
+        column,
+    ~uint8
+        data
+}
+
+# Shortcut methods on the token struct
+/; method Token
+    
+    # Initialize the data buffer
+    /; start
+        self.data = _alloc(1)
+        self.data{0} = 0
+    ;/
+
+    # Append a character to the end of the token
+    /; append (uint8 ch)
+        int ln = cstr_len(self.data)
+        self.data = _realloc(self.data, ln + 2)
+        self.data{ln} = ch
+        self.data{ln + 1} = 0
+    ;/
+
+    # Remove the last character from this token
+    /; pop
+        int ln = cstr_len(self.data)
+        self.data = _realloc(self.data, ln)
+        self.data{ln - 1} = 0
+    ;/
+
+    # Copy another token to this token
+    /; copy (Token other)
+        self._type = other._type
+        self.line = other.line
+        self.column = other.column
+
+        self.data = _alloc(cstr_len(other.data) + 1)
+
+        cstr_copy(other.data, self.data)
+    ;/
+
+    # Delete the memory associated with this token
+    /; _del
+        _delete(self.data)
+    ;/
+
+    # length of the string that this token encodes
+    /; _len [int]
+        return cstr_len(self.data)
+    ;/
+;/
+
+{}uint8 tkn_ok = "OK \0", tkn_no = "NO \0", tkn_nl = "\n\0"
+
+/; print_token (Token tok, ~void file_out)
+    /; if (in_csv(~CSV_KEYWORDS, tok.data) == true)
+        write_to_file(file_out, ~tkn_ok{0})
+    ;; else
+        write_to_file(file_out, ~tkn_no{0})
+    ;/
+    write_to_file(file_out, tok.data)
+    write_to_file(file_out, ~tkn_nl{0})
+;/
+
+# Returns true if the character is whitespace
+/; is_whitespace(uint8 c) [bool]
+    return contains_char(~WHITESPACE, c)
+;/
+
+
+# Returns true if the character is reserved
+/; is_reserved (uint8 c) [bool]
+    return contains_char(~RESERVED, c)
+;/
+
+# Returns true if the token is a valid reserved token
+/; tok_reserved (Token tok) [bool]
+    log_one_nl('i')
+    /; if (tok._len() == 1)
+        return is_reserved(tok.data{0})
+    ;; else if (tok._len() == 2)
+        bool a = contains_char(~MULTI_PARENS, tok.data{0})
+        bool b = contains_char(~MULTI_PARENS, tok.data{1})
+        return a && b
+    ;/
+
+    return in_csv(~CSV_AUGMENTS, tok.data)
+;/
+
+# Returns true if the token is a valid literal value
+/; tok_literal (Token tok) [bool]
+    # TODO: implement literals
+    return false
+;/
+
+/; get_tok_type(Token tok) [uint]
+    log_one_nl('h')
+    /; if (tok_reserved(tok) == true)
+        /; if (tok._len() > 1)
+            bool a = contains_char(~MULTI_PARENS, tok.data{0})
+            bool b = contains_char(~MULTI_PARENS, tok.data{1})
+            /; if (a && b)
+                return TOKEN_TYPE.DELIMITER
+            ;/
+            return TOKEN_TYPE.AUGMENT
+        ;; else if (contains_char(~PARENS, tok.data{0}))
+            return TOKEN_TYPE.DELIMITER
+        ;; else if (contains_char(~SEPS, tok.data{0}))
+            return TOKEN_TYPE.SEPARATOR
+        ;; else if (contains_char(~AUGMENTS, tok.data{0}))
+            return TOKEN_TYPE.AUGMENT
+        ;/
+    ;; else if (tok_literal(tok) == true)
+        return TOKEN_TYPE.LITERAL
+    ;/
+
+    return TOKEN_TYPE.DEFWORD
+;/
+
+
+/; break_token(Token tok, uint8 c) [bool]
+    log_one('g')
+    log_one(' ')
+    log_one_nl(c)
+    # return true
+    uint type_before = get_tok_type(tok)
+    tok.append(c)
+    uint type_after = get_tok_type(tok)
+    tok.pop()
+    log_one_nl('g')
+    bool a = type_before !== TOKEN_TYPE.DEFWORD && type_after == TOKEN_TYPE.DEFWORD
+    bool b = type_after !== TOKEN_TYPE.LITERAL && is_whitespace(c) == true
+    return a || b
+;/
+
+/; tokenize_file (~void file_in, file_out)
+
+    Token tmp
+    tmp.start()
+
+    uint8 buf = 0
+    int read_count = 0
+    # Start reading at beginning of file
+    _read_byte(file_in, ~buf, ~read_count)
+    # Read loop.
+    /; loop (_read_byte(file_in, ~buf, ~read_count))
+        /; if (read_count == 0)
+            break
+        ;/
+        log_one_nl('b')
+        # /; if (break_token(tmp, buf) == true)
+        #     log_one_nl('c')
+        #     /; if (tmp._len() > 0)
+        #         log_one_nl('d')
+        #         print_token(tmp, file_out)
+        #     ;/
+        #     
+        #     tmp._del()
+        #     tmp.start()
+# 
+        #     /; if (is_whitespace(buf) == false)
+        #         log_one_nl('e')
+        #         tmp.append(buf)
+        #     ;; else if (buf == WHITESPACE{2})
+        #         log_one_nl('f')
+        #         tmp.append(WHITESPACE{2})
+        #         print_token(tmp, file_out)
+        #         tmp._del()
+        #         tmp.start()
+        #     ;/
+        #     log_one_nl('c')
+        # ;; else
+            tmp.append(buf)
+        # ;/
+        log_one_nl('b')
+        read_count = 0
+    ;/
+
+    /; if (tmp._len() > 0)
+        print_token(tmp, file_out)
+    ;/
+
+    tmp._del()
+;/
diff --git a/tnslc/utils.tnsl b/tnslc/utils.tnsl
index 6fe7769..9fe113a 100644
--- a/tnslc/utils.tnsl
+++ b/tnslc/utils.tnsl
@@ -17,4 +17,63 @@
     ;/
 
     return true
-;/
\ No newline at end of file
+;/
+
+{}uint8 csv_pr = "\0\0"
+
+/; in_csv ({}uint8 csv, ~uint8 cstr) [bool]
+    int sl = cstr_len(cstr)
+    int cl = len csv
+    int seen = 0
+
+    /; loop (int i = 0; i < cl) [i++]
+        
+        /; if (seen < 0)
+            /; if (csv{i} == ',')
+                seen = 0
+            ;/
+            continue
+        ;/
+
+        /; if (seen !< sl)
+            /; if (csv{i} == ',')
+                return true
+            ;/
+            seen = -1
+        ;; else if (cstr{seen} == csv{i})
+            csv_pr{0} = cstr{seen}
+            # _printf(~csv_pr{0})
+            seen++
+        ;; else if (csv{i} == ',')
+            seen = 0
+        ;; else
+            seen = -1
+        ;/
+    ;/
+
+    return seen == sl
+;/
+
+/; cstr_copy (~uint8 from, to)
+    int ln = cstr_len(from)
+
+    /; loop (int i = 0; i !> ln) [i++]
+        to{i} = from{i}
+    ;/
+;/
+
+/; contains_char ({}uint8 arr, uint8 c) [bool]
+    /; loop (int i = 0; i < len arr) [i++]
+        /; if (arr{i} == c)
+            return true
+        ;/
+    ;/
+    return false
+;/
+
+/; write_to_file(~void file, ~uint8 string)
+    int ln = cstr_len(string)
+    /; loop (int i = 0; i < ln) [i++]
+        _write_byte(file, string + i)
+    ;/
+;/