1 files changed, 0 insertions, 114 deletions
diff --git a/tnslc/parse/tokenizer.tnsl b/tnslc/parse/tokenizer.tnsl
deleted file mode 100644
index 30ccd4b..0000000
--- a/tnslc/parse/tokenizer.tnsl
+++ /dev/null
@@ -1,114 +0,0 @@
-/#
-	Copyright 2021-2022 Kyle Gunger
-
-	This file is licensed under the CDDL 1.0 (the License)
-	and may only be used in accordance with the License.
-	You should have received a copy of the License with this
-	software/source code. If you did not, a copy can be found
-	at the following URL:
-
-	https://opensource.org/licenses/CDDL-1.0
-
-	THIS SOFTWARE/SOURCE CODE IS PROVIDED "AS IS" WITH NO
-	WARRANTY, GUARANTEE, OR CLAIM OF FITNESS FOR ANY PURPOSE
-	EXPRESS OR IMPLIED
-#/
-
-/; is_float (~{}uint8 dat) [bool]
-	;return is_numeric_literal(dat) && is_in_string(dat, '.')
-;/
-
-/; break_token ({}uint8 dat, uint8 c) [bool]
-	/; if (len dat == 0)
-		;return false
-
-	;; else if (dat{0} == '"' || dat{0} == '\'')
-		;return string_closed(dat, c)
-
-	;; else if (is_in_string(~RESERVED, dat{len dat - 1}))
-
-		/; if (is_in_string(~RESERVED, c))
-			;dat.append(c)
-			;return get_token_type(~dat) == TOKEN_TYPE.DEFWORD
-
-		;; else if (len dat == 1 && dat{0} == '.')
-			;return !is_digit(c)
-
-		;/
-
-		;return true
-
-	;; else if (is_in_string(~RESERVED, c))
-
-		/; if (is_numeric_literal(~dat) && !is_float(~dat) && c == '.')
-			;return false
-
-		;/
-
-		;return true
-	;/
-	
-	;return is_whitespace(c)
-;/
-
-/; strip_and_expand (~{}Token dat) [{}Token]
-	;{}Token out = {}
-
-	;bool cblk = false
-
-	/; loop (int i = 0; i < len dat`) [i++]
-		/; if (!cblk)
-			/; if (string_equate(dat`{i}.data`, "/#"))
-				;cblk = true
-			;; else
-				;out.append(dat`{i})
-			;/
-
-		;; else if (string_equate(dat`{i}.data`, "#/"))
-			;cblk = false
-		;/
-	;/
-	
-	;return out
-;/
-
-/; tokenize (tnsl.io.File fstr) [~{}Token]
-	;{}Token out = {}
-	;{}uint8 tdat = {}
-	;bool comment = false
-	;int line = 1, col = 1
-
-	/; loop (int i = fstr.read(); i !== -1) [i = fstr.read()]
-		/; if (break_token(tdat, i) && !comment)
-			/; if (len tdat == 1 && tdat{0} == '#')
-				;tdat = {}
-				;comment = true
-			;; else if (len tdat > 0)
-				;{}uint8 tmp = tdat
-				;Token ttk = {get_token_type(~tmp), line, col, ~tmp}
-				;out.append(ttk)
-				;tdat = {}
-			;/
-		;/
-
-		/; if ( (!is_whitespace(i) || is_text_literal(~tdat)) && !comment )
-			;tdat.append(i)
-		;; else if (i == '\n')
-			;line++
-			;col = 0
-			/; if (comment)
-				;comment = false
-			;/
-		;/
-
-		;col++
-	;/
-
-	/; if (len tdat > 0)
-		;Token ttk = {get_token_type(~tdat), line, col, ~tdat}
-		;out.append(ttk)
-	;/
-
-	;out = strip_and_expand(~out)
-	;return ~out
-;/