Basic updates

- Move preprocessor to it's own file
- Begin working on new delimiters
- Begin working on blocks in the tree

1 files changed, 27 insertions, 7 deletions

diff --git a/src/tparse/tokenize.go b/src/tparse/tokenize.go
index 79a0605..f5e3f54 100644
--- a/src/tparse/tokenize.go
+++ b/src/tparse/tokenize.go
@@ -126,12 +126,32 @@ func stripBlockComments(t []Token) []Token {
 	out := []Token{}
 	bc := false
 	for _, tok := range t {
-		if tok.Type == DELIMIT && tok.Data == "/#" {
-			bc = true
-			continue
-		} else if tok.Type == DELIMIT && tok.Data == "#/" {
-			bc = false
-			continue
+
+		if tok.Type == DELIMIT {
+			switch tok.Data {
+			case ";#":
+				out = append(out, Token{DELIMIT, ";/", tok.Line, tok.Char})
+				bc = true
+				continue
+			case ":#":
+				out = append(out, Token{DELIMIT, ":/", tok.Line, tok.Char})
+				bc = true
+				continue
+			case "/#":
+				bc = true
+				continue
+			case "#;":
+				out = append(out, Token{DELIMIT, "/;", tok.Line, tok.Char})
+				bc = false
+				continue
+			case "#:":
+				out = append(out, Token{DELIMIT, "/:", tok.Line, tok.Char})
+				bc = false
+				continue
+			case "#/":
+				bc = false
+				continue
+			}
 		} else if bc {
 			continue
 		}
@@ -158,7 +178,7 @@ func TokenizeFile(path string) []Token {
 
 	max := maxResRunes()
 
-	ln, cn, last := int(0), int(-1), int(0)
+	ln, cn, last := int(1), int(-1), int(0)
 	sp := false
 
 	for r := rune(' '); ; r, _, err = read.ReadRune() {