Extra Numbers

+ Add line and character numbers to tokens
+ Impliment line and character numbers
~ Line and char nums start at 0 for now
~ There's this itch in my mind like something is broken

8 files changed, 152 insertions, 76 deletions

diff --git a/.gitignore b/.gitignore
index d163863..01f9cb9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
-build/
\ No newline at end of file
+build/
+.vscode/
\ No newline at end of file
diff --git a/examp.tnsl b/examp.tnsl
index 8d5e257..0c32f3f 100644
--- a/examp.tnsl
+++ b/examp.tnsl
@@ -1,26 +1,25 @@
 #Comment like this
 
-/#
+/##
   Or like this (blocks begin with /<symbol> and end with <symbol>/)
+  Block Comment
+#/
+/##
+  Doc Comment
 #/
-
-
-
 
 # Preprocessor directives are like this
 # Import from library using '
-:import 'what/what.tnsl'
+:import 'what/what.tnsl' a
 # Import from local file using "
 :import "what/what.tnsl"
 
-
-
 # Code lines start with ;
 
 # pass a variable
 ;int s = 3
 
-
+;byte bitbyte = .2
 
 
 # generic scope block
@@ -38,8 +37,6 @@
 # d does not exist
 
 
-
-
 # Function def:
 # Any non-reserved word
 # Sig: [output1, output2] (input1, input2)
@@ -53,13 +50,13 @@
   # ~ after a address var represents the data the address points at
   
   ;int i = 1
-  ;~int j = ~i # address of int j = address of i
+  ;~int j ~= i # address of int j = address of i
   j~ = 2 # i = 2 # data of j = 2
 
   # /;loop represents the only loop in tnsl
   # loop is followed by (init statements) [multi statements]
   # where the first statement in multi is the test, and the once statements are only run once at the beginning of the loop
-  /; loop [i!==1]
+  /;loop [i!==1]
     # Do something
     ; i = 1
   ;/
@@ -145,7 +142,10 @@
 
   # You can do stuff here too
   /; if [t == 2]
-    i = t - i
+    ;i = t - i
+
+  ;;else if [t==3]
+    ;i = t+i
   ;/
 
   # Second case block
@@ -155,17 +155,18 @@
 ;/
 
 
-/;(type T) # Generic type
-  ; struct [gen] {
-    T i
-  }
-;/
+# Dumb generic type struct
+; struct [gen] (type T) {
+  T i
+}
 
+# This seems dumb
 ;gen(int) j{2}
 
-;{}gen(int) j{
-  {1},
-  {2},
-  {3}
+# But this seems dumber
+;{}gen(gen(int)) j{
+  {{1}},
+  {{2}},
+  {{3}}
 }
 
diff --git a/out.tnp b/out.tnp
index 9d67720..bc7c455 100644
--- a/out.tnp
+++ b/out.tnp
@@ -1 +1 @@
-[{0 ;} {8 a} {3 .} {8 pl} {2 (} {2 )}]
\ No newline at end of file
+[{0 : 12 0} {6 import 12 1} {4 'what/what.tnsl' 12 8} {8 a 12 25} {0 : 14 0} {6 import 14 1} {4 "what/what.tnsl" 14 8} {0 ; 19 0} {5 int 19 1} {8 s 19 5} {3 = 19 7} {4 3 19 9} {0 ; 21 0} {5 byte 21 1} {8 bitbyte 21 6} {3 = 21 14} {3 . 21 16} {4 2 21 17} {2 /; 28 0} {0 ; 30 2} {5 int 30 3} {8 d 30 7} {3 = 30 9} {4 1 30 11} {0 ; 32 2} {8 s 32 3} {3 = 32 5} {8 d 32 7} {2 ;/ 34 0} {2 /; 43 0} {8 main 43 2} {2 ( 43 7} {2 { 43 8} {2 } 43 9} {8 string 43 10} {8 str 43 17} {2 ) 43 20} {2 [ 43 22} {5 int 43 23} {2 ] 43 26} {0 ; 51 2} {5 int 51 3} {8 i 51 7} {3 = 51 9} {4 1 51 11} {0 ; 52 2} {3 ~ 52 3} {5 int 52 4} {8 j 52 8} {3 ~= 52 10} {8 i 52 13} {8 j 53 2} {3 ~ 53 3} {3 = 53 5} {4 2 53 7} {2 /; 58 2} {7 loop 58 4} {2 [ 58 9} {8 i 58 10} {3 !== 58 11} {4 1 58 14} {2 ] 58 16} {0 ; 60 4} {8 i 60 6} {3 = 60 8} {4 1 60 10} {2 ;/ 61 2} {2 ;/ 63 0} {0 ; 68 0} {7 struct 68 1} {2 [ 68 8} {8 s1 68 9} {2 ] 68 11} {2 { 68 13} {8 string 68 14} {8 Name 68 21} {1 , 68 25} {8 string 68 27} {8 Message 68 34} {3 = 68 42} {4 "Default message (c-style strings)" 68 44} {2 } 68 79} {0 ; 71 0} {7 struct 71 1} {2 [ 71 8} {8 s1 71 9} {2 ] 71 11} {2 { 71 13} {8 string 72 4} {8 Name 72 11} {1 , 72 15} {8 string 73 4} {8 Message 73 11} {3 = 73 19} {4 "Default message (c-style strings)" 73 21} {2 } 74 0} {0 ; 77 0} {8 s1 77 1} {8 a 77 4} {3 = 77 6} {2 { 77 8} {2 } 77 9} {0 ; 79 0} {8 s1 79 1} {8 a 79 4} {2 { 79 5} {2 } 79 6} {0 ; 80 0} {8 a 80 1} {3 . 80 2} {8 Name 80 3} {3 = 80 8} {4 "Kyle Gunger" 80 10} {0 ; 82 0} {3 ~ 82 1} {8 s1 82 2} {8 b 82 5} {3 = 82 7} {3 ~ 82 9} {8 a 82 10} {0 ; 83 0} {8 b 83 1} {3 ~ 83 2} {3 . 83 3} {8 Name 83 4} {0 ; 86 0} {8 s1 86 1} {8 c 86 4} {3 = 86 6} {2 { 86 8} {4 "" 86 9} {1 , 86 11} {4 "" 86 13} {2 } 86 15} {0 ; 90 0} {8 s1 90 1} {8 d 90 4} {2 { 90 5} {8 Message 91 4} {3 = 91 12} {4 "Message" 91 14} {1 , 91 23} {8 Name 92 4} {3 = 92 9} {4 "Name" 92 11} {2 } 93 0} {0 ; 99 0} {2 { 99 1} {2 } 99 2} {5 int 99 3} {8 a 99 7} {3 = 99 9} {2 { 99 11} {4 1, 100 2} {4 2, 100 6} {4 3, 100 10} {4 4 100 14} {2 } 101 0} {0 ; 104 0} {2 { 104 1} {2 } 104 2} {5 int 104 3} {8 a 104 7} {2 { 104 8} {4 1, 105 2} {4 2, 105 6} {4 3, 105 10} {4 4 105 14} {2 } 106 0} {2 /; 110 0} {8 s1 110 2} {2 [ 110 5} {8 s1 110 6} {2 ] 110 8} {8 return 112 2} {8 new 112 9} {8 s1 112 13} {2 { 112 15} {4 "Kyle" 112 16} {1 , 112 22} {4 "TNSL Creator" 112 24} {2 } 112 38} {2 ;/ 113 0} {2 /; 115 0} {7 if 115 3} {2 ( 115 6} {8 i 115 7} {3 == 115 9} {4 3 115 12} {2 ) 115 14} {2 ;; 118 0} {7 else 118 2} {2 ;/ 120 0} {2 /; 122 0} {7 switch 122 3} {2 ( 122 10} {8 i 122 11} {2 ) 122 12} {0 ; 124 2} {5 int 124 3} {8 t 124 7} {3 = 124 9} {4 0 124 11} {2 /; 127 2} {7 case 127 4} {4 1 127 9} {0 ; 128 4} {8 i 128 5} {3 = 128 7} {4 0 128 9} {0 ; 129 4} {8 t 129 5} {3 = 129 7} {4 2 129 9} {0 ; 130 4} {7 break 130 5} {2 ;; 132 2} {7 case 132 4} {4 2 132 9} {0 ; 133 4} {8 i 133 5} {3 = 133 7} {4 1 133 9} {0 ; 134 4} {8 t 134 5} {3 = 134 7} {4 2 134 9} {0 ; 135 4} {7 break 135 5} {2 ;; 137 2} {7 default 137 4} {0 ; 138 4} {8 i 138 5} {3 = 138 7} {4 3 138 9} {0 ; 139 4} {7 break 139 5} {2 ;/ 140 2} {2 /; 143 2} {7 if 143 5} {2 [ 143 8} {8 t 143 9} {3 == 143 11} {4 2 143 14} {2 ] 143 16} {0 ; 144 4} {8 i 144 5} {3 = 144 7} {8 t 144 9} {3 - 144 11} {8 i 144 13} {2 ;; 146 2} {7 else 146 4} {7 if 146 9} {2 [ 146 12} {8 t 146 13} {3 == 146 14} {4 3 146 16} {2 ] 146 18} {0 ; 147 4} {8 i 147 5} {3 = 147 7} {8 t 147 9} {3 + 147 10} {8 i 147 11} {2 ;/ 148 2} {2 /; 151 2} {7 case 151 4} {4 1 151 9} {0 ; 152 4} {8 i 152 5} {3 = 152 7} {4 4 152 9} {2 ;/ 153 2} {2 ;/ 154 0} {0 ; 158 0} {7 struct 158 2} {2 [ 158 9} {8 gen 158 10} {2 ] 158 13} {2 ( 158 15} {7 type 158 16} {8 T 158 21} {2 ) 158 22} {2 { 158 24} {8 T 159 2} {8 i 159 4} {2 } 160 0} {0 ; 163 0} {8 gen 163 1} {2 ( 163 4} {5 int 163 5} {2 ) 163 8} {8 j 163 10} {2 { 163 11} {4 2 163 12} {2 } 163 14} {0 ; 166 0} {2 { 166 1} {2 } 166 2} {8 gen 166 3} {2 ( 166 6} {8 gen 166 7} {2 ( 166 10} {5 int 166 11} {2 ) 166 14} {2 ) 166 15} {8 j 166 17} {2 { 166 18} {2 { 167 2} {2 { 167 3} {4 1 167 4} {2 } 167 6} {2 } 167 7} {1 , 167 8} {2 { 168 2} {2 { 168 3} {4 2 168 4} {2 } 168 6} {2 } 168 7} {1 , 168 8} {2 { 169 2} {2 { 169 3} {4 3 169 4} {2 } 169 6} {2 } 169 7} {2 } 170 0}]
\ No newline at end of file
diff --git a/src/main.go b/src/main.go
index ec3b14b..8edd129 100644
--- a/src/main.go
+++ b/src/main.go
@@ -18,7 +18,7 @@ func main() {
 		return
 	}
 
-	fd.WriteString(fmt.Sprint(tparse.ParseFile(*inputFile)))
+	fd.WriteString(fmt.Sprint(tparse.TokenizeFile(*inputFile)))
 
 	fd.Close()
 }
diff --git a/src/tparse/token.go b/src/tparse/token.go
index 712b746..386d52b 100644
--- a/src/tparse/token.go
+++ b/src/tparse/token.go
@@ -4,10 +4,6 @@ package tparse
 type Token struct {
 	Type int
 	Data string
-}
-
-// Container represents a container of data
-type Container struct {
-	Data  []interface{}
-	Holds bool
+	Line int
+	Char int
 }
diff --git a/src/tparse/parse.go b/src/tparse/tokenize.go
index 4f25fe3..79a0605 100644
--- a/src/tparse/parse.go
+++ b/src/tparse/tokenize.go
@@ -10,10 +10,10 @@ import (
 )
 
 // Read in a number (may be a float)
-func numericLiteral(r *bufio.Reader) Token {
+func numericLiteral(r *bufio.Reader, line int, char *int) Token {
 	decimal := false
 	run, _, err := r.ReadRune()
-
+	last := *char
 	b := strings.Builder{}
 
 	for ; err == nil; run, _, err = r.ReadRune() {
@@ -22,18 +22,20 @@ func numericLiteral(r *bufio.Reader) Token {
 		} else if !unicode.IsNumber(run) {
 			break
 		}
+		*char++
 		b.WriteRune(run)
 	}
 
 	r.UnreadRune()
 
-	return Token{Type: LITERAL, Data: b.String()}
+	return Token{Type: LITERAL, Data: b.String(), Line: line, Char: last}
 }
 
 // Parse a string (will escape \" only in this stage)
-func stringLiteral(r *bufio.Reader) Token {
+func stringLiteral(r *bufio.Reader, line, char *int) Token {
 	escape := false
 	run, _, err := r.ReadRune()
+	last := *char
 
 	if run != '"' {
 		return Token{Type: LITERAL}
@@ -44,22 +46,28 @@ func stringLiteral(r *bufio.Reader) Token {
 	run, _, err = r.ReadRune()
 
 	for ; err == nil; run, _, err = r.ReadRune() {
+		*char++
 		b.WriteRune(run)
 		if run == '\\' && !escape {
 			escape = true
-		} else if run == '"' && !escape {
+		} else if (run == '"' || run == '\n') && !escape {
 			break
+		} else if escape {
+			if run == '\n' {
+				*line++
+			}
+			escape = false
 		}
-
 	}
 
-	return Token{Type: LITERAL, Data: b.String()}
+	return Token{Type: LITERAL, Data: b.String(), Line: *line, Char: last}
 }
 
 // Parse a character in (escape \\ or \')
-func charLiteral(r *bufio.Reader) Token {
+func charLiteral(r *bufio.Reader, line int, char *int) Token {
 	escape := false
 	run, _, err := r.ReadRune()
+	last := *char
 
 	if run != '\'' {
 		return Token{Type: LITERAL}
@@ -71,19 +79,21 @@ func charLiteral(r *bufio.Reader) Token {
 
 	for ; err == nil; run, _, err = r.ReadRune() {
 		b.WriteRune(run)
+		*char++
 		if run == '\\' && !escape {
 			escape = true
-		} else if run == '\'' && !escape {
+		} else if (run == '\'' && !escape) || run == '\n' {
 			break
+		} else if escape {
+			escape = false
 		}
-
 	}
 
-	return Token{Type: LITERAL, Data: b.String()}
+	return Token{Type: LITERAL, Data: b.String(), Line: line, Char: last}
 }
 
 // Split reserved runes into rune groups
-func splitResRunes(str string, max int) []Token {
+func splitResRunes(str string, max, line, start int) []Token {
 	out := []Token{}
 
 	rs := StringAsRunes(str)
@@ -96,7 +106,7 @@ func splitResRunes(str string, max int) []Token {
 	for e <= len(rs) && s < len(rs) {
 		if checkRuneGroup(RunesAsString(rs[s:e])) != -1 || e == s+1 {
 			tmp := RunesAsString(rs[s:e])
-			out = append(out, Token{Type: checkRuneGroup(tmp), Data: tmp})
+			out = append(out, Token{Type: checkRuneGroup(tmp), Data: tmp, Line: line, Char: start + s})
 			s = e
 			if s+max < len(rs) {
 				e = s + max
@@ -119,14 +129,10 @@ func stripBlockComments(t []Token) []Token {
 		if tok.Type == DELIMIT && tok.Data == "/#" {
 			bc = true
 			continue
-		}
-
-		if tok.Type == DELIMIT && tok.Data == "#/" {
+		} else if tok.Type == DELIMIT && tok.Data == "#/" {
 			bc = false
 			continue
-		}
-
-		if bc {
+		} else if bc {
 			continue
 		}
 
@@ -136,8 +142,8 @@ func stripBlockComments(t []Token) []Token {
 	return out
 }
 
-// ParseFile tries to read a file and turn it into a series of tokens
-func ParseFile(path string) []Token {
+// TokenizeFile tries to read a file and turn it into a series of tokens
+func TokenizeFile(path string) []Token {
 	out := []Token{}
 
 	fd, err := os.Open(path)
@@ -152,7 +158,11 @@ func ParseFile(path string) []Token {
 
 	max := maxResRunes()
 
+	ln, cn, last := int(0), int(-1), int(0)
+	sp := false
+
 	for r := rune(' '); ; r, _, err = read.ReadRune() {
+		cn++
 		// If error in stream or EOF, break
 		if err != nil {
 			if err != io.EOF {
@@ -163,28 +173,42 @@ func ParseFile(path string) []Token {
 
 		// Checking for a space
 		if unicode.IsSpace(r) {
+			sp = true
 			if b.String() != "" {
-				out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+				out = append(out, Token{Type: checkToken(b.String()), Data: b.String(), Line: ln, Char: last})
 				b.Reset()
 			}
+
+			// checking for a newline
+			if r == '\n' {
+				ln++
+				cn = -1
+				last = 0
+			}
+
 			continue
+		} else if sp {
+			last = cn
+			sp = false
 		}
 
 		if unicode.IsNumber(r) && b.String() == "" {
 			read.UnreadRune()
-			out = append(out, numericLiteral(read))
+			out = append(out, numericLiteral(read, ln, &cn))
+			sp = true
 
 			continue
 		}
 
 		if r == '\'' {
 			if b.String() != "" {
-				out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+				out = append(out, Token{Type: checkToken(b.String()), Data: b.String(), Line: ln, Char: last})
 				b.Reset()
 			}
 
 			read.UnreadRune()
-			out = append(out, charLiteral(read))
+			out = append(out, charLiteral(read, ln, &cn))
+			sp = true
 
 			continue
 		}
@@ -196,7 +220,8 @@ func ParseFile(path string) []Token {
 			}
 
 			read.UnreadRune()
-			out = append(out, stringLiteral(read))
+			out = append(out, stringLiteral(read, &ln, &cn))
+			sp = true
 
 			continue
 		}
@@ -204,26 +229,31 @@ func ParseFile(path string) []Token {
 		// Checking for a rune group
 		if checkResRune(r) != -1 {
 			if b.String() != "" {
-				out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+				out = append(out, Token{Type: checkToken(b.String()), Data: b.String(), Line: ln, Char: last})
 				b.Reset()
 			}
-
+			last = cn
 			for ; err == nil; r, _, err = read.ReadRune() {
 				if checkResRune(r) == -1 {
 					break
 				}
+				cn++
 				b.WriteRune(r)
 			}
+			cn--
 
 			read.UnreadRune()
 
-			rgs := splitResRunes(b.String(), max)
+			rgs := splitResRunes(b.String(), max, ln, last)
 
 			// Line Comments
 			for i, rg := range rgs {
 				if rg.Data == "#" {
 					rgs = rgs[:i]
 					read.ReadString('\n')
+					ln++
+					cn = -1
+					last = 0
 					break
 				}
 			}
@@ -232,6 +262,8 @@ func ParseFile(path string) []Token {
 
 			b.Reset()
 
+			sp = true
+
 			continue
 		}
 
diff --git a/src/tparse/tree.go b/src/tparse/tree.go
new file mode 100644
index 0000000..41896fc
--- /dev/null
+++ b/src/tparse/tree.go
@@ -0,0 +1,58 @@
+package tparse
+
+// Node represents a group of nodes or a directive
+type Node struct {
+	SubNodes []Node
+
+	Dir Directive
+}
+
+// Directive represents a block or single directive
+type Directive struct {
+	Type string
+	ID   string
+
+	Data []string
+}
+
+func handleCode(tokens *[]Token, start int) (Node, int) {
+	out := Node{}
+
+	return out, start
+}
+
+func handlePre(tokens *[]Token, start int) (Node, int) {
+	out := Node{}
+
+	return out, start
+}
+
+// CreateTree takes a series of tokens and converts them into an AST
+func CreateTree(tokens *[]Token, start int) Node {
+	out := Node{}
+	out.Dir = Directive{Type: "root", ID: "root"}
+
+	var tmp Node
+
+	for i, t := range *tokens {
+		switch t.Type {
+		case LINESEP:
+			if t.Data == ";" {
+				tmp, i = handleCode(tokens, i)
+			} else if t.Data == ":" {
+				tmp, i = handlePre(tokens, i)
+			}
+			break
+		case DELIMIT:
+			if t.Data == "/;" {
+				tmp, i = handleCode(tokens, i)
+			} else if t.Data == "/:" {
+				tmp, i = handlePre(tokens, i)
+			}
+			break
+		}
+		out.SubNodes = append(out.SubNodes, tmp)
+	}
+
+	return out
+}
diff --git a/src/tparse/type.go b/src/tparse/type.go
index 867f14c..dc0df10 100644
--- a/src/tparse/type.go
+++ b/src/tparse/type.go
@@ -33,10 +33,12 @@ const DEFWORD = 8
 var RESWORD = map[string]int{
 	"import": PREWORD,
 
+	"bool": KEYTYPE,
+	"byte": KEYTYPE,
+	"char": KEYTYPE,
+
 	"int":   KEYTYPE,
-	"bool":  KEYTYPE,
 	"float": KEYTYPE,
-	"char":  KEYTYPE,
 
 	"struct": KEYWORD,
 	"type":   KEYWORD,
@@ -146,9 +148,6 @@ func checkResRune(r rune) int {
 
 // RESRUNES Reserved sets of reserved runes which mean something
 var RESRUNES = map[string]int{
-	// Pre-processor block
-	"/:": DELIMIT,
-	":/": DELIMIT,
 	// Code block
 	"/;": DELIMIT,
 	";/": DELIMIT,
@@ -156,18 +155,7 @@ var RESRUNES = map[string]int{
 	"/#": DELIMIT,
 	"#/": DELIMIT,
 
-	// Quick chain
-	"::": DELIMIT,
-	":;": DELIMIT,
-	":#": DELIMIT,
-
 	";;": DELIMIT,
-	";:": DELIMIT,
-	";#": DELIMIT,
-
-	"##": DELIMIT,
-	"#:": DELIMIT,
-	"#;": DELIMIT,
 
 	// Boolean equ
 	"==": AUGMENT,