[Bugfix] Literals and Comments

+ Fix literal value parsing
+ Remove comments from the tokenized result

2 files changed, 75 insertions, 8 deletions

diff --git a/src/tparse/parse.go b/src/tparse/parse.go
index e9e1ee6..4f25fe3 100644
--- a/src/tparse/parse.go
+++ b/src/tparse/parse.go
@@ -40,6 +40,8 @@ func stringLiteral(r *bufio.Reader) Token {
 	}
 
 	b := strings.Builder{}
+	b.WriteRune(run)
+	run, _, err = r.ReadRune()
 
 	for ; err == nil; run, _, err = r.ReadRune() {
 		b.WriteRune(run)
@@ -64,6 +66,8 @@ func charLiteral(r *bufio.Reader) Token {
 	}
 
 	b := strings.Builder{}
+	b.WriteRune(run)
+	run, _, err = r.ReadRune()
 
 	for ; err == nil; run, _, err = r.ReadRune() {
 		b.WriteRune(run)
@@ -107,6 +111,31 @@ func splitResRunes(str string, max int) []Token {
 	return out
 }
 
+// Remove block comments
+func stripBlockComments(t []Token) []Token {
+	out := []Token{}
+	bc := false
+	for _, tok := range t {
+		if tok.Type == DELIMIT && tok.Data == "/#" {
+			bc = true
+			continue
+		}
+
+		if tok.Type == DELIMIT && tok.Data == "#/" {
+			bc = false
+			continue
+		}
+
+		if bc {
+			continue
+		}
+
+		out = append(out, tok)
+	}
+
+	return out
+}
+
 // ParseFile tries to read a file and turn it into a series of tokens
 func ParseFile(path string) []Token {
 	out := []Token{}
@@ -123,7 +152,7 @@ func ParseFile(path string) []Token {
 
 	max := maxResRunes()
 
-	for r := ' '; ; r, _, err = read.ReadRune() {
+	for r := rune(' '); ; r, _, err = read.ReadRune() {
 		// If error in stream or EOF, break
 		if err != nil {
 			if err != io.EOF {
@@ -141,6 +170,37 @@ func ParseFile(path string) []Token {
 			continue
 		}
 
+		if unicode.IsNumber(r) && b.String() == "" {
+			read.UnreadRune()
+			out = append(out, numericLiteral(read))
+
+			continue
+		}
+
+		if r == '\'' {
+			if b.String() != "" {
+				out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+				b.Reset()
+			}
+
+			read.UnreadRune()
+			out = append(out, charLiteral(read))
+
+			continue
+		}
+
+		if r == '"' {
+			if b.String() != "" {
+				out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+				b.Reset()
+			}
+
+			read.UnreadRune()
+			out = append(out, stringLiteral(read))
+
+			continue
+		}
+
 		// Checking for a rune group
 		if checkResRune(r) != -1 {
 			if b.String() != "" {
@@ -157,7 +217,18 @@ func ParseFile(path string) []Token {
 
 			read.UnreadRune()
 
-			out = append(out, splitResRunes(b.String(), max)...)
+			rgs := splitResRunes(b.String(), max)
+
+			// Line Comments
+			for i, rg := range rgs {
+				if rg.Data == "#" {
+					rgs = rgs[:i]
+					read.ReadString('\n')
+					break
+				}
+			}
+
+			out = append(out, rgs...)
 
 			b.Reset()
 
@@ -168,7 +239,7 @@ func ParseFile(path string) []Token {
 		b.WriteRune(r)
 	}
 
-	return out
+	return stripBlockComments(out)
 }
 
 // StringAsRunes returns a string as a rune slice
diff --git a/src/tparse/type.go b/src/tparse/type.go
index ee8b5e7..867f14c 100644
--- a/src/tparse/type.go
+++ b/src/tparse/type.go
@@ -87,16 +87,12 @@ var RESRUNE = map[rune]int{
 	'{': DELIMIT,
 	// Array mark close
 	'}': DELIMIT,
-	// String literal
-	'\'': DELIMIT,
-	// String
-	'"': DELIMIT,
 
 	// Start of pre-proc directive
 	':': LINESEP,
 	// Start of line
 	';': LINESEP,
-	// Start of comment
+	// Comment line
 	'#': LINESEP,
 
 	// Seperate arguments