[Bugfix] Literals and Comments

+ Fix literal value parsing
+ Remove comments from the tokenized result

4 files changed, 77 insertions, 11 deletions

diff --git a/base.tnsl b/base.tnsl
index f275a82..a74997f 100644
--- a/base.tnsl
+++ b/base.tnsl
@@ -1,3 +1,2 @@
 
-[a!==b]
-!=!==
+;a.pl()
diff --git a/out.tnp b/out.tnp
index b4932e3..9d67720 100644
--- a/out.tnp
+++ b/out.tnp
@@ -1 +1 @@
-[{0 #} {8 Comment} {8 like} {8 this} {2 /#} {8 Or} {8 like} {8 this} {2 (} {8 blocks} {8 begin} {8 with} {3 /} {3 <} {8 symbol} {3 >} {8 and} {8 end} {8 with} {3 <} {8 symbol} {3 >} {3 /} {2 )} {2 #/} {0 #} {8 Preprocessor} {8 directives} {8 are} {8 like} {8 this} {0 :} {8 pack} {8 main} {0 #} {8 You} {8 can} {8 also} {8 create} {8 a} {8 block} {8 of} {8 them} {3 .} {2 /:} {6 import} {8 what} {3 .} {8 tnsl} {6 import} {2 "} {8 what} {3 .} {8 tnsl} {2 "} {2 :/} {0 #} {8 Code} {8 lines} {8 start} {8 with} {0 ;} {0 #} {8 pass} {8 a} {8 variable} {0 ;} {5 int} {8 s} {3 =} {8 3} {0 #} {8 generic} {8 scope} {8 block} {0 #} {8 d} {8 does} {8 not} {8 exist} {2 /;} {0 #} {8 Scope} {0 ;} {5 int} {8 d} {3 =} {8 1} {0 #} {8 d} {8 exists} {0 ;} {8 s} {3 =} {8 d} {2 ;/} {0 #} {8 Scope} {8 end} {0 #} {8 d} {8 does} {8 not} {8 exist} {0 #} {8 Function} {8 def} {0 :} {0 #} {8 Any} {8 non} {3 -} {8 reserved} {8 word} {0 #} {8 Sig} {0 :} {2 [} {8 output1} {1 ,} {8 output2} {2 ]} {2 (} {8 input1} {1 ,} {8 input2} {2 )} {0 #} {8 Main} {8 may} {8 have} {2 /;} {8 main} {2 (} {2 {} {2 }} {8 string} {8 str} {2 )} {2 [} {5 int} {2 ]} {0 #} {8 Doesn} {2 '} {8 t} {8 matter} {8 what} {8 order} {8 the} {8 sig} {8 is} {8 in} {0 #} {8 Main} {8 may} {8 also} {8 omit} {8 either} {8 for} {8 void} {8 sig} {0 #} {2 {} {2 }} {8 represents} {8 a} {8 tnsl} {8 style} {8 array} {0 #} {3 ~} {8 before} {8 var} {8 represents} {8 address} {8 var} {0 #} {3 ~} {8 after} {8 a} {8 address} {8 var} {8 represents} {8 the} {8 data} {8 the} {8 address} {8 points} {8 at} {0 ;} {5 int} {8 i} {3 =} {8 1} {0 ;} {3 ~} {5 int} {8 j} {3 =} {3 ~} {8 i} {0 #} {8 address} {8 of} {5 int} {8 j} {3 =} {8 address} {8 of} {8 i} {8 j} {3 ~} {3 =} {8 2} {0 #} {8 i} {3 =} {8 2} {0 #} {8 data} {8 of} {8 j} {3 =} {8 2} {0 #} {3 /} {7 loop} {8 represents} {8 the} {8 only} {7 loop} {8 in} {8 tnsl} {0 #} {7 loop} {8 is} {8 followed} {8 by} {2 (} {8 init} {8 statements} {2 )} {2 [} {8 multi} {8 statements} {2 ]} {0 #} {8 where} {8 the} {8 first} {8 statement} {8 in} {8 multi} {8 is} {8 the} {8 test} {1 ,} {8 and} {8 the} {8 once} {8 statements} {8 are} {8 only} {8 run} {8 once} {8 at} {8 the} {8 beginning} {8 of} {8 the} {7 loop} {2 /;} {7 loop} {2 [} {8 i} {3 !==} {8 1} {2 ]} {0 #} {8 Do} {8 something} {0 ;} {8 i} {3 =} {8 1} {2 ;/} {2 ;/} {0 #} {8 End} {8 main} {0 #} {8 The} {7 struct} {8 keyword} {8 is} {8 followed} {8 by} {2 [} {8 name} {2 ]} {2 {} {8 values} {2 }} {0 ;} {7 struct} {2 [} {8 s1} {2 ]} {2 {} {8 string} {8 Name} {1 ,} {8 string} {8 Message} {3 =} {2 "} {8 Default} {8 message} {2 (} {8 c} {3 -} {8 style} {8 strings} {2 )} {2 "} {2 }} {0 #} {8 Most} {8 people} {8 should} {8 declare} {8 as} {8 such} {0 :} {0 ;} {7 struct} {2 [} {8 s1} {2 ]} {2 {} {8 string} {8 Name} {1 ,} {8 string} {8 Message} {3 =} {2 "} {8 Default} {8 message} {2 (} {8 c} {3 -} {8 style} {8 strings} {2 )} {2 "} {2 }} {0 #} {8 When} {8 defining} {8 a} {8 new} {7 struct} {1 ,} {8 use} {2 {} {2 }} {0 ;} {8 s1} {8 a} {3 =} {2 {} {2 }} {0 ;} {8 a} {3 .} {8 Name} {3 =} {2 "} {8 Kyle} {8 Gunger} {2 "} {0 ;} {3 ~} {8 s1} {8 b} {3 =} {3 ~} {8 a} {0 ;} {8 b} {3 ~} {3 .} {8 Name} {0 #} {2 "} {8 Kyle} {8 Gunger} {2 "} {0 #} {8 Quick} {8 initialization} {0 ;} {8 s1} {8 c} {3 =} {2 {} {2 "} {2 "} {1 ,} {2 "} {2 "} {2 }} {0 #} {8 These} {8 come} {8 in} {8 the} {8 same} {8 order} {8 that} {8 they} {8 do} {8 in} {8 the} {7 struct} {1 ,} {8 so} {2 {} {8 Name} {1 ,} {8 Message} {2 }} {8 in} {8 this} {7 case} {3 .} {0 #} {8 You} {8 can} {8 also} {8 specify} {0 ;} {8 s1} {8 d} {3 =} {2 {} {8 Message} {3 =} {2 "} {8 Message} {2 "} {1 ,} {8 Name} {3 =} {2 "} {8 Name} {2 "} {2 }} {0 #} {8 This} {8 is} {8 how} {8 arrays} {8 are} {8 defined} {8 as} {8 well} {3 .} {0 ;} {2 {} {2 }} {5 int} {8 a} {3 =} {2 {} {8 1} {1 ,} {8 2} {1 ,} {8 3} {1 ,} {8 4} {2 }} {0 #} {8 You} {8 may} {8 also} {8 define} {8 an} {8 initializer} {8 like} {8 such} {0 :} {2 /;} {8 s1} {2 [} {8 s1} {2 ]} {0 #} {8 Initializer} {8 must} {8 be} {8 named} {8 same} {8 as} {7 struct} {1 ,} {8 and} {8 must} {8 return} {8 one} {8 of} {8 the} {8 structs} {8 as} {8 its} {8 only} {8 output} {8 return} {8 new} {8 s1} {2 {} {2 "} {8 Kyle} {2 "} {1 ,} {2 "} {8 TNSL} {8 Creator} {2 "} {2 }} {2 ;/} {2 /;} {7 if} {2 (} {8 i} {3 ==} {8 3} {2 )} {0 #} {8 Quick} {8 define} {8 new} {8 block} {2 ;;} {7 else} {2 ;/} {2 /;} {7 switch} {2 (} {8 i} {2 )} {0 #} {8 You} {8 can} {8 do} {8 stuff} {8 here} {8 as} {8 well} {0 ;} {5 int} {8 t} {3 =} {8 0} {0 #} {8 Case} {8 block} {2 /;} {7 case} {8 1} {0 ;} {8 i} {3 =} {8 0} {0 ;} {8 t} {3 =} {8 2} {0 ;} {7 break} {2 ;;} {7 case} {8 2} {0 ;} {8 i} {3 =} {8 1} {0 ;} {8 t} {3 =} {8 2} {0 ;} {7 break} {2 ;;} {7 default} {0 ;} {8 i} {3 =} {8 3} {0 ;} {7 break} {2 ;/} {0 #} {8 You} {8 can} {8 do} {8 stuff} {8 here} {8 too} {2 /;} {7 if} {2 [} {8 t} {3 ==} {8 2} {2 ]} {8 i} {3 =} {8 t} {3 -} {8 i} {2 ;/} {0 #} {8 Second} {7 case} {8 block} {2 /;} {7 case} {8 1} {0 ;} {8 i} {3 =} {8 4} {2 ;/} {2 ;/} {2 /;} {2 (} {7 type} {8 T} {2 )} {0 #} {8 Generic} {7 type} {0 ;} {7 struct} {2 [} {8 gen} {2 ]} {2 {} {8 T} {8 i} {2 }} {2 ;/} {0 ;} {8 gen} {2 (} {5 int} {2 )} {8 j} {2 {} {8 2} {2 }} {0 ;} {2 {} {2 }} {8 gen} {2 (} {5 int} {2 )} {8 j} {2 {} {2 {} {8 1} {2 }} {1 ,} {2 {} {8 2} {2 }} {1 ,} {2 {} {8 3} {2 }} {2 }}]
\ No newline at end of file
+[{0 ;} {8 a} {3 .} {8 pl} {2 (} {2 )}]
\ No newline at end of file
diff --git a/src/tparse/parse.go b/src/tparse/parse.go
index e9e1ee6..4f25fe3 100644
--- a/src/tparse/parse.go
+++ b/src/tparse/parse.go
@@ -40,6 +40,8 @@ func stringLiteral(r *bufio.Reader) Token {
 	}
 
 	b := strings.Builder{}
+	b.WriteRune(run)
+	run, _, err = r.ReadRune()
 
 	for ; err == nil; run, _, err = r.ReadRune() {
 		b.WriteRune(run)
@@ -64,6 +66,8 @@ func charLiteral(r *bufio.Reader) Token {
 	}
 
 	b := strings.Builder{}
+	b.WriteRune(run)
+	run, _, err = r.ReadRune()
 
 	for ; err == nil; run, _, err = r.ReadRune() {
 		b.WriteRune(run)
@@ -107,6 +111,31 @@ func splitResRunes(str string, max int) []Token {
 	return out
 }
 
+// Remove block comments
+func stripBlockComments(t []Token) []Token {
+	out := []Token{}
+	bc := false
+	for _, tok := range t {
+		if tok.Type == DELIMIT && tok.Data == "/#" {
+			bc = true
+			continue
+		}
+
+		if tok.Type == DELIMIT && tok.Data == "#/" {
+			bc = false
+			continue
+		}
+
+		if bc {
+			continue
+		}
+
+		out = append(out, tok)
+	}
+
+	return out
+}
+
 // ParseFile tries to read a file and turn it into a series of tokens
 func ParseFile(path string) []Token {
 	out := []Token{}
@@ -123,7 +152,7 @@ func ParseFile(path string) []Token {
 
 	max := maxResRunes()
 
-	for r := ' '; ; r, _, err = read.ReadRune() {
+	for r := rune(' '); ; r, _, err = read.ReadRune() {
 		// If error in stream or EOF, break
 		if err != nil {
 			if err != io.EOF {
@@ -141,6 +170,37 @@ func ParseFile(path string) []Token {
 			continue
 		}
 
+		if unicode.IsNumber(r) && b.String() == "" {
+			read.UnreadRune()
+			out = append(out, numericLiteral(read))
+
+			continue
+		}
+
+		if r == '\'' {
+			if b.String() != "" {
+				out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+				b.Reset()
+			}
+
+			read.UnreadRune()
+			out = append(out, charLiteral(read))
+
+			continue
+		}
+
+		if r == '"' {
+			if b.String() != "" {
+				out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+				b.Reset()
+			}
+
+			read.UnreadRune()
+			out = append(out, stringLiteral(read))
+
+			continue
+		}
+
 		// Checking for a rune group
 		if checkResRune(r) != -1 {
 			if b.String() != "" {
@@ -157,7 +217,18 @@ func ParseFile(path string) []Token {
 
 			read.UnreadRune()
 
-			out = append(out, splitResRunes(b.String(), max)...)
+			rgs := splitResRunes(b.String(), max)
+
+			// Line Comments
+			for i, rg := range rgs {
+				if rg.Data == "#" {
+					rgs = rgs[:i]
+					read.ReadString('\n')
+					break
+				}
+			}
+
+			out = append(out, rgs...)
 
 			b.Reset()
 
@@ -168,7 +239,7 @@ func ParseFile(path string) []Token {
 		b.WriteRune(r)
 	}
 
-	return out
+	return stripBlockComments(out)
 }
 
 // StringAsRunes returns a string as a rune slice
diff --git a/src/tparse/type.go b/src/tparse/type.go
index ee8b5e7..867f14c 100644
--- a/src/tparse/type.go
+++ b/src/tparse/type.go
@@ -87,16 +87,12 @@ var RESRUNE = map[rune]int{
 	'{': DELIMIT,
 	// Array mark close
 	'}': DELIMIT,
-	// String literal
-	'\'': DELIMIT,
-	// String
-	'"': DELIMIT,
 
 	// Start of pre-proc directive
 	':': LINESEP,
 	// Start of line
 	';': LINESEP,
-	// Start of comment
+	// Comment line
 	'#': LINESEP,
 
 	// Seperate arguments