From 5a022193a96e72fa5144755938e6a575aba165b0 Mon Sep 17 00:00:00 2001 From: Kyle Gunger Date: Fri, 3 Jul 2020 13:17:26 -0400 Subject: Extra Numbers + Add line and character numbers to tokens + Impliment line and character numbers ~ Line and char nums start at 0 for now ~ There's this itch in my mind like something is broken --- .gitignore | 3 +- examp.tnsl | 45 ++++---- out.tnp | 2 +- src/main.go | 2 +- src/tparse/parse.go | 274 ------------------------------------------- src/tparse/token.go | 8 +- src/tparse/tokenize.go | 306 +++++++++++++++++++++++++++++++++++++++++++++++++ src/tparse/tree.go | 58 ++++++++++ src/tparse/type.go | 20 +--- 9 files changed, 397 insertions(+), 321 deletions(-) delete mode 100644 src/tparse/parse.go create mode 100644 src/tparse/tokenize.go create mode 100644 src/tparse/tree.go diff --git a/.gitignore b/.gitignore index d163863..01f9cb9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -build/ \ No newline at end of file +build/ +.vscode/ \ No newline at end of file diff --git a/examp.tnsl b/examp.tnsl index 8d5e257..0c32f3f 100644 --- a/examp.tnsl +++ b/examp.tnsl @@ -1,26 +1,25 @@ #Comment like this -/# +/## Or like this (blocks begin with / and end with /) + Block Comment +#/ +/## + Doc Comment #/ - - - # Preprocessor directives are like this # Import from library using ' -:import 'what/what.tnsl' +:import 'what/what.tnsl' a # Import from local file using " :import "what/what.tnsl" - - # Code lines start with ; # pass a variable ;int s = 3 - +;byte bitbyte = .2 # generic scope block @@ -38,8 +37,6 @@ # d does not exist - - # Function def: # Any non-reserved word # Sig: [output1, output2] (input1, input2) @@ -53,13 +50,13 @@ # ~ after a address var represents the data the address points at ;int i = 1 - ;~int j = ~i # address of int j = address of i + ;~int j ~= i # address of int j = address of i j~ = 2 # i = 2 # data of j = 2 # /;loop represents the only loop in tnsl # loop is followed by (init statements) [multi statements] # where the first statement in multi is the test, and the once statements are only run once at the beginning of the loop - /; loop [i!==1] + /;loop [i!==1] # Do something ; i = 1 ;/ @@ -145,7 +142,10 @@ # You can do stuff here too /; if [t == 2] - i = t - i + ;i = t - i + + ;;else if [t==3] + ;i = t+i ;/ # Second case block @@ -155,17 +155,18 @@ ;/ -/;(type T) # Generic type - ; struct [gen] { - T i - } -;/ +# Dumb generic type struct +; struct [gen] (type T) { + T i +} +# This seems dumb ;gen(int) j{2} -;{}gen(int) j{ - {1}, - {2}, - {3} +# But this seems dumber +;{}gen(gen(int)) j{ + {{1}}, + {{2}}, + {{3}} } diff --git a/out.tnp b/out.tnp index 9d67720..bc7c455 100644 --- a/out.tnp +++ b/out.tnp @@ -1 +1 @@ -[{0 ;} {8 a} {3 .} {8 pl} {2 (} {2 )}] \ No newline at end of file +[{0 : 12 0} {6 import 12 1} {4 'what/what.tnsl' 12 8} {8 a 12 25} {0 : 14 0} {6 import 14 1} {4 "what/what.tnsl" 14 8} {0 ; 19 0} {5 int 19 1} {8 s 19 5} {3 = 19 7} {4 3 19 9} {0 ; 21 0} {5 byte 21 1} {8 bitbyte 21 6} {3 = 21 14} {3 . 21 16} {4 2 21 17} {2 /; 28 0} {0 ; 30 2} {5 int 30 3} {8 d 30 7} {3 = 30 9} {4 1 30 11} {0 ; 32 2} {8 s 32 3} {3 = 32 5} {8 d 32 7} {2 ;/ 34 0} {2 /; 43 0} {8 main 43 2} {2 ( 43 7} {2 { 43 8} {2 } 43 9} {8 string 43 10} {8 str 43 17} {2 ) 43 20} {2 [ 43 22} {5 int 43 23} {2 ] 43 26} {0 ; 51 2} {5 int 51 3} {8 i 51 7} {3 = 51 9} {4 1 51 11} {0 ; 52 2} {3 ~ 52 3} {5 int 52 4} {8 j 52 8} {3 ~= 52 10} {8 i 52 13} {8 j 53 2} {3 ~ 53 3} {3 = 53 5} {4 2 53 7} {2 /; 58 2} {7 loop 58 4} {2 [ 58 9} {8 i 58 10} {3 !== 58 11} {4 1 58 14} {2 ] 58 16} {0 ; 60 4} {8 i 60 6} {3 = 60 8} {4 1 60 10} {2 ;/ 61 2} {2 ;/ 63 0} {0 ; 68 0} {7 struct 68 1} {2 [ 68 8} {8 s1 68 9} {2 ] 68 11} {2 { 68 13} {8 string 68 14} {8 Name 68 21} {1 , 68 25} {8 string 68 27} {8 Message 68 34} {3 = 68 42} {4 "Default message (c-style strings)" 68 44} {2 } 68 79} {0 ; 71 0} {7 struct 71 1} {2 [ 71 8} {8 s1 71 9} {2 ] 71 11} {2 { 71 13} {8 string 72 4} {8 Name 72 11} {1 , 72 15} {8 string 73 4} {8 Message 73 11} {3 = 73 19} {4 "Default message (c-style strings)" 73 21} {2 } 74 0} {0 ; 77 0} {8 s1 77 1} {8 a 77 4} {3 = 77 6} {2 { 77 8} {2 } 77 9} {0 ; 79 0} {8 s1 79 1} {8 a 79 4} {2 { 79 5} {2 } 79 6} {0 ; 80 0} {8 a 80 1} {3 . 80 2} {8 Name 80 3} {3 = 80 8} {4 "Kyle Gunger" 80 10} {0 ; 82 0} {3 ~ 82 1} {8 s1 82 2} {8 b 82 5} {3 = 82 7} {3 ~ 82 9} {8 a 82 10} {0 ; 83 0} {8 b 83 1} {3 ~ 83 2} {3 . 83 3} {8 Name 83 4} {0 ; 86 0} {8 s1 86 1} {8 c 86 4} {3 = 86 6} {2 { 86 8} {4 "" 86 9} {1 , 86 11} {4 "" 86 13} {2 } 86 15} {0 ; 90 0} {8 s1 90 1} {8 d 90 4} {2 { 90 5} {8 Message 91 4} {3 = 91 12} {4 "Message" 91 14} {1 , 91 23} {8 Name 92 4} {3 = 92 9} {4 "Name" 92 11} {2 } 93 0} {0 ; 99 0} {2 { 99 1} {2 } 99 2} {5 int 99 3} {8 a 99 7} {3 = 99 9} {2 { 99 11} {4 1, 100 2} {4 2, 100 6} {4 3, 100 10} {4 4 100 14} {2 } 101 0} {0 ; 104 0} {2 { 104 1} {2 } 104 2} {5 int 104 3} {8 a 104 7} {2 { 104 8} {4 1, 105 2} {4 2, 105 6} {4 3, 105 10} {4 4 105 14} {2 } 106 0} {2 /; 110 0} {8 s1 110 2} {2 [ 110 5} {8 s1 110 6} {2 ] 110 8} {8 return 112 2} {8 new 112 9} {8 s1 112 13} {2 { 112 15} {4 "Kyle" 112 16} {1 , 112 22} {4 "TNSL Creator" 112 24} {2 } 112 38} {2 ;/ 113 0} {2 /; 115 0} {7 if 115 3} {2 ( 115 6} {8 i 115 7} {3 == 115 9} {4 3 115 12} {2 ) 115 14} {2 ;; 118 0} {7 else 118 2} {2 ;/ 120 0} {2 /; 122 0} {7 switch 122 3} {2 ( 122 10} {8 i 122 11} {2 ) 122 12} {0 ; 124 2} {5 int 124 3} {8 t 124 7} {3 = 124 9} {4 0 124 11} {2 /; 127 2} {7 case 127 4} {4 1 127 9} {0 ; 128 4} {8 i 128 5} {3 = 128 7} {4 0 128 9} {0 ; 129 4} {8 t 129 5} {3 = 129 7} {4 2 129 9} {0 ; 130 4} {7 break 130 5} {2 ;; 132 2} {7 case 132 4} {4 2 132 9} {0 ; 133 4} {8 i 133 5} {3 = 133 7} {4 1 133 9} {0 ; 134 4} {8 t 134 5} {3 = 134 7} {4 2 134 9} {0 ; 135 4} {7 break 135 5} {2 ;; 137 2} {7 default 137 4} {0 ; 138 4} {8 i 138 5} {3 = 138 7} {4 3 138 9} {0 ; 139 4} {7 break 139 5} {2 ;/ 140 2} {2 /; 143 2} {7 if 143 5} {2 [ 143 8} {8 t 143 9} {3 == 143 11} {4 2 143 14} {2 ] 143 16} {0 ; 144 4} {8 i 144 5} {3 = 144 7} {8 t 144 9} {3 - 144 11} {8 i 144 13} {2 ;; 146 2} {7 else 146 4} {7 if 146 9} {2 [ 146 12} {8 t 146 13} {3 == 146 14} {4 3 146 16} {2 ] 146 18} {0 ; 147 4} {8 i 147 5} {3 = 147 7} {8 t 147 9} {3 + 147 10} {8 i 147 11} {2 ;/ 148 2} {2 /; 151 2} {7 case 151 4} {4 1 151 9} {0 ; 152 4} {8 i 152 5} {3 = 152 7} {4 4 152 9} {2 ;/ 153 2} {2 ;/ 154 0} {0 ; 158 0} {7 struct 158 2} {2 [ 158 9} {8 gen 158 10} {2 ] 158 13} {2 ( 158 15} {7 type 158 16} {8 T 158 21} {2 ) 158 22} {2 { 158 24} {8 T 159 2} {8 i 159 4} {2 } 160 0} {0 ; 163 0} {8 gen 163 1} {2 ( 163 4} {5 int 163 5} {2 ) 163 8} {8 j 163 10} {2 { 163 11} {4 2 163 12} {2 } 163 14} {0 ; 166 0} {2 { 166 1} {2 } 166 2} {8 gen 166 3} {2 ( 166 6} {8 gen 166 7} {2 ( 166 10} {5 int 166 11} {2 ) 166 14} {2 ) 166 15} {8 j 166 17} {2 { 166 18} {2 { 167 2} {2 { 167 3} {4 1 167 4} {2 } 167 6} {2 } 167 7} {1 , 167 8} {2 { 168 2} {2 { 168 3} {4 2 168 4} {2 } 168 6} {2 } 168 7} {1 , 168 8} {2 { 169 2} {2 { 169 3} {4 3 169 4} {2 } 169 6} {2 } 169 7} {2 } 170 0}] \ No newline at end of file diff --git a/src/main.go b/src/main.go index ec3b14b..8edd129 100644 --- a/src/main.go +++ b/src/main.go @@ -18,7 +18,7 @@ func main() { return } - fd.WriteString(fmt.Sprint(tparse.ParseFile(*inputFile))) + fd.WriteString(fmt.Sprint(tparse.TokenizeFile(*inputFile))) fd.Close() } diff --git a/src/tparse/parse.go b/src/tparse/parse.go deleted file mode 100644 index 4f25fe3..0000000 --- a/src/tparse/parse.go +++ /dev/null @@ -1,274 +0,0 @@ -package tparse - -import ( - "bufio" - "io" - "os" - "strings" - "unicode" - "unicode/utf8" -) - -// Read in a number (may be a float) -func numericLiteral(r *bufio.Reader) Token { - decimal := false - run, _, err := r.ReadRune() - - b := strings.Builder{} - - for ; err == nil; run, _, err = r.ReadRune() { - if (run == '.' || run == ',') && !decimal { - decimal = true - } else if !unicode.IsNumber(run) { - break - } - b.WriteRune(run) - } - - r.UnreadRune() - - return Token{Type: LITERAL, Data: b.String()} -} - -// Parse a string (will escape \" only in this stage) -func stringLiteral(r *bufio.Reader) Token { - escape := false - run, _, err := r.ReadRune() - - if run != '"' { - return Token{Type: LITERAL} - } - - b := strings.Builder{} - b.WriteRune(run) - run, _, err = r.ReadRune() - - for ; err == nil; run, _, err = r.ReadRune() { - b.WriteRune(run) - if run == '\\' && !escape { - escape = true - } else if run == '"' && !escape { - break - } - - } - - return Token{Type: LITERAL, Data: b.String()} -} - -// Parse a character in (escape \\ or \') -func charLiteral(r *bufio.Reader) Token { - escape := false - run, _, err := r.ReadRune() - - if run != '\'' { - return Token{Type: LITERAL} - } - - b := strings.Builder{} - b.WriteRune(run) - run, _, err = r.ReadRune() - - for ; err == nil; run, _, err = r.ReadRune() { - b.WriteRune(run) - if run == '\\' && !escape { - escape = true - } else if run == '\'' && !escape { - break - } - - } - - return Token{Type: LITERAL, Data: b.String()} -} - -// Split reserved runes into rune groups -func splitResRunes(str string, max int) []Token { - out := []Token{} - - rs := StringAsRunes(str) - s, e := 0, max - - if max > len(rs) { - e = len(rs) - } - - for e <= len(rs) && s < len(rs) { - if checkRuneGroup(RunesAsString(rs[s:e])) != -1 || e == s+1 { - tmp := RunesAsString(rs[s:e]) - out = append(out, Token{Type: checkRuneGroup(tmp), Data: tmp}) - s = e - if s+max < len(rs) { - e = s + max - } else { - e = len(rs) - } - } else if e != s+1 { - e-- - } - } - - return out -} - -// Remove block comments -func stripBlockComments(t []Token) []Token { - out := []Token{} - bc := false - for _, tok := range t { - if tok.Type == DELIMIT && tok.Data == "/#" { - bc = true - continue - } - - if tok.Type == DELIMIT && tok.Data == "#/" { - bc = false - continue - } - - if bc { - continue - } - - out = append(out, tok) - } - - return out -} - -// ParseFile tries to read a file and turn it into a series of tokens -func ParseFile(path string) []Token { - out := []Token{} - - fd, err := os.Open(path) - - if err != nil { - return out - } - - read := bufio.NewReader(fd) - - b := strings.Builder{} - - max := maxResRunes() - - for r := rune(' '); ; r, _, err = read.ReadRune() { - // If error in stream or EOF, break - if err != nil { - if err != io.EOF { - out = append(out, Token{Type: -1}) - } - break - } - - // Checking for a space - if unicode.IsSpace(r) { - if b.String() != "" { - out = append(out, Token{Type: checkToken(b.String()), Data: b.String()}) - b.Reset() - } - continue - } - - if unicode.IsNumber(r) && b.String() == "" { - read.UnreadRune() - out = append(out, numericLiteral(read)) - - continue - } - - if r == '\'' { - if b.String() != "" { - out = append(out, Token{Type: checkToken(b.String()), Data: b.String()}) - b.Reset() - } - - read.UnreadRune() - out = append(out, charLiteral(read)) - - continue - } - - if r == '"' { - if b.String() != "" { - out = append(out, Token{Type: checkToken(b.String()), Data: b.String()}) - b.Reset() - } - - read.UnreadRune() - out = append(out, stringLiteral(read)) - - continue - } - - // Checking for a rune group - if checkResRune(r) != -1 { - if b.String() != "" { - out = append(out, Token{Type: checkToken(b.String()), Data: b.String()}) - b.Reset() - } - - for ; err == nil; r, _, err = read.ReadRune() { - if checkResRune(r) == -1 { - break - } - b.WriteRune(r) - } - - read.UnreadRune() - - rgs := splitResRunes(b.String(), max) - - // Line Comments - for i, rg := range rgs { - if rg.Data == "#" { - rgs = rgs[:i] - read.ReadString('\n') - break - } - } - - out = append(out, rgs...) - - b.Reset() - - continue - } - - // Accumulate - b.WriteRune(r) - } - - return stripBlockComments(out) -} - -// StringAsRunes returns a string as a rune slice -func StringAsRunes(s string) []rune { - out := []rune{} - for i, j := 0, 0; i < len(s); i += j { - r, w := utf8.DecodeRuneInString(s[i:]) - out = append(out, r) - j = w - } - return out -} - -// BytesAsRunes returns a byte slice as a rune slice -func BytesAsRunes(b []byte) []rune { - out := []rune{} - for i, j := 0, 0; i < len(b); i += j { - r, w := utf8.DecodeRune(b[i:]) - out = append(out, r) - j = w - } - return out -} - -// RunesAsString returns a string from a slice of runes -func RunesAsString(rs []rune) string { - b := strings.Builder{} - for _, r := range rs { - b.WriteRune(r) - } - return b.String() -} diff --git a/src/tparse/token.go b/src/tparse/token.go index 712b746..386d52b 100644 --- a/src/tparse/token.go +++ b/src/tparse/token.go @@ -4,10 +4,6 @@ package tparse type Token struct { Type int Data string -} - -// Container represents a container of data -type Container struct { - Data []interface{} - Holds bool + Line int + Char int } diff --git a/src/tparse/tokenize.go b/src/tparse/tokenize.go new file mode 100644 index 0000000..79a0605 --- /dev/null +++ b/src/tparse/tokenize.go @@ -0,0 +1,306 @@ +package tparse + +import ( + "bufio" + "io" + "os" + "strings" + "unicode" + "unicode/utf8" +) + +// Read in a number (may be a float) +func numericLiteral(r *bufio.Reader, line int, char *int) Token { + decimal := false + run, _, err := r.ReadRune() + last := *char + b := strings.Builder{} + + for ; err == nil; run, _, err = r.ReadRune() { + if (run == '.' || run == ',') && !decimal { + decimal = true + } else if !unicode.IsNumber(run) { + break + } + *char++ + b.WriteRune(run) + } + + r.UnreadRune() + + return Token{Type: LITERAL, Data: b.String(), Line: line, Char: last} +} + +// Parse a string (will escape \" only in this stage) +func stringLiteral(r *bufio.Reader, line, char *int) Token { + escape := false + run, _, err := r.ReadRune() + last := *char + + if run != '"' { + return Token{Type: LITERAL} + } + + b := strings.Builder{} + b.WriteRune(run) + run, _, err = r.ReadRune() + + for ; err == nil; run, _, err = r.ReadRune() { + *char++ + b.WriteRune(run) + if run == '\\' && !escape { + escape = true + } else if (run == '"' || run == '\n') && !escape { + break + } else if escape { + if run == '\n' { + *line++ + } + escape = false + } + } + + return Token{Type: LITERAL, Data: b.String(), Line: *line, Char: last} +} + +// Parse a character in (escape \\ or \') +func charLiteral(r *bufio.Reader, line int, char *int) Token { + escape := false + run, _, err := r.ReadRune() + last := *char + + if run != '\'' { + return Token{Type: LITERAL} + } + + b := strings.Builder{} + b.WriteRune(run) + run, _, err = r.ReadRune() + + for ; err == nil; run, _, err = r.ReadRune() { + b.WriteRune(run) + *char++ + if run == '\\' && !escape { + escape = true + } else if (run == '\'' && !escape) || run == '\n' { + break + } else if escape { + escape = false + } + } + + return Token{Type: LITERAL, Data: b.String(), Line: line, Char: last} +} + +// Split reserved runes into rune groups +func splitResRunes(str string, max, line, start int) []Token { + out := []Token{} + + rs := StringAsRunes(str) + s, e := 0, max + + if max > len(rs) { + e = len(rs) + } + + for e <= len(rs) && s < len(rs) { + if checkRuneGroup(RunesAsString(rs[s:e])) != -1 || e == s+1 { + tmp := RunesAsString(rs[s:e]) + out = append(out, Token{Type: checkRuneGroup(tmp), Data: tmp, Line: line, Char: start + s}) + s = e + if s+max < len(rs) { + e = s + max + } else { + e = len(rs) + } + } else if e != s+1 { + e-- + } + } + + return out +} + +// Remove block comments +func stripBlockComments(t []Token) []Token { + out := []Token{} + bc := false + for _, tok := range t { + if tok.Type == DELIMIT && tok.Data == "/#" { + bc = true + continue + } else if tok.Type == DELIMIT && tok.Data == "#/" { + bc = false + continue + } else if bc { + continue + } + + out = append(out, tok) + } + + return out +} + +// TokenizeFile tries to read a file and turn it into a series of tokens +func TokenizeFile(path string) []Token { + out := []Token{} + + fd, err := os.Open(path) + + if err != nil { + return out + } + + read := bufio.NewReader(fd) + + b := strings.Builder{} + + max := maxResRunes() + + ln, cn, last := int(0), int(-1), int(0) + sp := false + + for r := rune(' '); ; r, _, err = read.ReadRune() { + cn++ + // If error in stream or EOF, break + if err != nil { + if err != io.EOF { + out = append(out, Token{Type: -1}) + } + break + } + + // Checking for a space + if unicode.IsSpace(r) { + sp = true + if b.String() != "" { + out = append(out, Token{Type: checkToken(b.String()), Data: b.String(), Line: ln, Char: last}) + b.Reset() + } + + // checking for a newline + if r == '\n' { + ln++ + cn = -1 + last = 0 + } + + continue + } else if sp { + last = cn + sp = false + } + + if unicode.IsNumber(r) && b.String() == "" { + read.UnreadRune() + out = append(out, numericLiteral(read, ln, &cn)) + sp = true + + continue + } + + if r == '\'' { + if b.String() != "" { + out = append(out, Token{Type: checkToken(b.String()), Data: b.String(), Line: ln, Char: last}) + b.Reset() + } + + read.UnreadRune() + out = append(out, charLiteral(read, ln, &cn)) + sp = true + + continue + } + + if r == '"' { + if b.String() != "" { + out = append(out, Token{Type: checkToken(b.String()), Data: b.String()}) + b.Reset() + } + + read.UnreadRune() + out = append(out, stringLiteral(read, &ln, &cn)) + sp = true + + continue + } + + // Checking for a rune group + if checkResRune(r) != -1 { + if b.String() != "" { + out = append(out, Token{Type: checkToken(b.String()), Data: b.String(), Line: ln, Char: last}) + b.Reset() + } + last = cn + for ; err == nil; r, _, err = read.ReadRune() { + if checkResRune(r) == -1 { + break + } + cn++ + b.WriteRune(r) + } + cn-- + + read.UnreadRune() + + rgs := splitResRunes(b.String(), max, ln, last) + + // Line Comments + for i, rg := range rgs { + if rg.Data == "#" { + rgs = rgs[:i] + read.ReadString('\n') + ln++ + cn = -1 + last = 0 + break + } + } + + out = append(out, rgs...) + + b.Reset() + + sp = true + + continue + } + + // Accumulate + b.WriteRune(r) + } + + return stripBlockComments(out) +} + +// StringAsRunes returns a string as a rune slice +func StringAsRunes(s string) []rune { + out := []rune{} + for i, j := 0, 0; i < len(s); i += j { + r, w := utf8.DecodeRuneInString(s[i:]) + out = append(out, r) + j = w + } + return out +} + +// BytesAsRunes returns a byte slice as a rune slice +func BytesAsRunes(b []byte) []rune { + out := []rune{} + for i, j := 0, 0; i < len(b); i += j { + r, w := utf8.DecodeRune(b[i:]) + out = append(out, r) + j = w + } + return out +} + +// RunesAsString returns a string from a slice of runes +func RunesAsString(rs []rune) string { + b := strings.Builder{} + for _, r := range rs { + b.WriteRune(r) + } + return b.String() +} diff --git a/src/tparse/tree.go b/src/tparse/tree.go new file mode 100644 index 0000000..41896fc --- /dev/null +++ b/src/tparse/tree.go @@ -0,0 +1,58 @@ +package tparse + +// Node represents a group of nodes or a directive +type Node struct { + SubNodes []Node + + Dir Directive +} + +// Directive represents a block or single directive +type Directive struct { + Type string + ID string + + Data []string +} + +func handleCode(tokens *[]Token, start int) (Node, int) { + out := Node{} + + return out, start +} + +func handlePre(tokens *[]Token, start int) (Node, int) { + out := Node{} + + return out, start +} + +// CreateTree takes a series of tokens and converts them into an AST +func CreateTree(tokens *[]Token, start int) Node { + out := Node{} + out.Dir = Directive{Type: "root", ID: "root"} + + var tmp Node + + for i, t := range *tokens { + switch t.Type { + case LINESEP: + if t.Data == ";" { + tmp, i = handleCode(tokens, i) + } else if t.Data == ":" { + tmp, i = handlePre(tokens, i) + } + break + case DELIMIT: + if t.Data == "/;" { + tmp, i = handleCode(tokens, i) + } else if t.Data == "/:" { + tmp, i = handlePre(tokens, i) + } + break + } + out.SubNodes = append(out.SubNodes, tmp) + } + + return out +} diff --git a/src/tparse/type.go b/src/tparse/type.go index 867f14c..dc0df10 100644 --- a/src/tparse/type.go +++ b/src/tparse/type.go @@ -33,10 +33,12 @@ const DEFWORD = 8 var RESWORD = map[string]int{ "import": PREWORD, + "bool": KEYTYPE, + "byte": KEYTYPE, + "char": KEYTYPE, + "int": KEYTYPE, - "bool": KEYTYPE, "float": KEYTYPE, - "char": KEYTYPE, "struct": KEYWORD, "type": KEYWORD, @@ -146,9 +148,6 @@ func checkResRune(r rune) int { // RESRUNES Reserved sets of reserved runes which mean something var RESRUNES = map[string]int{ - // Pre-processor block - "/:": DELIMIT, - ":/": DELIMIT, // Code block "/;": DELIMIT, ";/": DELIMIT, @@ -156,18 +155,7 @@ var RESRUNES = map[string]int{ "/#": DELIMIT, "#/": DELIMIT, - // Quick chain - "::": DELIMIT, - ":;": DELIMIT, - ":#": DELIMIT, - ";;": DELIMIT, - ";:": DELIMIT, - ";#": DELIMIT, - - "##": DELIMIT, - "#:": DELIMIT, - "#;": DELIMIT, // Boolean equ "==": AUGMENT, -- cgit v1.2.3