summaryrefslogtreecommitdiff
path: root/src/tparse
diff options
context:
space:
mode:
Diffstat (limited to 'src/tparse')
-rw-r--r--src/tparse/token.go8
-rw-r--r--src/tparse/tokenize.go (renamed from src/tparse/parse.go)90
-rw-r--r--src/tparse/tree.go58
-rw-r--r--src/tparse/type.go20
4 files changed, 125 insertions, 51 deletions
diff --git a/src/tparse/token.go b/src/tparse/token.go
index 712b746..386d52b 100644
--- a/src/tparse/token.go
+++ b/src/tparse/token.go
@@ -4,10 +4,6 @@ package tparse
type Token struct {
Type int
Data string
-}
-
-// Container represents a container of data
-type Container struct {
- Data []interface{}
- Holds bool
+ Line int
+ Char int
}
diff --git a/src/tparse/parse.go b/src/tparse/tokenize.go
index 4f25fe3..79a0605 100644
--- a/src/tparse/parse.go
+++ b/src/tparse/tokenize.go
@@ -10,10 +10,10 @@ import (
)
// Read in a number (may be a float)
-func numericLiteral(r *bufio.Reader) Token {
+func numericLiteral(r *bufio.Reader, line int, char *int) Token {
decimal := false
run, _, err := r.ReadRune()
-
+ last := *char
b := strings.Builder{}
for ; err == nil; run, _, err = r.ReadRune() {
@@ -22,18 +22,20 @@ func numericLiteral(r *bufio.Reader) Token {
} else if !unicode.IsNumber(run) {
break
}
+ *char++
b.WriteRune(run)
}
r.UnreadRune()
- return Token{Type: LITERAL, Data: b.String()}
+ return Token{Type: LITERAL, Data: b.String(), Line: line, Char: last}
}
// Parse a string (will escape \" only in this stage)
-func stringLiteral(r *bufio.Reader) Token {
+func stringLiteral(r *bufio.Reader, line, char *int) Token {
escape := false
run, _, err := r.ReadRune()
+ last := *char
if run != '"' {
return Token{Type: LITERAL}
@@ -44,22 +46,28 @@ func stringLiteral(r *bufio.Reader) Token {
run, _, err = r.ReadRune()
for ; err == nil; run, _, err = r.ReadRune() {
+ *char++
b.WriteRune(run)
if run == '\\' && !escape {
escape = true
- } else if run == '"' && !escape {
+ } else if (run == '"' || run == '\n') && !escape {
break
+ } else if escape {
+ if run == '\n' {
+ *line++
+ }
+ escape = false
}
-
}
- return Token{Type: LITERAL, Data: b.String()}
+ return Token{Type: LITERAL, Data: b.String(), Line: *line, Char: last}
}
// Parse a character in (escape \\ or \')
-func charLiteral(r *bufio.Reader) Token {
+func charLiteral(r *bufio.Reader, line int, char *int) Token {
escape := false
run, _, err := r.ReadRune()
+ last := *char
if run != '\'' {
return Token{Type: LITERAL}
@@ -71,19 +79,21 @@ func charLiteral(r *bufio.Reader) Token {
for ; err == nil; run, _, err = r.ReadRune() {
b.WriteRune(run)
+ *char++
if run == '\\' && !escape {
escape = true
- } else if run == '\'' && !escape {
+ } else if (run == '\'' && !escape) || run == '\n' {
break
+ } else if escape {
+ escape = false
}
-
}
- return Token{Type: LITERAL, Data: b.String()}
+ return Token{Type: LITERAL, Data: b.String(), Line: line, Char: last}
}
// Split reserved runes into rune groups
-func splitResRunes(str string, max int) []Token {
+func splitResRunes(str string, max, line, start int) []Token {
out := []Token{}
rs := StringAsRunes(str)
@@ -96,7 +106,7 @@ func splitResRunes(str string, max int) []Token {
for e <= len(rs) && s < len(rs) {
if checkRuneGroup(RunesAsString(rs[s:e])) != -1 || e == s+1 {
tmp := RunesAsString(rs[s:e])
- out = append(out, Token{Type: checkRuneGroup(tmp), Data: tmp})
+ out = append(out, Token{Type: checkRuneGroup(tmp), Data: tmp, Line: line, Char: start + s})
s = e
if s+max < len(rs) {
e = s + max
@@ -119,14 +129,10 @@ func stripBlockComments(t []Token) []Token {
if tok.Type == DELIMIT && tok.Data == "/#" {
bc = true
continue
- }
-
- if tok.Type == DELIMIT && tok.Data == "#/" {
+ } else if tok.Type == DELIMIT && tok.Data == "#/" {
bc = false
continue
- }
-
- if bc {
+ } else if bc {
continue
}
@@ -136,8 +142,8 @@ func stripBlockComments(t []Token) []Token {
return out
}
-// ParseFile tries to read a file and turn it into a series of tokens
-func ParseFile(path string) []Token {
+// TokenizeFile tries to read a file and turn it into a series of tokens
+func TokenizeFile(path string) []Token {
out := []Token{}
fd, err := os.Open(path)
@@ -152,7 +158,11 @@ func ParseFile(path string) []Token {
max := maxResRunes()
+ ln, cn, last := int(0), int(-1), int(0)
+ sp := false
+
for r := rune(' '); ; r, _, err = read.ReadRune() {
+ cn++
// If error in stream or EOF, break
if err != nil {
if err != io.EOF {
@@ -163,28 +173,42 @@ func ParseFile(path string) []Token {
// Checking for a space
if unicode.IsSpace(r) {
+ sp = true
if b.String() != "" {
- out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+ out = append(out, Token{Type: checkToken(b.String()), Data: b.String(), Line: ln, Char: last})
b.Reset()
}
+
+ // checking for a newline
+ if r == '\n' {
+ ln++
+ cn = -1
+ last = 0
+ }
+
continue
+ } else if sp {
+ last = cn
+ sp = false
}
if unicode.IsNumber(r) && b.String() == "" {
read.UnreadRune()
- out = append(out, numericLiteral(read))
+ out = append(out, numericLiteral(read, ln, &cn))
+ sp = true
continue
}
if r == '\'' {
if b.String() != "" {
- out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+ out = append(out, Token{Type: checkToken(b.String()), Data: b.String(), Line: ln, Char: last})
b.Reset()
}
read.UnreadRune()
- out = append(out, charLiteral(read))
+ out = append(out, charLiteral(read, ln, &cn))
+ sp = true
continue
}
@@ -196,7 +220,8 @@ func ParseFile(path string) []Token {
}
read.UnreadRune()
- out = append(out, stringLiteral(read))
+ out = append(out, stringLiteral(read, &ln, &cn))
+ sp = true
continue
}
@@ -204,26 +229,31 @@ func ParseFile(path string) []Token {
// Checking for a rune group
if checkResRune(r) != -1 {
if b.String() != "" {
- out = append(out, Token{Type: checkToken(b.String()), Data: b.String()})
+ out = append(out, Token{Type: checkToken(b.String()), Data: b.String(), Line: ln, Char: last})
b.Reset()
}
-
+ last = cn
for ; err == nil; r, _, err = read.ReadRune() {
if checkResRune(r) == -1 {
break
}
+ cn++
b.WriteRune(r)
}
+ cn--
read.UnreadRune()
- rgs := splitResRunes(b.String(), max)
+ rgs := splitResRunes(b.String(), max, ln, last)
// Line Comments
for i, rg := range rgs {
if rg.Data == "#" {
rgs = rgs[:i]
read.ReadString('\n')
+ ln++
+ cn = -1
+ last = 0
break
}
}
@@ -232,6 +262,8 @@ func ParseFile(path string) []Token {
b.Reset()
+ sp = true
+
continue
}
diff --git a/src/tparse/tree.go b/src/tparse/tree.go
new file mode 100644
index 0000000..41896fc
--- /dev/null
+++ b/src/tparse/tree.go
@@ -0,0 +1,58 @@
+package tparse
+
+// Node represents a group of nodes or a directive
+type Node struct {
+ SubNodes []Node
+
+ Dir Directive
+}
+
+// Directive represents a block or single directive
+type Directive struct {
+ Type string
+ ID string
+
+ Data []string
+}
+
+func handleCode(tokens *[]Token, start int) (Node, int) {
+ out := Node{}
+
+ return out, start
+}
+
+func handlePre(tokens *[]Token, start int) (Node, int) {
+ out := Node{}
+
+ return out, start
+}
+
+// CreateTree takes a series of tokens and converts them into an AST
+func CreateTree(tokens *[]Token, start int) Node {
+ out := Node{}
+ out.Dir = Directive{Type: "root", ID: "root"}
+
+ var tmp Node
+
+ for i, t := range *tokens {
+ switch t.Type {
+ case LINESEP:
+ if t.Data == ";" {
+ tmp, i = handleCode(tokens, i)
+ } else if t.Data == ":" {
+ tmp, i = handlePre(tokens, i)
+ }
+ break
+ case DELIMIT:
+ if t.Data == "/;" {
+ tmp, i = handleCode(tokens, i)
+ } else if t.Data == "/:" {
+ tmp, i = handlePre(tokens, i)
+ }
+ break
+ }
+ out.SubNodes = append(out.SubNodes, tmp)
+ }
+
+ return out
+}
diff --git a/src/tparse/type.go b/src/tparse/type.go
index 867f14c..dc0df10 100644
--- a/src/tparse/type.go
+++ b/src/tparse/type.go
@@ -33,10 +33,12 @@ const DEFWORD = 8
var RESWORD = map[string]int{
"import": PREWORD,
+ "bool": KEYTYPE,
+ "byte": KEYTYPE,
+ "char": KEYTYPE,
+
"int": KEYTYPE,
- "bool": KEYTYPE,
"float": KEYTYPE,
- "char": KEYTYPE,
"struct": KEYWORD,
"type": KEYWORD,
@@ -146,9 +148,6 @@ func checkResRune(r rune) int {
// RESRUNES Reserved sets of reserved runes which mean something
var RESRUNES = map[string]int{
- // Pre-processor block
- "/:": DELIMIT,
- ":/": DELIMIT,
// Code block
"/;": DELIMIT,
";/": DELIMIT,
@@ -156,18 +155,7 @@ var RESRUNES = map[string]int{
"/#": DELIMIT,
"#/": DELIMIT,
- // Quick chain
- "::": DELIMIT,
- ":;": DELIMIT,
- ":#": DELIMIT,
-
";;": DELIMIT,
- ";:": DELIMIT,
- ";#": DELIMIT,
-
- "##": DELIMIT,
- "#:": DELIMIT,
- "#;": DELIMIT,
// Boolean equ
"==": AUGMENT,
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-07 14:48:43 +0000