diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | base.tnsl | 3 | ||||
| -rw-r--r-- | examp.tnsl | 170 | ||||
| -rwxr-xr-x | gobuild.sh | 9 | ||||
| -rw-r--r-- | out.tnp | 1 | ||||
| -rw-r--r-- | src/main.go | 24 | ||||
| -rw-r--r-- | src/tparse/parse.go | 203 | ||||
| -rw-r--r-- | src/tparse/token.go | 13 | ||||
| -rw-r--r-- | src/tparse/type.go | 267 |
9 files changed, 691 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d163863 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +build/
\ No newline at end of file diff --git a/base.tnsl b/base.tnsl new file mode 100644 index 0000000..f275a82 --- /dev/null +++ b/base.tnsl @@ -0,0 +1,3 @@ + +[a!==b] +!=!== diff --git a/examp.tnsl b/examp.tnsl new file mode 100644 index 0000000..03c7135 --- /dev/null +++ b/examp.tnsl @@ -0,0 +1,170 @@ +#Comment like this + +/# + Or like this (blocks begin with /<symbol> and end with <symbol>/) +#/ + + + + +# Preprocessor directives are like this +:pack main + +# You can also create a block of them. +/: + import what.tnsl + import "what.tnsl" +:/ + + + + +# Code lines start with ; + +# pass a variable +;int s = 3 + + + + +# generic scope block + +# d does not exist + +/; # Scope + + ;int d = 1 + # d exists + ;s = d + +;/ # Scope end + +# d does not exist + + + + +# Function def: +# Any non-reserved word +# Sig: [output1, output2] (input1, input2) +# Main may have +/;main ({}string str) [int] # Doesn't matter what order the sig is in + # Main may also omit either for void sig + + + # {} represents a tnsl style array + # ~ before var represents address var + # ~ after a address var represents the data the address points at + + ;int i = 1 + ;~int j = ~i # address of int j = address of i + j~ = 2 # i = 2 # data of j = 2 + + # /loop represents the only loop in tnsl + # loop is followed by (init statements) [multi statements] + # where the first statement in multi is the test, and the once statements are only run once at the beginning of the loop + /; loop [i!==1] + # Do something + ; i = 1 + ;/ + +;/ # End main + + + +# The struct keyword is followed by [name] {values} +;struct [s1] {string Name, string Message = "Default message (c-style strings)"} + +# Most people should declare as such: +;struct [s1] { + string Name, + string Message = "Default message (c-style strings)" +} + +# When defining a new struct, use {} +;s1 a = {} +;a.Name = "Kyle Gunger" + +;~s1 b = ~a +;b~.Name # "Kyle Gunger" + +# Quick initialization +;s1 c = {"", ""} +# These come in the same order that they do in the struct, so {Name, Message} in this case. + +# You can also specify +;s1 d = { + Message = "Message", + Name = "Name" +} + + + + +# This is how arrays are defined as well. +;{}int a = { + 1, 2, 3, 4 +} + + + + +# You may also define an initializer like such: +/;s1 [s1] + # Initializer must be named same as struct, and must return one of the structs as its only output + return new s1{"Kyle", "TNSL Creator"} +;/ + +/; if (i == 3) + +# Quick define new block +;;else + +;/ + +/; switch (i) + # You can do stuff here as well + ;int t = 0 + + # Case block + /;case 1 + ;i = 0 + ;t = 2 + ;break + + ;;case 2 + ;i = 1 + ;t = 2 + ;break + + ;;default + ;i = 3 + ;break + ;/ + + # You can do stuff here too + /; if [t == 2] + i = t - i + ;/ + + # Second case block + /;case 1 + ;i = 4 + ;/ +;/ + + +/;(type T) # Generic type + ; struct [gen] { + T i + } +;/ + +;gen(int) j{2} + +;{}gen(int) j{ + {1}, + {2}, + {3} +} + diff --git a/gobuild.sh b/gobuild.sh new file mode 100755 index 0000000..604554b --- /dev/null +++ b/gobuild.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +SRCDIR=$(pwd) + +GOPATH="$GOPATH:$SRCDIR" + +go env -w GOPATH=$GOPATH + +go build -o build/parse src/main.go @@ -0,0 +1 @@ +[{0 #} {8 Comment} {8 like} {8 this} {2 /#} {8 Or} {8 like} {8 this} {2 (} {8 blocks} {8 begin} {8 with} {3 /} {3 <} {8 symbol} {3 >} {8 and} {8 end} {8 with} {3 <} {8 symbol} {3 >} {3 /} {2 )} {2 #/} {0 #} {8 Preprocessor} {8 directives} {8 are} {8 like} {8 this} {0 :} {8 pack} {8 main} {0 #} {8 You} {8 can} {8 also} {8 create} {8 a} {8 block} {8 of} {8 them} {3 .} {2 /:} {6 import} {8 what} {3 .} {8 tnsl} {6 import} {2 "} {8 what} {3 .} {8 tnsl} {2 "} {2 :/} {0 #} {8 Code} {8 lines} {8 start} {8 with} {0 ;} {0 #} {8 pass} {8 a} {8 variable} {0 ;} {5 int} {8 s} {3 =} {8 3} {0 #} {8 generic} {8 scope} {8 block} {0 #} {8 d} {8 does} {8 not} {8 exist} {2 /;} {0 #} {8 Scope} {0 ;} {5 int} {8 d} {3 =} {8 1} {0 #} {8 d} {8 exists} {0 ;} {8 s} {3 =} {8 d} {2 ;/} {0 #} {8 Scope} {8 end} {0 #} {8 d} {8 does} {8 not} {8 exist} {0 #} {8 Function} {8 def} {0 :} {0 #} {8 Any} {8 non} {3 -} {8 reserved} {8 word} {0 #} {8 Sig} {0 :} {2 [} {8 output1} {1 ,} {8 output2} {2 ]} {2 (} {8 input1} {1 ,} {8 input2} {2 )} {0 #} {8 Main} {8 may} {8 have} {2 /;} {8 main} {2 (} {2 {} {2 }} {8 string} {8 str} {2 )} {2 [} {5 int} {2 ]} {0 #} {8 Doesn} {2 '} {8 t} {8 matter} {8 what} {8 order} {8 the} {8 sig} {8 is} {8 in} {0 #} {8 Main} {8 may} {8 also} {8 omit} {8 either} {8 for} {8 void} {8 sig} {0 #} {2 {} {2 }} {8 represents} {8 a} {8 tnsl} {8 style} {8 array} {0 #} {3 ~} {8 before} {8 var} {8 represents} {8 address} {8 var} {0 #} {3 ~} {8 after} {8 a} {8 address} {8 var} {8 represents} {8 the} {8 data} {8 the} {8 address} {8 points} {8 at} {0 ;} {5 int} {8 i} {3 =} {8 1} {0 ;} {3 ~} {5 int} {8 j} {3 =} {3 ~} {8 i} {0 #} {8 address} {8 of} {5 int} {8 j} {3 =} {8 address} {8 of} {8 i} {8 j} {3 ~} {3 =} {8 2} {0 #} {8 i} {3 =} {8 2} {0 #} {8 data} {8 of} {8 j} {3 =} {8 2} {0 #} {3 /} {7 loop} {8 represents} {8 the} {8 only} {7 loop} {8 in} {8 tnsl} {0 #} {7 loop} {8 is} {8 followed} {8 by} {2 (} {8 init} {8 statements} {2 )} {2 [} {8 multi} {8 statements} {2 ]} {0 #} {8 where} {8 the} {8 first} {8 statement} {8 in} {8 multi} {8 is} {8 the} {8 test} {1 ,} {8 and} {8 the} {8 once} {8 statements} {8 are} {8 only} {8 run} {8 once} {8 at} {8 the} {8 beginning} {8 of} {8 the} {7 loop} {2 /;} {7 loop} {2 [} {8 i} {3 !==} {8 1} {2 ]} {0 #} {8 Do} {8 something} {0 ;} {8 i} {3 =} {8 1} {2 ;/} {2 ;/} {0 #} {8 End} {8 main} {0 #} {8 The} {7 struct} {8 keyword} {8 is} {8 followed} {8 by} {2 [} {8 name} {2 ]} {2 {} {8 values} {2 }} {0 ;} {7 struct} {2 [} {8 s1} {2 ]} {2 {} {8 string} {8 Name} {1 ,} {8 string} {8 Message} {3 =} {2 "} {8 Default} {8 message} {2 (} {8 c} {3 -} {8 style} {8 strings} {2 )} {2 "} {2 }} {0 #} {8 Most} {8 people} {8 should} {8 declare} {8 as} {8 such} {0 :} {0 ;} {7 struct} {2 [} {8 s1} {2 ]} {2 {} {8 string} {8 Name} {1 ,} {8 string} {8 Message} {3 =} {2 "} {8 Default} {8 message} {2 (} {8 c} {3 -} {8 style} {8 strings} {2 )} {2 "} {2 }} {0 #} {8 When} {8 defining} {8 a} {8 new} {7 struct} {1 ,} {8 use} {2 {} {2 }} {0 ;} {8 s1} {8 a} {3 =} {2 {} {2 }} {0 ;} {8 a} {3 .} {8 Name} {3 =} {2 "} {8 Kyle} {8 Gunger} {2 "} {0 ;} {3 ~} {8 s1} {8 b} {3 =} {3 ~} {8 a} {0 ;} {8 b} {3 ~} {3 .} {8 Name} {0 #} {2 "} {8 Kyle} {8 Gunger} {2 "} {0 #} {8 Quick} {8 initialization} {0 ;} {8 s1} {8 c} {3 =} {2 {} {2 "} {2 "} {1 ,} {2 "} {2 "} {2 }} {0 #} {8 These} {8 come} {8 in} {8 the} {8 same} {8 order} {8 that} {8 they} {8 do} {8 in} {8 the} {7 struct} {1 ,} {8 so} {2 {} {8 Name} {1 ,} {8 Message} {2 }} {8 in} {8 this} {7 case} {3 .} {0 #} {8 You} {8 can} {8 also} {8 specify} {0 ;} {8 s1} {8 d} {3 =} {2 {} {8 Message} {3 =} {2 "} {8 Message} {2 "} {1 ,} {8 Name} {3 =} {2 "} {8 Name} {2 "} {2 }} {0 #} {8 This} {8 is} {8 how} {8 arrays} {8 are} {8 defined} {8 as} {8 well} {3 .} {0 ;} {2 {} {2 }} {5 int} {8 a} {3 =} {2 {} {8 1} {1 ,} {8 2} {1 ,} {8 3} {1 ,} {8 4} {2 }} {0 #} {8 You} {8 may} {8 also} {8 define} {8 an} {8 initializer} {8 like} {8 such} {0 :} {2 /;} {8 s1} {2 [} {8 s1} {2 ]} {0 #} {8 Initializer} {8 must} {8 be} {8 named} {8 same} {8 as} {7 struct} {1 ,} {8 and} {8 must} {8 return} {8 one} {8 of} {8 the} {8 structs} {8 as} {8 its} {8 only} {8 output} {8 return} {8 new} {8 s1} {2 {} {2 "} {8 Kyle} {2 "} {1 ,} {2 "} {8 TNSL} {8 Creator} {2 "} {2 }} {2 ;/} {2 /;} {7 if} {2 (} {8 i} {3 ==} {8 3} {2 )} {0 #} {8 Quick} {8 define} {8 new} {8 block} {2 ;;} {7 else} {2 ;/} {2 /;} {7 switch} {2 (} {8 i} {2 )} {0 #} {8 You} {8 can} {8 do} {8 stuff} {8 here} {8 as} {8 well} {0 ;} {5 int} {8 t} {3 =} {8 0} {0 #} {8 Case} {8 block} {2 /;} {7 case} {8 1} {0 ;} {8 i} {3 =} {8 0} {0 ;} {8 t} {3 =} {8 2} {0 ;} {7 break} {2 ;;} {7 case} {8 2} {0 ;} {8 i} {3 =} {8 1} {0 ;} {8 t} {3 =} {8 2} {0 ;} {7 break} {2 ;;} {7 default} {0 ;} {8 i} {3 =} {8 3} {0 ;} {7 break} {2 ;/} {0 #} {8 You} {8 can} {8 do} {8 stuff} {8 here} {8 too} {2 /;} {7 if} {2 [} {8 t} {3 ==} {8 2} {2 ]} {8 i} {3 =} {8 t} {3 -} {8 i} {2 ;/} {0 #} {8 Second} {7 case} {8 block} {2 /;} {7 case} {8 1} {0 ;} {8 i} {3 =} {8 4} {2 ;/} {2 ;/} {2 /;} {2 (} {7 type} {8 T} {2 )} {0 #} {8 Generic} {7 type} {0 ;} {7 struct} {2 [} {8 gen} {2 ]} {2 {} {8 T} {8 i} {2 }} {2 ;/} {0 ;} {8 gen} {2 (} {5 int} {2 )} {8 j} {2 {} {8 2} {2 }} {0 ;} {2 {} {2 }} {8 gen} {2 (} {5 int} {2 )} {8 j} {2 {} {2 {} {8 1} {2 }} {1 ,} {2 {} {8 2} {2 }} {1 ,} {2 {} {8 3} {2 }} {2 }}]
\ No newline at end of file diff --git a/src/main.go b/src/main.go new file mode 100644 index 0000000..ec3b14b --- /dev/null +++ b/src/main.go @@ -0,0 +1,24 @@ +package main + +import "fmt" +import "tparse" +import "flag" +import "os" + +func main() { + inputFile := flag.String("in", "", "The file to parse") + outputFile := flag.String("out", "out.tnp", "The file to store the parse in") + + flag.Parse() + + fd, err := os.Create(*outputFile) + + if err != nil { + fmt.Println(err.Error()) + return + } + + fd.WriteString(fmt.Sprint(tparse.ParseFile(*inputFile))) + + fd.Close() +} diff --git a/src/tparse/parse.go b/src/tparse/parse.go new file mode 100644 index 0000000..e9e1ee6 --- /dev/null +++ b/src/tparse/parse.go @@ -0,0 +1,203 @@ +package tparse + +import ( + "bufio" + "io" + "os" + "strings" + "unicode" + "unicode/utf8" +) + +// Read in a number (may be a float) +func numericLiteral(r *bufio.Reader) Token { + decimal := false + run, _, err := r.ReadRune() + + b := strings.Builder{} + + for ; err == nil; run, _, err = r.ReadRune() { + if (run == '.' || run == ',') && !decimal { + decimal = true + } else if !unicode.IsNumber(run) { + break + } + b.WriteRune(run) + } + + r.UnreadRune() + + return Token{Type: LITERAL, Data: b.String()} +} + +// Parse a string (will escape \" only in this stage) +func stringLiteral(r *bufio.Reader) Token { + escape := false + run, _, err := r.ReadRune() + + if run != '"' { + return Token{Type: LITERAL} + } + + b := strings.Builder{} + + for ; err == nil; run, _, err = r.ReadRune() { + b.WriteRune(run) + if run == '\\' && !escape { + escape = true + } else if run == '"' && !escape { + break + } + + } + + return Token{Type: LITERAL, Data: b.String()} +} + +// Parse a character in (escape \\ or \') +func charLiteral(r *bufio.Reader) Token { + escape := false + run, _, err := r.ReadRune() + + if run != '\'' { + return Token{Type: LITERAL} + } + + b := strings.Builder{} + + for ; err == nil; run, _, err = r.ReadRune() { + b.WriteRune(run) + if run == '\\' && !escape { + escape = true + } else if run == '\'' && !escape { + break + } + + } + + return Token{Type: LITERAL, Data: b.String()} +} + +// Split reserved runes into rune groups +func splitResRunes(str string, max int) []Token { + out := []Token{} + + rs := StringAsRunes(str) + s, e := 0, max + + if max > len(rs) { + e = len(rs) + } + + for e <= len(rs) && s < len(rs) { + if checkRuneGroup(RunesAsString(rs[s:e])) != -1 || e == s+1 { + tmp := RunesAsString(rs[s:e]) + out = append(out, Token{Type: checkRuneGroup(tmp), Data: tmp}) + s = e + if s+max < len(rs) { + e = s + max + } else { + e = len(rs) + } + } else if e != s+1 { + e-- + } + } + + return out +} + +// ParseFile tries to read a file and turn it into a series of tokens +func ParseFile(path string) []Token { + out := []Token{} + + fd, err := os.Open(path) + + if err != nil { + return out + } + + read := bufio.NewReader(fd) + + b := strings.Builder{} + + max := maxResRunes() + + for r := ' '; ; r, _, err = read.ReadRune() { + // If error in stream or EOF, break + if err != nil { + if err != io.EOF { + out = append(out, Token{Type: -1}) + } + break + } + + // Checking for a space + if unicode.IsSpace(r) { + if b.String() != "" { + out = append(out, Token{Type: checkToken(b.String()), Data: b.String()}) + b.Reset() + } + continue + } + + // Checking for a rune group + if checkResRune(r) != -1 { + if b.String() != "" { + out = append(out, Token{Type: checkToken(b.String()), Data: b.String()}) + b.Reset() + } + + for ; err == nil; r, _, err = read.ReadRune() { + if checkResRune(r) == -1 { + break + } + b.WriteRune(r) + } + + read.UnreadRune() + + out = append(out, splitResRunes(b.String(), max)...) + + b.Reset() + + continue + } + + // Accumulate + b.WriteRune(r) + } + + return out +} + +// StringAsRunes returns a string as a rune slice +func StringAsRunes(s string) []rune { + out := []rune{} + for i, j := 0, 0; i < len(s); i += j { + r, w := utf8.DecodeRuneInString(s[i:]) + out = append(out, r) + j = w + } + return out +} + +// BytesAsRunes returns a byte slice as a rune slice +func BytesAsRunes(b []byte) []rune { + out := []rune{} + for i, j := 0, 0; i < len(b); i += j { + r, w := utf8.DecodeRune(b[i:]) + out = append(out, r) + j = w + } + return out +} + +// RunesAsString returns a string from a slice of runes +func RunesAsString(rs []rune) string { + b := strings.Builder{} + for _, r := range rs { + b.WriteRune(r) + } + return b.String() +} diff --git a/src/tparse/token.go b/src/tparse/token.go new file mode 100644 index 0000000..712b746 --- /dev/null +++ b/src/tparse/token.go @@ -0,0 +1,13 @@ +package tparse + +// Token represents a token in a program +type Token struct { + Type int + Data string +} + +// Container represents a container of data +type Container struct { + Data []interface{} + Holds bool +} diff --git a/src/tparse/type.go b/src/tparse/type.go new file mode 100644 index 0000000..ee8b5e7 --- /dev/null +++ b/src/tparse/type.go @@ -0,0 +1,267 @@ +package tparse + +import () + +// LINESEP represents a line seperator +const LINESEP = 0 + +// ARGNSEP represents an inline seperator +const ARGNSEP = 1 + +// DELIMIT represents an opening or closing delimiter +const DELIMIT = 2 + +// AUGMENT represents an augmentation +const AUGMENT = 3 + +// LITERAL represents a literal value +const LITERAL = 4 + +// KEYTYPE represents a built in type +const KEYTYPE = 5 + +// PREWORD represents a reserved pre-processor directive +const PREWORD = 6 + +// KEYWORD represents a reserved word +const KEYWORD = 7 + +// DEFWORD represents a user-defined word such as a variable, method, or struct +const DEFWORD = 8 + +// RESWORD represents all the reserved words and what type of tokens they are. +var RESWORD = map[string]int{ + "import": PREWORD, + + "int": KEYTYPE, + "bool": KEYTYPE, + "float": KEYTYPE, + "char": KEYTYPE, + + "struct": KEYWORD, + "type": KEYWORD, + + "loop": KEYWORD, + "continue": KEYWORD, + "break": KEYWORD, + + "switch": KEYWORD, + "case": KEYWORD, + "default": KEYWORD, + + "label": KEYWORD, + "goto": KEYWORD, + + "if": KEYWORD, + "else": KEYWORD, + + "const": KEYWORD, + "static": KEYWORD, + "volatile": KEYWORD, + + "true": LITERAL, + "false": LITERAL, + + "null": LITERAL, +} + +func checkResWord(s string) int { + out, prs := RESWORD[s] + if !prs { + return -1 + } + return out +} + +// RESRUNE represents all the reserved runes +var RESRUNE = map[rune]int{ + // Starting condition open + '(': DELIMIT, + // Starting condition close + ')': DELIMIT, + // Ending condition open + '[': DELIMIT, + // Ending condition close + ']': DELIMIT, + // Array mark open + '{': DELIMIT, + // Array mark close + '}': DELIMIT, + // String literal + '\'': DELIMIT, + // String + '"': DELIMIT, + + // Start of pre-proc directive + ':': LINESEP, + // Start of line + ';': LINESEP, + // Start of comment + '#': LINESEP, + + // Seperate arguments + ',': ARGNSEP, + + // Assignment + '=': AUGMENT, + + // Get + '.': AUGMENT, + + // Bitwise and + '&': AUGMENT, + // Bitwise or + '|': AUGMENT, + // Bitwise xor + '^': AUGMENT, + + // Greater than + '>': AUGMENT, + // Less than + '<': AUGMENT, + + // Not (prefix any bool or bitwise) + '!': AUGMENT, + + // Addition + '+': AUGMENT, + // Subtraction + '-': AUGMENT, + // Multiplication + '*': AUGMENT, + // Division + '/': AUGMENT, + // Mod + '%': AUGMENT, + + // Address of + '~': AUGMENT, + // De-ref + '_': AUGMENT, +} + +func checkResRune(r rune) int { + out, prs := RESRUNE[r] + if !prs { + return -1 + } + return out +} + +// RESRUNES Reserved sets of reserved runes which mean something +var RESRUNES = map[string]int{ + // Pre-processor block + "/:": DELIMIT, + ":/": DELIMIT, + // Code block + "/;": DELIMIT, + ";/": DELIMIT, + // Comment block + "/#": DELIMIT, + "#/": DELIMIT, + + // Quick chain + "::": DELIMIT, + ":;": DELIMIT, + ":#": DELIMIT, + + ";;": DELIMIT, + ";:": DELIMIT, + ";#": DELIMIT, + + "##": DELIMIT, + "#:": DELIMIT, + "#;": DELIMIT, + + // Boolean equ + "==": AUGMENT, + // Boolean and + "&&": AUGMENT, + // Boolean or + "||": AUGMENT, + + // Bitwise l-shift + "<<": AUGMENT, + // Bitwise r-shift + ">>": AUGMENT, + + // PREaugmented augmentors + "&=": AUGMENT, + "|=": AUGMENT, + "^=": AUGMENT, + "!=": AUGMENT, + "+=": AUGMENT, + "-=": AUGMENT, + "*=": AUGMENT, + "/=": AUGMENT, + "%=": AUGMENT, + "~=": AUGMENT, + "_=": AUGMENT, + + // POSTaugmented augmentors + "!&": AUGMENT, + "!|": AUGMENT, + "!^": AUGMENT, + "!==": AUGMENT, + "!&&": AUGMENT, + "!||": AUGMENT, + "!>": AUGMENT, + "!<": AUGMENT, + ">==": AUGMENT, + "<==": AUGMENT, +} + +func maxResRunes() int { + max := 0 + + for k := range RESRUNES { + if len(k) > max { + max = len(k) + } + } + + return max +} + +func checkRuneGroup(s string) int { + rs := StringAsRunes(s) + + if len(rs) == 1 { + return checkResRune(rs[0]) + } + + out, prs := RESRUNES[s] + if !prs { + return -1 + } + return out +} + +func checkToken(s string) int { + rs := StringAsRunes(s) + + if len(rs) == 0 { + return -1 + } + + if len(rs) == 1 { + o := checkResRune(rs[0]) + if o > -1 { + return o + } + } + + o := checkResWord(s) + + if o > -1 { + return o + } + + o = checkRuneGroup(s) + + if o > -1 { + return o + } + + return DEFWORD +} |