summaryrefslogtreecommitdiff
path: root/src/tparse
diff options
context:
space:
mode:
Diffstat (limited to 'src/tparse')
-rw-r--r--src/tparse/tokenize.go2
1 files changed, 2 insertions, 0 deletions
diff --git a/src/tparse/tokenize.go b/src/tparse/tokenize.go
index 7f57576..2d49376 100644
--- a/src/tparse/tokenize.go
+++ b/src/tparse/tokenize.go
@@ -235,6 +235,8 @@ func TokenizeFile(path string) []Token {
if err != nil {
if err != io.EOF {
out = append(out, Token{Type: -1})
+ } else {
+ out = append(out, Token{Type: checkToken(b.String(), pre), Data: b.String(), Line: ln, Char: last})
}
break
}