diff options
Diffstat (limited to 'src/tparse/tokenize.go')
| -rw-r--r-- | src/tparse/tokenize.go | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/src/tparse/tokenize.go b/src/tparse/tokenize.go index 7f57576..2d49376 100644 --- a/src/tparse/tokenize.go +++ b/src/tparse/tokenize.go @@ -235,6 +235,8 @@ func TokenizeFile(path string) []Token { if err != nil { if err != io.EOF { out = append(out, Token{Type: -1}) + } else { + out = append(out, Token{Type: checkToken(b.String(), pre), Data: b.String(), Line: ln, Char: last}) } break } |