diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2022-04-04 00:56:10 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2022-04-04 00:56:10 -0400 |
| commit | 9d2b6e985b008492fc99a79f1f674cc2ea83be2f (patch) | |
| tree | dbb6b292502961f1d58cca66543537685f1ca608 /src/tparse | |
| parent | 903db77b64d9ad4754af367d9d12cc5ee6fc07c4 (diff) | |
Fix tokenizer bug relating to end of files
Diffstat (limited to 'src/tparse')
| -rw-r--r-- | src/tparse/tokenize.go | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/src/tparse/tokenize.go b/src/tparse/tokenize.go index 2d49376..8eab15a 100644 --- a/src/tparse/tokenize.go +++ b/src/tparse/tokenize.go @@ -235,7 +235,7 @@ func TokenizeFile(path string) []Token { if err != nil { if err != io.EOF { out = append(out, Token{Type: -1}) - } else { + } else if b.String() != "" { out = append(out, Token{Type: checkToken(b.String(), pre), Data: b.String(), Line: ln, Char: last}) } break |