diff options
| author | Kyle Gunger <kgunger12@gmail.com> | 2022-03-15 15:26:35 -0400 |
|---|---|---|
| committer | Kyle Gunger <kgunger12@gmail.com> | 2022-03-15 15:26:35 -0400 |
| commit | d581c7503f20a8b560a98e521daec632c763c56c (patch) | |
| tree | f6e2e5f8d0f40734fb3a124aee4734a0c99e4bd1 /src/tparse/tokenize.go | |
| parent | a2f399fdd922495dcac60f4f4601a9729b10e114 (diff) | |
[HOTFIX] Fix tokens not generating at eof
Diffstat (limited to 'src/tparse/tokenize.go')
| -rw-r--r-- | src/tparse/tokenize.go | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/src/tparse/tokenize.go b/src/tparse/tokenize.go index 7f57576..2d49376 100644 --- a/src/tparse/tokenize.go +++ b/src/tparse/tokenize.go @@ -235,6 +235,8 @@ func TokenizeFile(path string) []Token { if err != nil { if err != io.EOF { out = append(out, Token{Type: -1}) + } else { + out = append(out, Token{Type: checkToken(b.String(), pre), Data: b.String(), Line: ln, Char: last}) } break } |