parser: lazy support

This commit is contained in:
alex committed 2023-10-14 03:28:07 -07:00
1 parent e96bcd8754
commit 9338b01b48
25 files changed
+159 -118

No files matched your search

@@ -15,7 +15,6 @@ class TokenizeCommand : CliktCommand(help = "Tokenize Compilation Unit", name =
while (true) {
val token = tokenSource.next()
println("${token.sourceIndex} ${token.type.name} '${sanitize(token.text)}'")
tokenSource.peekTypeAhead(5)
if (token.type == TokenType.EndOfFile) {
break
}