strips/pork
2
0
Fork 0
mirror of https://github.com/GayPizzaSpecifications/pork.git synced 2025-08-03 13:11:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

gradle: 8.4 and parser: lazy tokenization

Browse Source
This commit is contained in:
Alex Zenla
2023-10-13 08:56:04 -07:00
parent 5078f38f61
commit e96bcd8754
8 changed files with 59 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt
View File

@ -4,15 +4,21 @@ import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument
import gay.pizza.dough.fs.PlatformFsProvider
import gay.pizza.pork.minimal.FileTool
import gay.pizza.pork.parser.TokenType
class TokenizeCommand : CliktCommand(help = "Tokenize Compilation Unit", name = "tokenize") {
val path by argument("file")
override fun run() {
val tool = FileTool(PlatformFsProvider.resolve(path))
val tokenStream = tool.tokenize()
for (token in tokenStream.tokens) {
println("${token.sourceIndex.index} ${token.type.name} '${sanitize(token.text)}'")
val tokenSource = tool.tokenize()
while (true) {
val token = tokenSource.next()
println("${token.sourceIndex} ${token.type.name} '${sanitize(token.text)}'")
tokenSource.peekTypeAhead(5)
if (token.type == TokenType.EndOfFile) {
break
}
}
}