diff --git a/build.gradle.kts b/build.gradle.kts index 65d0a5c..315b81b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,5 +3,5 @@ plugins { } tasks.withType { - gradleVersion = "8.3" + gradleVersion = "8.4" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 033e24c..7f93135 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ac72c34..3fa8f86 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index fcb6fca..0adc8e1 100755 --- a/gradlew +++ b/gradlew @@ -83,7 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt b/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt index d98c8a3..7106945 100644 --- a/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt +++ b/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt @@ -22,14 +22,14 @@ abstract class Tool { val rootImportLocator: ImportLocator get() = ImportLocator("local", rootFilePath()) - fun tokenize(): TokenStream = - Tokenizer(createCharSource()).stream() + fun tokenize(): LazyTokenSource = + LazyTokenSource(Tokenizer(createCharSource())) fun parse(attribution: NodeAttribution = DiscardNodeAttribution): CompilationUnit = - Parser(TokenStreamSource(tokenize()), attribution).parseCompilationUnit() + Parser(TokenStreamSource(tokenize().streamAllRemainingTokens()), attribution).parseCompilationUnit() fun highlight(scheme: HighlightScheme): List = - Highlighter(scheme).highlight(tokenize()) + Highlighter(scheme).highlight(tokenize().streamAllRemainingTokens()) fun reprint(): String = buildString { visit(Printer(this)) } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/LazyTokenSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/LazyTokenSource.kt new file mode 100644 index 0000000..39eba6a --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/LazyTokenSource.kt @@ -0,0 +1,38 @@ +package gay.pizza.pork.parser + +class LazyTokenSource(val tokenizer: Tokenizer) : TokenSource { + private val queue = mutableListOf() + private var index = 0 + override val currentIndex: Int + get() = index + + override fun next(): Token { + index++ + if (queue.isNotEmpty()) { + return queue.removeFirst() + } + return tokenizer.next() + } + + override fun peek(): Token { + if (queue.isNotEmpty()) { + return queue.first() + } + val token = tokenizer.next() + queue.add(token) + return token + } + + override fun peekTypeAhead(ahead: Int): TokenType { + wantAtLeast(ahead + 1) + return queue[ahead].type + } + + private fun wantAtLeast(ahead: Int) { + if (queue.size < ahead) { + for (i in 1..ahead) { + queue.add(tokenizer.next()) + } + } + } +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt index 75977a0..221d999 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt @@ -14,6 +14,9 @@ interface TokenSource : PeekableSource { return tokens } + fun streamAllRemainingTokens(): TokenStream = + TokenStream(consumeAllRemainingTokens().filter { !TokenType.ParserIgnoredTypes.contains(it.type) }) + fun ignoringParserIgnoredTypes(): TokenSource = - TokenStreamSource(TokenStream(consumeAllRemainingTokens().filter { !TokenType.ParserIgnoredTypes.contains(it.type) })) + TokenStreamSource(streamAllRemainingTokens()) } diff --git a/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt b/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt index 40825ae..d5b6a13 100644 --- a/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt +++ b/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt @@ -4,15 +4,21 @@ import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import gay.pizza.dough.fs.PlatformFsProvider import gay.pizza.pork.minimal.FileTool +import gay.pizza.pork.parser.TokenType class TokenizeCommand : CliktCommand(help = "Tokenize Compilation Unit", name = "tokenize") { val path by argument("file") override fun run() { val tool = FileTool(PlatformFsProvider.resolve(path)) - val tokenStream = tool.tokenize() - for (token in tokenStream.tokens) { - println("${token.sourceIndex.index} ${token.type.name} '${sanitize(token.text)}'") + val tokenSource = tool.tokenize() + while (true) { + val token = tokenSource.next() + println("${token.sourceIndex} ${token.type.name} '${sanitize(token.text)}'") + tokenSource.peekTypeAhead(5) + if (token.type == TokenType.EndOfFile) { + break + } } }