diff --git a/examples/gameoflife/gameoflife.pork b/examples/gameoflife/gameoflife.pork index c465a8a..e982000 100644 --- a/examples/gameoflife/gameoflife.pork +++ b/examples/gameoflife/gameoflife.pork @@ -40,7 +40,7 @@ func drawCells(renderer, cells, swap) { var ix = 0 while ix < gridWidth { let mask = if swap { 2 } else { 1 } - if (java_util_ArrayList_get(cells, i) & mask) == mask { + if (cells[i] & mask) == mask { let x = ix * cellSize let y = iy * cellSize SDL_RenderDrawLine(renderer, x, y, x + cellSize, y) @@ -66,7 +66,7 @@ func createCellGrid() { func getCell(cells, swap, x, y) { if (x >= 0) and (y >= 0) and (x < gridWidth) and (y < gridHeight) { let mask = if swap { 2 } else { 1 } - (java_util_ArrayList_get(cells, x + y * gridWidth) & mask) != 0 + (cells[x + y * gridWidth] & mask) != 0 } else { false } @@ -76,7 +76,7 @@ func setCell(cells, swap, x, y, state) { if (x >= 0) and (y >= 0) and (x < gridWidth) and (y < gridHeight) { let mask = if swap { 2 } else { 1 } let idx = x + y * gridWidth - let value = java_util_ArrayList_get(cells, idx) + let value = cells[idx] if state { java_util_ArrayList_set(cells, idx, value | mask) } else { java_util_ArrayList_set(cells, idx, value & (~mask)) } } @@ -122,8 +122,8 @@ func createGosperGun(cells, swap, x, y) { [25, 7], [35, 3], [36, 3], [35, 4], [36, 4] ] { setCell(cells, false, - x + java_util_List_get(i, 0), - y + java_util_List_get(i, 1), + x + i[0], + y + i[1], true) } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/BadCharacterError.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/BadCharacterError.kt new file mode 100644 index 0000000..4674ade --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/BadCharacterError.kt @@ -0,0 +1,5 @@ +package gay.pizza.pork.parser + +class BadCharacterError(val char: Char, sourceIndex: SourceIndex) : ParseError( + "Failed to produce token for '${char}' at $sourceIndex" +) diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ExpectedTokenError.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ExpectedTokenError.kt new file mode 100644 index 0000000..4be4f1e --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ExpectedTokenError.kt @@ -0,0 +1,28 @@ +package gay.pizza.pork.parser + +class ExpectedTokenError(got: Token, sourceIndex: SourceIndex, vararg expectedTypes: TokenType) : ParseError( + message(got, sourceIndex, expectedTypes) +) { + companion object { + fun message(got: Token, sourceIndex: SourceIndex, expectedTypes: Array): String { + val tokenTypeMessages = expectedTypes.map { + if (it.simpleWantString != null) + "${it.name} '${it.simpleWantString}'" + else + it.name + } + + val expected = if (expectedTypes.size > 1) { + "one of " + tokenTypeMessages.joinToString(", ") + } else tokenTypeMessages.firstOrNull() ?: "unknown" + + val friendlyIndex = if (sourceIndex.locationReliable) { + "line ${sourceIndex.line} column ${sourceIndex.column}" + } else { + "index ${sourceIndex.index}" + } + + return "Expected $expected at $friendlyIndex but got ${got.type} '${got.text}'" + } + } +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParseError.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParseError.kt index e6004a2..ab529ae 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParseError.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParseError.kt @@ -1,3 +1,6 @@ package gay.pizza.pork.parser -open class ParseError(val error: String) : RuntimeException(error) +open class ParseError(val error: String) : RuntimeException() { + override val message: String + get() = "${error}\nDescent path: ${ParserStackAnalysis(this).findDescentPath().joinToString(", ")}" +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt index ce0cd6f..2d016bb 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt @@ -1,6 +1,7 @@ package gay.pizza.pork.parser import gay.pizza.pork.ast.* +import kotlin.math.exp class Parser(source: TokenSource, attribution: NodeAttribution) : ParserBase(source, attribution) { @@ -18,7 +19,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : override fun parseExpression(): Expression = guarded { val token = peek() - val expression = when (token.type) { + var expression = when (token.type) { TokenType.NumberLiteral -> parseNumberLiteral() TokenType.StringLiteral -> parseStringLiteral() TokenType.True, TokenType.False -> parseBooleanLiteral() @@ -45,43 +46,35 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : } if (expression is SymbolReference && peek(TokenType.Equals)) { - return@guarded guarded(NodeType.SetAssignment) { + val symbolReference = expression as SymbolReference + expression = guarded(NodeType.SetAssignment) { attribution.adopt(expression) expect(TokenType.Equals) val value = parseExpression() - SetAssignment(expression.symbol, value) + SetAssignment(symbolReference.symbol, value) } } - return@guarded if (peek( - TokenType.Plus, - TokenType.Minus, - TokenType.Multiply, - TokenType.Divide, - TokenType.Ampersand, - TokenType.Pipe, - TokenType.Caret, - TokenType.Equality, - TokenType.Inequality, - TokenType.Mod, - TokenType.Rem, - TokenType.Lesser, - TokenType.Greater, - TokenType.LesserEqual, - TokenType.GreaterEqual, - TokenType.And, - TokenType.Or - ) - ) { + if (peek(TokenType.LeftBracket)) { + expression = guarded(NodeType.IndexedBy) { + attribution.adopt(expression) + expect(TokenType.LeftBracket) + val index = parseExpression() + expect(TokenType.RightBracket) + IndexedBy(expression, index) + } + } + + if (peek( + TokenType.Plus, TokenType.Minus, TokenType.Multiply, TokenType.Divide, TokenType.Ampersand, + TokenType.Pipe, TokenType.Caret, TokenType.Equality, TokenType.Inequality, TokenType.Mod, + TokenType.Rem, TokenType.Lesser, TokenType.Greater, TokenType.LesserEqual, TokenType.GreaterEqual, + TokenType.And, TokenType.Or)) { guarded(NodeType.InfixOperation) { val infixToken = next() val infixOperator = ParserHelpers.convertInfixOperator(infixToken) InfixOperation(expression, infixOperator, parseExpression()) } - } else if (next(TokenType.LeftBracket)) { - val index = parseExpression() - expect(TokenType.RightBracket) - IndexedBy(expression, index) } else expression } @@ -91,7 +84,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : } else if (next(TokenType.False)) { BooleanLiteral(false) } else { - throw ParseError("Expected ") + expectedTokenError(source.peek(), TokenType.True, TokenType.False) } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt index 0698c02..fd490c8 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt @@ -16,7 +16,7 @@ data class ParserAttributes(val tokens: List) { } } coalescer.visit(node) - all.sortBy { it.start } + all.sortBy { it.sourceIndex.index } return all } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt index 96b8127..bbe5e60 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt @@ -5,12 +5,8 @@ import gay.pizza.pork.ast.NodeParser import gay.pizza.pork.ast.NodeType abstract class ParserBase(val source: TokenSource, val attribution: NodeAttribution) : NodeParser { - class ExpectedTokenError(got: Token, vararg expectedTypes: TokenType) : ParseError( - "Expected one of ${expectedTypes.joinToString(", ")}" + - " but got type ${got.type} '${got.text}'" - ) - - protected fun guarded(type: NodeType? = null, block: () -> T): T = + @Suppress("NOTHING_TO_INLINE") + protected inline fun guarded(type: NodeType? = null, noinline block: () -> T): T = attribution.guarded(type, block) protected fun collect( @@ -56,7 +52,7 @@ abstract class ParserBase(val source: TokenSource, val attribution: NodeAttribut protected fun expect(vararg types: TokenType): Token { val token = next() if (!types.contains(token.type)) { - throw ExpectedTokenError(token, *types) + expectedTokenError(token, *types) } return token } @@ -64,6 +60,10 @@ abstract class ParserBase(val source: TokenSource, val attribution: NodeAttribut protected fun expect(vararg types: TokenType, consume: (Token) -> T): T = consume(expect(*types)) + protected fun expectedTokenError(token: Token, vararg types: TokenType): Nothing { + throw ExpectedTokenError(token, token.sourceIndex, *types) + } + protected fun next(): Token { while (true) { val token = source.next() diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserStackAnalysis.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserStackAnalysis.kt new file mode 100644 index 0000000..899904e --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserStackAnalysis.kt @@ -0,0 +1,27 @@ +package gay.pizza.pork.parser + +import gay.pizza.pork.ast.NodeType + +class ParserStackAnalysis(private val stack: Array) { + constructor(throwable: Throwable) : this(throwable.stackTrace) + + fun findDescentPath(): List { + val parseDescentPaths = mutableListOf() + for (element in stack) { + if (element.className != Parser::class.java.name) { + continue + } + + if (!element.methodName.startsWith("parse")) { + continue + } + + val nodeTypeString = element.methodName.substring(5) + val type = NodeType.entries.firstOrNull { it.name == nodeTypeString } + if (type != null) { + parseDescentPaths.add(type) + } + } + return parseDescentPaths.reversed() + } +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndex.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndex.kt new file mode 100644 index 0000000..6e94c00 --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndex.kt @@ -0,0 +1,10 @@ +package gay.pizza.pork.parser + +data class SourceIndex(val index: Int, val line: Int, val column: Int, val locationReliable: Boolean = true) { + companion object { + fun zero(): SourceIndex = SourceIndex(0, 1, 0) + fun indexOnly(index: Int) = SourceIndex(index, 0, 0, locationReliable = false) + } + + override fun toString(): String = if (locationReliable) "${line}:${column}" else "$index" +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Token.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/Token.kt index a6c8683..514ac1f 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Token.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/Token.kt @@ -1,11 +1,13 @@ package gay.pizza.pork.parser -class Token(val type: TokenType, val start: Int, val text: String) { +class Token(val type: TokenType, val sourceIndex: SourceIndex, val text: String) { override fun toString(): String = - "$start ${type.name} '${text.replace("\n", "\\n")}'" + "$sourceIndex ${type.name} '${text.replace("\n", "\\n")}'" companion object { - fun endOfFile(size: Int): Token = - Token(TokenType.EndOfFile, size, "") + fun endOfFile(sourceIndex: SourceIndex): Token = + Token(TokenType.EndOfFile, sourceIndex, "") } + + fun upgrade(upgradedType: TokenType): Token = Token(upgradedType, sourceIndex, text) } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt index 969762b..c29ca09 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt @@ -7,7 +7,7 @@ class TokenStreamSource(val stream: TokenStream) : TokenSource { override fun next(): Token { if (index == stream.tokens.size) { - return Token.endOfFile(stream.tokens.size) + return stream.tokens.last() } val char = stream.tokens[index] index++ @@ -16,7 +16,7 @@ class TokenStreamSource(val stream: TokenStream) : TokenSource { override fun peek(): Token { if (index == stream.tokens.size) { - return Token.endOfFile(stream.tokens.size) + return stream.tokens.last() } return stream.tokens[index] } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt index fd4fa52..5a14606 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt @@ -82,6 +82,8 @@ enum class TokenType(vararg properties: TokenTypeProperty) { val tokenUpgrader: TokenUpgrader? = properties.filterIsInstance().singleOrNull() + val simpleWantString: String? = manyChars?.text ?: singleChar?.char?.toString() + companion object { val AnyOf = entries.filter { item -> item.anyOf != null } val ManyChars = entries.filter { item -> item.manyChars != null } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt index fe0fa62..87b68b7 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt @@ -13,15 +13,15 @@ interface TokenTypeProperty { var upgraded: Token? = null for (item in TokenType.ManyChars) { if (item.manyChars != null && token.text == item.manyChars.text) { - upgraded = Token(item, token.start, token.text) + upgraded = token.upgrade(item) break } } if (upgraded == null) { - for (item in TokenType.AnyOf) { + for (item in TokenType.AnyOf) { if (item.anyOf != null && item.anyOf.strings.contains(token.text)) { - upgraded = Token(item, token.start, token.text) + upgraded = token.upgrade(item) break } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt index 6d47b34..59522aa 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt @@ -1,16 +1,18 @@ package gay.pizza.pork.parser class Tokenizer(val source: CharSource) { - private var tokenStart: Int = 0 + private var startIndex: SourceIndex = SourceIndex.zero() + private var currentLineIndex = 1 + private var currentLineColumn = 0 private fun readBlockComment(firstChar: Char): Token { val comment = buildString { append(firstChar) var endOfComment = false while (true) { - val char = source.next() + val char = nextChar() if (char == CharSource.NullChar) { - throw ParseError("Unterminated block comment") + throw UnterminatedTokenError("block comment", currentSourceIndex()) } append(char) @@ -27,7 +29,7 @@ class Tokenizer(val source: CharSource) { } } } - return Token(TokenType.BlockComment, tokenStart, comment) + return produceToken(TokenType.BlockComment, comment) } private fun readLineComment(firstChar: Char): Token { @@ -38,10 +40,10 @@ class Tokenizer(val source: CharSource) { if (char == CharSource.NullChar || char == '\n') { break } - append(source.next()) + append(nextChar()) } } - return Token(TokenType.LineComment, tokenStart, comment) + return produceToken(TokenType.LineComment, comment) } private fun readStringLiteral(firstChar: Char): Token { @@ -50,21 +52,21 @@ class Tokenizer(val source: CharSource) { while (true) { val char = source.peek() if (char == CharSource.NullChar) { - throw ParseError("Unterminated string.") + throw UnterminatedTokenError("string", currentSourceIndex()) } - append(source.next()) + append(nextChar()) if (char == '"') { break } } } - return Token(TokenType.StringLiteral, tokenStart, string) + return produceToken(TokenType.StringLiteral, string) } fun next(): Token { while (source.peek() != CharSource.NullChar) { - tokenStart = source.currentIndex - val char = source.next() + startIndex = currentSourceIndex() + val char = nextChar() if (char == '/' && source.peek() == '*') { return readBlockComment(char) @@ -89,13 +91,13 @@ class Tokenizer(val source: CharSource) { if (source.peek() != promotion.nextChar) { continue } - val nextChar = source.next() + val nextChar = nextChar() type = promotion.type text += nextChar promoted = true } } - return Token(type, tokenStart, text) + return produceToken(type, text) } var index = 0 @@ -121,10 +123,10 @@ class Tokenizer(val source: CharSource) { else item.charIndexConsumer!!.isValid(source.peek(), ++index) ) { - append(source.next()) + append(nextChar()) } } - var token = Token(item, tokenStart, text) + var token = produceToken(item, text) val tokenUpgrader = item.tokenUpgrader if (tokenUpgrader != null) { token = tokenUpgrader.maybeUpgrade(token) ?: token @@ -136,9 +138,9 @@ class Tokenizer(val source: CharSource) { return readStringLiteral(char) } - throw ParseError("Failed to parse: (${char}) next ${source.peek()}") + throw BadCharacterError(char, startIndex) } - return Token.endOfFile(source.currentIndex) + return Token.endOfFile(startIndex.copy(index = source.currentIndex)) } fun tokenize(): TokenStream { @@ -152,4 +154,19 @@ class Tokenizer(val source: CharSource) { } return TokenStream(tokens) } + + private fun produceToken(type: TokenType, text: String) = + Token(type, startIndex, text) + + private fun nextChar(): Char { + val char = source.next() + if (char == '\n') { + currentLineIndex++ + currentLineColumn = 0 + } + currentLineColumn++ + return char + } + + private fun currentSourceIndex(): SourceIndex = SourceIndex(source.currentIndex, currentLineIndex, currentLineColumn) } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/UnterminatedTokenError.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/UnterminatedTokenError.kt new file mode 100644 index 0000000..65fdba3 --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/UnterminatedTokenError.kt @@ -0,0 +1,5 @@ +package gay.pizza.pork.parser + +class UnterminatedTokenError(what: String, sourceIndex: SourceIndex) : ParseError( + "Unterminated $what at $sourceIndex" +) diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt index 98a2ad0..1223691 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt @@ -57,8 +57,8 @@ class PorkLexer : LexerBase() { try { val currentToken = tokenizer.next() currentTokenType = PorkElementTypes.elementTypeFor(currentToken.type) - internalTokenStart = currentToken.start - internalTokenEnd = currentToken.start + currentToken.text.length + internalTokenStart = currentToken.sourceIndex.index + internalTokenEnd = currentToken.sourceIndex.index + currentToken.text.length } catch (e: ProcessCanceledException) { throw e } catch (e: Throwable) { diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderMarkAttribution.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderMarkAttribution.kt index aed09fc..212c02d 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderMarkAttribution.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderMarkAttribution.kt @@ -29,12 +29,8 @@ class PsiBuilderMarkAttribution(val builder: PsiBuilder) : ParserNodeAttribution } throw PorkParser.ExitParser() } catch (e: PorkParser.ExitParser) { - if (e.error != null) { - marker.error(e.error) - } else { - marker.done(PorkElementTypes.FailedToParse) - } - throw PorkParser.ExitParser() + marker.done(PorkElementTypes.FailedToParse) + throw e } if (map[result] != null) { marker.drop() diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt index 3a027d2..fd7274c 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt @@ -1,6 +1,7 @@ package gay.pizza.pork.idea import com.intellij.lang.PsiBuilder +import gay.pizza.pork.parser.SourceIndex import gay.pizza.pork.parser.Token import gay.pizza.pork.parser.TokenSource import com.intellij.psi.TokenType as PsiTokenType @@ -17,15 +18,15 @@ class PsiBuilderTokenSource(val builder: PsiBuilder) : TokenSource { override fun peek(): Token { if (builder.eof()) { - return Token.endOfFile(builder.currentOffset) + return Token.endOfFile(SourceIndex.indexOnly(builder.currentOffset)) } val elementType = builder.tokenType!! if (elementType == PsiTokenType.BAD_CHARACTER) { - throw BadCharacterError("Invalid character.") + throw BadCharacterError("Invalid character") } val tokenType = PorkElementTypes.tokenTypeFor(elementType) ?: throw RuntimeException("Lexing failure: ${elementType.debugName}") - return Token(tokenType, builder.currentOffset, builder.tokenText!!) + return Token(tokenType, SourceIndex.indexOnly(builder.currentOffset), builder.tokenText!!) } class BadCharacterError(error: String) : RuntimeException(error) diff --git a/tool/build.gradle.kts b/tool/build.gradle.kts index 6c7c56e..1ac8a38 100644 --- a/tool/build.gradle.kts +++ b/tool/build.gradle.kts @@ -13,6 +13,7 @@ dependencies { application { applicationName = "pork" mainClass.set("gay.pizza.pork.tool.MainKt") + applicationDefaultJvmArgs += "-XstartOnFirstThread" } for (task in arrayOf(tasks.shadowDistTar, tasks.shadowDistZip, tasks.shadowJar)) { diff --git a/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt b/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt index ffecc0e..40825ae 100644 --- a/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt +++ b/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt @@ -12,7 +12,7 @@ class TokenizeCommand : CliktCommand(help = "Tokenize Compilation Unit", name = val tool = FileTool(PlatformFsProvider.resolve(path)) val tokenStream = tool.tokenize() for (token in tokenStream.tokens) { - println("${token.start} ${token.type.name} '${sanitize(token.text)}'") + println("${token.sourceIndex.index} ${token.type.name} '${sanitize(token.text)}'") } }