diff --git a/src/main/kotlin/gay/pizza/pork/main.kt b/src/main/kotlin/gay/pizza/pork/main.kt index d78b8d5..a48d452 100644 --- a/src/main/kotlin/gay/pizza/pork/main.kt +++ b/src/main/kotlin/gay/pizza/pork/main.kt @@ -15,17 +15,33 @@ fun eval(ast: Program) { println("> ${scope.call("main", Arguments.Zero)}") } +fun validateTokenSoundness(input: String, stream: TokenStream) { + var expectedIndex = 0 + for (token in stream.tokens) { + if (token.start != expectedIndex) { + throw RuntimeException("Expected token to be at index $expectedIndex but was ${token.start}") + } + val slice = input.slice(token.start until token.start + token.text.length) + if (slice != token.text) { + throw RuntimeException( + "Expected index ${token.start} for length ${token.text.length} to" + + " equal '${token.text}' but was '$slice'") + } + expectedIndex += token.text.length + } +} + fun main(args: Array) { val code = Path(args[0]).readText() - val stream = tokenize(code).excludeAllWhitespace() + val stream = tokenize(code) println(stream.tokens.joinToString("\n")) val program = parse(stream) eval(program) - val exactStream = tokenize(code) - val exactCode = exactStream.tokens.joinToString("") { it.text } + val exactCode = stream.tokens.joinToString("") { it.text } println(exactCode) println(code == exactCode) + validateTokenSoundness(code, stream) } fun tokenize(input: String): TokenStream = diff --git a/src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt b/src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt index 6d7d996..5de7aa1 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt @@ -2,7 +2,8 @@ package gay.pizza.pork.parse class StringCharSource(val input: String) : CharSource { private var index = 0 - override val currentIndex: Int = index + override val currentIndex: Int + get() = index override fun next(): Char { if (index == input.length) { diff --git a/src/main/kotlin/gay/pizza/pork/parse/Token.kt b/src/main/kotlin/gay/pizza/pork/parse/Token.kt index 184151a..e6a6a23 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/Token.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/Token.kt @@ -1,7 +1,7 @@ package gay.pizza.pork.parse class Token(val type: TokenType, val start: Int, val text: String) { - override fun toString(): String = "${type.name} $text" + override fun toString(): String = "$start ${type.name} '${text.replace("\n", "\\n")}'" companion object { fun endOfFile(size: Int): Token = diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt index 85940cc..778eee2 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt @@ -2,7 +2,8 @@ package gay.pizza.pork.parse class TokenStreamSource(val stream: TokenStream) : TokenSource { private var index = 0 - override val currentIndex: Int = index + override val currentIndex: Int + get() = index override fun next(): Token { if (index == stream.tokens.size) {