Validate token soundness.

src/main/kotlin/gay/pizza/pork/main.kt

@@ -15,17 +15,33 @@ fun eval(ast: Program) {
   println("> ${scope.call("main", Arguments.Zero)}")
 }
 
+fun validateTokenSoundness(input: String, stream: TokenStream) {
+  var expectedIndex = 0
+  for (token in stream.tokens) {
+    if (token.start != expectedIndex) {
+      throw RuntimeException("Expected token to be at index $expectedIndex but was ${token.start}")
+    }
+    val slice = input.slice(token.start until token.start + token.text.length)
+    if (slice != token.text) {
+      throw RuntimeException(
+        "Expected index ${token.start} for length ${token.text.length} to" +
+        " equal '${token.text}' but was '$slice'")
+    }
+    expectedIndex += token.text.length
+  }
+}
+
 fun main(args: Array<String>) {
   val code = Path(args[0]).readText()
-  val stream = tokenize(code).excludeAllWhitespace()
+  val stream = tokenize(code)
   println(stream.tokens.joinToString("\n"))
   val program = parse(stream)
   eval(program)
 
-  val exactStream = tokenize(code)
+  val exactCode = stream.tokens.joinToString("") { it.text }
-  val exactCode = exactStream.tokens.joinToString("") { it.text }
   println(exactCode)
   println(code == exactCode)
+  validateTokenSoundness(code, stream)
 }
 
 fun tokenize(input: String): TokenStream =

src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt

@@ -2,7 +2,8 @@ package gay.pizza.pork.parse
 
 class StringCharSource(val input: String) : CharSource {
   private var index = 0
-  override val currentIndex: Int = index
+  override val currentIndex: Int
+    get() = index
 
   override fun next(): Char {
     if (index == input.length) {

src/main/kotlin/gay/pizza/pork/parse/Token.kt

@@ -1,7 +1,7 @@
 package gay.pizza.pork.parse
 
 class Token(val type: TokenType, val start: Int, val text: String) {
-  override fun toString(): String = "${type.name} $text"
+  override fun toString(): String = "$start ${type.name} '${text.replace("\n", "\\n")}'"
 
   companion object {
     fun endOfFile(size: Int): Token =

src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt

@@ -2,7 +2,8 @@ package gay.pizza.pork.parse
 
 class TokenStreamSource(val stream: TokenStream) : TokenSource {
   private var index = 0
-  override val currentIndex: Int = index
+  override val currentIndex: Int
+    get() = index
 
   override fun next(): Token {
     if (index == stream.tokens.size) {