Ensure that whitespace preservation in the tokenizer produces exact results.

src/main/kotlin/gay/pizza/pork/main.kt

@@ -1,11 +1,13 @@
 package gay.pizza.pork
 
-import gay.pizza.pork.ast.*
+import gay.pizza.pork.ast.Program
-import gay.pizza.pork.compiler.KotlinCompiler
 import gay.pizza.pork.eval.Arguments
-import gay.pizza.pork.eval.Scope
 import gay.pizza.pork.eval.PorkEvaluator
-import gay.pizza.pork.parse.*
+import gay.pizza.pork.eval.Scope
+import gay.pizza.pork.parse.PorkParser
+import gay.pizza.pork.parse.PorkTokenizer
+import gay.pizza.pork.parse.StringCharSource
+import gay.pizza.pork.parse.TokenStreamSource
 import kotlin.io.path.Path
 import kotlin.io.path.readText
 
@@ -18,13 +20,13 @@ fun main(args: Array<String>) {
   }
 
   val code = Path(args[0]).readText()
-  val tokenizer = PorkTokenizer(StringCharSource(code))
+  val stream = PorkTokenizer(StringCharSource(code)).tokenize()
-  val stream = tokenizer.tokenize()
   println(stream.tokens.joinToString("\n"))
   val parser = PorkParser(TokenStreamSource(stream))
   val program = parser.readProgram()
   eval(program)
-  val kotlinCompiler = KotlinCompiler()
+
-  val kotlinCode =  kotlinCompiler.visit(program)
+  val exactStream = PorkTokenizer(StringCharSource(code), preserveWhitespace = true).tokenize()
-  println(kotlinCode)
+  val exactCode = exactStream.tokens.joinToString("") { it.text }
+  println(exactCode)
 }

src/main/kotlin/gay/pizza/pork/parse/PeekableSource.kt

@@ -2,7 +2,6 @@ package gay.pizza.pork.parse
 
 interface PeekableSource<T> {
   val currentIndex: Int
-  fun back()
   fun next(): T
   fun peek(): T
 }

src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt

@@ -35,10 +35,6 @@ class PorkParser(val source: PeekableSource<Token>) {
       FunctionCall(symbol, arguments)
     } else if (peekType(TokenType.Equals)) {
       expect(TokenType.Equals)
-      if (peekType(TokenType.Equals)) {
-        source.back()
-        return SymbolReference(symbol)
-      }
       Define(symbol, readExpression())
     } else {
       SymbolReference(symbol)

src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt

@@ -40,8 +40,9 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa
     return Token(TokenType.IntLiteral, number)
   }
 
-  private fun readWhitespace(): Token {
+  private fun readWhitespace(firstChar: Char): Token {
     val whitespace = buildString {
+      append(firstChar)
       while (isWhitespace(source.peek())) {
         val char = source.next()
         append(char)
@@ -74,7 +75,7 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa
       }
 
       if (isWhitespace(char)) {
-        val whitespace = readWhitespace()
+        val whitespace = readWhitespace(char)
         if (preserveWhitespace) {
           return whitespace
         }

src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt

@@ -3,9 +3,6 @@ package gay.pizza.pork.parse
 class StringCharSource(val input: String) : CharSource {
   private var index = 0
   override val currentIndex: Int = index
-  override fun back() {
-    index--
-  }
 
   override fun next(): Char {
     if (index == input.length) {

src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt

@@ -12,4 +12,4 @@ abstract class TokenProcessor(val source: TokenSource) {
   }
 
   abstract fun process(token: Token)
-}
+}

src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt

@@ -3,9 +3,6 @@ package gay.pizza.pork.parse
 class TokenStreamSource(val stream: TokenStream) : TokenSource {
   private var index = 0
   override val currentIndex: Int = index
-  override fun back() {
-    index--
-  }
 
   override fun next(): Token {
     if (index == stream.tokens.size) {