From c5d0881893329f762a0232cc91e5bd47a44b763a Mon Sep 17 00:00:00 2001
From: Alex Zenla <alex@fork.run>
Date: Sun, 20 Aug 2023 00:54:28 -0700
Subject: [PATCH] Ensure that whitespace preservation in the tokenizer produces
 exact results.

---
 src/main/kotlin/gay/pizza/pork/main.kt        | 20 ++++++++++---------
 .../gay/pizza/pork/parse/PeekableSource.kt    |  1 -
 .../kotlin/gay/pizza/pork/parse/PorkParser.kt |  4 ----
 .../gay/pizza/pork/parse/PorkTokenizer.kt     |  5 +++--
 .../gay/pizza/pork/parse/StringCharSource.kt  |  3 ---
 .../gay/pizza/pork/parse/TokenProcessor.kt    |  2 +-
 .../gay/pizza/pork/parse/TokenStreamSource.kt |  3 ---
 7 files changed, 15 insertions(+), 23 deletions(-)

diff --git a/src/main/kotlin/gay/pizza/pork/main.kt b/src/main/kotlin/gay/pizza/pork/main.kt
index 3bbf575..b2d0ae8 100644
--- a/src/main/kotlin/gay/pizza/pork/main.kt
+++ b/src/main/kotlin/gay/pizza/pork/main.kt
@@ -1,11 +1,13 @@
 package gay.pizza.pork
 
-import gay.pizza.pork.ast.*
-import gay.pizza.pork.compiler.KotlinCompiler
+import gay.pizza.pork.ast.Program
 import gay.pizza.pork.eval.Arguments
-import gay.pizza.pork.eval.Scope
 import gay.pizza.pork.eval.PorkEvaluator
-import gay.pizza.pork.parse.*
+import gay.pizza.pork.eval.Scope
+import gay.pizza.pork.parse.PorkParser
+import gay.pizza.pork.parse.PorkTokenizer
+import gay.pizza.pork.parse.StringCharSource
+import gay.pizza.pork.parse.TokenStreamSource
 import kotlin.io.path.Path
 import kotlin.io.path.readText
 
@@ -18,13 +20,13 @@ fun main(args: Array<String>) {
   }
 
   val code = Path(args[0]).readText()
-  val tokenizer = PorkTokenizer(StringCharSource(code))
-  val stream = tokenizer.tokenize()
+  val stream = PorkTokenizer(StringCharSource(code)).tokenize()
   println(stream.tokens.joinToString("\n"))
   val parser = PorkParser(TokenStreamSource(stream))
   val program = parser.readProgram()
   eval(program)
-  val kotlinCompiler = KotlinCompiler()
-  val kotlinCode =  kotlinCompiler.visit(program)
-  println(kotlinCode)
+
+  val exactStream = PorkTokenizer(StringCharSource(code), preserveWhitespace = true).tokenize()
+  val exactCode = exactStream.tokens.joinToString("") { it.text }
+  println(exactCode)
 }
diff --git a/src/main/kotlin/gay/pizza/pork/parse/PeekableSource.kt b/src/main/kotlin/gay/pizza/pork/parse/PeekableSource.kt
index c8c2ece..6335c4c 100644
--- a/src/main/kotlin/gay/pizza/pork/parse/PeekableSource.kt
+++ b/src/main/kotlin/gay/pizza/pork/parse/PeekableSource.kt
@@ -2,7 +2,6 @@ package gay.pizza.pork.parse
 
 interface PeekableSource<T> {
   val currentIndex: Int
-  fun back()
   fun next(): T
   fun peek(): T
 }
diff --git a/src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt b/src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt
index 45bd7c6..828e395 100644
--- a/src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt
+++ b/src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt
@@ -35,10 +35,6 @@ class PorkParser(val source: PeekableSource<Token>) {
       FunctionCall(symbol, arguments)
     } else if (peekType(TokenType.Equals)) {
       expect(TokenType.Equals)
-      if (peekType(TokenType.Equals)) {
-        source.back()
-        return SymbolReference(symbol)
-      }
       Define(symbol, readExpression())
     } else {
       SymbolReference(symbol)
diff --git a/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt b/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt
index 90271ff..304417b 100644
--- a/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt
+++ b/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt
@@ -40,8 +40,9 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa
     return Token(TokenType.IntLiteral, number)
   }
 
-  private fun readWhitespace(): Token {
+  private fun readWhitespace(firstChar: Char): Token {
     val whitespace = buildString {
+      append(firstChar)
       while (isWhitespace(source.peek())) {
         val char = source.next()
         append(char)
@@ -74,7 +75,7 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa
       }
 
       if (isWhitespace(char)) {
-        val whitespace = readWhitespace()
+        val whitespace = readWhitespace(char)
         if (preserveWhitespace) {
           return whitespace
         }
diff --git a/src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt b/src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt
index 540c57d..6d7d996 100644
--- a/src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt
+++ b/src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt
@@ -3,9 +3,6 @@ package gay.pizza.pork.parse
 class StringCharSource(val input: String) : CharSource {
   private var index = 0
   override val currentIndex: Int = index
-  override fun back() {
-    index--
-  }
 
   override fun next(): Char {
     if (index == input.length) {
diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt
index c9a62e6..78f7099 100644
--- a/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt
+++ b/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt
@@ -12,4 +12,4 @@ abstract class TokenProcessor(val source: TokenSource) {
   }
 
   abstract fun process(token: Token)
-}
\ No newline at end of file
+}
diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt
index b2ea3a7..3719415 100644
--- a/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt
+++ b/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt
@@ -3,9 +3,6 @@ package gay.pizza.pork.parse
 class TokenStreamSource(val stream: TokenStream) : TokenSource {
   private var index = 0
   override val currentIndex: Int = index
-  override fun back() {
-    index--
-  }
 
   override fun next(): Token {
     if (index == stream.tokens.size) {