From 77a9136be71645e69ca89d66f79143251d8ec38c Mon Sep 17 00:00:00 2001
From: Alex Zenla <alex@fork.run>
Date: Sat, 19 Aug 2023 22:20:45 -0700
Subject: [PATCH] Optional whitespace preservation.

---
 .../gay/pizza/pork/parse/Highlighter.kt       | 19 +++++++++++++++++++
 .../gay/pizza/pork/parse/PorkTokenizer.kt     | 17 ++++++++++++-----
 .../gay/pizza/pork/parse/TokenProcessor.kt    | 15 +++++++++++++++
 .../kotlin/gay/pizza/pork/parse/TokenType.kt  |  1 +
 4 files changed, 47 insertions(+), 5 deletions(-)
 create mode 100644 src/main/kotlin/gay/pizza/pork/parse/Highlighter.kt
 create mode 100644 src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt

diff --git a/src/main/kotlin/gay/pizza/pork/parse/Highlighter.kt b/src/main/kotlin/gay/pizza/pork/parse/Highlighter.kt
new file mode 100644
index 0000000..1f5370d
--- /dev/null
+++ b/src/main/kotlin/gay/pizza/pork/parse/Highlighter.kt
@@ -0,0 +1,19 @@
+package gay.pizza.pork.parse
+
+abstract class Highlighter(source: TokenSource) : TokenProcessor(source) {
+  override fun process(token: Token) {
+    when {
+      token.type.keyword != null -> {
+        keyword(token)
+      }
+      token.type == TokenType.Symbol -> {
+        symbol(token)
+      }
+      else -> other(token)
+    }
+  }
+
+  abstract fun keyword(token: Token)
+  abstract fun symbol(token: Token)
+  abstract fun other(token: Token)
+}
\ No newline at end of file
diff --git a/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt b/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt
index e656eac..8dc871e 100644
--- a/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt
+++ b/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt
@@ -1,6 +1,6 @@
 package gay.pizza.pork.parse
 
-class PorkTokenizer(val source: CharSource) {
+class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = false) {
   private var tokenStart: Int = 0
 
   private fun isSymbol(c: Char): Boolean =
@@ -40,10 +40,14 @@ class PorkTokenizer(val source: CharSource) {
     return Token(TokenType.IntLiteral, number)
   }
 
-  private fun skipWhitespace() {
-    while (isWhitespace(source.peek())) {
-      source.next()
+  private fun readWhitespace(): Token {
+    val whitespace = buildString {
+      while (isWhitespace(source.peek())) {
+        val char = source.next()
+        append(char)
+      }
     }
+    return Token(TokenType.Whitespace, whitespace)
   }
 
   fun next(): Token {
@@ -57,7 +61,10 @@ class PorkTokenizer(val source: CharSource) {
       }
 
       if (isWhitespace(char)) {
-        skipWhitespace()
+        val whitespace = readWhitespace()
+        if (preserveWhitespace) {
+          return whitespace
+        }
         continue
       }
 
diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt
new file mode 100644
index 0000000..c9a62e6
--- /dev/null
+++ b/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt
@@ -0,0 +1,15 @@
+package gay.pizza.pork.parse
+
+abstract class TokenProcessor(val source: TokenSource) {
+  fun processAll() {
+    while (true) {
+      val token = source.next()
+      process(token)
+      if (token.type == TokenType.EndOfFile) {
+        break
+      }
+    }
+  }
+
+  abstract fun process(token: Token)
+}
\ No newline at end of file
diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt
index 83d7c75..80a600d 100644
--- a/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt
+++ b/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt
@@ -21,6 +21,7 @@ enum class TokenType(val char: Char? = null, val keyword: String? = null) {
   If(keyword = "if"),
   Then(keyword = "then"),
   Else(keyword = "else"),
+  Whitespace,
   EndOfFile;
 
   companion object {