tokenizer: remove TokenStream, use sequences instead

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt

@@ -1,6 +1,6 @@
 package gay.pizza.pork.tokenizer
 
 class Highlighter(val scheme: HighlightScheme) {
-  fun highlight(stream: TokenStream): List<Highlight> =
-    stream.tokens.map { scheme.highlight(it) }
+  fun highlight(source: TokenSource): Sequence<Highlight> =
+    source.sequence().map { scheme.highlight(it) }
 }

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ListTokenSource.kt

@@ -0,0 +1,32 @@
+package gay.pizza.pork.tokenizer
+
+class ListTokenSource(val tokens: List<Token>) : TokenSource {
+  private var index = 0
+
+  override val currentIndex: Int
+    get() = index
+
+  override fun next(): Token {
+    if (index == tokens.size) {
+      return tokens.last()
+    }
+    val char = tokens[index]
+    index++
+    return char
+  }
+
+  override fun peek(): Token {
+    if (index == tokens.size) {
+      return tokens.last()
+    }
+    return tokens[index]
+  }
+
+  override fun peekTypeAhead(ahead: Int): TokenType {
+    val calculated = index + ahead
+    if (calculated >= tokens.size) {
+      return tokens.last().type
+    }
+    return tokens[calculated].type
+  }
+}

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt

@@ -15,5 +15,17 @@ interface TokenSource : PeekableSource<Token> {
     return tokens
   }
 
-  fun stream(): TokenStream = TokenStream(consumeAllRemainingTokens())
+  fun sequence(): Sequence<Token> {
+    var endFlag = false
+    return generateSequence {
+      if (endFlag) {
+        return@generateSequence null
+      }
+      val token = next()
+      if (token.type == TokenType.EndOfFile) {
+        endFlag = true
+        token
+      } else token
+    }
+  }
 }

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt

@@ -1,5 +0,0 @@
-package gay.pizza.pork.tokenizer
-
-class TokenStream(val tokens: List<Token>) {
-  override fun toString(): String = tokens.toString()
-}

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt

@@ -1,32 +0,0 @@
-package gay.pizza.pork.tokenizer
-
-class TokenStreamSource(val stream: TokenStream) : TokenSource {
-  private var index = 0
-
-  override val currentIndex: Int
-    get() = index
-
-  override fun next(): Token {
-    if (index == stream.tokens.size) {
-      return stream.tokens.last()
-    }
-    val char = stream.tokens[index]
-    index++
-    return char
-  }
-
-  override fun peek(): Token {
-    if (index == stream.tokens.size) {
-      return stream.tokens.last()
-    }
-    return stream.tokens[index]
-  }
-
-  override fun peekTypeAhead(ahead: Int): TokenType {
-    val calculated = index + ahead
-    if (calculated >= stream.tokens.size) {
-      return stream.tokens.last().type
-    }
-    return stream.tokens[calculated].type
-  }
-}