tokenizer: remove TokenStream, use sequences instead

minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt

@@ -29,8 +29,8 @@ abstract class Tool {
   fun parse(attribution: NodeAttribution = DiscardNodeAttribution): CompilationUnit =
     Parser(tokenize(), attribution).parseCompilationUnit()
 
-  fun highlight(scheme: HighlightScheme): List<Highlight> =
+  fun highlight(scheme: HighlightScheme): Sequence<Highlight> =
-    Highlighter(scheme).highlight(tokenize().stream())
+    Highlighter(scheme).highlight(tokenize())
 
   fun reprint(): String = buildString { visit(Printer(this)) }
 
@@ -44,6 +44,13 @@ abstract class Tool {
     return resultingScope.value("main") as CallableFunction
   }
 
+  fun loadMainFunctionStandard(scope: Scope, quiet: Boolean = false): CallableFunction =
+    loadMainFunction(scope, setupEvaluator = {
+      addNativeProvider("internal", InternalNativeProvider(quiet = quiet))
+      addNativeProvider("ffi", FfiNativeProvider())
+      addNativeProvider("java", JavaNativeProvider())
+    })
+
   fun buildWorld(): World {
     val fileContentSource = createContentSource()
     val dynamicImportSource = DynamicImportSource()
@@ -54,11 +61,7 @@ abstract class Tool {
   }
 
   fun run(scope: Scope, quiet: Boolean = false) {
-    val main = loadMainFunction(scope, setupEvaluator = {
+    val main = loadMainFunctionStandard(scope, quiet = quiet)
-      addNativeProvider("internal", InternalNativeProvider(quiet = quiet))
-      addNativeProvider("ffi", FfiNativeProvider())
-      addNativeProvider("java", JavaNativeProvider())
-    })
     main.call(emptyList(), CallStack())
   }
 }

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt

@@ -1,6 +1,6 @@
 package gay.pizza.pork.tokenizer
 
 class Highlighter(val scheme: HighlightScheme) {
-  fun highlight(stream: TokenStream): List<Highlight> =
+  fun highlight(source: TokenSource): Sequence<Highlight> =
-    stream.tokens.map { scheme.highlight(it) }
+    source.sequence().map { scheme.highlight(it) }
 }

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ListTokenSource.kt

@@ -0,0 +1,32 @@
+package gay.pizza.pork.tokenizer
+
+class ListTokenSource(val tokens: List<Token>) : TokenSource {
+  private var index = 0
+
+  override val currentIndex: Int
+    get() = index
+
+  override fun next(): Token {
+    if (index == tokens.size) {
+      return tokens.last()
+    }
+    val char = tokens[index]
+    index++
+    return char
+  }
+
+  override fun peek(): Token {
+    if (index == tokens.size) {
+      return tokens.last()
+    }
+    return tokens[index]
+  }
+
+  override fun peekTypeAhead(ahead: Int): TokenType {
+    val calculated = index + ahead
+    if (calculated >= tokens.size) {
+      return tokens.last().type
+    }
+    return tokens[calculated].type
+  }
+}

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt

@@ -15,5 +15,17 @@ interface TokenSource : PeekableSource<Token> {
     return tokens
   }
 
-  fun stream(): TokenStream = TokenStream(consumeAllRemainingTokens())
+  fun sequence(): Sequence<Token> {
+    var endFlag = false
+    return generateSequence {
+      if (endFlag) {
+        return@generateSequence null
+      }
+      val token = next()
+      if (token.type == TokenType.EndOfFile) {
+        endFlag = true
+        token
+      } else token
+    }
+  }
 }

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt

@@ -1,5 +0,0 @@
-package gay.pizza.pork.tokenizer
-
-class TokenStream(val tokens: List<Token>) {
-  override fun toString(): String = tokens.toString()
-}

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt

@@ -1,32 +0,0 @@
-package gay.pizza.pork.tokenizer
-
-class TokenStreamSource(val stream: TokenStream) : TokenSource {
-  private var index = 0
-
-  override val currentIndex: Int
-    get() = index
-
-  override fun next(): Token {
-    if (index == stream.tokens.size) {
-      return stream.tokens.last()
-    }
-    val char = stream.tokens[index]
-    index++
-    return char
-  }
-
-  override fun peek(): Token {
-    if (index == stream.tokens.size) {
-      return stream.tokens.last()
-    }
-    return stream.tokens[index]
-  }
-
-  override fun peekTypeAhead(ahead: Int): TokenType {
-    val calculated = index + ahead
-    if (calculated >= stream.tokens.size) {
-      return stream.tokens.last().type
-    }
-    return stream.tokens[calculated].type
-  }
-}

tool/src/main/kotlin/gay/pizza/pork/tool/RunCommand.kt

@@ -20,11 +20,7 @@ class RunCommand : CliktCommand(help = "Run Program", name = "run") {
   override fun run() {
     val tool = FileTool(PlatformFsProvider.resolve(path))
     val scope = Scope.root()
-    val main = tool.loadMainFunction(scope, setupEvaluator = {
+    val main = tool.loadMainFunctionStandard(scope, quiet = quiet)
-      addNativeProvider("internal", InternalNativeProvider(quiet = quiet))
-      addNativeProvider("ffi", FfiNativeProvider())
-      addNativeProvider("java", JavaNativeProvider())
-    })
 
     if (dumpScope) {
       val functionContext = main as FunctionContext