Multi-module arrangement and the start of AST generation.

core/build.gradle.kts

@@ -0,0 +1,27 @@
+plugins {
+  application
+  pork_module
+  id("com.github.johnrengelman.shadow") version "8.1.1"
+  id("org.graalvm.buildtools.native") version "0.9.25"
+}
+
+dependencies {
+  api(project(":ast"))
+  implementation(libs.clikt)
+}
+
+application {
+  mainClass.set("gay.pizza.pork.cli.MainKt")
+}
+
+graalvmNative {
+  binaries {
+    named("main") {
+      imageName.set("pork")
+      mainClass.set("gay.pizza.pork.cli.MainKt")
+      sharedLibrary.set(false)
+    }
+  }
+}
+
+tasks.run.get().outputs.upToDateWhen { false }

core/src/main/kotlin/gay/pizza/pork/PorkLanguage.kt

@@ -0,0 +1,3 @@
+package gay.pizza.pork
+
+object PorkLanguage

core/src/main/kotlin/gay/pizza/pork/cli/AstCommand.kt

@@ -0,0 +1,24 @@
+package gay.pizza.pork.cli
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.types.path
+import gay.pizza.pork.ast.Node
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.json.Json
+
+@OptIn(ExperimentalSerializationApi::class)
+class AstCommand : CliktCommand(help = "Print AST", name = "ast") {
+  val path by argument("file").path(mustExist = true, canBeDir = false)
+
+  private val json = Json {
+    prettyPrint = true
+    prettyPrintIndent = "  "
+    classDiscriminator = "\$"
+  }
+
+  override fun run() {
+    val tool = FileTool(path)
+    println(json.encodeToString(Node.serializer(), tool.parse()))
+  }
+}

core/src/main/kotlin/gay/pizza/pork/cli/AttributeCommand.kt

@@ -0,0 +1,26 @@
+package gay.pizza.pork.cli
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.types.path
+import gay.pizza.pork.ast.NodeCoalescer
+import gay.pizza.pork.parse.TokenNodeAttribution
+
+class AttributeCommand : CliktCommand(help = "Attribute AST", name = "attribute") {
+  val path by argument("file").path(mustExist = true, canBeDir = false)
+
+  override fun run() {
+    val tool = FileTool(path)
+    val attribution = TokenNodeAttribution()
+    val compilationUnit = tool.parse(attribution)
+
+    val coalescer = NodeCoalescer { node ->
+      val tokens = attribution.assembleTokens(node)
+      println("node ${node.toString().replace("\n", "^")}")
+      for (token in tokens) {
+        println("token $token")
+      }
+    }
+    coalescer.visit(compilationUnit)
+  }
+}

core/src/main/kotlin/gay/pizza/pork/cli/FileTool.kt

@@ -0,0 +1,14 @@
+package gay.pizza.pork.cli
+
+import gay.pizza.pork.frontend.ContentSource
+import gay.pizza.pork.frontend.FsContentSource
+import gay.pizza.pork.parse.CharSource
+import gay.pizza.pork.parse.StringCharSource
+import java.nio.file.Path
+import kotlin.io.path.readText
+
+class FileTool(val path: Path) : Tool() {
+  override fun createCharSource(): CharSource = StringCharSource(path.readText())
+  override fun createContentSource(): ContentSource = FsContentSource(path.parent)
+  override fun rootFilePath(): String = path.fileName.toString()
+}

core/src/main/kotlin/gay/pizza/pork/cli/HighlightCommand.kt

@@ -0,0 +1,15 @@
+package gay.pizza.pork.cli
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.types.path
+import gay.pizza.pork.parse.AnsiHighlightScheme
+
+class HighlightCommand : CliktCommand(help = "Syntax Highlighter", name = "highlight") {
+  val path by argument("file").path(mustExist = true, canBeDir = false)
+
+  override fun run() {
+    val tool = FileTool(path)
+    print(tool.highlight(AnsiHighlightScheme()).joinToString(""))
+  }
+}

core/src/main/kotlin/gay/pizza/pork/cli/ReprintCommand.kt

@@ -0,0 +1,14 @@
+package gay.pizza.pork.cli
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.types.path
+
+class ReprintCommand : CliktCommand(help = "Reprint Parsed Compilation Unit", name = "reprint") {
+  val path by argument("file").path(mustExist = true, canBeDir = false)
+
+  override fun run() {
+    val tool = FileTool(path)
+    print(tool.reprint())
+  }
+}

core/src/main/kotlin/gay/pizza/pork/cli/RootCommand.kt

@@ -0,0 +1,22 @@
+package gay.pizza.pork.cli
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.core.subcommands
+
+class RootCommand : CliktCommand(
+  help = "Pork - The BBQ Language",
+  name = "pork"
+) {
+  init {
+    subcommands(
+      RunCommand(),
+      HighlightCommand(),
+      TokenizeCommand(),
+      ReprintCommand(),
+      AstCommand(),
+      AttributeCommand()
+    )
+  }
+
+  override fun run() {}
+}

core/src/main/kotlin/gay/pizza/pork/cli/RunCommand.kt

@@ -0,0 +1,22 @@
+package gay.pizza.pork.cli
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.types.path
+import gay.pizza.pork.eval.CallableFunction
+import gay.pizza.pork.eval.Scope
+
+class RunCommand : CliktCommand(help = "Run Program", name = "run") {
+  val path by argument("file").path(mustExist = true, canBeDir = false)
+
+  override fun run() {
+    val tool = FileTool(path)
+    val scope = Scope()
+    scope.define("println", CallableFunction { arguments ->
+      for (argument in arguments.values) {
+        println(argument)
+      }
+    })
+    tool.evaluate(scope)
+  }
+}

core/src/main/kotlin/gay/pizza/pork/cli/TokenizeCommand.kt

@@ -0,0 +1,22 @@
+package gay.pizza.pork.cli
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.types.path
+
+class TokenizeCommand : CliktCommand(help = "Tokenize Compilation Unit", name = "tokenize") {
+  val path by argument("file").path(mustExist = true, canBeDir = false)
+
+  override fun run() {
+    val tool = FileTool(path)
+    val tokenStream = tool.tokenize()
+    for (token in tokenStream.tokens) {
+      println("${token.start} ${token.type.name} '${sanitize(token.text)}'")
+    }
+  }
+
+  private fun sanitize(input: String): String =
+    input
+      .replace("\n", "\\n")
+      .replace("\r", "\\r")
+}

core/src/main/kotlin/gay/pizza/pork/cli/Tool.kt

@@ -0,0 +1,36 @@
+package gay.pizza.pork.cli
+
+import gay.pizza.pork.ast.NodeVisitor
+import gay.pizza.pork.parse.Printer
+import gay.pizza.pork.ast.CompilationUnit
+import gay.pizza.pork.eval.*
+import gay.pizza.pork.frontend.ContentSource
+import gay.pizza.pork.frontend.World
+import gay.pizza.pork.parse.*
+
+abstract class Tool {
+  abstract fun createCharSource(): CharSource
+  abstract fun createContentSource(): ContentSource
+  abstract fun rootFilePath(): String
+
+  fun tokenize(): TokenStream =
+    Tokenizer(createCharSource()).tokenize()
+
+  fun parse(attribution: NodeAttribution = DiscardNodeAttribution): CompilationUnit =
+    Parser(TokenStreamSource(tokenize()), attribution).readCompilationUnit()
+
+  fun highlight(scheme: HighlightScheme): List<Highlight> =
+    Highlighter(scheme).highlight(tokenize())
+
+  fun reprint(): String = buildString { visit(Printer(this)) }
+
+  fun <T> visit(visitor: NodeVisitor<T>): T = visitor.visit(parse())
+
+  fun evaluate(scope: Scope) {
+    val contentSource = createContentSource()
+    val world = World(contentSource)
+    val evaluator = Evaluator(world, scope)
+    val resultingScope = evaluator.evaluate(rootFilePath())
+    resultingScope.call("main", Arguments(emptyList()))
+  }
+}

core/src/main/kotlin/gay/pizza/pork/cli/main.kt

@@ -0,0 +1,3 @@
+package gay.pizza.pork.cli
+
+fun main(args: Array<String>) = RootCommand().main(args)

core/src/main/kotlin/gay/pizza/pork/eval/Arguments.kt

@@ -0,0 +1,3 @@
+package gay.pizza.pork.eval
+
+class Arguments(val values: List<Any>)

core/src/main/kotlin/gay/pizza/pork/eval/BlockFunction.kt

@@ -0,0 +1,5 @@
+package gay.pizza.pork.eval
+
+fun interface BlockFunction {
+  fun call(): Any
+}

core/src/main/kotlin/gay/pizza/pork/eval/CallableFunction.kt

@@ -0,0 +1,5 @@
+package gay.pizza.pork.eval
+
+fun interface CallableFunction {
+  fun call(arguments: Arguments): Any
+}

core/src/main/kotlin/gay/pizza/pork/eval/EvaluationContext.kt

@@ -0,0 +1,36 @@
+package gay.pizza.pork.eval
+
+import gay.pizza.pork.ast.CompilationUnit
+import gay.pizza.pork.ast.ImportDeclaration
+
+class EvaluationContext(
+  val compilationUnit: CompilationUnit,
+  val evaluationContextProvider: EvaluationContextProvider,
+  rootScope: Scope
+) {
+  private var isAlreadySetup = false
+
+  val internalScope = rootScope.fork()
+  val externalScope = rootScope.fork()
+
+  private val evaluationVisitor = EvaluationVisitor(internalScope)
+
+  fun setup() {
+    if (isAlreadySetup) {
+      return
+    }
+    isAlreadySetup = true
+    val imports = compilationUnit.declarations.filterIsInstance<ImportDeclaration>()
+    for (import in imports) {
+      val evaluationContext = evaluationContextProvider.provideEvaluationContext(import.path.text)
+      internalScope.inherit(evaluationContext.externalScope)
+    }
+
+    for (definition in compilationUnit.definitions) {
+      evaluationVisitor.visit(definition)
+      if (definition.modifiers.export) {
+        externalScope.define(definition.symbol.id, internalScope.value(definition.symbol.id))
+      }
+    }
+  }
+}

core/src/main/kotlin/gay/pizza/pork/eval/EvaluationContextProvider.kt

@@ -0,0 +1,5 @@
+package gay.pizza.pork.eval
+
+interface EvaluationContextProvider {
+  fun provideEvaluationContext(path: String): EvaluationContext
+}

core/src/main/kotlin/gay/pizza/pork/eval/EvaluationVisitor.kt

@@ -0,0 +1,142 @@
+package gay.pizza.pork.eval
+
+import gay.pizza.pork.ast.*
+
+class EvaluationVisitor(root: Scope) : NodeVisitor<Any> {
+  private var currentScope: Scope = root
+
+  override fun visitIntLiteral(node: IntLiteral): Any = node.value
+  override fun visitStringLiteral(node: StringLiteral): Any = node.text
+  override fun visitBooleanLiteral(node: BooleanLiteral): Any = node.value
+  override fun visitListLiteral(node: ListLiteral): Any = node.items.map { visit(it) }
+
+  override fun visitSymbol(node: Symbol): Any = None
+
+  override fun visitFunctionCall(node: FunctionCall): Any {
+    val arguments = node.arguments.map { visit(it) }
+    return currentScope.call(node.symbol.id, Arguments(arguments))
+  }
+
+  override fun visitDefine(node: Assignment): Any {
+    val value = visit(node.value)
+    currentScope.define(node.symbol.id, value)
+    return value
+  }
+
+  override fun visitSymbolReference(node: SymbolReference): Any =
+    currentScope.value(node.symbol.id)
+
+  override fun visitLambda(node: Lambda): CallableFunction {
+    return CallableFunction { arguments ->
+      currentScope = currentScope.fork()
+      for ((index, argumentSymbol) in node.arguments.withIndex()) {
+        currentScope.define(argumentSymbol.id, arguments.values[index])
+      }
+      try {
+        var value: Any? = null
+        for (expression in node.expressions) {
+          value = visit(expression)
+        }
+        value ?: None
+      } finally {
+        currentScope = currentScope.leave()
+      }
+    }
+  }
+
+  override fun visitParentheses(node: Parentheses): Any = visit(node.expression)
+
+  override fun visitPrefixOperation(node: PrefixOperation): Any {
+    val value = visit(node.expression)
+    return when (node.op) {
+      PrefixOperator.Negate -> {
+        if (value !is Boolean) {
+          throw RuntimeException("Cannot negate a value which is not a boolean.")
+        }
+        !value
+      }
+    }
+  }
+
+  override fun visitIf(node: If): Any {
+    val condition = visit(node.condition)
+    return if (condition == true) {
+      visit(node.thenExpression)
+    } else {
+      if (node.elseExpression != null) {
+        visit(node.elseExpression!!)
+      } else {
+        None
+      }
+    }
+  }
+
+  override fun visitInfixOperation(node: InfixOperation): Any {
+    val left = visit(node.left)
+    val right = visit(node.right)
+
+    when (node.op) {
+      InfixOperator.Equals -> {
+        return left == right
+      }
+      InfixOperator.NotEquals -> {
+        return left != right
+      }
+      else -> {}
+    }
+
+    if (left !is Number || right !is Number) {
+      throw RuntimeException("Failed to evaluate infix operation, bad types.")
+    }
+
+    val leftInt = left.toInt()
+    val rightInt = right.toInt()
+
+    return when (node.op) {
+      InfixOperator.Plus -> leftInt + rightInt
+      InfixOperator.Minus -> leftInt - rightInt
+      InfixOperator.Multiply -> leftInt * rightInt
+      InfixOperator.Divide -> leftInt / rightInt
+      else -> throw RuntimeException("Unable to handle operation ${node.op}")
+    }
+  }
+
+  override fun visitFunctionDeclaration(node: FunctionDefinition): Any {
+    val blockFunction = visitBlock(node.block) as BlockFunction
+    val function = CallableFunction { arguments ->
+      currentScope = currentScope.fork()
+      for ((index, argumentSymbol) in node.arguments.withIndex()) {
+        currentScope.define(argumentSymbol.id, arguments.values[index])
+      }
+      try {
+        return@CallableFunction blockFunction.call()
+      } finally {
+        currentScope = currentScope.leave()
+      }
+    }
+    currentScope.define(node.symbol.id, function)
+    return None
+  }
+
+  override fun visitBlock(node: Block): Any = BlockFunction {
+    var value: Any? = null
+    for (expression in node.expressions) {
+      value = visit(expression)
+    }
+    value ?: None
+  }
+
+  override fun visitImportDeclaration(node: ImportDeclaration): Any {
+    throw RuntimeException(
+      "Import declarations cannot be visited in an EvaluationVisitor. " +
+      "Utilize an EvaluationContext."
+    )
+  }
+
+  override fun visitCompilationUnit(node: CompilationUnit): Any {
+    throw RuntimeException(
+      "Compilation units cannot be visited in an EvaluationVisitor. " +
+        "Utilize an EvaluationContext."
+    )
+  }
+}

core/src/main/kotlin/gay/pizza/pork/eval/Evaluator.kt

@@ -0,0 +1,22 @@
+package gay.pizza.pork.eval
+
+import gay.pizza.pork.frontend.World
+
+class Evaluator(val world: World, val scope: Scope) : EvaluationContextProvider {
+  private val contexts = mutableMapOf<String, EvaluationContext>()
+
+  fun evaluate(path: String): Scope {
+    val context = provideEvaluationContext(path)
+    return context.externalScope
+  }
+
+  override fun provideEvaluationContext(path: String): EvaluationContext {
+    val unit = world.load(path)
+    val identity = world.contentSource.stableContentIdentity(path)
+    val context = contexts.computeIfAbsent(identity) {
+      EvaluationContext(unit, this, scope)
+    }
+    context.setup()
+    return context
+  }
+}

core/src/main/kotlin/gay/pizza/pork/eval/None.kt

@@ -0,0 +1,3 @@
+package gay.pizza.pork.eval
+
+data object None

core/src/main/kotlin/gay/pizza/pork/eval/Scope.kt

@@ -0,0 +1,60 @@
+package gay.pizza.pork.eval
+
+class Scope(val parent: Scope? = null, inherits: List<Scope> = emptyList()) {
+  private val inherited = inherits.toMutableList()
+  private val variables = mutableMapOf<String, Any>()
+
+  fun has(name: String): Boolean =
+    variables.containsKey(name) ||
+      (parent?.has(name) ?: false) ||
+      inherited.any { inherit -> inherit.has(name) }
+
+  fun define(name: String, value: Any) {
+    if (variables.containsKey(name)) {
+      throw RuntimeException("Variable '${name}' is already defined")
+    }
+    variables[name] = value
+  }
+
+  fun value(name: String): Any {
+    val value = variables[name]
+    if (value == null) {
+      if (parent != null) {
+        if (parent.has(name)) {
+          return parent.value(name)
+        }
+      }
+
+      for (inherit in inherited) {
+        if (inherit.has(name)) {
+          return inherit.value(name)
+        }
+      }
+      throw RuntimeException("Variable '${name}' not defined.")
+    }
+    return value
+  }
+
+  fun call(name: String, arguments: Arguments): Any {
+    val value = value(name)
+    if (value !is CallableFunction) {
+      throw RuntimeException("$value is not callable")
+    }
+    return value.call(arguments)
+  }
+
+  fun fork(): Scope {
+    return Scope(this)
+  }
+
+  fun leave(): Scope {
+    if (parent == null) {
+      throw RuntimeException("Parent context not found")
+    }
+    return parent
+  }
+
+  internal fun inherit(scope: Scope) {
+    inherited.add(scope)
+  }
+}

core/src/main/kotlin/gay/pizza/pork/frontend/ContentSource.kt

@@ -0,0 +1,8 @@
+package gay.pizza.pork.frontend
+
+import gay.pizza.pork.parse.CharSource
+
+interface ContentSource {
+  fun loadAsCharSource(path: String): CharSource
+  fun stableContentIdentity(path: String): String
+}

core/src/main/kotlin/gay/pizza/pork/frontend/FsContentSource.kt

@@ -0,0 +1,24 @@
+package gay.pizza.pork.frontend
+
+import gay.pizza.pork.parse.CharSource
+import gay.pizza.pork.parse.StringCharSource
+import java.nio.file.Path
+import kotlin.io.path.absolutePathString
+import kotlin.io.path.readText
+
+class FsContentSource(val root: Path) : ContentSource {
+  override fun loadAsCharSource(path: String): CharSource =
+    StringCharSource(asFsPath(path).readText())
+
+  override fun stableContentIdentity(path: String): String =
+    asFsPath(path).absolutePathString()
+
+  private fun asFsPath(path: String): Path {
+    val fsPath = root.resolve(path)
+    val absoluteRootPath = root.absolutePathString() + root.fileSystem.separator
+    if (!fsPath.absolutePathString().startsWith(absoluteRootPath)) {
+      throw RuntimeException("Unable to load path outside of the root: $fsPath (root is ${root})")
+    }
+    return fsPath
+  }
+}

core/src/main/kotlin/gay/pizza/pork/frontend/World.kt

@@ -0,0 +1,42 @@
+package gay.pizza.pork.frontend
+
+import gay.pizza.pork.ast.CompilationUnit
+import gay.pizza.pork.ast.ImportDeclaration
+import gay.pizza.pork.parse.DiscardNodeAttribution
+import gay.pizza.pork.parse.Parser
+import gay.pizza.pork.parse.TokenStreamSource
+import gay.pizza.pork.parse.Tokenizer
+
+class World(val contentSource: ContentSource) {
+  private val units = mutableMapOf<String, CompilationUnit>()
+
+  private fun loadOneUnit(path: String): CompilationUnit {
+    val stableIdentity = contentSource.stableContentIdentity(path)
+    val cached = units[stableIdentity]
+    if (cached != null) {
+      return cached
+    }
+    val charSource = contentSource.loadAsCharSource(path)
+    val tokenizer = Tokenizer(charSource)
+    val tokenStream = tokenizer.tokenize()
+    val parser = Parser(TokenStreamSource(tokenStream), DiscardNodeAttribution)
+    return parser.readCompilationUnit()
+  }
+
+  private fun resolveAllImports(unit: CompilationUnit): Set<CompilationUnit> {
+    val units = mutableSetOf<CompilationUnit>()
+    for (declaration in unit.declarations.filterIsInstance<ImportDeclaration>()) {
+      val importedUnit = loadOneUnit(declaration.path.text)
+      units.add(importedUnit)
+    }
+    return units
+  }
+
+  fun load(path: String): CompilationUnit {
+    val unit = loadOneUnit(path)
+    resolveAllImports(unit)
+    return unit
+  }
+
+  fun units(path: String): Set<CompilationUnit> = resolveAllImports(loadOneUnit(path))
+}

core/src/main/kotlin/gay/pizza/pork/parse/AnsiHighlightScheme.kt

@@ -0,0 +1,31 @@
+package gay.pizza.pork.parse
+
+open class AnsiHighlightScheme : HighlightScheme {
+  override fun highlight(token: Token): Highlight {
+    val attributes = when (token.type.family) {
+      TokenFamily.StringLiteralFamily -> string()
+      TokenFamily.OperatorFamily -> operator()
+      TokenFamily.KeywordFamily -> keyword()
+      TokenFamily.SymbolFamily -> symbol()
+      TokenFamily.CommentFamily -> comment()
+      else -> null
+    }
+
+    return if (attributes != null) {
+      Highlight(token, ansi(attributes, token.text))
+    } else Highlight(token)
+  }
+
+  open fun string(): AnsiAttributes = AnsiAttributes("32m")
+  open fun symbol(): AnsiAttributes = AnsiAttributes("33m")
+  open fun operator(): AnsiAttributes = AnsiAttributes("34m")
+  open fun keyword(): AnsiAttributes = AnsiAttributes("35m")
+  open fun comment(): AnsiAttributes = AnsiAttributes("37m")
+
+  private fun ansi(attributes: AnsiAttributes, text: String): String =
+    "\u001b[${attributes.color}${text}\u001b[0m"
+
+  class AnsiAttributes(
+    val color: String
+  )
+}

core/src/main/kotlin/gay/pizza/pork/parse/CharSource.kt

@@ -0,0 +1,7 @@
+package gay.pizza.pork.parse
+
+interface CharSource : PeekableSource<Char> {
+  companion object {
+    const val NullChar = 0.toChar()
+  }
+}

core/src/main/kotlin/gay/pizza/pork/parse/DiscardNodeAttribution.kt

@@ -0,0 +1,9 @@
+package gay.pizza.pork.parse
+
+import gay.pizza.pork.ast.Node
+
+object DiscardNodeAttribution : NodeAttribution {
+  override fun enter() {}
+  override fun push(token: Token) {}
+  override fun <T : Node> exit(node: T): T = node
+}

core/src/main/kotlin/gay/pizza/pork/parse/Highlight.kt

@@ -0,0 +1,5 @@
+package gay.pizza.pork.parse
+
+class Highlight(val token: Token, val text: String? = null) {
+  override fun toString(): String = text ?: token.text
+}

core/src/main/kotlin/gay/pizza/pork/parse/HighlightScheme.kt

@@ -0,0 +1,5 @@
+package gay.pizza.pork.parse
+
+interface HighlightScheme {
+  fun highlight(token: Token): Highlight
+}

core/src/main/kotlin/gay/pizza/pork/parse/Highlighter.kt

@@ -0,0 +1,6 @@
+package gay.pizza.pork.parse
+
+class Highlighter(val scheme: HighlightScheme) {
+  fun highlight(stream: TokenStream): List<Highlight> =
+    stream.tokens.map { scheme.highlight(it) }
+}

core/src/main/kotlin/gay/pizza/pork/parse/NodeAttribution.kt

@@ -0,0 +1,9 @@
+package gay.pizza.pork.parse
+
+import gay.pizza.pork.ast.Node
+
+interface NodeAttribution {
+  fun enter()
+  fun push(token: Token)
+  fun <T: Node> exit(node: T): T
+}

core/src/main/kotlin/gay/pizza/pork/parse/Parser.kt

@@ -0,0 +1,338 @@
+package gay.pizza.pork.parse
+
+import gay.pizza.pork.ast.*
+import gay.pizza.pork.util.StringEscape
+
+class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
+  private val unsanitizedSource = source
+
+  private fun readIntLiteral(): IntLiteral = within {
+    expect(TokenType.IntLiteral) { IntLiteral(it.text.toInt()) }
+  }
+
+  private fun readStringLiteral(): StringLiteral = within {
+    expect(TokenType.StringLiteral) {
+      val content = StringEscape.unescape(StringEscape.unquote(it.text))
+      StringLiteral(content)
+    }
+  }
+
+  private fun readBooleanLiteral(): BooleanLiteral = within {
+    expect(TokenType.True, TokenType.False) {
+      BooleanLiteral(it.type == TokenType.True)
+    }
+  }
+
+  private fun readListLiteral(): ListLiteral = within {
+    expect(TokenType.LeftBracket)
+    val items = collect(TokenType.RightBracket, TokenType.Comma) {
+      readExpression()
+    }
+    expect(TokenType.RightBracket)
+    ListLiteral(items)
+  }
+
+  private fun readSymbolRaw(): Symbol = within {
+    expect(TokenType.Symbol) { Symbol(it.text) }
+  }
+
+  private fun readSymbolCases(): Expression = within {
+    val symbol = readSymbolRaw()
+    if (next(TokenType.LeftParentheses)) {
+      val arguments = collect(TokenType.RightParentheses, TokenType.Comma) {
+        readExpression()
+      }
+      expect(TokenType.RightParentheses)
+      FunctionCall(symbol, arguments)
+    } else if (next(TokenType.Equals)) {
+      Assignment(symbol, readExpression())
+    } else {
+      SymbolReference(symbol)
+    }
+  }
+
+  private fun readLambda(): Lambda = within {
+    expect(TokenType.LeftCurly)
+    val arguments = mutableListOf<Symbol>()
+    while (!peek(TokenType.In)) {
+      val symbol = readSymbolRaw()
+      arguments.add(symbol)
+      if (next(TokenType.Comma)) {
+        continue
+      } else {
+        break
+      }
+    }
+    expect(TokenType.In)
+    val items = collect(TokenType.RightCurly) {
+      readExpression()
+    }
+    expect(TokenType.RightCurly)
+    Lambda(arguments, items)
+  }
+
+  private fun readParentheses(): Parentheses = within {
+    expect(TokenType.LeftParentheses)
+    val expression = readExpression()
+    expect(TokenType.RightParentheses)
+    Parentheses(expression)
+  }
+
+  private fun readPrefixOperation(): PrefixOperation = within {
+    expect(TokenType.Negation) {
+      PrefixOperation(PrefixOperator.Negate, readExpression())
+    }
+  }
+
+  private fun readIf(): If = within {
+    expect(TokenType.If)
+    val condition = readExpression()
+    expect(TokenType.Then)
+    val thenExpression = readExpression()
+    var elseExpression: Expression? = null
+    if (next(TokenType.Else)) {
+      elseExpression = readExpression()
+    }
+    If(condition, thenExpression, elseExpression)
+  }
+
+  fun readExpression(): Expression {
+    val token = peek()
+    val expression = when (token.type) {
+      TokenType.IntLiteral -> {
+        readIntLiteral()
+      }
+
+      TokenType.StringLiteral -> {
+        readStringLiteral()
+      }
+
+      TokenType.True, TokenType.False -> {
+        readBooleanLiteral()
+      }
+
+      TokenType.LeftBracket -> {
+        readListLiteral()
+      }
+
+      TokenType.Symbol -> {
+        readSymbolCases()
+      }
+
+      TokenType.LeftCurly -> {
+        readLambda()
+      }
+
+      TokenType.LeftParentheses -> {
+        readParentheses()
+      }
+
+      TokenType.Negation -> {
+        readPrefixOperation()
+      }
+
+      TokenType.If -> {
+        readIf()
+      }
+
+      else -> {
+        throw RuntimeException(
+          "Failed to parse token: ${token.type} '${token.text}' as" +
+            " expression (index ${unsanitizedSource.currentIndex})"
+        )
+      }
+    }
+
+    return if (peek(
+        TokenType.Plus,
+        TokenType.Minus,
+        TokenType.Multiply,
+        TokenType.Divide,
+        TokenType.Equality,
+        TokenType.Inequality
+      )
+    ) {
+      within {
+        val infixToken = next()
+        val infixOperator = convertInfixOperator(infixToken)
+        InfixOperation(expression, infixOperator, readExpression())
+      }
+    } else expression
+  }
+
+  private fun readBlock(): Block = within {
+    expect(TokenType.LeftCurly)
+    val items = collect(TokenType.RightCurly) {
+      readExpression()
+    }
+    expect(TokenType.RightCurly)
+    Block(items)
+  }
+
+  private fun readImportDeclaration(): ImportDeclaration = within {
+    expect(TokenType.Import)
+    ImportDeclaration(readStringLiteral())
+  }
+
+  private fun readFunctionDeclaration(): FunctionDefinition = within {
+    val modifiers = DefinitionModifiers(export = false)
+    while (true) {
+      val token = peek()
+      when (token.type) {
+        TokenType.Export -> {
+          expect(TokenType.Export)
+          modifiers.export = true
+        }
+        else -> break
+      }
+    }
+    expect(TokenType.Func)
+    val name = readSymbolRaw()
+    expect(TokenType.LeftParentheses)
+    val arguments = collect(TokenType.RightParentheses, TokenType.Comma) { readSymbolRaw() }
+    expect(TokenType.RightParentheses)
+    FunctionDefinition(modifiers, name, arguments, readBlock())
+  }
+
+  private fun maybeReadDefinition(): Definition? {
+    val token = peek()
+    return when (token.type) {
+      TokenType.Export,
+      TokenType.Func -> readFunctionDeclaration()
+      else -> null
+    }
+  }
+
+  private fun readDefinition(): Definition {
+    val definition = maybeReadDefinition()
+    if (definition != null) {
+      return definition
+    }
+    val token = peek()
+    throw RuntimeException(
+      "Failed to parse token: ${token.type} '${token.text}' as" +
+        " definition (index ${unsanitizedSource.currentIndex})"
+    )
+  }
+
+  fun readDeclaration(): Declaration {
+    val token = peek()
+    return when (token.type) {
+      TokenType.Import -> readImportDeclaration()
+      else -> throw RuntimeException(
+        "Failed to parse token: ${token.type} '${token.text}' as" +
+          " declaration (index ${unsanitizedSource.currentIndex})"
+      )
+    }
+  }
+
+  private fun convertInfixOperator(token: Token): InfixOperator =
+    when (token.type) {
+      TokenType.Plus -> InfixOperator.Plus
+      TokenType.Minus -> InfixOperator.Minus
+      TokenType.Multiply -> InfixOperator.Multiply
+      TokenType.Divide -> InfixOperator.Divide
+      TokenType.Equality -> InfixOperator.Equals
+      TokenType.Inequality -> InfixOperator.NotEquals
+      else -> throw RuntimeException("Unknown Infix Operator")
+    }
+
+  fun readCompilationUnit(): CompilationUnit = within {
+    val declarations = mutableListOf<Declaration>()
+    val definitions = mutableListOf<Definition>()
+    var declarationAccepted = true
+
+    while (!peek(TokenType.EndOfFile)) {
+      if (declarationAccepted) {
+        val definition = maybeReadDefinition()
+        if (definition != null) {
+          declarationAccepted = false
+          definitions.add(definition)
+          continue
+        }
+        declarations.add(readDeclaration())
+      } else {
+        definitions.add(readDefinition())
+      }
+    }
+
+    CompilationUnit(declarations, definitions)
+  }
+
+  private fun <T> collect(
+    peeking: TokenType,
+    consuming: TokenType? = null,
+    read: () -> T
+  ): List<T> {
+    val items = mutableListOf<T>()
+    while (!peek(peeking)) {
+      val expression = read()
+      if (consuming != null) {
+        next(consuming)
+      }
+      items.add(expression)
+    }
+    return items
+  }
+
+  private fun peek(vararg types: TokenType): Boolean {
+    val token = peek()
+    return types.contains(token.type)
+  }
+
+  private fun next(type: TokenType): Boolean {
+    return if (peek(type)) {
+      expect(type)
+      true
+    } else false
+  }
+
+  private fun expect(vararg types: TokenType): Token {
+    val token = next()
+    if (!types.contains(token.type)) {
+      throw RuntimeException(
+        "Expected one of ${types.joinToString(", ")}" +
+          " but got type ${token.type} '${token.text}'"
+      )
+    }
+    return token
+  }
+
+  private fun <T: Node> expect(vararg types: TokenType, consume: (Token) -> T): T =
+    consume(expect(*types))
+
+  private fun next(): Token {
+    while (true) {
+      val token = unsanitizedSource.next()
+      attribution.push(token)
+      if (ignoredByParser(token.type)) {
+        continue
+      }
+      return token
+    }
+  }
+
+  private fun peek(): Token {
+    while (true) {
+      val token = unsanitizedSource.peek()
+      if (ignoredByParser(token.type)) {
+        attribution.push(token)
+        unsanitizedSource.next()
+        continue
+      }
+      return token
+    }
+  }
+
+  private fun <T: Node> within(block: () -> T): T {
+    attribution.enter()
+    return attribution.exit(block())
+  }
+
+  private fun ignoredByParser(type: TokenType): Boolean = when (type) {
+    TokenType.BlockComment -> true
+    TokenType.LineComment -> true
+    TokenType.Whitespace -> true
+    else -> false
+  }
+}

core/src/main/kotlin/gay/pizza/pork/parse/PeekableSource.kt

@@ -0,0 +1,7 @@
+package gay.pizza.pork.parse
+
+interface PeekableSource<T> {
+  val currentIndex: Int
+  fun next(): T
+  fun peek(): T
+}

core/src/main/kotlin/gay/pizza/pork/parse/Printer.kt

@@ -0,0 +1,198 @@
+package gay.pizza.pork.parse
+
+import gay.pizza.pork.ast.*
+import gay.pizza.pork.util.IndentPrinter
+import gay.pizza.pork.util.StringEscape
+
+class Printer(buffer: StringBuilder) : NodeVisitor<Unit> {
+  private val out = IndentPrinter(buffer)
+  private var autoIndentState = false
+
+  private fun append(text: String) {
+    if (autoIndentState) {
+      out.emitIndent()
+      autoIndentState = false
+    }
+    out.append(text)
+  }
+
+  private fun appendLine() {
+    out.appendLine()
+    autoIndentState = true
+  }
+
+  override fun visitIntLiteral(node: IntLiteral) {
+    append(node.value.toString())
+  }
+
+  override fun visitStringLiteral(node: StringLiteral) {
+    append("\"")
+    append(StringEscape.escape(node.text))
+    append("\"")
+  }
+
+  override fun visitBooleanLiteral(node: BooleanLiteral) {
+    if (node.value) {
+      append("true")
+    } else {
+      append("false")
+    }
+  }
+
+  override fun visitListLiteral(node: ListLiteral) {
+    append("[")
+    if (node.items.isNotEmpty()) {
+      out.increaseIndent()
+      appendLine()
+      for ((index, item) in node.items.withIndex()) {
+        visit(item)
+        if (index != node.items.size - 1) {
+          append(",")
+        }
+        appendLine()
+      }
+      out.decreaseIndent()
+    }
+    append("]")
+  }
+
+  override fun visitSymbol(node: Symbol) {
+    append(node.id)
+  }
+
+  override fun visitFunctionCall(node: FunctionCall) {
+    visit(node.symbol)
+    append("(")
+    for ((index, argument) in node.arguments.withIndex()) {
+      visit(argument)
+      if (index + 1 != node.arguments.size) {
+        append(", ")
+      }
+    }
+    append(")")
+  }
+
+  override fun visitDefine(node: Assignment) {
+    visit(node.symbol)
+    append(" = ")
+    visit(node.value)
+  }
+
+  override fun visitSymbolReference(node: SymbolReference) {
+    visit(node.symbol)
+  }
+
+  override fun visitLambda(node: Lambda) {
+    append("{")
+    if (node.arguments.isNotEmpty()) {
+      append(" ")
+      for ((index, argument) in node.arguments.withIndex()) {
+        visit(argument)
+        if (index + 1 != node.arguments.size) {
+          append(",")
+        }
+        append(" ")
+      }
+    } else {
+      append(" ")
+    }
+    append("in")
+    out.increaseIndent()
+    for (expression in node.expressions) {
+      appendLine()
+      visit(expression)
+    }
+
+    if (node.expressions.isNotEmpty()) {
+      appendLine()
+    }
+    out.decreaseIndent()
+    append("}")
+  }
+
+  override fun visitParentheses(node: Parentheses) {
+    append("(")
+    visit(node.expression)
+    append(")")
+  }
+
+  override fun visitPrefixOperation(node: PrefixOperation) {
+    append(node.op.token)
+    visit(node.expression)
+  }
+
+  override fun visitIf(node: If) {
+    append("if ")
+    visit(node.condition)
+    append(" then")
+    out.increaseIndent()
+    appendLine()
+    visit(node.thenExpression)
+    out.decreaseIndent()
+    if (node.elseExpression != null) {
+      appendLine()
+      append("else")
+      out.increaseIndent()
+      appendLine()
+      visit(node.elseExpression!!)
+      out.decreaseIndent()
+    }
+  }
+
+  override fun visitInfixOperation(node: InfixOperation) {
+    visit(node.left)
+    append(" ")
+    append(node.op.token)
+    append(" ")
+    visit(node.right)
+  }
+
+  override fun visitFunctionDeclaration(node: FunctionDefinition) {
+    append("fn ")
+    visit(node.symbol)
+    append("(")
+    for ((index, argument) in node.arguments.withIndex()) {
+      visit(argument)
+      if (index + 1 != node.arguments.size) {
+        append(", ")
+      }
+    }
+    append(") ")
+    visit(node.block)
+  }
+
+  override fun visitBlock(node: Block) {
+    append("{")
+    if (node.expressions.isNotEmpty()) {
+      out.increaseIndent()
+      for (expression in node.expressions) {
+        appendLine()
+        visit(expression)
+      }
+      out.decreaseIndent()
+      appendLine()
+    }
+    append("}")
+  }
+
+  override fun visitImportDeclaration(node: ImportDeclaration) {
+    append("import ")
+    visit(node.path)
+  }
+
+  override fun visitCompilationUnit(node: CompilationUnit) {
+    for (declaration in node.declarations) {
+      visit(declaration)
+      appendLine()
+    }
+
+    if (node.declarations.isNotEmpty()) {
+      appendLine()
+    }
+
+    for (definition in node.definitions) {
+      visit(definition)
+      appendLine()
+    }
+  }
+}

core/src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt

@@ -0,0 +1,23 @@
+package gay.pizza.pork.parse
+
+class StringCharSource(val input: String) : CharSource {
+  private var index = 0
+  override val currentIndex: Int
+    get() = index
+
+  override fun next(): Char {
+    if (index == input.length) {
+      return CharSource.NullChar
+    }
+    val char = input[index]
+    index++
+    return char
+  }
+
+  override fun peek(): Char {
+    if (index == input.length) {
+      return CharSource.NullChar
+    }
+    return input[index]
+  }
+}

core/src/main/kotlin/gay/pizza/pork/parse/Token.kt

@@ -0,0 +1,10 @@
+package gay.pizza.pork.parse
+
+class Token(val type: TokenType, val start: Int, val text: String) {
+  override fun toString(): String = "$start ${type.name} '${text.replace("\n", "\\n")}'"
+
+  companion object {
+    fun endOfFile(size: Int): Token =
+      Token(TokenType.EndOfFile, size, "")
+  }
+}

core/src/main/kotlin/gay/pizza/pork/parse/TokenFamily.kt

@@ -0,0 +1,11 @@
+package gay.pizza.pork.parse
+
+enum class TokenFamily : TokenTypeProperty {
+  OperatorFamily,
+  KeywordFamily,
+  SymbolFamily,
+  NumericLiteralFamily,
+  StringLiteralFamily,
+  CommentFamily,
+  OtherFamily
+}

core/src/main/kotlin/gay/pizza/pork/parse/TokenNodeAttribution.kt

@@ -0,0 +1,44 @@
+package gay.pizza.pork.parse
+
+import gay.pizza.pork.ast.NodeCoalescer
+import gay.pizza.pork.ast.Node
+import java.util.IdentityHashMap
+
+class TokenNodeAttribution : NodeAttribution {
+  val nodes: MutableMap<Node, List<Token>> = IdentityHashMap()
+
+  private val stack = mutableListOf<MutableList<Token>>()
+  private var current: MutableList<Token>? = null
+
+  override fun enter() {
+    val store = mutableListOf<Token>()
+    current = store
+    stack.add(store)
+  }
+
+  override fun push(token: Token) {
+    val store = current ?: throw RuntimeException("enter() not called!")
+    store.add(token)
+  }
+
+  override fun <T: Node> exit(node: T): T {
+    val store = stack.removeLast()
+    nodes[node] = store
+    current = stack.lastOrNull()
+    return node
+  }
+
+  fun tokensOf(node: Node): List<Token>? = nodes[node]
+
+  fun assembleTokens(node: Node): List<Token> {
+    val allTokens = mutableListOf<Token>()
+    val coalescer = NodeCoalescer { item ->
+      val tokens = tokensOf(item)
+      if (tokens != null) {
+        allTokens.addAll(tokens)
+      }
+    }
+    coalescer.visit(node)
+    return allTokens.asSequence().distinct().sortedBy { it.start }.toList()
+  }
+}

core/src/main/kotlin/gay/pizza/pork/parse/TokenSource.kt

@@ -0,0 +1,3 @@
+package gay.pizza.pork.parse
+
+interface TokenSource : PeekableSource<Token>

core/src/main/kotlin/gay/pizza/pork/parse/TokenStream.kt

@@ -0,0 +1,5 @@
+package gay.pizza.pork.parse
+
+class TokenStream(val tokens: List<Token>) {
+  override fun toString(): String = tokens.toString()
+}

core/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt

@@ -0,0 +1,23 @@
+package gay.pizza.pork.parse
+
+class TokenStreamSource(val stream: TokenStream) : TokenSource {
+  private var index = 0
+  override val currentIndex: Int
+    get() = index
+
+  override fun next(): Token {
+    if (index == stream.tokens.size) {
+      return Token.endOfFile(stream.tokens.size)
+    }
+    val char = stream.tokens[index]
+    index++
+    return char
+  }
+
+  override fun peek(): Token {
+    if (index == stream.tokens.size) {
+      return Token.endOfFile(stream.tokens.size)
+    }
+    return stream.tokens[index]
+  }
+}

core/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt

@@ -0,0 +1,52 @@
+package gay.pizza.pork.parse
+
+import gay.pizza.pork.parse.TokenTypeProperty.*
+import gay.pizza.pork.parse.TokenFamily.*
+
+enum class TokenType(vararg properties: TokenTypeProperty) {
+  Symbol(SymbolFamily, CharConsumer { (it in 'a'..'z') || (it in 'A'..'Z') || it == '_' }, KeywordUpgrader),
+  IntLiteral(NumericLiteralFamily, CharConsumer { it in '0'..'9' }),
+  StringLiteral(StringLiteralFamily),
+  Equality(OperatorFamily),
+  Inequality(OperatorFamily),
+  Equals(SingleChar('='), Promotion('=', Equality)),
+  Plus(SingleChar('+'), OperatorFamily),
+  Minus(SingleChar('-'), OperatorFamily),
+  Multiply(SingleChar('*'), OperatorFamily),
+  Divide(SingleChar('/'), OperatorFamily),
+  LeftCurly(SingleChar('{')),
+  RightCurly(SingleChar('}')),
+  LeftBracket(SingleChar('[')),
+  RightBracket(SingleChar(']')),
+  LeftParentheses(SingleChar('(')),
+  RightParentheses(SingleChar(')')),
+  Negation(SingleChar('!'), Promotion('=', Inequality), OperatorFamily),
+  Comma(SingleChar(',')),
+  False(Keyword("false"), KeywordFamily),
+  True(Keyword("true"), KeywordFamily),
+  In(Keyword("in"), KeywordFamily),
+  If(Keyword("if"), KeywordFamily),
+  Then(Keyword("then"), KeywordFamily),
+  Else(Keyword("else"), KeywordFamily),
+  Import(Keyword("import"), KeywordFamily),
+  Export(Keyword("export"), KeywordFamily),
+  Func(Keyword("func"), KeywordFamily),
+  Whitespace(CharConsumer { it == ' ' || it == '\r' || it == '\n' || it == '\t' }),
+  BlockComment(CommentFamily),
+  LineComment(CommentFamily),
+  EndOfFile;
+
+  val promotions: List<Promotion> = properties.filterIsInstance<Promotion>()
+  val keyword: Keyword? = properties.filterIsInstance<Keyword>().singleOrNull()
+  val singleChar: SingleChar? = properties.filterIsInstance<SingleChar>().singleOrNull()
+  val family: TokenFamily =
+    properties.filterIsInstance<TokenFamily>().singleOrNull() ?: OtherFamily
+  val charConsumer: CharConsumer? = properties.filterIsInstance<CharConsumer>().singleOrNull()
+  val tokenUpgrader: TokenUpgrader? = properties.filterIsInstance<TokenUpgrader>().singleOrNull()
+
+  companion object {
+    val Keywords = entries.filter { item -> item.keyword != null }
+    val SingleChars = entries.filter { item -> item.singleChar != null }
+    val CharConsumers = entries.filter { item -> item.charConsumer != null }
+  }
+}

core/src/main/kotlin/gay/pizza/pork/parse/TokenTypeProperty.kt

@@ -0,0 +1,20 @@
+package gay.pizza.pork.parse
+
+interface TokenTypeProperty {
+  class SingleChar(val char: Char) : TokenTypeProperty
+  class Promotion(val nextChar: Char, val type: TokenType) : TokenTypeProperty
+  class Keyword(val text: String) : TokenTypeProperty
+  class CharConsumer(val isValid: (Char) -> Boolean) : TokenTypeProperty
+  open class TokenUpgrader(val maybeUpgrade: (Token) -> Token?) : TokenTypeProperty
+
+  object KeywordUpgrader : TokenUpgrader({ token ->
+    var upgraded: Token? = null
+    for (item in TokenType.Keywords) {
+      if (item.keyword != null && token.text == item.keyword.text) {
+        upgraded = Token(item, token.start, token.text)
+        break
+      }
+    }
+    upgraded
+  })
+}

core/src/main/kotlin/gay/pizza/pork/parse/Tokenizer.kt

@@ -0,0 +1,133 @@
+package gay.pizza.pork.parse
+
+class Tokenizer(val source: CharSource) {
+  private var tokenStart: Int = 0
+
+  private fun readBlockComment(firstChar: Char): Token {
+    val comment = buildString {
+      append(firstChar)
+      var endOfComment = false
+      while (true) {
+        val char = source.next()
+        append(char)
+
+        if (endOfComment) {
+          if (char != '/') {
+            endOfComment = false
+            continue
+          }
+          break
+        }
+
+        if (char == '*') {
+          endOfComment = true
+        }
+      }
+    }
+    return Token(TokenType.BlockComment, tokenStart, comment)
+  }
+
+  private fun readLineComment(firstChar: Char): Token {
+    val comment = buildString {
+      append(firstChar)
+      while (true) {
+        val char = source.peek()
+        if (char == CharSource.NullChar || char == '\n') {
+          break
+        }
+        append(source.next())
+      }
+    }
+    return Token(TokenType.LineComment, tokenStart, comment)
+  }
+
+  private fun readStringLiteral(firstChar: Char): Token {
+    val string = buildString {
+      append(firstChar)
+      while (true) {
+        val char = source.peek()
+        if (char == CharSource.NullChar) {
+          throw RuntimeException("Unterminated string.")
+        }
+        append(source.next())
+        if (char == '"') {
+          break
+        }
+      }
+    }
+    return Token(TokenType.StringLiteral, tokenStart, string)
+  }
+
+  fun next(): Token {
+    while (source.peek() != CharSource.NullChar) {
+      tokenStart = source.currentIndex
+      val char = source.next()
+
+      if (char == '/' && source.peek() == '*') {
+        return readBlockComment(char)
+      }
+
+      if (char == '/' && source.peek() == '/') {
+        return readLineComment(char)
+      }
+
+      for (item in TokenType.SingleChars) {
+        val itemChar = item.singleChar!!.char
+        if (itemChar != char) {
+          continue
+        }
+
+        var type = item
+        var text = itemChar.toString()
+        for (promotion in item.promotions) {
+          if (source.peek() != promotion.nextChar) {
+            continue
+          }
+          val nextChar = source.next()
+          type = promotion.type
+          text += nextChar
+        }
+        return Token(type, tokenStart, text)
+      }
+
+      for (item in TokenType.CharConsumers) {
+        val consumer = item.charConsumer ?: continue
+        if (!consumer.isValid(char)) {
+          continue
+        }
+
+        val text = buildString {
+          append(char)
+          while (consumer.isValid(source.peek())) {
+            append(source.next())
+          }
+        }
+        var token = Token(item, tokenStart, text)
+        val tokenUpgrader = item.tokenUpgrader
+        if (tokenUpgrader != null) {
+          token = tokenUpgrader.maybeUpgrade(token) ?: token
+        }
+        return token
+      }
+
+      if (char == '"') {
+        return readStringLiteral(char)
+      }
+
+      throw RuntimeException("Failed to parse: (${char}) next ${source.peek()}")
+    }
+    return Token.endOfFile(source.currentIndex)
+  }
+
+  fun tokenize(): TokenStream {
+    val tokens = mutableListOf<Token>()
+    while (true) {
+      val token = next()
+      tokens.add(token)
+      if (token.type == TokenType.EndOfFile) {
+        break
+      }
+    }
+    return TokenStream(tokens)
+  }
+}

core/src/main/kotlin/gay/pizza/pork/util/IndentPrinter.kt

@@ -0,0 +1,25 @@
+package gay.pizza.pork.util
+
+class IndentPrinter(
+  val buffer: StringBuilder = StringBuilder(),
+  val indent: String = "  "
+) : Appendable by buffer, CharSequence by buffer {
+  private var indentLevel: Int = 0
+  private var indentLevelText: String = ""
+
+  fun emitIndent() {
+    append(indentLevelText)
+  }
+
+  fun increaseIndent() {
+    indentLevel++
+    indentLevelText = indent.repeat(indentLevel)
+  }
+
+  fun decreaseIndent() {
+    indentLevel--
+    indentLevelText = indent.repeat(indentLevel)
+  }
+
+  override fun toString(): String = buffer.toString()
+}

core/src/main/kotlin/gay/pizza/pork/util/StringEscape.kt

@@ -0,0 +1,8 @@
+package gay.pizza.pork.util
+
+object StringEscape {
+  fun escape(input: String): String = input.replace("\n", "\\n")
+  fun unescape(input: String): String = input.replace("\\n", "\n")
+
+  fun unquote(input: String): String = input.substring(1, input.length - 1)
+}