add a standard library, and introduce formed imports (import std "myfile.pork")

ast/src/main/ast/pork.yml

@@ -110,6 +110,8 @@ types:
   ImportDeclaration:
     parent: Declaration
     values:
+    - name: form
+      type: Symbol?
     - name: path
       type: StringLiteral
   IntLiteral:

ast/src/main/kotlin/gay/pizza/pork/ast/ImportDeclaration.kt

@@ -6,22 +6,23 @@ import kotlinx.serialization.Serializable
 
 @Serializable
 @SerialName("importDeclaration")
-class ImportDeclaration(val path: StringLiteral) : Declaration() {
+class ImportDeclaration(val form: Symbol?, val path: StringLiteral) : Declaration() {
   override val type: NodeType = NodeType.ImportDeclaration
 
   override fun <T> visitChildren(visitor: NodeVisitor<T>): List<T> =
-    visitor.visitNodes(path)
+    visitor.visitNodes(form, path)
 
   override fun <T> visit(visitor: NodeVisitor<T>): T =
     visitor.visitImportDeclaration(this)
 
   override fun equals(other: Any?): Boolean {
     if (other !is ImportDeclaration) return false
-    return other.path == path
+    return other.form == form && other.path == path
   }
 
   override fun hashCode(): Int {
-    var result = path.hashCode()
+    var result = form.hashCode()
+    result = 31 * result + path.hashCode()
     result = 31 * result + type.hashCode()
     return result
   }

evaluator/src/main/kotlin/gay/pizza/pork/evaluator/CompilationUnitContext.kt

@@ -4,6 +4,7 @@ import gay.pizza.pork.ast.CompilationUnit
 import gay.pizza.pork.ast.Definition
 import gay.pizza.pork.ast.FunctionDefinition
 import gay.pizza.pork.ast.ImportDeclaration
+import gay.pizza.pork.frontend.ImportLocator
 
 class CompilationUnitContext(
   val compilationUnit: CompilationUnit,
@@ -50,8 +51,8 @@ class CompilationUnitContext(
   }
 
   private fun processImport(import: ImportDeclaration) {
-    val path = import.path.text
+    val importLocator = ImportLocator(import.path.text, import.form?.id)
-    val evaluationContext = evaluator.context(path)
+    val evaluationContext = evaluator.context(importLocator)
     internalScope.inherit(evaluationContext.externalScope)
   }
 }

evaluator/src/main/kotlin/gay/pizza/pork/evaluator/Evaluator.kt

@@ -1,17 +1,18 @@
 package gay.pizza.pork.evaluator
 
+import gay.pizza.pork.frontend.ImportLocator
 import gay.pizza.pork.frontend.World
 
 class Evaluator(val world: World, val scope: Scope) {
   private val contexts = mutableMapOf<String, CompilationUnitContext>()
   private val nativeFunctionProviders = mutableMapOf<String, NativeFunctionProvider>()
 
-  fun evaluate(path: String): Scope =
+  fun evaluate(locator: ImportLocator): Scope =
-    context(path).externalScope
+    context(locator).externalScope
 
-  fun context(path: String): CompilationUnitContext {
+  fun context(locator: ImportLocator): CompilationUnitContext {
-    val unit = world.load(path)
+    val unit = world.load(locator)
-    val identity = world.contentSource.stableContentIdentity(path)
+    val identity = world.stableIdentity(locator)
     val context = contexts.computeIfAbsent(identity) {
       CompilationUnitContext(unit, this, scope)
     }

examples/ffi.pork

@@ -1,8 +1,4 @@
-func malloc(size)
+import std "ffi/malloc.pork"
-  native ffi "c:malloc:void*"
-
-func free(pointer)
-  native ffi "c:free:void"
 
 export func main() {
   while true {

frontend/src/main/kotlin/gay/pizza/pork/frontend/ImportLocator.kt

@@ -0,0 +1,3 @@
+package gay.pizza.pork.frontend
+
+data class ImportLocator(val path: String, val form: String? = null)

frontend/src/main/kotlin/gay/pizza/pork/frontend/ImportSource.kt

@@ -0,0 +1,7 @@
+package gay.pizza.pork.frontend
+
+interface ImportSource {
+  val fileContentSource: ContentSource
+
+  fun provideContentSource(form: String): ContentSource
+}

frontend/src/main/kotlin/gay/pizza/pork/frontend/StandardImportSource.kt

@@ -0,0 +1,14 @@
+package gay.pizza.pork.frontend
+
+class StandardImportSource(override val fileContentSource: ContentSource) : ImportSource {
+  private val providers = mutableMapOf<String,ContentSource>()
+
+  override fun provideContentSource(form: String): ContentSource {
+    return providers[form] ?:
+      throw RuntimeException("Unknown import form: $form")
+  }
+
+  fun addContentSource(form: String, contentSource: ContentSource) {
+    providers[form] = contentSource
+  }
+}

frontend/src/main/kotlin/gay/pizza/pork/frontend/World.kt

@@ -7,37 +7,57 @@ import gay.pizza.pork.parser.Parser
 import gay.pizza.pork.parser.TokenStreamSource
 import gay.pizza.pork.parser.Tokenizer
 
-class World(val contentSource: ContentSource) {
+class World(val importSource: ImportSource) {
   private val internalUnits = mutableMapOf<String, CompilationUnit>()
 
   val units: List<CompilationUnit>
     get() = internalUnits.values.toList()
 
-  private fun loadOneUnit(path: String): CompilationUnit {
+  private fun loadOneUnit(importLocator: ImportLocator): CompilationUnit {
-    val stableIdentity = contentSource.stableContentIdentity(path)
+    val contentSource = pickContentSource(importLocator.form)
-    val cached = internalUnits[stableIdentity]
+    val stableKey = stableIdentity(importLocator, contentSource = contentSource)
+    val cached = internalUnits[stableKey]
     if (cached != null) {
       return cached
     }
-    val charSource = contentSource.loadAsCharSource(path)
+    val charSource = contentSource.loadAsCharSource(importLocator.path)
     val tokenizer = Tokenizer(charSource)
     val tokenStream = tokenizer.tokenize()
     val parser = Parser(TokenStreamSource(tokenStream), DiscardNodeAttribution)
-    return parser.readCompilationUnit()
+    val unit = parser.readCompilationUnit()
+    internalUnits[stableKey] = unit
+    return unit
   }
 
   private fun resolveAllImports(unit: CompilationUnit): Set<CompilationUnit> {
     val units = mutableSetOf<CompilationUnit>()
     for (declaration in unit.declarations.filterIsInstance<ImportDeclaration>()) {
-      val importedUnit = loadOneUnit(declaration.path.text)
+      val importLocator = ImportLocator(declaration.path.text, form = declaration.form?.id)
+      val importedUnit = loadOneUnit(importLocator)
       units.add(importedUnit)
     }
     return units
   }
 
-  fun load(path: String): CompilationUnit {
+  fun load(importLocator: ImportLocator): CompilationUnit {
-    val unit = loadOneUnit(path)
+    val unit = loadOneUnit(importLocator)
     resolveAllImports(unit)
     return unit
   }
+
+  private fun pickContentSource(form: String? = null): ContentSource {
+    if (form != null) {
+      return importSource.provideContentSource(form)
+    }
+    return importSource.fileContentSource
+  }
+
+  fun stableIdentity(
+    importLocator: ImportLocator,
+    contentSource: ContentSource = pickContentSource(importLocator.form)
+  ): String {
+    val formKey = importLocator.form ?: "file"
+    val stableIdentity = contentSource.stableContentIdentity(importLocator.path)
+    return "[${formKey}][${stableIdentity}]"
+  }
 }

parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt

@@ -183,7 +183,11 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
 
   private fun readImportDeclaration(): ImportDeclaration = within {
     expect(TokenType.Import)
-    ImportDeclaration(readStringLiteral())
+    var form: Symbol? = null
+    if (peek(TokenType.Symbol)) {
+      form = readSymbolRaw()
+    }
+    ImportDeclaration(form, readStringLiteral())
   }
 
   private fun readFunctionDeclaration(): FunctionDefinition = within {

settings.gradle.kts

@@ -8,6 +8,7 @@ include(
   ":parser",
   ":frontend",
   ":evaluator",
+  ":stdlib",
   ":ffi",
   ":tool"
 )

stdlib/build.gradle.kts

@@ -0,0 +1,13 @@
+plugins {
+  id("gay.pizza.pork.module")
+}
+
+tasks.processResources {
+  from("src/main/pork") {
+    into("pork/stdlib")
+  }
+}
+
+dependencies {
+  implementation(project(":frontend"))
+}

stdlib/src/main/kotlin/gay/pizza/pork/stdlib/PorkStdlib.kt

@@ -0,0 +1,33 @@
+package gay.pizza.pork.stdlib
+
+import gay.pizza.pork.frontend.ContentSource
+import gay.pizza.pork.parser.CharSource
+import gay.pizza.pork.parser.StringCharSource
+
+object PorkStdlib : ContentSource {
+  private val stdlibClass = PorkStdlib::class.java
+
+  private fun readManifestFiles(): List<String> {
+    val manifestContent = read("stdlib.manifest")
+    return manifestContent.split("\n").filter { line ->
+      val trimmed = line.trim()
+      trimmed.isNotEmpty() && !trimmed.startsWith("#")
+    }
+  }
+
+  val files: List<String> = readManifestFiles()
+
+  private fun read(path: String): String {
+    val stream = stdlibClass.getResourceAsStream("/pork/stdlib/${path}")
+      ?: throw RuntimeException("Stdlib does not contain file '${path}'")
+    return String(stream.readAllBytes())
+  }
+
+  override fun loadAsCharSource(path: String): CharSource {
+    return StringCharSource(read(path))
+  }
+
+  override fun stableContentIdentity(path: String): String {
+    return path
+  }
+}

stdlib/src/main/pork/ffi/malloc.pork

@@ -0,0 +1,5 @@
+export func malloc(size)
+  native ffi "c:malloc:void*"
+
+export func free(pointer)
+  native ffi "c:free:void"

stdlib/src/main/pork/numbers.pork

@@ -0,0 +1,15 @@
+export func add(a, b) {
+  a + b
+}
+
+export func subtract(a, b) {
+  a - b
+}
+
+export func multiply(a, b) {
+  a * b
+}
+
+export func divide(a, b) {
+  a / b
+}

stdlib/src/main/pork/stdlib.manifest

@@ -0,0 +1 @@
+numbers.pork

tool/build.gradle.kts

@@ -10,6 +10,7 @@ dependencies {
   api(project(":parser"))
   api(project(":frontend"))
   api(project(":evaluator"))
+  api(project(":stdlib"))
   api(project(":ffi"))
   api("com.github.ajalt.clikt:clikt:4.2.0")
   implementation(project(":common"))

tool/src/main/kotlin/gay/pizza/pork/tool/Tool.kt

@@ -1,16 +1,17 @@
 package gay.pizza.pork.tool
 
-import gay.pizza.pork.ast.NodeVisitor
-import gay.pizza.pork.parser.Printer
 import gay.pizza.pork.ast.CompilationUnit
+import gay.pizza.pork.ast.NodeVisitor
 import gay.pizza.pork.ast.visit
-import gay.pizza.pork.evaluator.Arguments
 import gay.pizza.pork.evaluator.CallableFunction
 import gay.pizza.pork.evaluator.Evaluator
 import gay.pizza.pork.evaluator.Scope
 import gay.pizza.pork.frontend.ContentSource
+import gay.pizza.pork.frontend.ImportLocator
+import gay.pizza.pork.frontend.StandardImportSource
 import gay.pizza.pork.frontend.World
 import gay.pizza.pork.parser.*
+import gay.pizza.pork.stdlib.PorkStdlib
 
 abstract class Tool {
   abstract fun createCharSource(): CharSource
@@ -31,11 +32,13 @@ abstract class Tool {
   fun <T> visit(visitor: NodeVisitor<T>): T = visitor.visit(parse())
 
   fun loadMainFunction(scope: Scope, setupEvaluator: Evaluator.() -> Unit = {}): CallableFunction {
-    val contentSource = createContentSource()
+    val fileContentSource = createContentSource()
-    val world = World(contentSource)
+    val standardImportSource = StandardImportSource(fileContentSource)
+    standardImportSource.addContentSource("std", PorkStdlib)
+    val world = World(standardImportSource)
     val evaluator = Evaluator(world, scope)
     setupEvaluator(evaluator)
-    val resultingScope = evaluator.evaluate(rootFilePath())
+    val resultingScope = evaluator.evaluate(ImportLocator(rootFilePath()))
     return resultingScope.value("main") as CallableFunction
   }
 }