pork: it's got it all, ffi, state machine tokenizer, and better IDE support

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiMacPlatform.kt

@@ -1,6 +1,5 @@
 package gay.pizza.pork.ffi
 
-import java.nio.file.Path
 import kotlin.io.path.*
 
 object FfiMacPlatform : FfiPlatform {
@@ -8,18 +7,23 @@ object FfiMacPlatform : FfiPlatform {
     "/Library/Frameworks"
   )
 
-  override fun findLibrary(name: String): Path? {
+  override fun findLibrary(name: String): String? {
     val frameworksToCheck = frameworksDirectories.map { frameworkDirectory ->
       Path("$frameworkDirectory/$name.framework/$name")
     }
     for (framework in frameworksToCheck) {
       if (!framework.exists()) continue
       return if (framework.isSymbolicLink()) {
-        return framework.parent.resolve(framework.readSymbolicLink()).absolute()
+        return framework.parent.resolve(framework.readSymbolicLink()).absolutePathString()
       } else {
-        framework.absolute()
+        framework.absolutePathString()
       }
     }
+
+    if (name == "c") {
+      return "libSystem.dylib"
+    }
+
     return null
   }
 }

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiNativeProvider.kt

@@ -3,104 +3,153 @@ package gay.pizza.pork.ffi
 import com.kenai.jffi.*
 import com.kenai.jffi.Function
 import gay.pizza.pork.ast.gen.ArgumentSpec
-import gay.pizza.pork.evaluator.CallableFunction
-import gay.pizza.pork.evaluator.NativeProvider
-import gay.pizza.pork.evaluator.None
-import java.nio.file.Path
+import gay.pizza.pork.evaluator.*
 import kotlin.io.path.Path
 import kotlin.io.path.absolutePathString
 import kotlin.io.path.exists
 
 class FfiNativeProvider : NativeProvider {
-  private val ffiTypeRegistry = FfiTypeRegistry()
+  private val internalFunctions = mutableMapOf<String, (ArgumentList) -> Any>(
+    "ffiStructDefine" to ::ffiStructDefine,
+    "ffiStructAllocate" to ::ffiStructAllocate,
+    "ffiStructValue" to ::ffiStructValue,
+    "ffiStructBytes" to ::ffiStructBytes
+  )
+
+  private val rootTypeRegistry = FfiTypeRegistry()
+
+  init {
+    rootTypeRegistry.registerPrimitiveTypes()
+  }
+
+  override fun provideNativeFunction(
+    definitions: List<String>,
+    arguments: List<ArgumentSpec>,
+    inside: CompilationUnitContext
+  ): CallableFunction {
+    if (definitions[0] == "internal") {
+      val internal = internalFunctions[definitions[1]] ?:
+        throw RuntimeException("Unknown internal function: ${definitions[1]}")
+      return CallableFunction { functionArguments, _ ->
+        internal(functionArguments)
+      }
+    }
 
-  override fun provideNativeFunction(definitions: List<String>, arguments: List<ArgumentSpec>): CallableFunction {
     val functionDefinition = FfiFunctionDefinition.parse(definitions[0], definitions[1])
     val functionAddress = lookupSymbol(functionDefinition)
 
+    val ffiTypeRegistry = rootTypeRegistry.fork()
+
+    addStructDefs(ffiTypeRegistry, functionDefinition.parameters, inside)
+
     val parameters = functionDefinition.parameters.map { id ->
-      ffiTypeRegistry.lookup(id) ?: throw RuntimeException("Unknown ffi type: $id")
+      ffiTypeRegistry.required(id)
     }
 
     val returnTypeId = functionDefinition.returnType
-    val returnType = ffiTypeRegistry.lookup(returnTypeId) ?:
-      throw RuntimeException("Unknown ffi return type: $returnTypeId")
+    val returnType = ffiTypeRegistry.required(returnTypeId)
     val returnTypeFfi = typeConversion(returnType)
     val parameterArray = parameters.map { typeConversion(it) }.toTypedArray()
     val function = Function(functionAddress, returnTypeFfi, *parameterArray)
     val context = function.callContext
     val invoker = Invoker.getInstance()
     return CallableFunction { functionArguments, _ ->
-      val buffer = HeapInvocationBuffer(context)
       val freeStringList = mutableListOf<FfiString>()
-      for ((index, spec) in arguments.withIndex()) {
-        val ffiType = ffiTypeRegistry.lookup(functionDefinition.parameters[index]) ?:
-          throw RuntimeException("Unknown ffi type: ${functionDefinition.parameters[index]}")
-        if (spec.multiple) {
-          val variableArguments = functionArguments
-            .subList(index, functionArguments.size)
-          variableArguments.forEach {
-            var value = it
-            if (value is String) {
-              value = FfiString.allocate(value)
-              freeStringList.add(value)
-            }
-            FfiPrimitiveType.push(buffer, value)
-          }
-          break
-        } else {
-          var argumentValue = functionArguments[index]
-          if (argumentValue is String) {
-            argumentValue = FfiString.allocate(argumentValue)
-            freeStringList.add(argumentValue)
-          }
-          ffiType.put(buffer, argumentValue)
-        }
-      }
-
       try {
+        val buffer = buildArgumentList(
+          context,
+          arguments,
+          functionArguments,
+          ffiTypeRegistry,
+          functionDefinition,
+          freeStringList
+        )
         return@CallableFunction invoke(invoker, function, buffer, returnType)
       } finally {
-        freeStringList.forEach { it.free() }
+        for (ffiString in freeStringList) {
+          ffiString.free()
+        }
       }
     }
   }
 
+  private fun addStructDefs(ffiTypeRegistry: FfiTypeRegistry, types: List<String>, inside: CompilationUnitContext) {
+    for (parameter in types) {
+      if (!parameter.startsWith("struct ")) {
+        continue
+      }
+
+      var structureName = parameter.substring(7)
+      if (structureName.endsWith("*")) {
+        structureName = structureName.substring(0, structureName.length - 1)
+      }
+      val structureDefinitionValue = inside.internalScope.value(structureName)
+      if (structureDefinitionValue !is FfiStructDefinition) {
+        throw RuntimeException("Structure '${structureName}' was not an FfiStructDefinition.")
+      }
+      val struct = FfiStruct(ffiTypeRegistry)
+      for ((name, type) in structureDefinitionValue.values) {
+        struct.add(name, type)
+      }
+      ffiTypeRegistry.add("struct $structureName", struct)
+    }
+  }
+
+  private fun buildArgumentList(
+    context: CallContext,
+    functionArgumentSpecs: List<ArgumentSpec>,
+    functionArguments: List<Any>,
+    ffiTypeRegistry: FfiTypeRegistry,
+    functionDefinition: FfiFunctionDefinition,
+    freeStringList: MutableList<FfiString>
+  ): HeapInvocationBuffer {
+    val buffer = HeapInvocationBuffer(context)
+    for ((index, spec) in functionArgumentSpecs.withIndex()) {
+      val ffiType = ffiTypeRegistry.lookup(functionDefinition.parameters[index]) ?:
+      throw RuntimeException("Unknown ffi type: ${functionDefinition.parameters[index]}")
+      if (spec.multiple) {
+        val variableArguments = functionArguments
+          .subList(index, functionArguments.size)
+        for (variableArgument in variableArguments) {
+          var value = variableArgument
+          if (value is String) {
+            value = FfiString.allocate(value)
+            freeStringList.add(value)
+          }
+          FfiPrimitiveType.push(buffer, value)
+        }
+        break
+      } else {
+        var argumentValue = functionArguments[index]
+        if (argumentValue is String) {
+          argumentValue = FfiString.allocate(argumentValue)
+          freeStringList.add(argumentValue)
+        }
+        ffiType.put(buffer, argumentValue)
+      }
+    }
+    return buffer
+  }
+
   private fun lookupSymbol(functionDefinition: FfiFunctionDefinition): Long {
     val actualLibraryPath = findLibraryPath(functionDefinition.library)
-    val library = Library.getCachedInstance(actualLibraryPath.absolutePathString(), Library.NOW)
+    val library = Library.getCachedInstance(actualLibraryPath, Library.NOW)
       ?: throw RuntimeException("Failed to load library $actualLibraryPath")
     val functionAddress = library.getSymbolAddress(functionDefinition.function)
     if (functionAddress == 0L) {
       throw RuntimeException(
         "Failed to find symbol ${functionDefinition.function} in " +
-        "library ${actualLibraryPath.absolutePathString()}")
+        "library $actualLibraryPath")
     }
     return functionAddress
   }
 
-  private fun typeConversion(type: FfiType): Type = when (type) {
-    FfiPrimitiveType.UnsignedByte -> Type.UINT8
-    FfiPrimitiveType.Byte -> Type.SINT8
-    FfiPrimitiveType.UnsignedInt -> Type.UINT32
-    FfiPrimitiveType.Int -> Type.SINT32
-    FfiPrimitiveType.UnsignedShort -> Type.UINT16
-    FfiPrimitiveType.Short -> Type.SINT16
-    FfiPrimitiveType.UnsignedLong -> Type.UINT64
-    FfiPrimitiveType.Long -> Type.SINT64
-    FfiPrimitiveType.String -> Type.POINTER
-    FfiPrimitiveType.Pointer -> Type.POINTER
-    FfiPrimitiveType.Void -> Type.VOID
-    else -> throw RuntimeException("Unknown ffi type: $type")
-  }
-
-  private fun findLibraryPath(name: String): Path {
+  private fun findLibraryPath(name: String): String {
     val initialPath = Path(name)
     if (initialPath.exists()) {
-      return initialPath
+      return initialPath.absolutePathString()
     }
-    return FfiPlatforms.current.platform.findLibrary(name)
-      ?: throw RuntimeException("Unable to find library: $name")
+    return FfiPlatforms.current.platform.findLibrary(name) ?: name
   }
 
   private fun invoke(invoker: Invoker, function: Function, buffer: HeapInvocationBuffer, type: FfiType): Any = when (type) {
@@ -113,4 +162,65 @@ class FfiNativeProvider : NativeProvider {
     FfiPrimitiveType.String -> invoker.invokeAddress(function, buffer)
     else -> throw RuntimeException("Unsupported ffi return type: $type")
   } ?: None
+
+  private fun ffiStructDefine(arguments: ArgumentList): Any {
+    val copy = arguments.toMutableList()
+    val fields = LinkedHashMap<String, String>()
+    while (copy.isNotEmpty()) {
+      val type = copy.removeAt(0)
+      fields[copy.removeAt(0).toString()] = type.toString()
+    }
+    return FfiStructDefinition(fields)
+  }
+
+  private fun ffiStructAllocate(arguments: ArgumentList): Any {
+    val ffiTypeRegistry = rootTypeRegistry.fork()
+    val structDefinition = arguments[0] as FfiStructDefinition
+    val structType = FfiStruct(ffiTypeRegistry)
+    for ((name, type) in structDefinition.values) {
+      structType.add(name, type)
+    }
+    return FfiAddress(MemoryIO.getInstance().allocateMemory(structType.size, true))
+  }
+
+  private fun ffiStructValue(arguments: ArgumentList): Any {
+    val structDefinition = arguments[0] as FfiStructDefinition
+    val field = arguments[1] as String
+    val value = arguments[2] as FfiAddress
+    val ffiTypeRegistry = rootTypeRegistry.fork()
+    val structType = FfiStruct(ffiTypeRegistry)
+    for ((name, type) in structDefinition.values) {
+      structType.add(name, type)
+    }
+    return structType.get(field, value)
+  }
+
+  private fun ffiStructBytes(arguments: ArgumentList): Any {
+    val structDefinition = arguments[0] as FfiStructDefinition
+    val address = arguments[1] as FfiAddress
+    val ffiTypeRegistry = rootTypeRegistry.fork()
+    val structType = FfiStruct(ffiTypeRegistry)
+    for ((name, type) in structDefinition.values) {
+      structType.add(name, type)
+    }
+    return structType.read(address, 0)
+  }
+
+  companion object {
+    fun typeConversion(type: FfiType): Type = when (type) {
+      FfiPrimitiveType.UnsignedByte -> Type.UINT8
+      FfiPrimitiveType.Byte -> Type.SINT8
+      FfiPrimitiveType.UnsignedInt -> Type.UINT32
+      FfiPrimitiveType.Int -> Type.SINT32
+      FfiPrimitiveType.UnsignedShort -> Type.UINT16
+      FfiPrimitiveType.Short -> Type.SINT16
+      FfiPrimitiveType.UnsignedLong -> Type.UINT64
+      FfiPrimitiveType.Long -> Type.SINT64
+      FfiPrimitiveType.String -> Type.POINTER
+      FfiPrimitiveType.Pointer -> Type.POINTER
+      FfiPrimitiveType.Void -> Type.VOID
+      is FfiStruct -> type.ffiStruct
+      else -> throw RuntimeException("Unknown ffi type: $type")
+    }
+  }
 }

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiPlatform.kt

@@ -1,7 +1,5 @@
 package gay.pizza.pork.ffi
 
-import java.nio.file.Path
-
 enum class FfiPlatforms(val id: String, val platform: FfiPlatform) {
   Mac("macOS", FfiMacPlatform),
   Windows("Windows", FfiWindowsPlatform),
@@ -20,5 +18,5 @@ enum class FfiPlatforms(val id: String, val platform: FfiPlatform) {
 }
 
 interface FfiPlatform {
-  fun findLibrary(name: String): Path?
+  fun findLibrary(name: String): String?
 }

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiPrimitiveType.kt

@@ -1,6 +1,7 @@
 package gay.pizza.pork.ffi
 
 import com.kenai.jffi.InvocationBuffer
+import com.kenai.jffi.MemoryIO
 import gay.pizza.pork.evaluator.None
 
 enum class FfiPrimitiveType(
@@ -89,6 +90,21 @@ enum class FfiPrimitiveType(
     return ffi
   }
 
+  override fun read(address: FfiAddress, offset: kotlin.Int): Any {
+    val actual = address.location + offset
+    return when (this) {
+      UnsignedByte, Byte -> MemoryIO.getInstance().getByte(actual)
+      UnsignedShort, Short -> MemoryIO.getInstance().getShort(actual)
+      UnsignedInt, Int -> MemoryIO.getInstance().getInt(actual)
+      UnsignedLong, Long -> MemoryIO.getInstance().getLong(actual)
+      Float -> MemoryIO.getInstance().getFloat(actual)
+      Double -> MemoryIO.getInstance().getDouble(actual)
+      Pointer -> MemoryIO.getInstance().getAddress(actual)
+      String -> FfiString(FfiAddress(actual))
+      Void -> None
+    }
+  }
+
   companion object {
     fun push(buffer: InvocationBuffer, value: Any): Unit = when (value) {
       is kotlin.Byte -> buffer.putByte(value.toInt())

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiStruct.kt

@@ -1,36 +1,61 @@
 package gay.pizza.pork.ffi
 
 import com.kenai.jffi.InvocationBuffer
+import com.kenai.jffi.MemoryIO
+import com.kenai.jffi.Struct
 import gay.pizza.pork.evaluator.None
-import java.util.*
 
-class FfiStruct : FfiType {
-  private val fields = TreeMap<String, FfiStructField>()
+class FfiStruct(val ffiTypeRegistry: FfiTypeRegistry) : FfiType {
+  private val fields = LinkedHashMap<String, FfiStructField>()
+  private var internalStructType: Struct? = null
+  val ffiStruct: Struct
+    get() {
+      if (internalStructType == null) {
+        internalStructType = Struct.newStruct(*ffiTypes.map {
+          FfiNativeProvider.typeConversion(it)
+        }.toTypedArray())
+      }
+      return internalStructType!!
+    }
+  private var internalTypes: List<FfiType>? = null
+  val ffiTypes: List<FfiType>
+    get() {
+      if (internalTypes == null) {
+        internalTypes = fields.values.map { ffiTypeRegistry.required(it.type) }
+      }
+      return internalTypes!!
+    }
 
-  data class FfiStructField(val name: String, val type: FfiType)
+  data class FfiStructField(val name: String, val type: String)
 
-  fun add(field: String, type: FfiType) {
+  fun add(field: String, type: String) {
     fields[field] = FfiStructField(field, type)
+    internalStructType = null
+    internalTypes = null
   }
 
   override val size: Long
-    get() = fields.values.sumOf { it.type.size }
+    get() = ffiStruct.size().toLong()
 
   override fun put(buffer: InvocationBuffer, value: Any?) {
     when (value) {
       is Map<*, *> -> {
         for (field in fields.values) {
           val item = value[field.name] ?: None
-          field.type.put(buffer, item)
+          val itemType = ffiTypeRegistry.required(field.type)
+          itemType.put(buffer, item)
         }
       }
 
       is List<*> -> {
         for ((index, field) in fields.values.withIndex()) {
-          field.type.put(buffer, value[index])
+          val itemType = ffiTypeRegistry.required(field.type)
+          itemType.put(buffer, value[index])
         }
       }
 
+      is None -> {}
+
       else -> {
         throw RuntimeException("Unknown value type: $value")
       }
@@ -40,4 +65,28 @@ class FfiStruct : FfiType {
   override fun value(ffi: Any?): Any {
     return None
   }
+
+  override fun read(address: FfiAddress, offset: Int): Any {
+    val bytes = ByteArray(size.toInt()) { 0 }
+    MemoryIO.getInstance().getByteArray(address.location, bytes, offset, size.toInt())
+    return bytes
+  }
+
+  fun get(field: String, address: FfiAddress): Any {
+    var indexWithoutAlignment = 0L
+    var type: FfiType? = null
+    for ((index, key) in fields.keys.withIndex()) {
+      if (key == field) {
+        type = ffiTypes[index]
+        break
+      }
+      indexWithoutAlignment += ffiTypes[index].size
+    }
+
+    if (type == null) {
+      throw RuntimeException("Unable to read unknown field $field from struct.")
+    }
+
+    return type.read(address, indexWithoutAlignment.toInt())
+  }
 }

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiStructDefinition.kt

@@ -0,0 +1,3 @@
+package gay.pizza.pork.ffi
+
+data class FfiStructDefinition(val values: LinkedHashMap<String, String>)

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiType.kt

@@ -7,4 +7,5 @@ interface FfiType {
 
   fun put(buffer: InvocationBuffer, value: Any?)
   fun value(ffi: Any?): Any
+  fun read(address: FfiAddress, offset: Int): Any
 }

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiTypeRegistry.kt

@@ -1,9 +1,9 @@
 package gay.pizza.pork.ffi
 
-class FfiTypeRegistry {
+class FfiTypeRegistry(val parent: FfiTypeRegistry? = null) {
   private val types = mutableMapOf<String, FfiType>()
 
-  init {
+  fun registerPrimitiveTypes() {
     for (type in FfiPrimitiveType.entries) {
       add(type.id, type)
     }
@@ -12,7 +12,14 @@ class FfiTypeRegistry {
 
   fun add(name: String, type: FfiType) {
     types[name] = type
+
+    if (type is FfiStruct) {
+      types["${name}*"] = FfiPrimitiveType.Pointer
+    }
   }
 
-  fun lookup(name: String): FfiType? = types[name]
+  fun lookup(name: String): FfiType? = types[name] ?: parent?.lookup(name)
+  fun required(name: String): FfiType = lookup(name) ?: throw RuntimeException("Unknown ffi type: $name")
+
+  fun fork(): FfiTypeRegistry = FfiTypeRegistry(this)
 }

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiUnixPlatform.kt

@@ -3,5 +3,5 @@ package gay.pizza.pork.ffi
 import java.nio.file.Path
 
 object FfiUnixPlatform : FfiPlatform {
-  override fun findLibrary(name: String): Path? = null
+  override fun findLibrary(name: String): String? = null
 }

ffi/src/main/kotlin/gay/pizza/pork/ffi/FfiWindowsPlatform.kt

@@ -3,5 +3,5 @@ package gay.pizza.pork.ffi
 import java.nio.file.Path
 
 object FfiWindowsPlatform : FfiPlatform {
-  override fun findLibrary(name: String): Path? = null
+  override fun findLibrary(name: String): String? = null
 }

ffi/src/main/kotlin/gay/pizza/pork/ffi/JavaNativeProvider.kt

@@ -2,6 +2,7 @@ package gay.pizza.pork.ffi
 
 import gay.pizza.pork.ast.gen.ArgumentSpec
 import gay.pizza.pork.evaluator.CallableFunction
+import gay.pizza.pork.evaluator.CompilationUnitContext
 import gay.pizza.pork.evaluator.NativeProvider
 import gay.pizza.pork.evaluator.None
 import java.lang.invoke.MethodHandles
@@ -10,7 +11,11 @@ import java.lang.invoke.MethodType
 class JavaNativeProvider : NativeProvider {
   private val lookup = MethodHandles.lookup()
 
-  override fun provideNativeFunction(definitions: List<String>, arguments: List<ArgumentSpec>): CallableFunction {
+  override fun provideNativeFunction(
+    definitions: List<String>,
+    arguments: List<ArgumentSpec>,
+    inside: CompilationUnitContext
+  ): CallableFunction {
     val functionDefinition = JavaFunctionDefinition.parse(definitions)
     val javaClass = lookupClass(functionDefinition.type)
     val returnTypeClass = lookupClass(functionDefinition.returnType)