Significant progress on AST codegen.

This commit is contained in:
alex committed 2023-09-04 18:38:09 -07:00
1 parent edec706ed4
commit f06ea93dc4
24 files changed
+670 -55

No files matched your search

+122 -1
View File
@@ -4,6 +4,9 @@ types:
parent: Node parent: Node
Symbol: Symbol:
parent: Node parent: Node
values:
- name: id
type: String
Declaration: Declaration:
parent: Node parent: Node
Definition: Definition:
@@ -11,8 +14,10 @@ types:
values: values:
- name: symbol - name: symbol
type: Symbol type: Symbol
required: true
- name: modifiers - name: modifiers
type: DefinitionModifiers type: DefinitionModifiers
required: true
DefinitionModifiers: DefinitionModifiers:
values: values:
- name: export - name: export
@@ -29,10 +34,126 @@ types:
type: List<Declaration> type: List<Declaration>
- name: definitions - name: definitions
type: List<Declaration> type: List<Declaration>
Assignment: LetAssignment:
parent: Expression parent: Expression
values: values:
- name: symbol - name: symbol
type: Symbol type: Symbol
- name: value - name: value
type: Expression type: Expression
InfixOperator:
values:
- name: token
type: String
enums:
- name: Plus
values:
token: "+"
- name: Minus
values:
token: "-"
- name: Multiply
values:
token: "*"
- name: Divide
values:
token: "/"
- name: Equals
values:
token: "=="
- name: NotEquals
values:
token: "!="
InfixOperation:
parent: Expression
values:
- name: left
type: Expression
- name: op
type: InfixOperator
- name: right
type: Expression
BooleanLiteral:
parent: Expression
values:
- name: value
type: Boolean
FunctionCall:
parent: Expression
values:
- name: symbol
type: Symbol
- name: arguments
type: List<Expression>
FunctionDefinition:
parent: Definition
values:
- name: modifiers
type: DefinitionModifiers
- name: symbol
type: Symbol
- name: arguments
type: List<Symbol>
- name: block
type: Block
If:
parent: Expression
values:
- name: condition
type: Expression
- name: thenExpression
type: Expression
- name: elseExpression
type: Expression?
ImportDeclaration:
parent: Declaration
values:
- name: path
type: StringLiteral
IntLiteral:
parent: Expression
values:
- name: value
type: Int
Lambda:
parent: Expression
values:
- name: arguments
type: List<Symbol>
- name: expressions
type: List<Expression>
ListLiteral:
parent: Expression
values:
- name: items
type: List<Expression>
Parentheses:
parent: Expression
values:
- name: expression
type: Expression
PrefixOperator:
values:
- name: token
type: String
enums:
- name: Negate
values:
token: "!"
PrefixOperation:
parent: Expression
values:
- name: op
type: PrefixOperator
- name: expression
type: Expression
StringLiteral:
parent: Expression
values:
- name: text
type: String
SymbolReference:
parent: Expression
values:
- name: symbol
type: Symbol
@@ -21,21 +21,5 @@ enum class NodeType(val parent: NodeType? = null) {
FunctionCall(Expression), FunctionCall(Expression),
If(Expression), If(Expression),
ImportDeclaration(Declaration), ImportDeclaration(Declaration),
FunctionDefinition(Definition); FunctionDefinition(Definition)
val parents: Set<NodeType>
init {
val calculatedParents = mutableListOf<NodeType>()
var self = this
while (true) {
calculatedParents.add(self)
if (self.parent != null) {
self = self.parent!!
} else {
break
}
}
parents = calculatedParents.toSet()
}
} }
@@ -1,13 +1,11 @@
package gay.pizza.pork.gradle package gay.pizza.pork.gradle
import com.fasterxml.jackson.databind.ObjectMapper import gay.pizza.pork.gradle.ast.AstCodegen
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.kotlin.KotlinModule
import gay.pizza.pork.gradle.ast.AstDescription
import org.gradle.api.DefaultTask import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.TaskAction import org.gradle.api.tasks.TaskAction
import gay.pizza.pork.gradle.ast.AstWorld import org.gradle.api.tasks.OutputDirectory
import java.io.File import java.io.File
open class GenerateAstCode : DefaultTask() { open class GenerateAstCode : DefaultTask() {
@@ -16,15 +14,16 @@ open class GenerateAstCode : DefaultTask() {
} }
@get:InputFile @get:InputFile
val astDescriptionFile: File = project.file("src/main/ast/pork.yml") var astDescriptionFile: File = project.file("src/main/ast/pork.yml")
@get:Input
var codePackage: String = "gay.pizza.pork.gen"
@get:OutputDirectory
var outputDirectory: File = project.file("src/main/kotlin/gay/pizza/pork/gen")
@TaskAction @TaskAction
fun generate() { fun generate() {
val astYamlText = astDescriptionFile.readText() AstCodegen.run(codePackage, astDescriptionFile.toPath(), outputDirectory.toPath())
val mapper = ObjectMapper(YAMLFactory())
mapper.registerModules(KotlinModule.Builder().build())
val astDescription = mapper.readValue(astYamlText, AstDescription::class.java)
val world = AstWorld()
world.build(astDescription)
} }
} }
@@ -0,0 +1,182 @@
package gay.pizza.pork.gradle.ast
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.kotlin.KotlinModule
import gay.pizza.pork.gradle.codegen.*
import java.nio.charset.StandardCharsets
import java.nio.file.Path
import kotlin.io.path.*
class AstCodegen(val pkg: String, val outputDirectory: Path, val world: AstWorld) {
private fun deleteAllContents() {
for (child in outputDirectory.listDirectoryEntries("*.kt")) {
child.deleteExisting()
}
}
fun generate() {
deleteAllContents()
for (type in world.typeRegistry.types) {
writeAstType(type)
}
writeNodeType()
}
private fun writeNodeType() {
val enumClass = KotlinEnum(pkg, "NodeType")
val parentMember = KotlinMember("parent", "NodeType?", value = "null")
enumClass.members.add(parentMember)
val typesInNameOrder = world.typeRegistry.types.sortedBy { it.name }
val typesInDependencyOrder = mutableListOf<AstType>()
for (type in typesInNameOrder) {
if (type.parent != null) {
if (!typesInDependencyOrder.contains(type.parent)) {
typesInDependencyOrder.add(type.parent!!)
}
}
if (!typesInDependencyOrder.contains(type)) {
typesInDependencyOrder.add(type)
}
}
for (type in typesInDependencyOrder) {
val role = world.typeRegistry.roleOfType(type)
if (role == AstTypeRole.ValueHolder || role == AstTypeRole.Enum) {
println(type)
continue
}
val entry = KotlinEnumEntry(type.name)
if (type.parent != null) {
entry.parameters.add(type.parent!!.name)
}
enumClass.entries.add(entry)
}
write("NodeType.kt", KotlinWriter(enumClass))
}
private fun writeAstType(type: AstType) {
val role = world.typeRegistry.roleOfType(type)
val kotlinClassLike: KotlinClassLike
if (role == AstTypeRole.Enum) {
val kotlinEnum = KotlinEnum(pkg, type.name)
kotlinClassLike = kotlinEnum
} else {
val kotlinClass = KotlinClass(pkg, type.name)
kotlinClassLike = kotlinClass
if (role == AstTypeRole.RootNode || role == AstTypeRole.HierarchyNode) {
kotlinClass.sealed = true
}
}
if (role == AstTypeRole.RootNode) {
val typeMember = KotlinMember(
"type",
"NodeType",
abstract = true
)
kotlinClassLike.members.add(typeMember)
} else if (role == AstTypeRole.AstNode) {
val typeMember = KotlinMember(
"type",
"NodeType",
overridden = true,
value = "NodeType.${type.name}"
)
kotlinClassLike.members.add(typeMember)
}
if (type.parent != null) {
val parentName = type.parent!!.name
kotlinClassLike.inherits.add("$parentName()")
}
for (value in type.values) {
val member = KotlinMember(value.name, toKotlinType(value.typeRef))
member.abstract = value.abstract
if (type.isParentAbstract(value)) {
member.overridden = true
}
kotlinClassLike.members.add(member)
}
if (role == AstTypeRole.Enum) {
val kotlinEnum = kotlinClassLike as KotlinEnum
for (entry in type.enums) {
val orderOfKeys = entry.values.keys.sortedBy { key ->
kotlinClassLike.members.indexOfFirst { it.name == key }
}
val parameters = mutableListOf<String>()
for (key in orderOfKeys) {
val value = entry.values[key] ?: continue
parameters.add("\"${value}\"")
}
val enumEntry = KotlinEnumEntry(entry.name, parameters)
kotlinEnum.entries.add(enumEntry)
}
}
if (role == AstTypeRole.AstNode) {
val equalsAndHashCodeFields = kotlinClassLike.members.map { it.name }
val equalsFunction = KotlinFunction(
"equals",
returnType = "Boolean",
overridden = true
)
equalsFunction.parameters.add(KotlinParameter(
"other",
"Any?"
))
equalsFunction.body.add("if (other !is ${type.name}) return false")
val predicate = equalsAndHashCodeFields.joinToString(" && ") {
"other.${it} == $it"
}
equalsFunction.body.add("return $predicate")
kotlinClassLike.functions.add(equalsFunction)
}
val serialName = kotlinClassLike.name[0].lowercase() +
kotlinClassLike.name.substring(1)
kotlinClassLike.imports.add("kotlinx.serialization.SerialName")
kotlinClassLike.imports.add("kotlinx.serialization.Serializable")
kotlinClassLike.annotations.add("Serializable")
kotlinClassLike.annotations.add("SerialName(\"$serialName\")")
write("${type.name}.kt", KotlinWriter(kotlinClassLike))
}
private fun toKotlinType(typeRef: AstTypeRef): String {
val baseType = typeRef.type?.name ?: typeRef.primitive?.id
?: throw RuntimeException("Unable to determine base type.")
return when (typeRef.form) {
AstTypeRefForm.Single -> baseType
AstTypeRefForm.Nullable -> "${baseType}?"
AstTypeRefForm.List -> "List<${baseType}>"
}
}
private fun write(fileName: String, writer: KotlinWriter) {
val path = outputDirectory.resolve(fileName)
path.deleteIfExists()
path.writeText(writer.toString(), StandardCharsets.UTF_8)
}
companion object {
fun run(pkg: String, astDescriptionFile: Path, outputDirectory: Path) {
val astYamlText = astDescriptionFile.readText()
val mapper = ObjectMapper(YAMLFactory())
mapper.registerModules(KotlinModule.Builder().build())
val astDescription = mapper.readValue(astYamlText, AstDescription::class.java)
val world = AstWorld.build(astDescription)
val codegen = AstCodegen(pkg, outputDirectory, world)
codegen.generate()
}
}
}
@@ -0,0 +1,6 @@
package gay.pizza.pork.gradle.ast
class AstEnum(
val name: String,
val values: Map<String, String>
)
@@ -0,0 +1,6 @@
package gay.pizza.pork.gradle.ast
class AstEnumDescription(
val name: String,
val values: Map<String, String>
)
@@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
enum class AstPrimitive(val id: kotlin.String) { enum class AstPrimitive(val id: kotlin.String) {
Boolean("Boolean"), Boolean("Boolean"),
String("String") String("String"),
Int("Int")
} }
@@ -2,11 +2,39 @@ package gay.pizza.pork.gradle.ast
class AstType(val name: String, var parent: AstType? = null) { class AstType(val name: String, var parent: AstType? = null) {
private val internalValues = mutableListOf<AstValue>() private val internalValues = mutableListOf<AstValue>()
private val internalEnums = mutableListOf<AstEnum>()
val values: List<AstValue> val values: List<AstValue>
get() = internalValues get() = internalValues
val enums: List<AstEnum>
get() = internalEnums
internal fun addValue(value: AstValue) { internal fun addValue(value: AstValue) {
internalValues.add(value) internalValues.add(value)
} }
internal fun addEnum(enum: AstEnum) {
internalEnums.add(enum)
}
fun isParentAbstract(value: AstValue): Boolean {
if (parent == null) {
return false
}
var current = parent
while (current != null) {
val abstract = current.values.firstOrNull {
it.name == value.name && it.abstract
}
if (abstract != null) {
return true
}
current = current.parent
}
return false
}
override fun toString(): String = "AstType(${name})"
} }
@@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
data class AstTypeDescription( data class AstTypeDescription(
val parent: String? = null, val parent: String? = null,
val values: List<AstValueDescription> = emptyList() val values: List<AstValueDescription> = emptyList(),
val enums: List<AstEnumDescription> = emptyList()
) )
@@ -17,15 +17,22 @@ class AstTypeRef(
) )
} }
val primitive = AstPrimitive.values().firstOrNull { it.name == input } var form = AstTypeRefForm.Single
var typeName: String = input
if (input.endsWith("?")) {
form = AstTypeRefForm.Nullable
typeName = input.substring(0, input.length - 1)
}
val primitive = AstPrimitive.values().firstOrNull { it.name == typeName }
if (primitive != null) { if (primitive != null) {
return AstTypeRef( return AstTypeRef(
primitive = primitive, primitive = primitive,
form = AstTypeRefForm.Single form = form
) )
} }
return AstTypeRef(type = registry.lookup(input), form = AstTypeRefForm.Single) return AstTypeRef(type = registry.lookup(typeName), form = form)
} }
} }
} }
@@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
enum class AstTypeRefForm { enum class AstTypeRefForm {
Single, Single,
List List,
Nullable
} }
@@ -19,7 +19,9 @@ class AstTypeRegistry {
fun roleOfType(type: AstType): AstTypeRole = fun roleOfType(type: AstType): AstTypeRole =
when { when {
type.parent == null && type.values.isNotEmpty() -> type.enums.isNotEmpty() ->
AstTypeRole.Enum
type.parent == null && type.values.isEmpty() ->
AstTypeRole.RootNode AstTypeRole.RootNode
type.parent != null && type.values.all { it.abstract } -> type.parent != null && type.values.all { it.abstract } ->
AstTypeRole.HierarchyNode AstTypeRole.HierarchyNode
@@ -4,5 +4,6 @@ enum class AstTypeRole {
RootNode, RootNode,
HierarchyNode, HierarchyNode,
AstNode, AstNode,
ValueHolder ValueHolder,
Enum
} }
@@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
data class AstValueDescription( data class AstValueDescription(
val name: String, val name: String,
val type: String val type: String,
val required: Boolean = false
) )
@@ -3,26 +3,37 @@ package gay.pizza.pork.gradle.ast
class AstWorld { class AstWorld {
val typeRegistry: AstTypeRegistry = AstTypeRegistry() val typeRegistry: AstTypeRegistry = AstTypeRegistry()
fun build(description: AstDescription) { companion object {
val rootType = typeRegistry.add(AstType(description.root)) fun build(description: AstDescription): AstWorld {
for (typeName in description.types.keys) { val world = AstWorld()
if (typeName == rootType.name) { val rootType = world.typeRegistry.add(AstType(description.root))
throw RuntimeException("Cannot have type with the same name as the root type.") for (typeName in description.types.keys) {
if (typeName == rootType.name) {
throw RuntimeException("Cannot have type with the same name as the root type.")
}
world.typeRegistry.add(AstType(typeName))
} }
typeRegistry.add(AstType(typeName)) for ((typeName, typeDescription) in description.types) {
} val type = world.typeRegistry.lookup(typeName)
for ((typeName, typeDescription) in description.types) { if (typeDescription.parent != null) {
val type = typeRegistry.lookup(typeName) type.parent = world.typeRegistry.lookup(typeDescription.parent)
if (typeDescription.parent != null) { }
type.parent = typeRegistry.lookup(typeDescription.parent)
} for (value in typeDescription.values) {
for (value in typeDescription.values) { val typeRef = AstTypeRef.parse(value.type, world.typeRegistry)
val typeRef = AstTypeRef.parse(value.type, typeRegistry) val typeValue = AstValue(value.name, typeRef, abstract = value.required)
val typeValue = AstValue(value.name, typeRef) type.addValue(typeValue)
type.addValue(typeValue) }
for (enum in typeDescription.enums) {
val astEnum = AstEnum(enum.name, enum.values)
type.addEnum(astEnum)
}
} }
return world
} }
} }
} }
@@ -0,0 +1,14 @@
package gay.pizza.pork.gradle.ast
import kotlin.io.path.Path
object RunCodegenIde {
@JvmStatic
fun main(args: Array<String>) {
AstCodegen.run(
pkg = "gay.pizza.pork.gen",
astDescriptionFile = Path("src/main/ast/pork.yml"),
outputDirectory = Path("src/main/kotlin/gay/pizza/pork/gen")
)
}
}
@@ -0,0 +1,12 @@
package gay.pizza.pork.gradle.codegen
class KotlinClass(
override val pkg: String,
override var name: String,
var sealed: Boolean = false,
override var inherits: MutableList<String> = mutableListOf(),
override var imports: MutableList<String> = mutableListOf(),
override var members: MutableList<KotlinMember> = mutableListOf(),
override var annotations: MutableList<String> = mutableListOf(),
override var functions: MutableList<KotlinFunction> = mutableListOf()
) : KotlinClassLike()
@@ -0,0 +1,11 @@
package gay.pizza.pork.gradle.codegen
abstract class KotlinClassLike {
abstract val pkg: String
abstract val name: String
abstract var imports: MutableList<String>
abstract var inherits: MutableList<String>
abstract var annotations: MutableList<String>
abstract var members: MutableList<KotlinMember>
abstract var functions: MutableList<KotlinFunction>
}
@@ -0,0 +1,12 @@
package gay.pizza.pork.gradle.codegen
class KotlinEnum(
override val pkg: String,
override val name: String,
override var imports: MutableList<String> = mutableListOf(),
override var inherits: MutableList<String> = mutableListOf(),
override var annotations: MutableList<String> = mutableListOf(),
override var members: MutableList<KotlinMember> = mutableListOf(),
override var functions: MutableList<KotlinFunction> = mutableListOf(),
var entries: MutableList<KotlinEnumEntry> = mutableListOf(),
) : KotlinClassLike()
@@ -0,0 +1,6 @@
package gay.pizza.pork.gradle.codegen
class KotlinEnumEntry(
val name: String,
var parameters: MutableList<String> = mutableListOf()
)
@@ -0,0 +1,11 @@
package gay.pizza.pork.gradle.codegen
class KotlinFunction(
val name: String,
var parameters: MutableList<KotlinParameter> = mutableListOf(),
var returnType: String? = null,
var abstract: Boolean = false,
var overridden: Boolean = false,
var isImmediateExpression: Boolean = false,
var body: MutableList<String> = mutableListOf()
)
@@ -0,0 +1,9 @@
package gay.pizza.pork.gradle.codegen
class KotlinMember(
var name: String,
var type: String,
var abstract: Boolean = false,
var overridden: Boolean = false,
var value: String? = null
)
@@ -0,0 +1,7 @@
package gay.pizza.pork.gradle.codegen
class KotlinParameter(
val name: String,
val type: String,
val defaultValue: String? = null
)
@@ -0,0 +1,182 @@
package gay.pizza.pork.gradle.codegen
class KotlinWriter {
private val buffer = StringBuilder()
constructor(kotlinClassLike: KotlinClassLike) {
write(kotlinClassLike)
}
fun writeClass(kotlinClass: KotlinClass): Unit = buffer.run {
val classType = if (kotlinClass.sealed) "sealed class" else "class"
writeClassLike(classType, kotlinClass)
val members = kotlinClass.members.filter {
it.abstract || (it.overridden && it.value != null)
}
if (members.isEmpty() && kotlinClass.functions.isEmpty()) {
appendLine()
} else {
appendLine(" {")
}
for (member in members) {
if (member.abstract) {
appendLine(" abstract val ${member.name}: ${member.type}")
} else {
if (member.overridden) {
append(" override ")
}
append("val ${member.name}: ${member.type}")
if (member.value != null) {
append(" = ")
append(member.value)
}
appendLine()
}
}
if (members.isNotEmpty() && kotlinClass.functions.isNotEmpty()) {
appendLine()
}
writeFunctions(kotlinClass)
if (members.isNotEmpty() || kotlinClass.functions.isNotEmpty()) {
appendLine("}")
}
}
fun writeEnum(kotlinEnum: KotlinEnum): Unit = buffer.run {
writeClassLike("enum class", kotlinEnum)
val membersNotCompatible = kotlinEnum.members.filter { it.abstract }
if (membersNotCompatible.isNotEmpty()) {
throw RuntimeException(
"Incompatible members in enum class " +
"${kotlinEnum.name}: $membersNotCompatible"
)
}
if (kotlinEnum.entries.isEmpty() && kotlinEnum.functions.isEmpty()) {
appendLine()
} else {
appendLine(" {")
}
for ((index, entry) in kotlinEnum.entries.withIndex()) {
append(" ${entry.name}")
if (entry.parameters.isNotEmpty()) {
append("(")
append(entry.parameters.joinToString(", "))
append(")")
}
if (index != kotlinEnum.entries.size - 1) {
append(",")
}
appendLine()
}
if (kotlinEnum.entries.isNotEmpty() && kotlinEnum.functions.isNotEmpty()) {
appendLine()
}
writeFunctions(kotlinEnum)
if (kotlinEnum.entries.isNotEmpty()) {
appendLine("}")
}
}
private fun writeClassLike(
classType: String,
kotlinClass: KotlinClassLike
): Unit = buffer.run {
appendLine("package ${kotlinClass.pkg}")
appendLine()
for (import in kotlinClass.imports) {
appendLine("import $import")
}
if (kotlinClass.imports.isNotEmpty()) {
appendLine()
}
for (annotation in kotlinClass.annotations) {
appendLine("@${annotation}")
}
append("$classType ${kotlinClass.name}")
val contructedMembers = kotlinClass.members.filter {
!it.abstract && !(it.overridden && it.value != null)
}
if (contructedMembers.isNotEmpty()) {
val constructor = contructedMembers.joinToString(", ") {
val prefix = if (it.overridden) "override " else ""
val start = "${prefix}val ${it.name}: ${it.type}"
if (it.value != null) {
"$start = ${it.value}"
} else start
}
append("(${constructor})")
}
if (kotlinClass.inherits.isNotEmpty()) {
append(" : ${kotlinClass.inherits.joinToString(", ")}")
}
}
private fun writeFunctions(kotlinClassLike: KotlinClassLike): Unit = buffer.run {
for (function in kotlinClassLike.functions) {
append(" ")
if (function.overridden) {
append("override ")
}
if (function.abstract) {
append("abstract ")
}
append("fun ${function.name}(")
append(function.parameters.joinToString(", ") {
val start = "${it.name}: ${it.type}"
if (it.defaultValue != null) {
start + " = ${it.defaultValue}"
} else start
})
append(")")
if (function.returnType != null) {
append(": ${function.returnType}")
}
if (!function.isImmediateExpression) {
append(" {")
} else {
appendLine(" =")
}
if (function.body.isNotEmpty()) {
appendLine()
for (item in function.body) {
appendLine(" $item")
}
}
if (!function.isImmediateExpression) {
appendLine(" }")
}
}
}
fun write(input: KotlinClassLike): Unit = when (input) {
is KotlinClass -> writeClass(input)
is KotlinEnum -> writeEnum(input)
else -> throw RuntimeException("Unknown Kotlin Class Type")
}
override fun toString(): String = buffer.toString()
}