Significant progress on AST codegen.

This commit is contained in:
Alex Zenla 2023-09-04 18:38:09 -07:00
parent edec706ed4
commit f06ea93dc4
Signed by: alex
GPG Key ID: C0780728420EBFE5
24 changed files with 670 additions and 55 deletions

View File

@ -4,6 +4,9 @@ types:
parent: Node parent: Node
Symbol: Symbol:
parent: Node parent: Node
values:
- name: id
type: String
Declaration: Declaration:
parent: Node parent: Node
Definition: Definition:
@ -11,8 +14,10 @@ types:
values: values:
- name: symbol - name: symbol
type: Symbol type: Symbol
required: true
- name: modifiers - name: modifiers
type: DefinitionModifiers type: DefinitionModifiers
required: true
DefinitionModifiers: DefinitionModifiers:
values: values:
- name: export - name: export
@ -29,10 +34,126 @@ types:
type: List<Declaration> type: List<Declaration>
- name: definitions - name: definitions
type: List<Declaration> type: List<Declaration>
Assignment: LetAssignment:
parent: Expression parent: Expression
values: values:
- name: symbol - name: symbol
type: Symbol type: Symbol
- name: value - name: value
type: Expression type: Expression
InfixOperator:
values:
- name: token
type: String
enums:
- name: Plus
values:
token: "+"
- name: Minus
values:
token: "-"
- name: Multiply
values:
token: "*"
- name: Divide
values:
token: "/"
- name: Equals
values:
token: "=="
- name: NotEquals
values:
token: "!="
InfixOperation:
parent: Expression
values:
- name: left
type: Expression
- name: op
type: InfixOperator
- name: right
type: Expression
BooleanLiteral:
parent: Expression
values:
- name: value
type: Boolean
FunctionCall:
parent: Expression
values:
- name: symbol
type: Symbol
- name: arguments
type: List<Expression>
FunctionDefinition:
parent: Definition
values:
- name: modifiers
type: DefinitionModifiers
- name: symbol
type: Symbol
- name: arguments
type: List<Symbol>
- name: block
type: Block
If:
parent: Expression
values:
- name: condition
type: Expression
- name: thenExpression
type: Expression
- name: elseExpression
type: Expression?
ImportDeclaration:
parent: Declaration
values:
- name: path
type: StringLiteral
IntLiteral:
parent: Expression
values:
- name: value
type: Int
Lambda:
parent: Expression
values:
- name: arguments
type: List<Symbol>
- name: expressions
type: List<Expression>
ListLiteral:
parent: Expression
values:
- name: items
type: List<Expression>
Parentheses:
parent: Expression
values:
- name: expression
type: Expression
PrefixOperator:
values:
- name: token
type: String
enums:
- name: Negate
values:
token: "!"
PrefixOperation:
parent: Expression
values:
- name: op
type: PrefixOperator
- name: expression
type: Expression
StringLiteral:
parent: Expression
values:
- name: text
type: String
SymbolReference:
parent: Expression
values:
- name: symbol
type: Symbol

View File

@ -21,21 +21,5 @@ enum class NodeType(val parent: NodeType? = null) {
FunctionCall(Expression), FunctionCall(Expression),
If(Expression), If(Expression),
ImportDeclaration(Declaration), ImportDeclaration(Declaration),
FunctionDefinition(Definition); FunctionDefinition(Definition)
val parents: Set<NodeType>
init {
val calculatedParents = mutableListOf<NodeType>()
var self = this
while (true) {
calculatedParents.add(self)
if (self.parent != null) {
self = self.parent!!
} else {
break
}
}
parents = calculatedParents.toSet()
}
} }

View File

@ -1,13 +1,11 @@
package gay.pizza.pork.gradle package gay.pizza.pork.gradle
import com.fasterxml.jackson.databind.ObjectMapper import gay.pizza.pork.gradle.ast.AstCodegen
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.kotlin.KotlinModule
import gay.pizza.pork.gradle.ast.AstDescription
import org.gradle.api.DefaultTask import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.TaskAction import org.gradle.api.tasks.TaskAction
import gay.pizza.pork.gradle.ast.AstWorld import org.gradle.api.tasks.OutputDirectory
import java.io.File import java.io.File
open class GenerateAstCode : DefaultTask() { open class GenerateAstCode : DefaultTask() {
@ -16,15 +14,16 @@ open class GenerateAstCode : DefaultTask() {
} }
@get:InputFile @get:InputFile
val astDescriptionFile: File = project.file("src/main/ast/pork.yml") var astDescriptionFile: File = project.file("src/main/ast/pork.yml")
@get:Input
var codePackage: String = "gay.pizza.pork.gen"
@get:OutputDirectory
var outputDirectory: File = project.file("src/main/kotlin/gay/pizza/pork/gen")
@TaskAction @TaskAction
fun generate() { fun generate() {
val astYamlText = astDescriptionFile.readText() AstCodegen.run(codePackage, astDescriptionFile.toPath(), outputDirectory.toPath())
val mapper = ObjectMapper(YAMLFactory())
mapper.registerModules(KotlinModule.Builder().build())
val astDescription = mapper.readValue(astYamlText, AstDescription::class.java)
val world = AstWorld()
world.build(astDescription)
} }
} }

View File

@ -0,0 +1,182 @@
package gay.pizza.pork.gradle.ast
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.kotlin.KotlinModule
import gay.pizza.pork.gradle.codegen.*
import java.nio.charset.StandardCharsets
import java.nio.file.Path
import kotlin.io.path.*
class AstCodegen(val pkg: String, val outputDirectory: Path, val world: AstWorld) {
private fun deleteAllContents() {
for (child in outputDirectory.listDirectoryEntries("*.kt")) {
child.deleteExisting()
}
}
fun generate() {
deleteAllContents()
for (type in world.typeRegistry.types) {
writeAstType(type)
}
writeNodeType()
}
private fun writeNodeType() {
val enumClass = KotlinEnum(pkg, "NodeType")
val parentMember = KotlinMember("parent", "NodeType?", value = "null")
enumClass.members.add(parentMember)
val typesInNameOrder = world.typeRegistry.types.sortedBy { it.name }
val typesInDependencyOrder = mutableListOf<AstType>()
for (type in typesInNameOrder) {
if (type.parent != null) {
if (!typesInDependencyOrder.contains(type.parent)) {
typesInDependencyOrder.add(type.parent!!)
}
}
if (!typesInDependencyOrder.contains(type)) {
typesInDependencyOrder.add(type)
}
}
for (type in typesInDependencyOrder) {
val role = world.typeRegistry.roleOfType(type)
if (role == AstTypeRole.ValueHolder || role == AstTypeRole.Enum) {
println(type)
continue
}
val entry = KotlinEnumEntry(type.name)
if (type.parent != null) {
entry.parameters.add(type.parent!!.name)
}
enumClass.entries.add(entry)
}
write("NodeType.kt", KotlinWriter(enumClass))
}
private fun writeAstType(type: AstType) {
val role = world.typeRegistry.roleOfType(type)
val kotlinClassLike: KotlinClassLike
if (role == AstTypeRole.Enum) {
val kotlinEnum = KotlinEnum(pkg, type.name)
kotlinClassLike = kotlinEnum
} else {
val kotlinClass = KotlinClass(pkg, type.name)
kotlinClassLike = kotlinClass
if (role == AstTypeRole.RootNode || role == AstTypeRole.HierarchyNode) {
kotlinClass.sealed = true
}
}
if (role == AstTypeRole.RootNode) {
val typeMember = KotlinMember(
"type",
"NodeType",
abstract = true
)
kotlinClassLike.members.add(typeMember)
} else if (role == AstTypeRole.AstNode) {
val typeMember = KotlinMember(
"type",
"NodeType",
overridden = true,
value = "NodeType.${type.name}"
)
kotlinClassLike.members.add(typeMember)
}
if (type.parent != null) {
val parentName = type.parent!!.name
kotlinClassLike.inherits.add("$parentName()")
}
for (value in type.values) {
val member = KotlinMember(value.name, toKotlinType(value.typeRef))
member.abstract = value.abstract
if (type.isParentAbstract(value)) {
member.overridden = true
}
kotlinClassLike.members.add(member)
}
if (role == AstTypeRole.Enum) {
val kotlinEnum = kotlinClassLike as KotlinEnum
for (entry in type.enums) {
val orderOfKeys = entry.values.keys.sortedBy { key ->
kotlinClassLike.members.indexOfFirst { it.name == key }
}
val parameters = mutableListOf<String>()
for (key in orderOfKeys) {
val value = entry.values[key] ?: continue
parameters.add("\"${value}\"")
}
val enumEntry = KotlinEnumEntry(entry.name, parameters)
kotlinEnum.entries.add(enumEntry)
}
}
if (role == AstTypeRole.AstNode) {
val equalsAndHashCodeFields = kotlinClassLike.members.map { it.name }
val equalsFunction = KotlinFunction(
"equals",
returnType = "Boolean",
overridden = true
)
equalsFunction.parameters.add(KotlinParameter(
"other",
"Any?"
))
equalsFunction.body.add("if (other !is ${type.name}) return false")
val predicate = equalsAndHashCodeFields.joinToString(" && ") {
"other.${it} == $it"
}
equalsFunction.body.add("return $predicate")
kotlinClassLike.functions.add(equalsFunction)
}
val serialName = kotlinClassLike.name[0].lowercase() +
kotlinClassLike.name.substring(1)
kotlinClassLike.imports.add("kotlinx.serialization.SerialName")
kotlinClassLike.imports.add("kotlinx.serialization.Serializable")
kotlinClassLike.annotations.add("Serializable")
kotlinClassLike.annotations.add("SerialName(\"$serialName\")")
write("${type.name}.kt", KotlinWriter(kotlinClassLike))
}
private fun toKotlinType(typeRef: AstTypeRef): String {
val baseType = typeRef.type?.name ?: typeRef.primitive?.id
?: throw RuntimeException("Unable to determine base type.")
return when (typeRef.form) {
AstTypeRefForm.Single -> baseType
AstTypeRefForm.Nullable -> "${baseType}?"
AstTypeRefForm.List -> "List<${baseType}>"
}
}
private fun write(fileName: String, writer: KotlinWriter) {
val path = outputDirectory.resolve(fileName)
path.deleteIfExists()
path.writeText(writer.toString(), StandardCharsets.UTF_8)
}
companion object {
fun run(pkg: String, astDescriptionFile: Path, outputDirectory: Path) {
val astYamlText = astDescriptionFile.readText()
val mapper = ObjectMapper(YAMLFactory())
mapper.registerModules(KotlinModule.Builder().build())
val astDescription = mapper.readValue(astYamlText, AstDescription::class.java)
val world = AstWorld.build(astDescription)
val codegen = AstCodegen(pkg, outputDirectory, world)
codegen.generate()
}
}
}

View File

@ -0,0 +1,6 @@
package gay.pizza.pork.gradle.ast
class AstEnum(
val name: String,
val values: Map<String, String>
)

View File

@ -0,0 +1,6 @@
package gay.pizza.pork.gradle.ast
class AstEnumDescription(
val name: String,
val values: Map<String, String>
)

View File

@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
enum class AstPrimitive(val id: kotlin.String) { enum class AstPrimitive(val id: kotlin.String) {
Boolean("Boolean"), Boolean("Boolean"),
String("String") String("String"),
Int("Int")
} }

View File

@ -2,11 +2,39 @@ package gay.pizza.pork.gradle.ast
class AstType(val name: String, var parent: AstType? = null) { class AstType(val name: String, var parent: AstType? = null) {
private val internalValues = mutableListOf<AstValue>() private val internalValues = mutableListOf<AstValue>()
private val internalEnums = mutableListOf<AstEnum>()
val values: List<AstValue> val values: List<AstValue>
get() = internalValues get() = internalValues
val enums: List<AstEnum>
get() = internalEnums
internal fun addValue(value: AstValue) { internal fun addValue(value: AstValue) {
internalValues.add(value) internalValues.add(value)
} }
internal fun addEnum(enum: AstEnum) {
internalEnums.add(enum)
}
fun isParentAbstract(value: AstValue): Boolean {
if (parent == null) {
return false
}
var current = parent
while (current != null) {
val abstract = current.values.firstOrNull {
it.name == value.name && it.abstract
}
if (abstract != null) {
return true
}
current = current.parent
}
return false
}
override fun toString(): String = "AstType(${name})"
} }

View File

@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
data class AstTypeDescription( data class AstTypeDescription(
val parent: String? = null, val parent: String? = null,
val values: List<AstValueDescription> = emptyList() val values: List<AstValueDescription> = emptyList(),
val enums: List<AstEnumDescription> = emptyList()
) )

View File

@ -17,15 +17,22 @@ class AstTypeRef(
) )
} }
val primitive = AstPrimitive.values().firstOrNull { it.name == input } var form = AstTypeRefForm.Single
var typeName: String = input
if (input.endsWith("?")) {
form = AstTypeRefForm.Nullable
typeName = input.substring(0, input.length - 1)
}
val primitive = AstPrimitive.values().firstOrNull { it.name == typeName }
if (primitive != null) { if (primitive != null) {
return AstTypeRef( return AstTypeRef(
primitive = primitive, primitive = primitive,
form = AstTypeRefForm.Single form = form
) )
} }
return AstTypeRef(type = registry.lookup(input), form = AstTypeRefForm.Single) return AstTypeRef(type = registry.lookup(typeName), form = form)
} }
} }
} }

View File

@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
enum class AstTypeRefForm { enum class AstTypeRefForm {
Single, Single,
List List,
Nullable
} }

View File

@ -19,7 +19,9 @@ class AstTypeRegistry {
fun roleOfType(type: AstType): AstTypeRole = fun roleOfType(type: AstType): AstTypeRole =
when { when {
type.parent == null && type.values.isNotEmpty() -> type.enums.isNotEmpty() ->
AstTypeRole.Enum
type.parent == null && type.values.isEmpty() ->
AstTypeRole.RootNode AstTypeRole.RootNode
type.parent != null && type.values.all { it.abstract } -> type.parent != null && type.values.all { it.abstract } ->
AstTypeRole.HierarchyNode AstTypeRole.HierarchyNode

View File

@ -4,5 +4,6 @@ enum class AstTypeRole {
RootNode, RootNode,
HierarchyNode, HierarchyNode,
AstNode, AstNode,
ValueHolder ValueHolder,
Enum
} }

View File

@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
data class AstValueDescription( data class AstValueDescription(
val name: String, val name: String,
val type: String val type: String,
val required: Boolean = false
) )

View File

@ -3,26 +3,37 @@ package gay.pizza.pork.gradle.ast
class AstWorld { class AstWorld {
val typeRegistry: AstTypeRegistry = AstTypeRegistry() val typeRegistry: AstTypeRegistry = AstTypeRegistry()
fun build(description: AstDescription) { companion object {
val rootType = typeRegistry.add(AstType(description.root)) fun build(description: AstDescription): AstWorld {
for (typeName in description.types.keys) { val world = AstWorld()
if (typeName == rootType.name) { val rootType = world.typeRegistry.add(AstType(description.root))
throw RuntimeException("Cannot have type with the same name as the root type.") for (typeName in description.types.keys) {
if (typeName == rootType.name) {
throw RuntimeException("Cannot have type with the same name as the root type.")
}
world.typeRegistry.add(AstType(typeName))
} }
typeRegistry.add(AstType(typeName)) for ((typeName, typeDescription) in description.types) {
} val type = world.typeRegistry.lookup(typeName)
for ((typeName, typeDescription) in description.types) { if (typeDescription.parent != null) {
val type = typeRegistry.lookup(typeName) type.parent = world.typeRegistry.lookup(typeDescription.parent)
if (typeDescription.parent != null) { }
type.parent = typeRegistry.lookup(typeDescription.parent)
} for (value in typeDescription.values) {
for (value in typeDescription.values) { val typeRef = AstTypeRef.parse(value.type, world.typeRegistry)
val typeRef = AstTypeRef.parse(value.type, typeRegistry) val typeValue = AstValue(value.name, typeRef, abstract = value.required)
val typeValue = AstValue(value.name, typeRef) type.addValue(typeValue)
type.addValue(typeValue) }
for (enum in typeDescription.enums) {
val astEnum = AstEnum(enum.name, enum.values)
type.addEnum(astEnum)
}
} }
return world
} }
} }
} }

View File

@ -0,0 +1,14 @@
package gay.pizza.pork.gradle.ast
import kotlin.io.path.Path
object RunCodegenIde {
@JvmStatic
fun main(args: Array<String>) {
AstCodegen.run(
pkg = "gay.pizza.pork.gen",
astDescriptionFile = Path("src/main/ast/pork.yml"),
outputDirectory = Path("src/main/kotlin/gay/pizza/pork/gen")
)
}
}

View File

@ -0,0 +1,12 @@
package gay.pizza.pork.gradle.codegen
class KotlinClass(
override val pkg: String,
override var name: String,
var sealed: Boolean = false,
override var inherits: MutableList<String> = mutableListOf(),
override var imports: MutableList<String> = mutableListOf(),
override var members: MutableList<KotlinMember> = mutableListOf(),
override var annotations: MutableList<String> = mutableListOf(),
override var functions: MutableList<KotlinFunction> = mutableListOf()
) : KotlinClassLike()

View File

@ -0,0 +1,11 @@
package gay.pizza.pork.gradle.codegen
abstract class KotlinClassLike {
abstract val pkg: String
abstract val name: String
abstract var imports: MutableList<String>
abstract var inherits: MutableList<String>
abstract var annotations: MutableList<String>
abstract var members: MutableList<KotlinMember>
abstract var functions: MutableList<KotlinFunction>
}

View File

@ -0,0 +1,12 @@
package gay.pizza.pork.gradle.codegen
class KotlinEnum(
override val pkg: String,
override val name: String,
override var imports: MutableList<String> = mutableListOf(),
override var inherits: MutableList<String> = mutableListOf(),
override var annotations: MutableList<String> = mutableListOf(),
override var members: MutableList<KotlinMember> = mutableListOf(),
override var functions: MutableList<KotlinFunction> = mutableListOf(),
var entries: MutableList<KotlinEnumEntry> = mutableListOf(),
) : KotlinClassLike()

View File

@ -0,0 +1,6 @@
package gay.pizza.pork.gradle.codegen
class KotlinEnumEntry(
val name: String,
var parameters: MutableList<String> = mutableListOf()
)

View File

@ -0,0 +1,11 @@
package gay.pizza.pork.gradle.codegen
class KotlinFunction(
val name: String,
var parameters: MutableList<KotlinParameter> = mutableListOf(),
var returnType: String? = null,
var abstract: Boolean = false,
var overridden: Boolean = false,
var isImmediateExpression: Boolean = false,
var body: MutableList<String> = mutableListOf()
)

View File

@ -0,0 +1,9 @@
package gay.pizza.pork.gradle.codegen
class KotlinMember(
var name: String,
var type: String,
var abstract: Boolean = false,
var overridden: Boolean = false,
var value: String? = null
)

View File

@ -0,0 +1,7 @@
package gay.pizza.pork.gradle.codegen
class KotlinParameter(
val name: String,
val type: String,
val defaultValue: String? = null
)

View File

@ -0,0 +1,182 @@
package gay.pizza.pork.gradle.codegen
class KotlinWriter {
private val buffer = StringBuilder()
constructor(kotlinClassLike: KotlinClassLike) {
write(kotlinClassLike)
}
fun writeClass(kotlinClass: KotlinClass): Unit = buffer.run {
val classType = if (kotlinClass.sealed) "sealed class" else "class"
writeClassLike(classType, kotlinClass)
val members = kotlinClass.members.filter {
it.abstract || (it.overridden && it.value != null)
}
if (members.isEmpty() && kotlinClass.functions.isEmpty()) {
appendLine()
} else {
appendLine(" {")
}
for (member in members) {
if (member.abstract) {
appendLine(" abstract val ${member.name}: ${member.type}")
} else {
if (member.overridden) {
append(" override ")
}
append("val ${member.name}: ${member.type}")
if (member.value != null) {
append(" = ")
append(member.value)
}
appendLine()
}
}
if (members.isNotEmpty() && kotlinClass.functions.isNotEmpty()) {
appendLine()
}
writeFunctions(kotlinClass)
if (members.isNotEmpty() || kotlinClass.functions.isNotEmpty()) {
appendLine("}")
}
}
fun writeEnum(kotlinEnum: KotlinEnum): Unit = buffer.run {
writeClassLike("enum class", kotlinEnum)
val membersNotCompatible = kotlinEnum.members.filter { it.abstract }
if (membersNotCompatible.isNotEmpty()) {
throw RuntimeException(
"Incompatible members in enum class " +
"${kotlinEnum.name}: $membersNotCompatible"
)
}
if (kotlinEnum.entries.isEmpty() && kotlinEnum.functions.isEmpty()) {
appendLine()
} else {
appendLine(" {")
}
for ((index, entry) in kotlinEnum.entries.withIndex()) {
append(" ${entry.name}")
if (entry.parameters.isNotEmpty()) {
append("(")
append(entry.parameters.joinToString(", "))
append(")")
}
if (index != kotlinEnum.entries.size - 1) {
append(",")
}
appendLine()
}
if (kotlinEnum.entries.isNotEmpty() && kotlinEnum.functions.isNotEmpty()) {
appendLine()
}
writeFunctions(kotlinEnum)
if (kotlinEnum.entries.isNotEmpty()) {
appendLine("}")
}
}
private fun writeClassLike(
classType: String,
kotlinClass: KotlinClassLike
): Unit = buffer.run {
appendLine("package ${kotlinClass.pkg}")
appendLine()
for (import in kotlinClass.imports) {
appendLine("import $import")
}
if (kotlinClass.imports.isNotEmpty()) {
appendLine()
}
for (annotation in kotlinClass.annotations) {
appendLine("@${annotation}")
}
append("$classType ${kotlinClass.name}")
val contructedMembers = kotlinClass.members.filter {
!it.abstract && !(it.overridden && it.value != null)
}
if (contructedMembers.isNotEmpty()) {
val constructor = contructedMembers.joinToString(", ") {
val prefix = if (it.overridden) "override " else ""
val start = "${prefix}val ${it.name}: ${it.type}"
if (it.value != null) {
"$start = ${it.value}"
} else start
}
append("(${constructor})")
}
if (kotlinClass.inherits.isNotEmpty()) {
append(" : ${kotlinClass.inherits.joinToString(", ")}")
}
}
private fun writeFunctions(kotlinClassLike: KotlinClassLike): Unit = buffer.run {
for (function in kotlinClassLike.functions) {
append(" ")
if (function.overridden) {
append("override ")
}
if (function.abstract) {
append("abstract ")
}
append("fun ${function.name}(")
append(function.parameters.joinToString(", ") {
val start = "${it.name}: ${it.type}"
if (it.defaultValue != null) {
start + " = ${it.defaultValue}"
} else start
})
append(")")
if (function.returnType != null) {
append(": ${function.returnType}")
}
if (!function.isImmediateExpression) {
append(" {")
} else {
appendLine(" =")
}
if (function.body.isNotEmpty()) {
appendLine()
for (item in function.body) {
appendLine(" $item")
}
}
if (!function.isImmediateExpression) {
appendLine(" }")
}
}
}
fun write(input: KotlinClassLike): Unit = when (input) {
is KotlinClass -> writeClass(input)
is KotlinEnum -> writeEnum(input)
else -> throw RuntimeException("Unknown Kotlin Class Type")
}
override fun toString(): String = buffer.toString()
}