Significant progress on AST codegen.

This commit is contained in:
Alex Zenla 2023-09-04 18:38:09 -07:00
parent edec706ed4
commit f06ea93dc4
Signed by: alex
GPG Key ID: C0780728420EBFE5
24 changed files with 670 additions and 55 deletions

View File

@ -4,6 +4,9 @@ types:
parent: Node
Symbol:
parent: Node
values:
- name: id
type: String
Declaration:
parent: Node
Definition:
@ -11,8 +14,10 @@ types:
values:
- name: symbol
type: Symbol
required: true
- name: modifiers
type: DefinitionModifiers
required: true
DefinitionModifiers:
values:
- name: export
@ -29,10 +34,126 @@ types:
type: List<Declaration>
- name: definitions
type: List<Declaration>
Assignment:
LetAssignment:
parent: Expression
values:
- name: symbol
type: Symbol
- name: value
type: Expression
InfixOperator:
values:
- name: token
type: String
enums:
- name: Plus
values:
token: "+"
- name: Minus
values:
token: "-"
- name: Multiply
values:
token: "*"
- name: Divide
values:
token: "/"
- name: Equals
values:
token: "=="
- name: NotEquals
values:
token: "!="
InfixOperation:
parent: Expression
values:
- name: left
type: Expression
- name: op
type: InfixOperator
- name: right
type: Expression
BooleanLiteral:
parent: Expression
values:
- name: value
type: Boolean
FunctionCall:
parent: Expression
values:
- name: symbol
type: Symbol
- name: arguments
type: List<Expression>
FunctionDefinition:
parent: Definition
values:
- name: modifiers
type: DefinitionModifiers
- name: symbol
type: Symbol
- name: arguments
type: List<Symbol>
- name: block
type: Block
If:
parent: Expression
values:
- name: condition
type: Expression
- name: thenExpression
type: Expression
- name: elseExpression
type: Expression?
ImportDeclaration:
parent: Declaration
values:
- name: path
type: StringLiteral
IntLiteral:
parent: Expression
values:
- name: value
type: Int
Lambda:
parent: Expression
values:
- name: arguments
type: List<Symbol>
- name: expressions
type: List<Expression>
ListLiteral:
parent: Expression
values:
- name: items
type: List<Expression>
Parentheses:
parent: Expression
values:
- name: expression
type: Expression
PrefixOperator:
values:
- name: token
type: String
enums:
- name: Negate
values:
token: "!"
PrefixOperation:
parent: Expression
values:
- name: op
type: PrefixOperator
- name: expression
type: Expression
StringLiteral:
parent: Expression
values:
- name: text
type: String
SymbolReference:
parent: Expression
values:
- name: symbol
type: Symbol

View File

@ -21,21 +21,5 @@ enum class NodeType(val parent: NodeType? = null) {
FunctionCall(Expression),
If(Expression),
ImportDeclaration(Declaration),
FunctionDefinition(Definition);
val parents: Set<NodeType>
init {
val calculatedParents = mutableListOf<NodeType>()
var self = this
while (true) {
calculatedParents.add(self)
if (self.parent != null) {
self = self.parent!!
} else {
break
}
}
parents = calculatedParents.toSet()
}
FunctionDefinition(Definition)
}

View File

@ -1,13 +1,11 @@
package gay.pizza.pork.gradle
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.kotlin.KotlinModule
import gay.pizza.pork.gradle.ast.AstDescription
import gay.pizza.pork.gradle.ast.AstCodegen
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.TaskAction
import gay.pizza.pork.gradle.ast.AstWorld
import org.gradle.api.tasks.OutputDirectory
import java.io.File
open class GenerateAstCode : DefaultTask() {
@ -16,15 +14,16 @@ open class GenerateAstCode : DefaultTask() {
}
@get:InputFile
val astDescriptionFile: File = project.file("src/main/ast/pork.yml")
var astDescriptionFile: File = project.file("src/main/ast/pork.yml")
@get:Input
var codePackage: String = "gay.pizza.pork.gen"
@get:OutputDirectory
var outputDirectory: File = project.file("src/main/kotlin/gay/pizza/pork/gen")
@TaskAction
fun generate() {
val astYamlText = astDescriptionFile.readText()
val mapper = ObjectMapper(YAMLFactory())
mapper.registerModules(KotlinModule.Builder().build())
val astDescription = mapper.readValue(astYamlText, AstDescription::class.java)
val world = AstWorld()
world.build(astDescription)
AstCodegen.run(codePackage, astDescriptionFile.toPath(), outputDirectory.toPath())
}
}

View File

@ -0,0 +1,182 @@
package gay.pizza.pork.gradle.ast
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.kotlin.KotlinModule
import gay.pizza.pork.gradle.codegen.*
import java.nio.charset.StandardCharsets
import java.nio.file.Path
import kotlin.io.path.*
class AstCodegen(val pkg: String, val outputDirectory: Path, val world: AstWorld) {
private fun deleteAllContents() {
for (child in outputDirectory.listDirectoryEntries("*.kt")) {
child.deleteExisting()
}
}
fun generate() {
deleteAllContents()
for (type in world.typeRegistry.types) {
writeAstType(type)
}
writeNodeType()
}
private fun writeNodeType() {
val enumClass = KotlinEnum(pkg, "NodeType")
val parentMember = KotlinMember("parent", "NodeType?", value = "null")
enumClass.members.add(parentMember)
val typesInNameOrder = world.typeRegistry.types.sortedBy { it.name }
val typesInDependencyOrder = mutableListOf<AstType>()
for (type in typesInNameOrder) {
if (type.parent != null) {
if (!typesInDependencyOrder.contains(type.parent)) {
typesInDependencyOrder.add(type.parent!!)
}
}
if (!typesInDependencyOrder.contains(type)) {
typesInDependencyOrder.add(type)
}
}
for (type in typesInDependencyOrder) {
val role = world.typeRegistry.roleOfType(type)
if (role == AstTypeRole.ValueHolder || role == AstTypeRole.Enum) {
println(type)
continue
}
val entry = KotlinEnumEntry(type.name)
if (type.parent != null) {
entry.parameters.add(type.parent!!.name)
}
enumClass.entries.add(entry)
}
write("NodeType.kt", KotlinWriter(enumClass))
}
private fun writeAstType(type: AstType) {
val role = world.typeRegistry.roleOfType(type)
val kotlinClassLike: KotlinClassLike
if (role == AstTypeRole.Enum) {
val kotlinEnum = KotlinEnum(pkg, type.name)
kotlinClassLike = kotlinEnum
} else {
val kotlinClass = KotlinClass(pkg, type.name)
kotlinClassLike = kotlinClass
if (role == AstTypeRole.RootNode || role == AstTypeRole.HierarchyNode) {
kotlinClass.sealed = true
}
}
if (role == AstTypeRole.RootNode) {
val typeMember = KotlinMember(
"type",
"NodeType",
abstract = true
)
kotlinClassLike.members.add(typeMember)
} else if (role == AstTypeRole.AstNode) {
val typeMember = KotlinMember(
"type",
"NodeType",
overridden = true,
value = "NodeType.${type.name}"
)
kotlinClassLike.members.add(typeMember)
}
if (type.parent != null) {
val parentName = type.parent!!.name
kotlinClassLike.inherits.add("$parentName()")
}
for (value in type.values) {
val member = KotlinMember(value.name, toKotlinType(value.typeRef))
member.abstract = value.abstract
if (type.isParentAbstract(value)) {
member.overridden = true
}
kotlinClassLike.members.add(member)
}
if (role == AstTypeRole.Enum) {
val kotlinEnum = kotlinClassLike as KotlinEnum
for (entry in type.enums) {
val orderOfKeys = entry.values.keys.sortedBy { key ->
kotlinClassLike.members.indexOfFirst { it.name == key }
}
val parameters = mutableListOf<String>()
for (key in orderOfKeys) {
val value = entry.values[key] ?: continue
parameters.add("\"${value}\"")
}
val enumEntry = KotlinEnumEntry(entry.name, parameters)
kotlinEnum.entries.add(enumEntry)
}
}
if (role == AstTypeRole.AstNode) {
val equalsAndHashCodeFields = kotlinClassLike.members.map { it.name }
val equalsFunction = KotlinFunction(
"equals",
returnType = "Boolean",
overridden = true
)
equalsFunction.parameters.add(KotlinParameter(
"other",
"Any?"
))
equalsFunction.body.add("if (other !is ${type.name}) return false")
val predicate = equalsAndHashCodeFields.joinToString(" && ") {
"other.${it} == $it"
}
equalsFunction.body.add("return $predicate")
kotlinClassLike.functions.add(equalsFunction)
}
val serialName = kotlinClassLike.name[0].lowercase() +
kotlinClassLike.name.substring(1)
kotlinClassLike.imports.add("kotlinx.serialization.SerialName")
kotlinClassLike.imports.add("kotlinx.serialization.Serializable")
kotlinClassLike.annotations.add("Serializable")
kotlinClassLike.annotations.add("SerialName(\"$serialName\")")
write("${type.name}.kt", KotlinWriter(kotlinClassLike))
}
private fun toKotlinType(typeRef: AstTypeRef): String {
val baseType = typeRef.type?.name ?: typeRef.primitive?.id
?: throw RuntimeException("Unable to determine base type.")
return when (typeRef.form) {
AstTypeRefForm.Single -> baseType
AstTypeRefForm.Nullable -> "${baseType}?"
AstTypeRefForm.List -> "List<${baseType}>"
}
}
private fun write(fileName: String, writer: KotlinWriter) {
val path = outputDirectory.resolve(fileName)
path.deleteIfExists()
path.writeText(writer.toString(), StandardCharsets.UTF_8)
}
companion object {
fun run(pkg: String, astDescriptionFile: Path, outputDirectory: Path) {
val astYamlText = astDescriptionFile.readText()
val mapper = ObjectMapper(YAMLFactory())
mapper.registerModules(KotlinModule.Builder().build())
val astDescription = mapper.readValue(astYamlText, AstDescription::class.java)
val world = AstWorld.build(astDescription)
val codegen = AstCodegen(pkg, outputDirectory, world)
codegen.generate()
}
}
}

View File

@ -0,0 +1,6 @@
package gay.pizza.pork.gradle.ast
class AstEnum(
val name: String,
val values: Map<String, String>
)

View File

@ -0,0 +1,6 @@
package gay.pizza.pork.gradle.ast
class AstEnumDescription(
val name: String,
val values: Map<String, String>
)

View File

@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
enum class AstPrimitive(val id: kotlin.String) {
Boolean("Boolean"),
String("String")
String("String"),
Int("Int")
}

View File

@ -2,11 +2,39 @@ package gay.pizza.pork.gradle.ast
class AstType(val name: String, var parent: AstType? = null) {
private val internalValues = mutableListOf<AstValue>()
private val internalEnums = mutableListOf<AstEnum>()
val values: List<AstValue>
get() = internalValues
val enums: List<AstEnum>
get() = internalEnums
internal fun addValue(value: AstValue) {
internalValues.add(value)
}
internal fun addEnum(enum: AstEnum) {
internalEnums.add(enum)
}
fun isParentAbstract(value: AstValue): Boolean {
if (parent == null) {
return false
}
var current = parent
while (current != null) {
val abstract = current.values.firstOrNull {
it.name == value.name && it.abstract
}
if (abstract != null) {
return true
}
current = current.parent
}
return false
}
override fun toString(): String = "AstType(${name})"
}

View File

@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
data class AstTypeDescription(
val parent: String? = null,
val values: List<AstValueDescription> = emptyList()
val values: List<AstValueDescription> = emptyList(),
val enums: List<AstEnumDescription> = emptyList()
)

View File

@ -17,15 +17,22 @@ class AstTypeRef(
)
}
val primitive = AstPrimitive.values().firstOrNull { it.name == input }
var form = AstTypeRefForm.Single
var typeName: String = input
if (input.endsWith("?")) {
form = AstTypeRefForm.Nullable
typeName = input.substring(0, input.length - 1)
}
val primitive = AstPrimitive.values().firstOrNull { it.name == typeName }
if (primitive != null) {
return AstTypeRef(
primitive = primitive,
form = AstTypeRefForm.Single
form = form
)
}
return AstTypeRef(type = registry.lookup(input), form = AstTypeRefForm.Single)
return AstTypeRef(type = registry.lookup(typeName), form = form)
}
}
}

View File

@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
enum class AstTypeRefForm {
Single,
List
List,
Nullable
}

View File

@ -19,7 +19,9 @@ class AstTypeRegistry {
fun roleOfType(type: AstType): AstTypeRole =
when {
type.parent == null && type.values.isNotEmpty() ->
type.enums.isNotEmpty() ->
AstTypeRole.Enum
type.parent == null && type.values.isEmpty() ->
AstTypeRole.RootNode
type.parent != null && type.values.all { it.abstract } ->
AstTypeRole.HierarchyNode

View File

@ -4,5 +4,6 @@ enum class AstTypeRole {
RootNode,
HierarchyNode,
AstNode,
ValueHolder
ValueHolder,
Enum
}

View File

@ -2,5 +2,6 @@ package gay.pizza.pork.gradle.ast
data class AstValueDescription(
val name: String,
val type: String
val type: String,
val required: Boolean = false
)

View File

@ -3,26 +3,37 @@ package gay.pizza.pork.gradle.ast
class AstWorld {
val typeRegistry: AstTypeRegistry = AstTypeRegistry()
fun build(description: AstDescription) {
val rootType = typeRegistry.add(AstType(description.root))
for (typeName in description.types.keys) {
if (typeName == rootType.name) {
throw RuntimeException("Cannot have type with the same name as the root type.")
companion object {
fun build(description: AstDescription): AstWorld {
val world = AstWorld()
val rootType = world.typeRegistry.add(AstType(description.root))
for (typeName in description.types.keys) {
if (typeName == rootType.name) {
throw RuntimeException("Cannot have type with the same name as the root type.")
}
world.typeRegistry.add(AstType(typeName))
}
typeRegistry.add(AstType(typeName))
}
for ((typeName, typeDescription) in description.types) {
val type = world.typeRegistry.lookup(typeName)
for ((typeName, typeDescription) in description.types) {
val type = typeRegistry.lookup(typeName)
if (typeDescription.parent != null) {
type.parent = typeRegistry.lookup(typeDescription.parent)
}
for (value in typeDescription.values) {
val typeRef = AstTypeRef.parse(value.type, typeRegistry)
val typeValue = AstValue(value.name, typeRef)
type.addValue(typeValue)
if (typeDescription.parent != null) {
type.parent = world.typeRegistry.lookup(typeDescription.parent)
}
for (value in typeDescription.values) {
val typeRef = AstTypeRef.parse(value.type, world.typeRegistry)
val typeValue = AstValue(value.name, typeRef, abstract = value.required)
type.addValue(typeValue)
}
for (enum in typeDescription.enums) {
val astEnum = AstEnum(enum.name, enum.values)
type.addEnum(astEnum)
}
}
return world
}
}
}

View File

@ -0,0 +1,14 @@
package gay.pizza.pork.gradle.ast
import kotlin.io.path.Path
object RunCodegenIde {
@JvmStatic
fun main(args: Array<String>) {
AstCodegen.run(
pkg = "gay.pizza.pork.gen",
astDescriptionFile = Path("src/main/ast/pork.yml"),
outputDirectory = Path("src/main/kotlin/gay/pizza/pork/gen")
)
}
}

View File

@ -0,0 +1,12 @@
package gay.pizza.pork.gradle.codegen
class KotlinClass(
override val pkg: String,
override var name: String,
var sealed: Boolean = false,
override var inherits: MutableList<String> = mutableListOf(),
override var imports: MutableList<String> = mutableListOf(),
override var members: MutableList<KotlinMember> = mutableListOf(),
override var annotations: MutableList<String> = mutableListOf(),
override var functions: MutableList<KotlinFunction> = mutableListOf()
) : KotlinClassLike()

View File

@ -0,0 +1,11 @@
package gay.pizza.pork.gradle.codegen
abstract class KotlinClassLike {
abstract val pkg: String
abstract val name: String
abstract var imports: MutableList<String>
abstract var inherits: MutableList<String>
abstract var annotations: MutableList<String>
abstract var members: MutableList<KotlinMember>
abstract var functions: MutableList<KotlinFunction>
}

View File

@ -0,0 +1,12 @@
package gay.pizza.pork.gradle.codegen
class KotlinEnum(
override val pkg: String,
override val name: String,
override var imports: MutableList<String> = mutableListOf(),
override var inherits: MutableList<String> = mutableListOf(),
override var annotations: MutableList<String> = mutableListOf(),
override var members: MutableList<KotlinMember> = mutableListOf(),
override var functions: MutableList<KotlinFunction> = mutableListOf(),
var entries: MutableList<KotlinEnumEntry> = mutableListOf(),
) : KotlinClassLike()

View File

@ -0,0 +1,6 @@
package gay.pizza.pork.gradle.codegen
class KotlinEnumEntry(
val name: String,
var parameters: MutableList<String> = mutableListOf()
)

View File

@ -0,0 +1,11 @@
package gay.pizza.pork.gradle.codegen
class KotlinFunction(
val name: String,
var parameters: MutableList<KotlinParameter> = mutableListOf(),
var returnType: String? = null,
var abstract: Boolean = false,
var overridden: Boolean = false,
var isImmediateExpression: Boolean = false,
var body: MutableList<String> = mutableListOf()
)

View File

@ -0,0 +1,9 @@
package gay.pizza.pork.gradle.codegen
class KotlinMember(
var name: String,
var type: String,
var abstract: Boolean = false,
var overridden: Boolean = false,
var value: String? = null
)

View File

@ -0,0 +1,7 @@
package gay.pizza.pork.gradle.codegen
class KotlinParameter(
val name: String,
val type: String,
val defaultValue: String? = null
)

View File

@ -0,0 +1,182 @@
package gay.pizza.pork.gradle.codegen
class KotlinWriter {
private val buffer = StringBuilder()
constructor(kotlinClassLike: KotlinClassLike) {
write(kotlinClassLike)
}
fun writeClass(kotlinClass: KotlinClass): Unit = buffer.run {
val classType = if (kotlinClass.sealed) "sealed class" else "class"
writeClassLike(classType, kotlinClass)
val members = kotlinClass.members.filter {
it.abstract || (it.overridden && it.value != null)
}
if (members.isEmpty() && kotlinClass.functions.isEmpty()) {
appendLine()
} else {
appendLine(" {")
}
for (member in members) {
if (member.abstract) {
appendLine(" abstract val ${member.name}: ${member.type}")
} else {
if (member.overridden) {
append(" override ")
}
append("val ${member.name}: ${member.type}")
if (member.value != null) {
append(" = ")
append(member.value)
}
appendLine()
}
}
if (members.isNotEmpty() && kotlinClass.functions.isNotEmpty()) {
appendLine()
}
writeFunctions(kotlinClass)
if (members.isNotEmpty() || kotlinClass.functions.isNotEmpty()) {
appendLine("}")
}
}
fun writeEnum(kotlinEnum: KotlinEnum): Unit = buffer.run {
writeClassLike("enum class", kotlinEnum)
val membersNotCompatible = kotlinEnum.members.filter { it.abstract }
if (membersNotCompatible.isNotEmpty()) {
throw RuntimeException(
"Incompatible members in enum class " +
"${kotlinEnum.name}: $membersNotCompatible"
)
}
if (kotlinEnum.entries.isEmpty() && kotlinEnum.functions.isEmpty()) {
appendLine()
} else {
appendLine(" {")
}
for ((index, entry) in kotlinEnum.entries.withIndex()) {
append(" ${entry.name}")
if (entry.parameters.isNotEmpty()) {
append("(")
append(entry.parameters.joinToString(", "))
append(")")
}
if (index != kotlinEnum.entries.size - 1) {
append(",")
}
appendLine()
}
if (kotlinEnum.entries.isNotEmpty() && kotlinEnum.functions.isNotEmpty()) {
appendLine()
}
writeFunctions(kotlinEnum)
if (kotlinEnum.entries.isNotEmpty()) {
appendLine("}")
}
}
private fun writeClassLike(
classType: String,
kotlinClass: KotlinClassLike
): Unit = buffer.run {
appendLine("package ${kotlinClass.pkg}")
appendLine()
for (import in kotlinClass.imports) {
appendLine("import $import")
}
if (kotlinClass.imports.isNotEmpty()) {
appendLine()
}
for (annotation in kotlinClass.annotations) {
appendLine("@${annotation}")
}
append("$classType ${kotlinClass.name}")
val contructedMembers = kotlinClass.members.filter {
!it.abstract && !(it.overridden && it.value != null)
}
if (contructedMembers.isNotEmpty()) {
val constructor = contructedMembers.joinToString(", ") {
val prefix = if (it.overridden) "override " else ""
val start = "${prefix}val ${it.name}: ${it.type}"
if (it.value != null) {
"$start = ${it.value}"
} else start
}
append("(${constructor})")
}
if (kotlinClass.inherits.isNotEmpty()) {
append(" : ${kotlinClass.inherits.joinToString(", ")}")
}
}
private fun writeFunctions(kotlinClassLike: KotlinClassLike): Unit = buffer.run {
for (function in kotlinClassLike.functions) {
append(" ")
if (function.overridden) {
append("override ")
}
if (function.abstract) {
append("abstract ")
}
append("fun ${function.name}(")
append(function.parameters.joinToString(", ") {
val start = "${it.name}: ${it.type}"
if (it.defaultValue != null) {
start + " = ${it.defaultValue}"
} else start
})
append(")")
if (function.returnType != null) {
append(": ${function.returnType}")
}
if (!function.isImmediateExpression) {
append(" {")
} else {
appendLine(" =")
}
if (function.body.isNotEmpty()) {
appendLine()
for (item in function.body) {
appendLine(" $item")
}
}
if (!function.isImmediateExpression) {
appendLine(" }")
}
}
}
fun write(input: KotlinClassLike): Unit = when (input) {
is KotlinClass -> writeClass(input)
is KotlinEnum -> writeEnum(input)
else -> throw RuntimeException("Unknown Kotlin Class Type")
}
override fun toString(): String = buffer.toString()
}