From 62ba662a911d5dc67aafd4c2c6313eadf7b274ee Mon Sep 17 00:00:00 2001 From: Alex Zenla Date: Sat, 2 Sep 2023 17:01:56 -0700 Subject: [PATCH] Fix attribution of tokens. --- examples/commented.pork | 17 ++++++++++ .../gay/pizza/pork/ast/NodeCoalescer.kt | 33 +++++++++++-------- .../gay/pizza/pork/cli/AttributeCommand.kt | 27 +++++++++++++++ .../kotlin/gay/pizza/pork/cli/RootCommand.kt | 1 + .../kotlin/gay/pizza/pork/parse/Parser.kt | 22 +++++++------ .../pizza/pork/parse/TokenNodeAttribution.kt | 9 ++--- 6 files changed, 81 insertions(+), 28 deletions(-) create mode 100644 examples/commented.pork create mode 100644 src/main/kotlin/gay/pizza/pork/cli/AttributeCommand.kt diff --git a/examples/commented.pork b/examples/commented.pork new file mode 100644 index 0000000..3912860 --- /dev/null +++ b/examples/commented.pork @@ -0,0 +1,17 @@ +/* fibonacci sequence */ +/** + * fib(n): calculate the fibonacci sequence. + * @input n the number to calculate fibonacci for + * @result the value of the fibonacci sequence for the number + */ +fib = { n in + if n == 0 // if n is zero, return zero + then 0 + else if n == 1 // if n is one, return one + then 1 + else fib(n - 1) + fib(n - 2) +} + +// result of fib(20) +result = fib(20) +println(result) diff --git a/src/main/kotlin/gay/pizza/pork/ast/NodeCoalescer.kt b/src/main/kotlin/gay/pizza/pork/ast/NodeCoalescer.kt index 3c21c2c..3b8d27a 100644 --- a/src/main/kotlin/gay/pizza/pork/ast/NodeCoalescer.kt +++ b/src/main/kotlin/gay/pizza/pork/ast/NodeCoalescer.kt @@ -3,18 +3,23 @@ package gay.pizza.pork.ast import gay.pizza.pork.ast.nodes.* class NodeCoalescer(val handler: (Node) -> Unit) : NodeVisitor { - override fun visitIntLiteral(node: IntLiteral): Unit = handler(node) - override fun visitStringLiteral(node: StringLiteral): Unit = handler(node) - override fun visitBooleanLiteral(node: BooleanLiteral): Unit = handler(node) - override fun visitListLiteral(node: ListLiteral): Unit = handler(node) - override fun visitSymbol(node: Symbol): Unit = handler(node) - override fun visitFunctionCall(node: FunctionCall): Unit = handler(node) - override fun visitDefine(node: Define): Unit = handler(node) - override fun visitSymbolReference(node: SymbolReference): Unit = handler(node) - override fun visitLambda(node: Lambda): Unit = handler(node) - override fun visitParentheses(node: Parentheses): Unit = handler(node) - override fun visitPrefixOperation(node: PrefixOperation): Unit = handler(node) - override fun visitIf(node: If): Unit = handler(node) - override fun visitInfixOperation(node: InfixOperation): Unit = handler(node) - override fun visitProgram(node: Program): Unit = handler(node) + override fun visitIntLiteral(node: IntLiteral): Unit = handle(node) + override fun visitStringLiteral(node: StringLiteral): Unit = handle(node) + override fun visitBooleanLiteral(node: BooleanLiteral): Unit = handle(node) + override fun visitListLiteral(node: ListLiteral): Unit = handle(node) + override fun visitSymbol(node: Symbol): Unit = handle(node) + override fun visitFunctionCall(node: FunctionCall): Unit = handle(node) + override fun visitDefine(node: Define): Unit = handle(node) + override fun visitSymbolReference(node: SymbolReference): Unit = handle(node) + override fun visitLambda(node: Lambda): Unit = handle(node) + override fun visitParentheses(node: Parentheses): Unit = handle(node) + override fun visitPrefixOperation(node: PrefixOperation): Unit = handle(node) + override fun visitIf(node: If): Unit = handle(node) + override fun visitInfixOperation(node: InfixOperation): Unit = handle(node) + override fun visitProgram(node: Program): Unit = handle(node) + + private fun handle(node: Node) { + handler(node) + node.visitChildren(this) + } } diff --git a/src/main/kotlin/gay/pizza/pork/cli/AttributeCommand.kt b/src/main/kotlin/gay/pizza/pork/cli/AttributeCommand.kt new file mode 100644 index 0000000..5b43584 --- /dev/null +++ b/src/main/kotlin/gay/pizza/pork/cli/AttributeCommand.kt @@ -0,0 +1,27 @@ +package gay.pizza.pork.cli + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.types.path +import gay.pizza.pork.ast.NodeCoalescer +import gay.pizza.pork.frontend.FileFrontend +import gay.pizza.pork.parse.TokenNodeAttribution + +class AttributeCommand : CliktCommand(help = "Attribute AST", name = "attribute") { + val path by argument("file").path(mustExist = true, canBeDir = false) + + override fun run() { + val frontend = FileFrontend(path) + val attribution = TokenNodeAttribution() + val program = frontend.parse(attribution) + + val coalescer = NodeCoalescer { node -> + val tokens = attribution.assembleTokens(node) + println("node ${node.toString().replace("\n", "^")}") + for (token in tokens) { + println("token $token") + } + } + coalescer.visit(program) + } +} diff --git a/src/main/kotlin/gay/pizza/pork/cli/RootCommand.kt b/src/main/kotlin/gay/pizza/pork/cli/RootCommand.kt index cf853bd..e069864 100644 --- a/src/main/kotlin/gay/pizza/pork/cli/RootCommand.kt +++ b/src/main/kotlin/gay/pizza/pork/cli/RootCommand.kt @@ -14,6 +14,7 @@ class RootCommand : CliktCommand( TokenizeCommand(), ReprintCommand(), AstCommand(), + AttributeCommand(), GenerateKotlinCommand(), GenerateDartCommand() ) diff --git a/src/main/kotlin/gay/pizza/pork/parse/Parser.kt b/src/main/kotlin/gay/pizza/pork/parse/Parser.kt index d362178..13fa65c 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/Parser.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/Parser.kt @@ -32,8 +32,9 @@ class Parser(source: PeekableSource, val attribution: NodeAttribution) { ListLiteral(items) } - private fun readSymbolRaw(): Symbol = + private fun readSymbolRaw(): Symbol = within { expect(TokenType.Symbol) { Symbol(it.text) } + } private fun readSymbolCases(): Expression = within { val symbol = readSymbolRaw() @@ -142,7 +143,7 @@ class Parser(source: PeekableSource, val attribution: NodeAttribution) { } } - if (peek( + return if (peek( TokenType.Plus, TokenType.Minus, TokenType.Multiply, @@ -151,12 +152,12 @@ class Parser(source: PeekableSource, val attribution: NodeAttribution) { TokenType.Inequality ) ) { - val infixToken = next() - val infixOperator = convertInfixOperator(infixToken) - return InfixOperation(expression, infixOperator, readExpression()) - } - - return expression + within { + val infixToken = next() + val infixOperator = convertInfixOperator(infixToken) + InfixOperation(expression, infixOperator, readExpression()) + } + } else expression } private fun convertInfixOperator(token: Token): InfixOperator = @@ -170,10 +171,10 @@ class Parser(source: PeekableSource, val attribution: NodeAttribution) { else -> throw RuntimeException("Unknown Infix Operator") } - fun readProgram(): Program { + fun readProgram(): Program = within { val items = collect(TokenType.EndOfFile) { readExpression() } expect(TokenType.EndOfFile) - return Program(items) + Program(items) } private fun collect( @@ -233,6 +234,7 @@ class Parser(source: PeekableSource, val attribution: NodeAttribution) { while (true) { val token = unsanitizedSource.peek() if (ignoredByParser(token.type)) { + attribution.push(token) unsanitizedSource.next() continue } diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenNodeAttribution.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenNodeAttribution.kt index 0edb5d1..8d4867b 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/TokenNodeAttribution.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/TokenNodeAttribution.kt @@ -5,7 +5,7 @@ import gay.pizza.pork.ast.nodes.Node import java.util.IdentityHashMap class TokenNodeAttribution : NodeAttribution { - private val map: MutableMap> = IdentityHashMap() + val nodes: MutableMap> = IdentityHashMap() private val stack = mutableListOf>() private var current: MutableList? = null @@ -23,11 +23,12 @@ class TokenNodeAttribution : NodeAttribution { override fun exit(node: T): T { val store = stack.removeLast() - map[node] = store + nodes[node] = store + current = stack.lastOrNull() return node } - fun tokensOf(node: Node): List? = map[node] + fun tokensOf(node: Node): List? = nodes[node] fun assembleTokens(node: Node): List { val allTokens = mutableListOf() @@ -38,6 +39,6 @@ class TokenNodeAttribution : NodeAttribution { } } coalescer.visit(node) - return allTokens + return allTokens.asSequence().distinct().sortedBy { it.start }.toList() } }