Fix attribution of tokens.

examples/commented.pork

@@ -0,0 +1,17 @@
+/* fibonacci sequence */
+/**
+ * fib(n): calculate the fibonacci sequence.
+ * @input n the number to calculate fibonacci for
+ * @result the value of the fibonacci sequence for the number
+ */
+fib = { n in
+  if n == 0 // if n is zero, return zero
+    then 0
+  else if n == 1 // if n is one, return one
+    then 1
+  else fib(n - 1) + fib(n - 2)
+}
+
+// result of fib(20)
+result = fib(20)
+println(result)

src/main/kotlin/gay/pizza/pork/ast/NodeCoalescer.kt

@@ -3,18 +3,23 @@ package gay.pizza.pork.ast
 import gay.pizza.pork.ast.nodes.*
 
 class NodeCoalescer(val handler: (Node) -> Unit) : NodeVisitor<Unit> {
-  override fun visitIntLiteral(node: IntLiteral): Unit = handler(node)
+  override fun visitIntLiteral(node: IntLiteral): Unit = handle(node)
-  override fun visitStringLiteral(node: StringLiteral): Unit = handler(node)
+  override fun visitStringLiteral(node: StringLiteral): Unit = handle(node)
-  override fun visitBooleanLiteral(node: BooleanLiteral): Unit = handler(node)
+  override fun visitBooleanLiteral(node: BooleanLiteral): Unit = handle(node)
-  override fun visitListLiteral(node: ListLiteral): Unit = handler(node)
+  override fun visitListLiteral(node: ListLiteral): Unit = handle(node)
-  override fun visitSymbol(node: Symbol): Unit = handler(node)
+  override fun visitSymbol(node: Symbol): Unit = handle(node)
-  override fun visitFunctionCall(node: FunctionCall): Unit = handler(node)
+  override fun visitFunctionCall(node: FunctionCall): Unit = handle(node)
-  override fun visitDefine(node: Define): Unit = handler(node)
+  override fun visitDefine(node: Define): Unit = handle(node)
-  override fun visitSymbolReference(node: SymbolReference): Unit = handler(node)
+  override fun visitSymbolReference(node: SymbolReference): Unit = handle(node)
-  override fun visitLambda(node: Lambda): Unit = handler(node)
+  override fun visitLambda(node: Lambda): Unit = handle(node)
-  override fun visitParentheses(node: Parentheses): Unit = handler(node)
+  override fun visitParentheses(node: Parentheses): Unit = handle(node)
-  override fun visitPrefixOperation(node: PrefixOperation): Unit = handler(node)
+  override fun visitPrefixOperation(node: PrefixOperation): Unit = handle(node)
-  override fun visitIf(node: If): Unit = handler(node)
+  override fun visitIf(node: If): Unit = handle(node)
-  override fun visitInfixOperation(node: InfixOperation): Unit = handler(node)
+  override fun visitInfixOperation(node: InfixOperation): Unit = handle(node)
-  override fun visitProgram(node: Program): Unit = handler(node)
+  override fun visitProgram(node: Program): Unit = handle(node)
+
+  private fun handle(node: Node) {
+    handler(node)
+    node.visitChildren(this)
+  }
 }

src/main/kotlin/gay/pizza/pork/cli/AttributeCommand.kt

@@ -0,0 +1,27 @@
+package gay.pizza.pork.cli
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.types.path
+import gay.pizza.pork.ast.NodeCoalescer
+import gay.pizza.pork.frontend.FileFrontend
+import gay.pizza.pork.parse.TokenNodeAttribution
+
+class AttributeCommand : CliktCommand(help = "Attribute AST", name = "attribute") {
+  val path by argument("file").path(mustExist = true, canBeDir = false)
+
+  override fun run() {
+    val frontend = FileFrontend(path)
+    val attribution = TokenNodeAttribution()
+    val program = frontend.parse(attribution)
+
+    val coalescer = NodeCoalescer { node ->
+      val tokens = attribution.assembleTokens(node)
+      println("node ${node.toString().replace("\n", "^")}")
+      for (token in tokens) {
+        println("token $token")
+      }
+    }
+    coalescer.visit(program)
+  }
+}

src/main/kotlin/gay/pizza/pork/cli/RootCommand.kt

@@ -14,6 +14,7 @@ class RootCommand : CliktCommand(
       TokenizeCommand(),
       ReprintCommand(),
       AstCommand(),
+      AttributeCommand(),
       GenerateKotlinCommand(),
       GenerateDartCommand()
     )

src/main/kotlin/gay/pizza/pork/parse/Parser.kt

@@ -32,8 +32,9 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
     ListLiteral(items)
   }
 
-  private fun readSymbolRaw(): Symbol =
+  private fun readSymbolRaw(): Symbol = within {
     expect(TokenType.Symbol) { Symbol(it.text) }
+  }
 
   private fun readSymbolCases(): Expression = within {
     val symbol = readSymbolRaw()
@@ -142,7 +143,7 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
       }
     }
 
-    if (peek(
+    return if (peek(
         TokenType.Plus,
         TokenType.Minus,
         TokenType.Multiply,
@@ -151,12 +152,12 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
         TokenType.Inequality
       )
     ) {
-      val infixToken = next()
+      within {
-      val infixOperator = convertInfixOperator(infixToken)
+        val infixToken = next()
-      return InfixOperation(expression, infixOperator, readExpression())
+        val infixOperator = convertInfixOperator(infixToken)
-    }
+        InfixOperation(expression, infixOperator, readExpression())
-
+      }
-    return expression
+    } else expression
   }
 
   private fun convertInfixOperator(token: Token): InfixOperator =
@@ -170,10 +171,10 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
       else -> throw RuntimeException("Unknown Infix Operator")
     }
 
-  fun readProgram(): Program {
+  fun readProgram(): Program = within {
     val items = collect(TokenType.EndOfFile) { readExpression() }
     expect(TokenType.EndOfFile)
-    return Program(items)
+    Program(items)
   }
 
   private fun <T> collect(
@@ -233,6 +234,7 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
     while (true) {
       val token = unsanitizedSource.peek()
       if (ignoredByParser(token.type)) {
+        attribution.push(token)
         unsanitizedSource.next()
         continue
       }

src/main/kotlin/gay/pizza/pork/parse/TokenNodeAttribution.kt

@@ -5,7 +5,7 @@ import gay.pizza.pork.ast.nodes.Node
 import java.util.IdentityHashMap
 
 class TokenNodeAttribution : NodeAttribution {
-  private val map: MutableMap<Node, List<Token>> = IdentityHashMap()
+  val nodes: MutableMap<Node, List<Token>> = IdentityHashMap()
 
   private val stack = mutableListOf<MutableList<Token>>()
   private var current: MutableList<Token>? = null
@@ -23,11 +23,12 @@ class TokenNodeAttribution : NodeAttribution {
 
   override fun <T: Node> exit(node: T): T {
     val store = stack.removeLast()
-    map[node] = store
+    nodes[node] = store
+    current = stack.lastOrNull()
     return node
   }
 
-  fun tokensOf(node: Node): List<Token>? = map[node]
+  fun tokensOf(node: Node): List<Token>? = nodes[node]
 
   fun assembleTokens(node: Node): List<Token> {
     val allTokens = mutableListOf<Token>()
@@ -38,6 +39,6 @@ class TokenNodeAttribution : NodeAttribution {
       }
     }
     coalescer.visit(node)
-    return allTokens
+    return allTokens.asSequence().distinct().sortedBy { it.start }.toList()
   }
 }