parser: various code and quality enhancements

ast/src/main/kotlin/gay/pizza/pork/ast/gen/NodeCoalescer.kt

@@ -1,7 +1,7 @@
 // GENERATED CODE FROM PORK AST CODEGEN
 package gay.pizza.pork.ast.gen
 
-class NodeCoalescer(val handler: (Node) -> Unit) : NodeVisitor<Unit> {
+class NodeCoalescer(val followChildren: Boolean = true, val handler: (Node) -> Unit) : NodeVisitor<Unit> {
   override fun visitArgumentSpec(node: ArgumentSpec): Unit =
     handle(node)
 
@@ -97,6 +97,8 @@ class NodeCoalescer(val handler: (Node) -> Unit) : NodeVisitor<Unit> {
 
   fun handle(node: Node) {
     handler(node)
-    node.visitChildren(this)
+    if (followChildren) {
+      node.visitChildren(this)
+    }
   }
 }

buildext/src/main/kotlin/gay/pizza/pork/buildext/ast/AstStandardCodegen.kt

@@ -193,6 +193,11 @@ class AstStandardCodegen(pkg: String, outputDirectory: Path, world: AstWorld) :
       "NodeCoalescer",
       inherits = mutableListOf("NodeVisitor<Unit>"),
       members = mutableListOf(
+        KotlinMember(
+          "followChildren",
+          "Boolean",
+          value = "true"
+        ),
         KotlinMember(
           "handler",
           "(Node) -> Unit"
@@ -227,7 +232,9 @@ class AstStandardCodegen(pkg: String, outputDirectory: Path, world: AstWorld) :
       )
     )
     handleFunction.body.add("handler(node)")
-    handleFunction.body.add("node.visitChildren(this)")
+    handleFunction.body.add("if (followChildren) {")
+    handleFunction.body.add("  node.visitChildren(this)")
+    handleFunction.body.add("}")
     coalescerClass.functions.add(handleFunction)
 
     write("NodeCoalescer.kt", KotlinWriter(coalescerClass))

common/src/main/kotlin/gay/pizza/pork/common/IndentBuffer.kt

@@ -0,0 +1,16 @@
+package gay.pizza.pork.common
+
+class IndentBuffer(
+  val buffer: StringBuilder = StringBuilder(),
+  indent: String = "  "
+) : IndentTracked(indent), Appendable by buffer, CharSequence by buffer {
+  override fun emit(text: String) {
+    append(text)
+  }
+
+  override fun emitLine(text: String) {
+    appendLine(text)
+  }
+
+  override fun toString(): String = buffer.toString()
+}

common/src/main/kotlin/gay/pizza/pork/common/IndentPrinter.kt

@@ -1,36 +1,11 @@
 package gay.pizza.pork.common
 
-class IndentPrinter(
+class IndentPrinter(indent: String = "  ") : IndentTracked(indent) {
-  val buffer: StringBuilder = StringBuilder(),
+  override fun emit(text: String) {
-  val indent: String = "  "
+    print(text)
-) : Appendable by buffer, CharSequence by buffer {
-  private var indentLevel: Int = 0
-  private var indentLevelText: String = ""
-
-  fun emitIndent() {
-    append(indentLevelText)
   }
 
-  fun emitIndentedLine(line: String) {
+  override fun emitLine(text: String) {
-    emitIndent()
+    println(text)
-    appendLine(line)
   }
-
-  fun increaseIndent() {
-    indentLevel++
-    indentLevelText += indent
-  }
-
-  fun decreaseIndent() {
-    indentLevel--
-    indentLevelText = indent.repeat(indentLevel)
-  }
-
-  inline fun indented(block: IndentPrinter.() -> Unit) {
-    increaseIndent()
-    block(this)
-    decreaseIndent()
-  }
-
-  override fun toString(): String = buffer.toString()
 }

common/src/main/kotlin/gay/pizza/pork/common/IndentTracked.kt

@@ -0,0 +1,37 @@
+package gay.pizza.pork.common
+
+abstract class IndentTracked(val indent: String) {
+  private var internalIndentLevel = 0
+  private var indentLevelText = ""
+
+  val indentLevel: Int
+    get() = internalIndentLevel
+
+  fun emitIndent() {
+    emit(indentLevelText)
+  }
+
+  fun emitIndentedLine(line: String) {
+    emitIndent()
+    emitLine(line)
+  }
+
+  fun increaseIndent() {
+    internalIndentLevel++
+    indentLevelText += indent
+  }
+
+  fun decreaseIndent() {
+    internalIndentLevel--
+    indentLevelText = indent.repeat(indentLevel)
+  }
+
+  inline fun indented(block: IndentTracked.() -> Unit) {
+    increaseIndent()
+    block(this)
+    decreaseIndent()
+  }
+
+  abstract fun emit(text: String)
+  abstract fun emitLine(text: String)
+}

evaluator/src/main/kotlin/gay/pizza/pork/evaluator/EvaluationVisitor.kt

@@ -337,7 +337,6 @@ class EvaluationVisitor(root: Scope, val stack: CallStack) : NodeVisitor<Any> {
       InfixOperator.Minus -> subtract(convert(left), convert(right))
       InfixOperator.Multiply -> multiply(convert(left), convert(right))
       InfixOperator.Divide -> divide(convert(left), convert(right))
-      InfixOperator.Equals, InfixOperator.NotEquals, InfixOperator.BooleanAnd, InfixOperator.BooleanOr -> throw RuntimeException("Unable to handle operation $op")
       InfixOperator.BinaryAnd -> binaryAnd(convert(left), convert(right))
       InfixOperator.BinaryOr -> binaryOr(convert(left), convert(right))
       InfixOperator.BinaryExclusiveOr -> binaryExclusiveOr(convert(left), convert(right))
@@ -347,6 +346,8 @@ class EvaluationVisitor(root: Scope, val stack: CallStack) : NodeVisitor<Any> {
       InfixOperator.Greater -> greater(convert(left), convert(right))
       InfixOperator.LesserEqual -> lesserEqual(convert(left), convert(right))
       InfixOperator.GreaterEqual -> greaterEqual(convert(left), convert(right))
+      InfixOperator.Equals, InfixOperator.NotEquals, InfixOperator.BooleanAnd, InfixOperator.BooleanOr ->
+        throw RuntimeException("Unable to handle operation $op")
     }
   }
 

parser/src/main/kotlin/gay/pizza/pork/parser/CharConsumer.kt

@@ -0,0 +1,5 @@
+package gay.pizza.pork.parser
+
+interface CharConsumer {
+  fun consume(type: TokenType, tokenizer: Tokenizer): String?
+}

parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt

@@ -1,8 +1,10 @@
 package gay.pizza.pork.parser
 
 interface CharSource : PeekableSource<Char> {
+  fun peek(index: Int): Char
+
   companion object {
     @Suppress("ConstPropertyName")
-    const val NullChar = 0.toChar()
+    const val EndOfFile = 0.toChar()
   }
 }

parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt

@@ -1,7 +1,7 @@
 package gay.pizza.pork.parser
 
 fun CharSource.readToString(): String = buildString {
-  while (peek() != CharSource.NullChar) {
+  while (peek() != CharSource.EndOfFile) {
     append(next())
   }
 }

parser/src/main/kotlin/gay/pizza/pork/parser/MatchedCharConsumer.kt

@@ -0,0 +1,48 @@
+package gay.pizza.pork.parser
+
+@Suppress("CanBeParameter")
+class MatchedCharConsumer(
+  val start: CharSequence,
+  val end: CharSequence,
+  vararg val options: Options
+) : CharConsumer {
+  private val eofTerminationAllowed = options.contains(Options.AllowEofTermination)
+
+  override fun consume(type: TokenType, tokenizer: Tokenizer): String? {
+    if (!tokenizer.peek(start)) {
+      return null
+    }
+    val buffer = StringBuilder()
+    tokenizer.read(start.length, buffer)
+    var endsNeededToTerminate = 1
+    while (true) {
+      if (tokenizer.peek(start)) {
+        endsNeededToTerminate++
+        tokenizer.read(start.length, buffer)
+        continue
+      }
+
+      if (tokenizer.peek(end)) {
+        endsNeededToTerminate--
+        tokenizer.read(end.length, buffer)
+      }
+
+      if (endsNeededToTerminate == 0) {
+        return buffer.toString()
+      }
+
+      val char = tokenizer.source.next()
+      if (char == CharSource.EndOfFile) {
+        if (eofTerminationAllowed) {
+          return buffer.toString()
+        }
+        throw UnterminatedTokenError(type.name, tokenizer.source.currentSourceIndex())
+      }
+      buffer.append(char)
+    }
+  }
+
+  enum class Options {
+    AllowEofTermination
+  }
+}

parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt

@@ -9,8 +9,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     ArgumentSpec(symbol, next(TokenType.DotDotDot))
   }
 
-  override fun parseBlock(): Block = guarded(NodeType.Block) {
+  override fun parseBlock(): Block = expect(NodeType.Block, TokenType.LeftCurly) {
-    expect(TokenType.LeftCurly)
     val items = collect(TokenType.RightCurly) {
       parseExpression()
     }
@@ -18,7 +17,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     Block(items)
   }
 
-  override fun parseExpression(): Expression = guarded {
+  override fun parseExpression(): Expression {
     val token = peek()
     var expression = when (token.type) {
       TokenType.NumberLiteral -> parseNumberLiteral()
@@ -66,7 +65,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
       }
     }
 
-    if (peek(
+    return if (peek(
         TokenType.Plus, TokenType.Minus, TokenType.Multiply, TokenType.Divide, TokenType.Ampersand,
         TokenType.Pipe, TokenType.Caret, TokenType.Equality, TokenType.Inequality, TokenType.Mod,
         TokenType.Rem, TokenType.Lesser, TokenType.Greater, TokenType.LesserEqual, TokenType.GreaterEqual,
@@ -176,12 +175,11 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
       )
     }
 
-  override fun parseDoubleLiteral(): DoubleLiteral = guarded(NodeType.DoubleLiteral) {
+  override fun parseDoubleLiteral(): DoubleLiteral = expect(NodeType.DoubleLiteral, TokenType.NumberLiteral) {
-    DoubleLiteral(expect(TokenType.NumberLiteral).text.toDouble())
+    DoubleLiteral(it.text.toDouble())
   }
 
-  override fun parseForIn(): ForIn = guarded(NodeType.ForIn) {
+  override fun parseForIn(): ForIn = expect(NodeType.ForIn, TokenType.For) {
-    expect(TokenType.For)
     val forInItem = parseForInItem()
     expect(TokenType.In)
     val value = parseExpression()
@@ -223,8 +221,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     FunctionDefinition(modifiers, name, arguments, block, native)
   }
 
-  override fun parseIf(): If = guarded(NodeType.If) {
+  override fun parseIf(): If = expect(NodeType.If, TokenType.If) {
-    expect(TokenType.If)
     val condition = parseExpression()
     val thenBlock = parseBlock()
     var elseBlock: Block? = null
@@ -234,8 +231,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     If(condition, thenBlock, elseBlock)
   }
 
-  override fun parseImportDeclaration(): ImportDeclaration = guarded(NodeType.ImportDeclaration) {
+  override fun parseImportDeclaration(): ImportDeclaration = expect(NodeType.ImportDeclaration, TokenType.Import) {
-    expect(TokenType.Import)
     val form = parseSymbol()
     val components = oneAndContinuedBy(TokenType.Dot) {
       parseSymbol()
@@ -257,13 +253,13 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     InfixOperation(parseExpression(), infixOperator, parseExpression())
   }
 
-  private fun parseNumberLiteral(): Expression = guarded {
+  private fun parseNumberLiteral(): Expression {
     val token = peek()
     if (token.type != TokenType.NumberLiteral) {
       expect(TokenType.NumberLiteral)
     }
 
-    when {
+    return when {
       token.text.contains(".") -> parseDoubleLiteral()
       token.text.toIntOrNull() != null -> parseIntegerLiteral()
       token.text.toLongOrNull() != null -> parseLongLiteral()
@@ -271,12 +267,11 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     }
   }
 
-  override fun parseIntegerLiteral(): IntegerLiteral = guarded(NodeType.IntegerLiteral) {
+  override fun parseIntegerLiteral(): IntegerLiteral = expect(NodeType.IntegerLiteral, TokenType.NumberLiteral) {
-    IntegerLiteral(expect(TokenType.NumberLiteral).text.toInt())
+    IntegerLiteral(it.text.toInt())
   }
 
-  override fun parseLetAssignment(): LetAssignment = guarded(NodeType.LetAssignment) {
+  override fun parseLetAssignment(): LetAssignment = expect(NodeType.LetAssignment, TokenType.Let) {
-    expect(TokenType.Let)
     val symbol = parseSymbol()
     expect(TokenType.Equals)
     val value = parseExpression()
@@ -292,8 +287,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     LetDefinition(definitionModifiers, name, value)
   }
 
-  override fun parseListLiteral(): ListLiteral = guarded(NodeType.ListLiteral) {
+  override fun parseListLiteral(): ListLiteral = expect(NodeType.ListLiteral, TokenType.LeftBracket) {
-    expect(TokenType.LeftBracket)
     val items = collect(TokenType.RightBracket, TokenType.Comma) {
       parseExpression()
     }
@@ -301,12 +295,11 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     ListLiteral(items)
   }
 
-  override fun parseLongLiteral(): LongLiteral = guarded(NodeType.LongLiteral) {
+  override fun parseLongLiteral(): LongLiteral = expect(NodeType.LongLiteral, TokenType.NumberLiteral) {
-    LongLiteral(expect(TokenType.NumberLiteral).text.toLong())
+    LongLiteral(it.text.toLong())
   }
 
-  override fun parseNative(): Native = guarded(NodeType.Native) {
+  override fun parseNative(): Native = expect(NodeType.Native, TokenType.Native) {
-    expect(TokenType.Native)
     val form = parseSymbol()
     val definitions = mutableListOf<StringLiteral>()
     while (peek(TokenType.StringLiteral)) {
@@ -315,22 +308,22 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     Native(form, definitions)
   }
 
-  override fun parseNoneLiteral(): NoneLiteral = guarded(NodeType.NoneLiteral) {
+  override fun parseNoneLiteral(): NoneLiteral = expect(NodeType.NoneLiteral, TokenType.None) {
-    expect(TokenType.None)
     NoneLiteral()
   }
 
-  override fun parseParentheses(): Parentheses = guarded(NodeType.Parentheses) {
+  override fun parseParentheses(): Parentheses = expect(NodeType.Parentheses, TokenType.LeftParentheses) {
-    expect(TokenType.LeftParentheses)
     val expression = parseExpression()
     expect(TokenType.RightParentheses)
     Parentheses(expression)
   }
 
-  override fun parsePrefixOperation(): PrefixOperation = guarded(NodeType.PrefixOperation) {
+  override fun parsePrefixOperation(): PrefixOperation = expect(
-    expect(TokenType.Not, TokenType.Plus, TokenType.Minus, TokenType.Tilde) {
+    NodeType.PrefixOperation,
-      PrefixOperation(ParserHelpers.convertPrefixOperator(it), parseExpression())
+    TokenType.Not, TokenType.Plus,
-    }
+    TokenType.Minus, TokenType.Tilde
+  ) {
+    PrefixOperation(ParserHelpers.convertPrefixOperator(it), parseExpression())
   }
 
   override fun parseSetAssignment(): SetAssignment = guarded(NodeType.SetAssignment) {
@@ -340,11 +333,9 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     SetAssignment(symbol, value)
   }
 
-  override fun parseStringLiteral(): StringLiteral = guarded(NodeType.StringLiteral) {
+  override fun parseStringLiteral(): StringLiteral = expect(NodeType.StringLiteral, TokenType.StringLiteral) {
-    expect(TokenType.StringLiteral) {
+    val content = StringEscape.unescape(StringEscape.unquote(it.text))
-      val content = StringEscape.unescape(StringEscape.unquote(it.text))
+    StringLiteral(content)
-      StringLiteral(content)
-    }
   }
 
   override fun parseSuffixOperation(): SuffixOperation = guarded(NodeType.SuffixOperation) {
@@ -354,32 +345,30 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     }
   }
 
-  private fun parseSymbolCases(): Expression = guarded {
+  private fun parseSymbolCases(): Expression {
-    if (peek(1, TokenType.LeftParentheses)) {
+    return if (peek(1, TokenType.LeftParentheses)) {
       parseFunctionCall()
     } else if (peek(1, TokenType.PlusPlus, TokenType.MinusMinus)) {
       parseSuffixOperation()
     } else parseSymbolReference()
   }
 
-  override fun parseSymbol(): Symbol = guarded(NodeType.Symbol) {
+  override fun parseSymbol(): Symbol = expect(NodeType.Symbol, TokenType.Symbol) {
-    expect(TokenType.Symbol) { Symbol(it.text) }
+    Symbol(it.text)
   }
 
   override fun parseSymbolReference(): SymbolReference = guarded(NodeType.SymbolReference) {
     SymbolReference(parseSymbol())
   }
 
-  override fun parseVarAssignment(): VarAssignment = guarded(NodeType.VarAssignment) {
+  override fun parseVarAssignment(): VarAssignment = expect(NodeType.VarAssignment, TokenType.Var) {
-    expect(TokenType.Var)
     val symbol = parseSymbol()
     expect(TokenType.Equals)
     val value = parseExpression()
     VarAssignment(symbol, value)
   }
 
-  override fun parseWhile(): While = guarded(NodeType.While) {
+  override fun parseWhile(): While = expect(NodeType.While, TokenType.While) {
-    expect(TokenType.While)
     val condition = parseExpression()
     val block = parseBlock()
     While(condition, block)

parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt

@@ -7,6 +7,14 @@ import gay.pizza.pork.ast.gen.visit
 
 data class ParserAttributes(val tokens: List<Token>) {
   companion object {
+    fun recallOwnedTokens(node: Node): List<Token> {
+      val attributes = node.data<ParserAttributes>()
+      if (attributes != null) {
+        return attributes.tokens
+      }
+      return emptyList()
+    }
+
     fun recallAllTokens(node: Node): List<Token> {
       val all = mutableListOf<Token>()
       val coalescer = NodeCoalescer { item ->

parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt

@@ -11,6 +11,12 @@ abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution)
   protected inline fun <T: Node> guarded(type: NodeType? = null, noinline block: () -> T): T =
     attribution.guarded(type, block)
 
+  @Suppress("NOTHING_TO_INLINE")
+  protected inline fun <T: Node> expect(type: NodeType? = null, vararg tokenTypes: TokenType, noinline block: (Token) -> T): T =
+    guarded(type) {
+      block(expect(*tokenTypes))
+    }
+
   protected fun <T> collect(
     peeking: TokenType,
     consuming: TokenType? = null,
@@ -66,7 +72,11 @@ abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution)
     throw ExpectedTokenError(token, token.sourceIndex, *types)
   }
 
-  protected fun next(): Token = source.next()
+  protected fun next(): Token {
+    val token = source.next()
+    attribution.push(token)
+    return token
+  }
   protected fun peek(): Token = source.peek()
   protected fun peek(ahead: Int): TokenType = source.peekTypeAhead(ahead)
   protected fun peek(ahead: Int, vararg types: TokenType): Boolean = types.contains(source.peekTypeAhead(ahead))

parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt

@@ -23,7 +23,7 @@ internal object ParserHelpers {
     TokenType.GreaterEqual -> InfixOperator.GreaterEqual
     TokenType.And -> InfixOperator.BooleanAnd
     TokenType.Or -> InfixOperator.BooleanOr
-    else -> throw ParseError("Unknown Infix Operator")
+    else -> throw ParseError("Unknown infix operator: ${token.type}")
   }
 
   fun convertPrefixOperator(token: Token): PrefixOperator = when (token.type) {
@@ -31,12 +31,12 @@ internal object ParserHelpers {
     TokenType.Plus -> PrefixOperator.UnaryPlus
     TokenType.Minus -> PrefixOperator.UnaryMinus
     TokenType.Tilde -> PrefixOperator.BinaryNot
-    else -> throw ParseError("Unknown Prefix Operator")
+    else -> throw ParseError("Unknown prefix operator: ${token.type}")
   }
 
   fun convertSuffixOperator(token: Token): SuffixOperator = when (token.type) {
     TokenType.PlusPlus -> SuffixOperator.Increment
     TokenType.MinusMinus -> SuffixOperator.Decrement
-    else -> throw ParseError("Unknown Suffix Operator")
+    else -> throw ParseError("Unknown suffix operator: ${token.type}")
   }
 }

parser/src/main/kotlin/gay/pizza/pork/parser/Printer.kt

@@ -1,10 +1,10 @@
 package gay.pizza.pork.parser
 
 import gay.pizza.pork.ast.gen.*
-import gay.pizza.pork.common.IndentPrinter
+import gay.pizza.pork.common.IndentBuffer
 
 class Printer(buffer: StringBuilder) : NodeVisitor<Unit> {
-  private val out = IndentPrinter(buffer)
+  private val out = IndentBuffer(buffer)
   private var autoIndentState = false
 
   private fun append(text: String) {

parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt

@@ -0,0 +1,19 @@
+package gay.pizza.pork.parser
+
+class SourceIndexCharSource(val delegate: CharSource) : CharSource by delegate {
+  private var currentLineIndex = 1
+  private var currentLineColumn = 0
+
+  override fun next(): Char {
+    val char = delegate.next()
+    if (char == '\n') {
+      currentLineIndex++
+      currentLineColumn = 0
+    }
+    currentLineColumn++
+    return char
+  }
+
+  fun currentSourceIndex(): SourceIndex =
+    SourceIndex(delegate.currentIndex, currentLineIndex, currentLineColumn)
+}

parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt

@@ -0,0 +1,29 @@
+package gay.pizza.pork.parser
+
+object StringCharConsumer : CharConsumer {
+  override fun consume(type: TokenType, tokenizer: Tokenizer): String? {
+    if (!tokenizer.peek("\"")) {
+      return null
+    }
+
+    val buffer = StringBuilder()
+    buffer.append(tokenizer.source.next())
+    var escape = false
+    while (true) {
+      val char = tokenizer.source.peek()
+
+      if (char == CharSource.EndOfFile) {
+        throw UnterminatedTokenError("String", tokenizer.source.currentSourceIndex())
+      }
+
+      buffer.append(tokenizer.source.next())
+
+      if (char == '\\') {
+        escape = true
+      } else if (char == '"' && !escape) {
+        break
+      }
+    }
+    return buffer.toString()
+  }
+}

parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt

@@ -6,12 +6,13 @@ class StringCharSource(
   val endIndex: Int = input.length - 1
 ) : CharSource {
   private var index = startIndex
+
   override val currentIndex: Int
     get() = index
 
   override fun next(): Char {
     if (index == endIndex) {
-      return CharSource.NullChar
+      return CharSource.EndOfFile
     }
     val char = input[index]
     index++
@@ -20,8 +21,16 @@ class StringCharSource(
 
   override fun peek(): Char {
     if (index == endIndex) {
-      return CharSource.NullChar
+      return CharSource.EndOfFile
     }
     return input[index]
   }
+
+  override fun peek(index: Int): Char {
+    val target = this.index + index
+    if (target >= endIndex) {
+      return CharSource.EndOfFile
+    }
+    return input[target]
+  }
 }

parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt

@@ -1,22 +1,23 @@
 package gay.pizza.pork.parser
 
 import gay.pizza.pork.parser.CharMatcher.*
+import gay.pizza.pork.parser.MatchedCharConsumer.Options.AllowEofTermination
 import gay.pizza.pork.parser.TokenTypeProperty.*
 import gay.pizza.pork.parser.TokenFamily.*
 import gay.pizza.pork.parser.TokenTypeProperty.AnyOf
 
 enum class TokenType(vararg properties: TokenTypeProperty) {
-  NumberLiteral(NumericLiteralFamily, CharConsumer(CharMatcher.AnyOf(
+  NumberLiteral(NumericLiteralFamily, CharMatch(CharMatcher.AnyOf(
     MatchRange('0'..'9'),
     NotAtIndex(0, MatchSingle('.'))
   ))),
-  Symbol(SymbolFamily, CharConsumer(CharMatcher.AnyOf(
+  Symbol(SymbolFamily, CharMatch(CharMatcher.AnyOf(
     MatchRange('a'..'z'),
     MatchRange('A'..'Z'),
     MatchRange('0' .. '9'),
     MatchSingle('_')
   )), KeywordUpgrader),
-  StringLiteral(StringLiteralFamily),
+  StringLiteral(StringLiteralFamily, CharConsume(StringCharConsumer)),
   Equality(OperatorFamily),
   Inequality(ManyChars("!="), OperatorFamily),
   ExclamationPoint(SingleChar('!'), Promotion('=', Inequality)),
@@ -66,14 +67,14 @@ enum class TokenType(vararg properties: TokenTypeProperty) {
   Native(ManyChars("native"), KeywordFamily),
   Let(ManyChars("let"), KeywordFamily),
   Var(ManyChars("var"), KeywordFamily),
-  Whitespace(CharConsumer(CharMatcher.AnyOf(
+  Whitespace(CharMatch(CharMatcher.AnyOf(
     MatchSingle(' '),
     MatchSingle('\r'),
     MatchSingle('\n'),
     MatchSingle('\t')
   ))),
-  BlockComment(CommentFamily),
+  BlockComment(CharConsume(MatchedCharConsumer("/*", "*/")), CommentFamily),
-  LineComment(CommentFamily),
+  LineComment(CharConsume(MatchedCharConsumer("//", "\n", AllowEofTermination)), CommentFamily),
   EndOfFile;
 
   val promotions: List<Promotion> =
@@ -86,7 +87,8 @@ enum class TokenType(vararg properties: TokenTypeProperty) {
     properties.filterIsInstance<SingleChar>().singleOrNull()
   val family: TokenFamily =
     properties.filterIsInstance<TokenFamily>().singleOrNull() ?: OtherFamily
-  val charConsumer: CharConsumer? = properties.filterIsInstance<CharConsumer>().singleOrNull()
+  val charMatch: CharMatch? = properties.filterIsInstance<CharMatch>().singleOrNull()
+  val charConsume: CharConsume? = properties.filterIsInstance<CharConsume>().singleOrNull()
   val tokenUpgrader: TokenUpgrader? =
     properties.filterIsInstance<TokenUpgrader>().singleOrNull()
 
@@ -96,8 +98,8 @@ enum class TokenType(vararg properties: TokenTypeProperty) {
     val AnyOf = entries.filter { item -> item.anyOf != null }
     val ManyChars = entries.filter { item -> item.manyChars != null }
     val SingleChars = entries.filter { item -> item.singleChar != null }
-    val CharConsumers = entries.filter { item ->
+    val CharMatches = entries.filter { item -> item.charMatch != null }
-      item.charConsumer != null }
+    val CharConsumes = entries.filter { item -> item.charConsume != null }
 
     val ParserIgnoredTypes: Array<TokenType> = arrayOf(
       Whitespace,

parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt

@@ -5,7 +5,8 @@ interface TokenTypeProperty {
   class Promotion(val nextChar: Char, val type: TokenType) : TokenTypeProperty
   class ManyChars(val text: String) : TokenTypeProperty
   class AnyOf(vararg val strings: String): TokenTypeProperty
-  open class CharConsumer(val matcher: CharMatcher) : TokenTypeProperty
+  open class CharMatch(val matcher: CharMatcher) : TokenTypeProperty
+  open class CharConsume(val consumer: CharConsumer) : TokenTypeProperty
   open class TokenUpgrader(val maybeUpgrade: (Token) -> Token?) : TokenTypeProperty
 
   object KeywordUpgrader : TokenUpgrader({ token ->

parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt

@@ -1,80 +1,22 @@
 package gay.pizza.pork.parser
 
-class Tokenizer(val source: CharSource) {
+class Tokenizer(source: CharSource) {
+  val source: SourceIndexCharSource = SourceIndexCharSource(source)
+
   private var startIndex: SourceIndex = SourceIndex.zero()
-  private var currentLineIndex = 1
-  private var currentLineColumn = 0
-
-  private fun readBlockComment(firstChar: Char): Token {
-    val comment = buildString {
-      append(firstChar)
-      var endOfComment = false
-      while (true) {
-        val char = nextChar()
-        if (char == CharSource.NullChar) {
-          throw UnterminatedTokenError("block comment", currentSourceIndex())
-        }
-        append(char)
-
-        if (endOfComment) {
-          if (char != '/') {
-            endOfComment = false
-            continue
-          }
-          break
-        }
-
-        if (char == '*') {
-          endOfComment = true
-        }
-      }
-    }
-    return produceToken(TokenType.BlockComment, comment)
-  }
-
-  private fun readLineComment(firstChar: Char): Token {
-    val comment = buildString {
-      append(firstChar)
-      while (true) {
-        val char = source.peek()
-        if (char == CharSource.NullChar || char == '\n') {
-          break
-        }
-        append(nextChar())
-      }
-    }
-    return produceToken(TokenType.LineComment, comment)
-  }
-
-  private fun readStringLiteral(firstChar: Char): Token {
-    val string = buildString {
-      append(firstChar)
-      while (true) {
-        val char = source.peek()
-        if (char == CharSource.NullChar) {
-          throw UnterminatedTokenError("string", currentSourceIndex())
-        }
-        append(nextChar())
-        if (char == '"') {
-          break
-        }
-      }
-    }
-    return produceToken(TokenType.StringLiteral, string)
-  }
 
   fun next(): Token {
-    while (source.peek() != CharSource.NullChar) {
+    while (source.peek() != CharSource.EndOfFile) {
-      startIndex = currentSourceIndex()
+      startIndex = source.currentSourceIndex()
-      val char = nextChar()
 
-      if (char == '/' && source.peek() == '*') {
+      for (item in TokenType.CharConsumes) {
-        return readBlockComment(char)
+        val text = item.charConsume!!.consumer.consume(item, this)
+        if (text != null) {
+          return produceToken(item, text)
+        }
       }
 
-      if (char == '/' && source.peek() == '/') {
+      val char = source.next()
-        return readLineComment(char)
-      }
 
       for (item in TokenType.SingleChars) {
         val itemChar = item.singleChar!!.char
@@ -91,7 +33,7 @@ class Tokenizer(val source: CharSource) {
             if (source.peek() != promotion.nextChar) {
               continue
             }
-            val nextChar = nextChar()
+            val nextChar = source.next()
             type = promotion.type
             text += nextChar
             promoted = true
@@ -101,16 +43,16 @@ class Tokenizer(val source: CharSource) {
       }
 
       var index = 0
-      for (item in TokenType.CharConsumers) {
+      for (item in TokenType.CharMatches) {
-        if (!item.charConsumer!!.matcher.valid(char, index)) {
+        if (!item.charMatch!!.matcher.valid(char, index)) {
           continue
         }
 
         val text = buildString {
           append(char)
 
-          while (item.charConsumer.matcher.valid(source.peek(), ++index)) {
+          while (item.charMatch.matcher.valid(source.peek(), ++index)) {
-            append(nextChar())
+            append(source.next())
           }
         }
         var token = produceToken(item, text)
@@ -121,10 +63,6 @@ class Tokenizer(val source: CharSource) {
         return token
       }
 
-      if (char == '"') {
-        return readStringLiteral(char)
-      }
-
       throw BadCharacterError(char, startIndex)
     }
     return Token.endOfFile(startIndex.copy(index = source.currentIndex))
@@ -142,19 +80,23 @@ class Tokenizer(val source: CharSource) {
     return TokenStream(tokens)
   }
 
-  private fun produceToken(type: TokenType, text: String) =
+  internal fun produceToken(type: TokenType, text: String) =
     Token(type, startIndex, text)
 
-  private fun nextChar(): Char {
+  internal fun peek(what: CharSequence): Boolean {
-    val char = source.next()
+    var current = 0
-    if (char == '\n') {
+    for (c in what) {
-      currentLineIndex++
+      if (source.peek(current) != c) {
-      currentLineColumn = 0
+        return false
+      }
+      current++
     }
-    currentLineColumn++
+    return true
-    return char
   }
 
-  private fun currentSourceIndex(): SourceIndex =
+  internal fun read(count: Int, buffer: StringBuilder) {
-    SourceIndex(source.currentIndex, currentLineIndex, currentLineColumn)
+    for (i in 1..count) {
+      buffer.append(source.next())
+    }
+  }
 }

tool/build.gradle.kts

@@ -8,6 +8,8 @@ plugins {
 dependencies {
   api(project(":minimal"))
   api("com.github.ajalt.clikt:clikt:4.2.0")
+
+  implementation(project(":common"))
 }
 
 application {

tool/src/main/kotlin/gay/pizza/pork/tool/AttributeCommand.kt

@@ -2,14 +2,19 @@ package gay.pizza.pork.tool
 
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.options.flag
+import com.github.ajalt.clikt.parameters.options.option
 import gay.pizza.dough.fs.PlatformFsProvider
+import gay.pizza.pork.ast.gen.Node
 import gay.pizza.pork.ast.gen.NodeCoalescer
 import gay.pizza.pork.ast.gen.visit
+import gay.pizza.pork.common.IndentPrinter
 import gay.pizza.pork.minimal.FileTool
 import gay.pizza.pork.parser.ParserAttributes
 import gay.pizza.pork.parser.ParserNodeAttribution
 
 class AttributeCommand : CliktCommand(help = "Attribute AST", name = "attribute") {
+  val hierarchical by option("--hierarchical", help = "Print Hierarchical Output").flag(default = true)
   val path by argument("file")
 
   override fun run() {
@@ -17,13 +22,27 @@ class AttributeCommand : CliktCommand(help = "Attribute AST", name = "attribute"
     val attribution = ParserNodeAttribution()
     val compilationUnit = tool.parse(attribution)
 
-    val coalescer = NodeCoalescer { node ->
+    if (hierarchical) {
-      val allTokens = ParserAttributes.recallAllTokens(node)
+      val output = IndentPrinter()
-      println("node ${node.type.name}")
+      fun visit(node: Node) {
-      for (token in allTokens) {
+        output.emitIndentedLine("${node.type.name} ->")
-        println("token $token")
+        output.indented {
+          for (token in ParserAttributes.recallOwnedTokens(node)) {
+            output.emitIndentedLine(token.toString())
+          }
+          node.visitChildren(NodeCoalescer(followChildren = false, handler = ::visit))
+        }
       }
+      visit(compilationUnit)
+    } else {
+      val coalescer = NodeCoalescer { node ->
+        val allTokens = ParserAttributes.recallAllTokens(node)
+        println("node ${node.type.name}")
+        for (token in allTokens) {
+          println("token $token")
+        }
+      }
+      coalescer.visit(compilationUnit)
     }
-    coalescer.visit(compilationUnit)
   }
 }