diff --git a/ast/src/main/kotlin/gay/pizza/pork/ast/gen/NodeCoalescer.kt b/ast/src/main/kotlin/gay/pizza/pork/ast/gen/NodeCoalescer.kt index e2b9dfa..8afc004 100644 --- a/ast/src/main/kotlin/gay/pizza/pork/ast/gen/NodeCoalescer.kt +++ b/ast/src/main/kotlin/gay/pizza/pork/ast/gen/NodeCoalescer.kt @@ -1,7 +1,7 @@ // GENERATED CODE FROM PORK AST CODEGEN package gay.pizza.pork.ast.gen -class NodeCoalescer(val handler: (Node) -> Unit) : NodeVisitor { +class NodeCoalescer(val followChildren: Boolean = true, val handler: (Node) -> Unit) : NodeVisitor { override fun visitArgumentSpec(node: ArgumentSpec): Unit = handle(node) @@ -97,6 +97,8 @@ class NodeCoalescer(val handler: (Node) -> Unit) : NodeVisitor { fun handle(node: Node) { handler(node) - node.visitChildren(this) + if (followChildren) { + node.visitChildren(this) + } } } diff --git a/buildext/src/main/kotlin/gay/pizza/pork/buildext/ast/AstStandardCodegen.kt b/buildext/src/main/kotlin/gay/pizza/pork/buildext/ast/AstStandardCodegen.kt index 075d0a1..65c67b2 100644 --- a/buildext/src/main/kotlin/gay/pizza/pork/buildext/ast/AstStandardCodegen.kt +++ b/buildext/src/main/kotlin/gay/pizza/pork/buildext/ast/AstStandardCodegen.kt @@ -193,6 +193,11 @@ class AstStandardCodegen(pkg: String, outputDirectory: Path, world: AstWorld) : "NodeCoalescer", inherits = mutableListOf("NodeVisitor"), members = mutableListOf( + KotlinMember( + "followChildren", + "Boolean", + value = "true" + ), KotlinMember( "handler", "(Node) -> Unit" @@ -227,7 +232,9 @@ class AstStandardCodegen(pkg: String, outputDirectory: Path, world: AstWorld) : ) ) handleFunction.body.add("handler(node)") - handleFunction.body.add("node.visitChildren(this)") + handleFunction.body.add("if (followChildren) {") + handleFunction.body.add(" node.visitChildren(this)") + handleFunction.body.add("}") coalescerClass.functions.add(handleFunction) write("NodeCoalescer.kt", KotlinWriter(coalescerClass)) diff --git a/common/src/main/kotlin/gay/pizza/pork/common/IndentBuffer.kt b/common/src/main/kotlin/gay/pizza/pork/common/IndentBuffer.kt new file mode 100644 index 0000000..4790d36 --- /dev/null +++ b/common/src/main/kotlin/gay/pizza/pork/common/IndentBuffer.kt @@ -0,0 +1,16 @@ +package gay.pizza.pork.common + +class IndentBuffer( + val buffer: StringBuilder = StringBuilder(), + indent: String = " " +) : IndentTracked(indent), Appendable by buffer, CharSequence by buffer { + override fun emit(text: String) { + append(text) + } + + override fun emitLine(text: String) { + appendLine(text) + } + + override fun toString(): String = buffer.toString() +} diff --git a/common/src/main/kotlin/gay/pizza/pork/common/IndentPrinter.kt b/common/src/main/kotlin/gay/pizza/pork/common/IndentPrinter.kt index bd30015..f64689a 100644 --- a/common/src/main/kotlin/gay/pizza/pork/common/IndentPrinter.kt +++ b/common/src/main/kotlin/gay/pizza/pork/common/IndentPrinter.kt @@ -1,36 +1,11 @@ package gay.pizza.pork.common -class IndentPrinter( - val buffer: StringBuilder = StringBuilder(), - val indent: String = " " -) : Appendable by buffer, CharSequence by buffer { - private var indentLevel: Int = 0 - private var indentLevelText: String = "" - - fun emitIndent() { - append(indentLevelText) +class IndentPrinter(indent: String = " ") : IndentTracked(indent) { + override fun emit(text: String) { + print(text) } - fun emitIndentedLine(line: String) { - emitIndent() - appendLine(line) + override fun emitLine(text: String) { + println(text) } - - fun increaseIndent() { - indentLevel++ - indentLevelText += indent - } - - fun decreaseIndent() { - indentLevel-- - indentLevelText = indent.repeat(indentLevel) - } - - inline fun indented(block: IndentPrinter.() -> Unit) { - increaseIndent() - block(this) - decreaseIndent() - } - - override fun toString(): String = buffer.toString() } diff --git a/common/src/main/kotlin/gay/pizza/pork/common/IndentTracked.kt b/common/src/main/kotlin/gay/pizza/pork/common/IndentTracked.kt new file mode 100644 index 0000000..03c86bc --- /dev/null +++ b/common/src/main/kotlin/gay/pizza/pork/common/IndentTracked.kt @@ -0,0 +1,37 @@ +package gay.pizza.pork.common + +abstract class IndentTracked(val indent: String) { + private var internalIndentLevel = 0 + private var indentLevelText = "" + + val indentLevel: Int + get() = internalIndentLevel + + fun emitIndent() { + emit(indentLevelText) + } + + fun emitIndentedLine(line: String) { + emitIndent() + emitLine(line) + } + + fun increaseIndent() { + internalIndentLevel++ + indentLevelText += indent + } + + fun decreaseIndent() { + internalIndentLevel-- + indentLevelText = indent.repeat(indentLevel) + } + + inline fun indented(block: IndentTracked.() -> Unit) { + increaseIndent() + block(this) + decreaseIndent() + } + + abstract fun emit(text: String) + abstract fun emitLine(text: String) +} diff --git a/evaluator/src/main/kotlin/gay/pizza/pork/evaluator/EvaluationVisitor.kt b/evaluator/src/main/kotlin/gay/pizza/pork/evaluator/EvaluationVisitor.kt index 234aeca..0133c38 100644 --- a/evaluator/src/main/kotlin/gay/pizza/pork/evaluator/EvaluationVisitor.kt +++ b/evaluator/src/main/kotlin/gay/pizza/pork/evaluator/EvaluationVisitor.kt @@ -337,7 +337,6 @@ class EvaluationVisitor(root: Scope, val stack: CallStack) : NodeVisitor { InfixOperator.Minus -> subtract(convert(left), convert(right)) InfixOperator.Multiply -> multiply(convert(left), convert(right)) InfixOperator.Divide -> divide(convert(left), convert(right)) - InfixOperator.Equals, InfixOperator.NotEquals, InfixOperator.BooleanAnd, InfixOperator.BooleanOr -> throw RuntimeException("Unable to handle operation $op") InfixOperator.BinaryAnd -> binaryAnd(convert(left), convert(right)) InfixOperator.BinaryOr -> binaryOr(convert(left), convert(right)) InfixOperator.BinaryExclusiveOr -> binaryExclusiveOr(convert(left), convert(right)) @@ -347,6 +346,8 @@ class EvaluationVisitor(root: Scope, val stack: CallStack) : NodeVisitor { InfixOperator.Greater -> greater(convert(left), convert(right)) InfixOperator.LesserEqual -> lesserEqual(convert(left), convert(right)) InfixOperator.GreaterEqual -> greaterEqual(convert(left), convert(right)) + InfixOperator.Equals, InfixOperator.NotEquals, InfixOperator.BooleanAnd, InfixOperator.BooleanOr -> + throw RuntimeException("Unable to handle operation $op") } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/CharConsumer.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/CharConsumer.kt new file mode 100644 index 0000000..da59bb6 --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/CharConsumer.kt @@ -0,0 +1,5 @@ +package gay.pizza.pork.parser + +interface CharConsumer { + fun consume(type: TokenType, tokenizer: Tokenizer): String? +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt index 36a7a7a..c095dea 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt @@ -1,8 +1,10 @@ package gay.pizza.pork.parser interface CharSource : PeekableSource { + fun peek(index: Int): Char + companion object { @Suppress("ConstPropertyName") - const val NullChar = 0.toChar() + const val EndOfFile = 0.toChar() } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt index df7fbfa..307a9de 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt @@ -1,7 +1,7 @@ package gay.pizza.pork.parser fun CharSource.readToString(): String = buildString { - while (peek() != CharSource.NullChar) { + while (peek() != CharSource.EndOfFile) { append(next()) } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/MatchedCharConsumer.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/MatchedCharConsumer.kt new file mode 100644 index 0000000..5f55127 --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/MatchedCharConsumer.kt @@ -0,0 +1,48 @@ +package gay.pizza.pork.parser + +@Suppress("CanBeParameter") +class MatchedCharConsumer( + val start: CharSequence, + val end: CharSequence, + vararg val options: Options +) : CharConsumer { + private val eofTerminationAllowed = options.contains(Options.AllowEofTermination) + + override fun consume(type: TokenType, tokenizer: Tokenizer): String? { + if (!tokenizer.peek(start)) { + return null + } + val buffer = StringBuilder() + tokenizer.read(start.length, buffer) + var endsNeededToTerminate = 1 + while (true) { + if (tokenizer.peek(start)) { + endsNeededToTerminate++ + tokenizer.read(start.length, buffer) + continue + } + + if (tokenizer.peek(end)) { + endsNeededToTerminate-- + tokenizer.read(end.length, buffer) + } + + if (endsNeededToTerminate == 0) { + return buffer.toString() + } + + val char = tokenizer.source.next() + if (char == CharSource.EndOfFile) { + if (eofTerminationAllowed) { + return buffer.toString() + } + throw UnterminatedTokenError(type.name, tokenizer.source.currentSourceIndex()) + } + buffer.append(char) + } + } + + enum class Options { + AllowEofTermination + } +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt index 0e7f99c..282eae5 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt @@ -9,8 +9,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : ArgumentSpec(symbol, next(TokenType.DotDotDot)) } - override fun parseBlock(): Block = guarded(NodeType.Block) { - expect(TokenType.LeftCurly) + override fun parseBlock(): Block = expect(NodeType.Block, TokenType.LeftCurly) { val items = collect(TokenType.RightCurly) { parseExpression() } @@ -18,7 +17,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : Block(items) } - override fun parseExpression(): Expression = guarded { + override fun parseExpression(): Expression { val token = peek() var expression = when (token.type) { TokenType.NumberLiteral -> parseNumberLiteral() @@ -66,7 +65,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : } } - if (peek( + return if (peek( TokenType.Plus, TokenType.Minus, TokenType.Multiply, TokenType.Divide, TokenType.Ampersand, TokenType.Pipe, TokenType.Caret, TokenType.Equality, TokenType.Inequality, TokenType.Mod, TokenType.Rem, TokenType.Lesser, TokenType.Greater, TokenType.LesserEqual, TokenType.GreaterEqual, @@ -176,12 +175,11 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : ) } - override fun parseDoubleLiteral(): DoubleLiteral = guarded(NodeType.DoubleLiteral) { - DoubleLiteral(expect(TokenType.NumberLiteral).text.toDouble()) + override fun parseDoubleLiteral(): DoubleLiteral = expect(NodeType.DoubleLiteral, TokenType.NumberLiteral) { + DoubleLiteral(it.text.toDouble()) } - override fun parseForIn(): ForIn = guarded(NodeType.ForIn) { - expect(TokenType.For) + override fun parseForIn(): ForIn = expect(NodeType.ForIn, TokenType.For) { val forInItem = parseForInItem() expect(TokenType.In) val value = parseExpression() @@ -223,8 +221,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : FunctionDefinition(modifiers, name, arguments, block, native) } - override fun parseIf(): If = guarded(NodeType.If) { - expect(TokenType.If) + override fun parseIf(): If = expect(NodeType.If, TokenType.If) { val condition = parseExpression() val thenBlock = parseBlock() var elseBlock: Block? = null @@ -234,8 +231,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : If(condition, thenBlock, elseBlock) } - override fun parseImportDeclaration(): ImportDeclaration = guarded(NodeType.ImportDeclaration) { - expect(TokenType.Import) + override fun parseImportDeclaration(): ImportDeclaration = expect(NodeType.ImportDeclaration, TokenType.Import) { val form = parseSymbol() val components = oneAndContinuedBy(TokenType.Dot) { parseSymbol() @@ -257,13 +253,13 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : InfixOperation(parseExpression(), infixOperator, parseExpression()) } - private fun parseNumberLiteral(): Expression = guarded { + private fun parseNumberLiteral(): Expression { val token = peek() if (token.type != TokenType.NumberLiteral) { expect(TokenType.NumberLiteral) } - when { + return when { token.text.contains(".") -> parseDoubleLiteral() token.text.toIntOrNull() != null -> parseIntegerLiteral() token.text.toLongOrNull() != null -> parseLongLiteral() @@ -271,12 +267,11 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : } } - override fun parseIntegerLiteral(): IntegerLiteral = guarded(NodeType.IntegerLiteral) { - IntegerLiteral(expect(TokenType.NumberLiteral).text.toInt()) + override fun parseIntegerLiteral(): IntegerLiteral = expect(NodeType.IntegerLiteral, TokenType.NumberLiteral) { + IntegerLiteral(it.text.toInt()) } - override fun parseLetAssignment(): LetAssignment = guarded(NodeType.LetAssignment) { - expect(TokenType.Let) + override fun parseLetAssignment(): LetAssignment = expect(NodeType.LetAssignment, TokenType.Let) { val symbol = parseSymbol() expect(TokenType.Equals) val value = parseExpression() @@ -292,8 +287,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : LetDefinition(definitionModifiers, name, value) } - override fun parseListLiteral(): ListLiteral = guarded(NodeType.ListLiteral) { - expect(TokenType.LeftBracket) + override fun parseListLiteral(): ListLiteral = expect(NodeType.ListLiteral, TokenType.LeftBracket) { val items = collect(TokenType.RightBracket, TokenType.Comma) { parseExpression() } @@ -301,12 +295,11 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : ListLiteral(items) } - override fun parseLongLiteral(): LongLiteral = guarded(NodeType.LongLiteral) { - LongLiteral(expect(TokenType.NumberLiteral).text.toLong()) + override fun parseLongLiteral(): LongLiteral = expect(NodeType.LongLiteral, TokenType.NumberLiteral) { + LongLiteral(it.text.toLong()) } - override fun parseNative(): Native = guarded(NodeType.Native) { - expect(TokenType.Native) + override fun parseNative(): Native = expect(NodeType.Native, TokenType.Native) { val form = parseSymbol() val definitions = mutableListOf() while (peek(TokenType.StringLiteral)) { @@ -315,22 +308,22 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : Native(form, definitions) } - override fun parseNoneLiteral(): NoneLiteral = guarded(NodeType.NoneLiteral) { - expect(TokenType.None) + override fun parseNoneLiteral(): NoneLiteral = expect(NodeType.NoneLiteral, TokenType.None) { NoneLiteral() } - override fun parseParentheses(): Parentheses = guarded(NodeType.Parentheses) { - expect(TokenType.LeftParentheses) + override fun parseParentheses(): Parentheses = expect(NodeType.Parentheses, TokenType.LeftParentheses) { val expression = parseExpression() expect(TokenType.RightParentheses) Parentheses(expression) } - override fun parsePrefixOperation(): PrefixOperation = guarded(NodeType.PrefixOperation) { - expect(TokenType.Not, TokenType.Plus, TokenType.Minus, TokenType.Tilde) { - PrefixOperation(ParserHelpers.convertPrefixOperator(it), parseExpression()) - } + override fun parsePrefixOperation(): PrefixOperation = expect( + NodeType.PrefixOperation, + TokenType.Not, TokenType.Plus, + TokenType.Minus, TokenType.Tilde + ) { + PrefixOperation(ParserHelpers.convertPrefixOperator(it), parseExpression()) } override fun parseSetAssignment(): SetAssignment = guarded(NodeType.SetAssignment) { @@ -340,11 +333,9 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : SetAssignment(symbol, value) } - override fun parseStringLiteral(): StringLiteral = guarded(NodeType.StringLiteral) { - expect(TokenType.StringLiteral) { - val content = StringEscape.unescape(StringEscape.unquote(it.text)) - StringLiteral(content) - } + override fun parseStringLiteral(): StringLiteral = expect(NodeType.StringLiteral, TokenType.StringLiteral) { + val content = StringEscape.unescape(StringEscape.unquote(it.text)) + StringLiteral(content) } override fun parseSuffixOperation(): SuffixOperation = guarded(NodeType.SuffixOperation) { @@ -354,32 +345,30 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : } } - private fun parseSymbolCases(): Expression = guarded { - if (peek(1, TokenType.LeftParentheses)) { + private fun parseSymbolCases(): Expression { + return if (peek(1, TokenType.LeftParentheses)) { parseFunctionCall() } else if (peek(1, TokenType.PlusPlus, TokenType.MinusMinus)) { parseSuffixOperation() } else parseSymbolReference() } - override fun parseSymbol(): Symbol = guarded(NodeType.Symbol) { - expect(TokenType.Symbol) { Symbol(it.text) } + override fun parseSymbol(): Symbol = expect(NodeType.Symbol, TokenType.Symbol) { + Symbol(it.text) } override fun parseSymbolReference(): SymbolReference = guarded(NodeType.SymbolReference) { SymbolReference(parseSymbol()) } - override fun parseVarAssignment(): VarAssignment = guarded(NodeType.VarAssignment) { - expect(TokenType.Var) + override fun parseVarAssignment(): VarAssignment = expect(NodeType.VarAssignment, TokenType.Var) { val symbol = parseSymbol() expect(TokenType.Equals) val value = parseExpression() VarAssignment(symbol, value) } - override fun parseWhile(): While = guarded(NodeType.While) { - expect(TokenType.While) + override fun parseWhile(): While = expect(NodeType.While, TokenType.While) { val condition = parseExpression() val block = parseBlock() While(condition, block) diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt index bac5a56..e50c021 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt @@ -7,6 +7,14 @@ import gay.pizza.pork.ast.gen.visit data class ParserAttributes(val tokens: List) { companion object { + fun recallOwnedTokens(node: Node): List { + val attributes = node.data() + if (attributes != null) { + return attributes.tokens + } + return emptyList() + } + fun recallAllTokens(node: Node): List { val all = mutableListOf() val coalescer = NodeCoalescer { item -> diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt index b51cb64..321612c 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt @@ -11,6 +11,12 @@ abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution) protected inline fun guarded(type: NodeType? = null, noinline block: () -> T): T = attribution.guarded(type, block) + @Suppress("NOTHING_TO_INLINE") + protected inline fun expect(type: NodeType? = null, vararg tokenTypes: TokenType, noinline block: (Token) -> T): T = + guarded(type) { + block(expect(*tokenTypes)) + } + protected fun collect( peeking: TokenType, consuming: TokenType? = null, @@ -66,7 +72,11 @@ abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution) throw ExpectedTokenError(token, token.sourceIndex, *types) } - protected fun next(): Token = source.next() + protected fun next(): Token { + val token = source.next() + attribution.push(token) + return token + } protected fun peek(): Token = source.peek() protected fun peek(ahead: Int): TokenType = source.peekTypeAhead(ahead) protected fun peek(ahead: Int, vararg types: TokenType): Boolean = types.contains(source.peekTypeAhead(ahead)) diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt index ca28be9..4fa5423 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt @@ -23,7 +23,7 @@ internal object ParserHelpers { TokenType.GreaterEqual -> InfixOperator.GreaterEqual TokenType.And -> InfixOperator.BooleanAnd TokenType.Or -> InfixOperator.BooleanOr - else -> throw ParseError("Unknown Infix Operator") + else -> throw ParseError("Unknown infix operator: ${token.type}") } fun convertPrefixOperator(token: Token): PrefixOperator = when (token.type) { @@ -31,12 +31,12 @@ internal object ParserHelpers { TokenType.Plus -> PrefixOperator.UnaryPlus TokenType.Minus -> PrefixOperator.UnaryMinus TokenType.Tilde -> PrefixOperator.BinaryNot - else -> throw ParseError("Unknown Prefix Operator") + else -> throw ParseError("Unknown prefix operator: ${token.type}") } fun convertSuffixOperator(token: Token): SuffixOperator = when (token.type) { TokenType.PlusPlus -> SuffixOperator.Increment TokenType.MinusMinus -> SuffixOperator.Decrement - else -> throw ParseError("Unknown Suffix Operator") + else -> throw ParseError("Unknown suffix operator: ${token.type}") } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Printer.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/Printer.kt index 83d4e4b..2da75a5 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Printer.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/Printer.kt @@ -1,10 +1,10 @@ package gay.pizza.pork.parser import gay.pizza.pork.ast.gen.* -import gay.pizza.pork.common.IndentPrinter +import gay.pizza.pork.common.IndentBuffer class Printer(buffer: StringBuilder) : NodeVisitor { - private val out = IndentPrinter(buffer) + private val out = IndentBuffer(buffer) private var autoIndentState = false private fun append(text: String) { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt new file mode 100644 index 0000000..fc58216 --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt @@ -0,0 +1,19 @@ +package gay.pizza.pork.parser + +class SourceIndexCharSource(val delegate: CharSource) : CharSource by delegate { + private var currentLineIndex = 1 + private var currentLineColumn = 0 + + override fun next(): Char { + val char = delegate.next() + if (char == '\n') { + currentLineIndex++ + currentLineColumn = 0 + } + currentLineColumn++ + return char + } + + fun currentSourceIndex(): SourceIndex = + SourceIndex(delegate.currentIndex, currentLineIndex, currentLineColumn) +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt new file mode 100644 index 0000000..933c0bf --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt @@ -0,0 +1,29 @@ +package gay.pizza.pork.parser + +object StringCharConsumer : CharConsumer { + override fun consume(type: TokenType, tokenizer: Tokenizer): String? { + if (!tokenizer.peek("\"")) { + return null + } + + val buffer = StringBuilder() + buffer.append(tokenizer.source.next()) + var escape = false + while (true) { + val char = tokenizer.source.peek() + + if (char == CharSource.EndOfFile) { + throw UnterminatedTokenError("String", tokenizer.source.currentSourceIndex()) + } + + buffer.append(tokenizer.source.next()) + + if (char == '\\') { + escape = true + } else if (char == '"' && !escape) { + break + } + } + return buffer.toString() + } +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt index a7aa156..a7188b9 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt @@ -6,12 +6,13 @@ class StringCharSource( val endIndex: Int = input.length - 1 ) : CharSource { private var index = startIndex + override val currentIndex: Int get() = index override fun next(): Char { if (index == endIndex) { - return CharSource.NullChar + return CharSource.EndOfFile } val char = input[index] index++ @@ -20,8 +21,16 @@ class StringCharSource( override fun peek(): Char { if (index == endIndex) { - return CharSource.NullChar + return CharSource.EndOfFile } return input[index] } + + override fun peek(index: Int): Char { + val target = this.index + index + if (target >= endIndex) { + return CharSource.EndOfFile + } + return input[target] + } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt index a141759..ca5db56 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt @@ -1,22 +1,23 @@ package gay.pizza.pork.parser import gay.pizza.pork.parser.CharMatcher.* +import gay.pizza.pork.parser.MatchedCharConsumer.Options.AllowEofTermination import gay.pizza.pork.parser.TokenTypeProperty.* import gay.pizza.pork.parser.TokenFamily.* import gay.pizza.pork.parser.TokenTypeProperty.AnyOf enum class TokenType(vararg properties: TokenTypeProperty) { - NumberLiteral(NumericLiteralFamily, CharConsumer(CharMatcher.AnyOf( + NumberLiteral(NumericLiteralFamily, CharMatch(CharMatcher.AnyOf( MatchRange('0'..'9'), NotAtIndex(0, MatchSingle('.')) ))), - Symbol(SymbolFamily, CharConsumer(CharMatcher.AnyOf( + Symbol(SymbolFamily, CharMatch(CharMatcher.AnyOf( MatchRange('a'..'z'), MatchRange('A'..'Z'), MatchRange('0' .. '9'), MatchSingle('_') )), KeywordUpgrader), - StringLiteral(StringLiteralFamily), + StringLiteral(StringLiteralFamily, CharConsume(StringCharConsumer)), Equality(OperatorFamily), Inequality(ManyChars("!="), OperatorFamily), ExclamationPoint(SingleChar('!'), Promotion('=', Inequality)), @@ -66,14 +67,14 @@ enum class TokenType(vararg properties: TokenTypeProperty) { Native(ManyChars("native"), KeywordFamily), Let(ManyChars("let"), KeywordFamily), Var(ManyChars("var"), KeywordFamily), - Whitespace(CharConsumer(CharMatcher.AnyOf( + Whitespace(CharMatch(CharMatcher.AnyOf( MatchSingle(' '), MatchSingle('\r'), MatchSingle('\n'), MatchSingle('\t') ))), - BlockComment(CommentFamily), - LineComment(CommentFamily), + BlockComment(CharConsume(MatchedCharConsumer("/*", "*/")), CommentFamily), + LineComment(CharConsume(MatchedCharConsumer("//", "\n", AllowEofTermination)), CommentFamily), EndOfFile; val promotions: List = @@ -86,7 +87,8 @@ enum class TokenType(vararg properties: TokenTypeProperty) { properties.filterIsInstance().singleOrNull() val family: TokenFamily = properties.filterIsInstance().singleOrNull() ?: OtherFamily - val charConsumer: CharConsumer? = properties.filterIsInstance().singleOrNull() + val charMatch: CharMatch? = properties.filterIsInstance().singleOrNull() + val charConsume: CharConsume? = properties.filterIsInstance().singleOrNull() val tokenUpgrader: TokenUpgrader? = properties.filterIsInstance().singleOrNull() @@ -96,8 +98,8 @@ enum class TokenType(vararg properties: TokenTypeProperty) { val AnyOf = entries.filter { item -> item.anyOf != null } val ManyChars = entries.filter { item -> item.manyChars != null } val SingleChars = entries.filter { item -> item.singleChar != null } - val CharConsumers = entries.filter { item -> - item.charConsumer != null } + val CharMatches = entries.filter { item -> item.charMatch != null } + val CharConsumes = entries.filter { item -> item.charConsume != null } val ParserIgnoredTypes: Array = arrayOf( Whitespace, diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt index 9473e4d..64f40aa 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt @@ -5,7 +5,8 @@ interface TokenTypeProperty { class Promotion(val nextChar: Char, val type: TokenType) : TokenTypeProperty class ManyChars(val text: String) : TokenTypeProperty class AnyOf(vararg val strings: String): TokenTypeProperty - open class CharConsumer(val matcher: CharMatcher) : TokenTypeProperty + open class CharMatch(val matcher: CharMatcher) : TokenTypeProperty + open class CharConsume(val consumer: CharConsumer) : TokenTypeProperty open class TokenUpgrader(val maybeUpgrade: (Token) -> Token?) : TokenTypeProperty object KeywordUpgrader : TokenUpgrader({ token -> diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt index fcb24c5..402ae8c 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt @@ -1,80 +1,22 @@ package gay.pizza.pork.parser -class Tokenizer(val source: CharSource) { +class Tokenizer(source: CharSource) { + val source: SourceIndexCharSource = SourceIndexCharSource(source) + private var startIndex: SourceIndex = SourceIndex.zero() - private var currentLineIndex = 1 - private var currentLineColumn = 0 - - private fun readBlockComment(firstChar: Char): Token { - val comment = buildString { - append(firstChar) - var endOfComment = false - while (true) { - val char = nextChar() - if (char == CharSource.NullChar) { - throw UnterminatedTokenError("block comment", currentSourceIndex()) - } - append(char) - - if (endOfComment) { - if (char != '/') { - endOfComment = false - continue - } - break - } - - if (char == '*') { - endOfComment = true - } - } - } - return produceToken(TokenType.BlockComment, comment) - } - - private fun readLineComment(firstChar: Char): Token { - val comment = buildString { - append(firstChar) - while (true) { - val char = source.peek() - if (char == CharSource.NullChar || char == '\n') { - break - } - append(nextChar()) - } - } - return produceToken(TokenType.LineComment, comment) - } - - private fun readStringLiteral(firstChar: Char): Token { - val string = buildString { - append(firstChar) - while (true) { - val char = source.peek() - if (char == CharSource.NullChar) { - throw UnterminatedTokenError("string", currentSourceIndex()) - } - append(nextChar()) - if (char == '"') { - break - } - } - } - return produceToken(TokenType.StringLiteral, string) - } fun next(): Token { - while (source.peek() != CharSource.NullChar) { - startIndex = currentSourceIndex() - val char = nextChar() + while (source.peek() != CharSource.EndOfFile) { + startIndex = source.currentSourceIndex() - if (char == '/' && source.peek() == '*') { - return readBlockComment(char) + for (item in TokenType.CharConsumes) { + val text = item.charConsume!!.consumer.consume(item, this) + if (text != null) { + return produceToken(item, text) + } } - if (char == '/' && source.peek() == '/') { - return readLineComment(char) - } + val char = source.next() for (item in TokenType.SingleChars) { val itemChar = item.singleChar!!.char @@ -91,7 +33,7 @@ class Tokenizer(val source: CharSource) { if (source.peek() != promotion.nextChar) { continue } - val nextChar = nextChar() + val nextChar = source.next() type = promotion.type text += nextChar promoted = true @@ -101,16 +43,16 @@ class Tokenizer(val source: CharSource) { } var index = 0 - for (item in TokenType.CharConsumers) { - if (!item.charConsumer!!.matcher.valid(char, index)) { + for (item in TokenType.CharMatches) { + if (!item.charMatch!!.matcher.valid(char, index)) { continue } val text = buildString { append(char) - while (item.charConsumer.matcher.valid(source.peek(), ++index)) { - append(nextChar()) + while (item.charMatch.matcher.valid(source.peek(), ++index)) { + append(source.next()) } } var token = produceToken(item, text) @@ -121,10 +63,6 @@ class Tokenizer(val source: CharSource) { return token } - if (char == '"') { - return readStringLiteral(char) - } - throw BadCharacterError(char, startIndex) } return Token.endOfFile(startIndex.copy(index = source.currentIndex)) @@ -142,19 +80,23 @@ class Tokenizer(val source: CharSource) { return TokenStream(tokens) } - private fun produceToken(type: TokenType, text: String) = + internal fun produceToken(type: TokenType, text: String) = Token(type, startIndex, text) - private fun nextChar(): Char { - val char = source.next() - if (char == '\n') { - currentLineIndex++ - currentLineColumn = 0 + internal fun peek(what: CharSequence): Boolean { + var current = 0 + for (c in what) { + if (source.peek(current) != c) { + return false + } + current++ } - currentLineColumn++ - return char + return true } - private fun currentSourceIndex(): SourceIndex = - SourceIndex(source.currentIndex, currentLineIndex, currentLineColumn) + internal fun read(count: Int, buffer: StringBuilder) { + for (i in 1..count) { + buffer.append(source.next()) + } + } } diff --git a/tool/build.gradle.kts b/tool/build.gradle.kts index fc257c7..a8de78f 100644 --- a/tool/build.gradle.kts +++ b/tool/build.gradle.kts @@ -8,6 +8,8 @@ plugins { dependencies { api(project(":minimal")) api("com.github.ajalt.clikt:clikt:4.2.0") + + implementation(project(":common")) } application { diff --git a/tool/src/main/kotlin/gay/pizza/pork/tool/AttributeCommand.kt b/tool/src/main/kotlin/gay/pizza/pork/tool/AttributeCommand.kt index 06db208..326c9bf 100644 --- a/tool/src/main/kotlin/gay/pizza/pork/tool/AttributeCommand.kt +++ b/tool/src/main/kotlin/gay/pizza/pork/tool/AttributeCommand.kt @@ -2,14 +2,19 @@ package gay.pizza.pork.tool import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option import gay.pizza.dough.fs.PlatformFsProvider +import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.NodeCoalescer import gay.pizza.pork.ast.gen.visit +import gay.pizza.pork.common.IndentPrinter import gay.pizza.pork.minimal.FileTool import gay.pizza.pork.parser.ParserAttributes import gay.pizza.pork.parser.ParserNodeAttribution class AttributeCommand : CliktCommand(help = "Attribute AST", name = "attribute") { + val hierarchical by option("--hierarchical", help = "Print Hierarchical Output").flag(default = true) val path by argument("file") override fun run() { @@ -17,13 +22,27 @@ class AttributeCommand : CliktCommand(help = "Attribute AST", name = "attribute" val attribution = ParserNodeAttribution() val compilationUnit = tool.parse(attribution) - val coalescer = NodeCoalescer { node -> - val allTokens = ParserAttributes.recallAllTokens(node) - println("node ${node.type.name}") - for (token in allTokens) { - println("token $token") + if (hierarchical) { + val output = IndentPrinter() + fun visit(node: Node) { + output.emitIndentedLine("${node.type.name} ->") + output.indented { + for (token in ParserAttributes.recallOwnedTokens(node)) { + output.emitIndentedLine(token.toString()) + } + node.visitChildren(NodeCoalescer(followChildren = false, handler = ::visit)) + } } + visit(compilationUnit) + } else { + val coalescer = NodeCoalescer { node -> + val allTokens = ParserAttributes.recallAllTokens(node) + println("node ${node.type.name}") + for (token in allTokens) { + println("token $token") + } + } + coalescer.visit(compilationUnit) } - coalescer.visit(compilationUnit) } }