From d36310e698cedc4a9fdd47ca209a740d37d4e8cc Mon Sep 17 00:00:00 2001 From: Alex Zenla Date: Fri, 6 Oct 2023 15:30:43 -0700 Subject: [PATCH] parser: switch to char matcher interface --- .../gay/pizza/pork/common/IndentPrinter.kt | 13 +++++++- .../gay/pizza/pork/parser/CharMatcher.kt | 26 ++++++++++++++++ .../kotlin/gay/pizza/pork/parser/TokenType.kt | 30 ++++++++++++------- .../pizza/pork/parser/TokenTypeProperty.kt | 3 +- .../kotlin/gay/pizza/pork/parser/Tokenizer.kt | 22 ++++---------- support/pork-idea/build.gradle.kts | 2 +- .../pizza/pork/idea/PorkSymbolDeclaration.kt | 3 +- 7 files changed, 65 insertions(+), 34 deletions(-) create mode 100644 parser/src/main/kotlin/gay/pizza/pork/parser/CharMatcher.kt diff --git a/common/src/main/kotlin/gay/pizza/pork/common/IndentPrinter.kt b/common/src/main/kotlin/gay/pizza/pork/common/IndentPrinter.kt index 5e68c70..bd30015 100644 --- a/common/src/main/kotlin/gay/pizza/pork/common/IndentPrinter.kt +++ b/common/src/main/kotlin/gay/pizza/pork/common/IndentPrinter.kt @@ -11,9 +11,14 @@ class IndentPrinter( append(indentLevelText) } + fun emitIndentedLine(line: String) { + emitIndent() + appendLine(line) + } + fun increaseIndent() { indentLevel++ - indentLevelText = indent.repeat(indentLevel) + indentLevelText += indent } fun decreaseIndent() { @@ -21,5 +26,11 @@ class IndentPrinter( indentLevelText = indent.repeat(indentLevel) } + inline fun indented(block: IndentPrinter.() -> Unit) { + increaseIndent() + block(this) + decreaseIndent() + } + override fun toString(): String = buffer.toString() } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/CharMatcher.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/CharMatcher.kt new file mode 100644 index 0000000..2044812 --- /dev/null +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/CharMatcher.kt @@ -0,0 +1,26 @@ +package gay.pizza.pork.parser + +fun interface CharMatcher { + fun valid(char: Char, index: Int): Boolean + + class AnyOf(vararg val filters: CharMatcher) : CharMatcher { + override fun valid(char: Char, index: Int): Boolean = + filters.any { it.valid(char, index) } + } + + class MatchSingle(val char: Char) : CharMatcher { + override fun valid(char: Char, index: Int): Boolean = + char == this.char + } + + class MatchRange(val charRange: CharRange) : CharMatcher { + override fun valid(char: Char, index: Int): Boolean = + charRange.contains(char) + } + + class NotAtIndex(val index: Int, val matcher: CharMatcher) : CharMatcher { + override fun valid(char: Char, index: Int): Boolean { + return this.index != index && matcher.valid(char, index) + } + } +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt index c1fbea5..a141759 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt @@ -1,16 +1,21 @@ package gay.pizza.pork.parser +import gay.pizza.pork.parser.CharMatcher.* import gay.pizza.pork.parser.TokenTypeProperty.* import gay.pizza.pork.parser.TokenFamily.* +import gay.pizza.pork.parser.TokenTypeProperty.AnyOf enum class TokenType(vararg properties: TokenTypeProperty) { - NumberLiteral(NumericLiteralFamily, CharIndexConsumer { it, index -> - (it in '0'..'9') || (index > 0 && it == '.') }), - Symbol(SymbolFamily, CharConsumer { - (it in 'a'..'z') || - (it in 'A'..'Z') || - (it == '_') || - (it in '0' .. '9')}, KeywordUpgrader), + NumberLiteral(NumericLiteralFamily, CharConsumer(CharMatcher.AnyOf( + MatchRange('0'..'9'), + NotAtIndex(0, MatchSingle('.')) + ))), + Symbol(SymbolFamily, CharConsumer(CharMatcher.AnyOf( + MatchRange('a'..'z'), + MatchRange('A'..'Z'), + MatchRange('0' .. '9'), + MatchSingle('_') + )), KeywordUpgrader), StringLiteral(StringLiteralFamily), Equality(OperatorFamily), Inequality(ManyChars("!="), OperatorFamily), @@ -61,7 +66,12 @@ enum class TokenType(vararg properties: TokenTypeProperty) { Native(ManyChars("native"), KeywordFamily), Let(ManyChars("let"), KeywordFamily), Var(ManyChars("var"), KeywordFamily), - Whitespace(CharConsumer { it == ' ' || it == '\r' || it == '\n' || it == '\t' }), + Whitespace(CharConsumer(CharMatcher.AnyOf( + MatchSingle(' '), + MatchSingle('\r'), + MatchSingle('\n'), + MatchSingle('\t') + ))), BlockComment(CommentFamily), LineComment(CommentFamily), EndOfFile; @@ -77,8 +87,6 @@ enum class TokenType(vararg properties: TokenTypeProperty) { val family: TokenFamily = properties.filterIsInstance().singleOrNull() ?: OtherFamily val charConsumer: CharConsumer? = properties.filterIsInstance().singleOrNull() - val charIndexConsumer: CharIndexConsumer? = - properties.filterIsInstance().singleOrNull() val tokenUpgrader: TokenUpgrader? = properties.filterIsInstance().singleOrNull() @@ -89,7 +97,7 @@ enum class TokenType(vararg properties: TokenTypeProperty) { val ManyChars = entries.filter { item -> item.manyChars != null } val SingleChars = entries.filter { item -> item.singleChar != null } val CharConsumers = entries.filter { item -> - item.charConsumer != null || item.charIndexConsumer != null } + item.charConsumer != null } val ParserIgnoredTypes: Array = arrayOf( Whitespace, diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt index 87b68b7..9473e4d 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt @@ -5,8 +5,7 @@ interface TokenTypeProperty { class Promotion(val nextChar: Char, val type: TokenType) : TokenTypeProperty class ManyChars(val text: String) : TokenTypeProperty class AnyOf(vararg val strings: String): TokenTypeProperty - class CharConsumer(val isValid: (Char) -> Boolean) : TokenTypeProperty - class CharIndexConsumer(val isValid: (Char, Int) -> Boolean) : TokenTypeProperty + open class CharConsumer(val matcher: CharMatcher) : TokenTypeProperty open class TokenUpgrader(val maybeUpgrade: (Token) -> Token?) : TokenTypeProperty object KeywordUpgrader : TokenUpgrader({ token -> diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt index 59522aa..fcb24c5 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt @@ -102,27 +102,14 @@ class Tokenizer(val source: CharSource) { var index = 0 for (item in TokenType.CharConsumers) { - if (item.charConsumer != null) { - if (!item.charConsumer.isValid(char)) { - continue - } - } else if (item.charIndexConsumer != null) { - if (!item.charIndexConsumer.isValid(char, index)) { - continue - } - } else { - throw ParseError("Unknown Char Consumer") + if (!item.charConsumer!!.matcher.valid(char, index)) { + continue } val text = buildString { append(char) - while ( - if (item.charConsumer != null) - item.charConsumer.isValid(source.peek()) - else - item.charIndexConsumer!!.isValid(source.peek(), ++index) - ) { + while (item.charConsumer.matcher.valid(source.peek(), ++index)) { append(nextChar()) } } @@ -168,5 +155,6 @@ class Tokenizer(val source: CharSource) { return char } - private fun currentSourceIndex(): SourceIndex = SourceIndex(source.currentIndex, currentLineIndex, currentLineColumn) + private fun currentSourceIndex(): SourceIndex = + SourceIndex(source.currentIndex, currentLineIndex, currentLineColumn) } diff --git a/support/pork-idea/build.gradle.kts b/support/pork-idea/build.gradle.kts index 95ec482..09fbf81 100644 --- a/support/pork-idea/build.gradle.kts +++ b/support/pork-idea/build.gradle.kts @@ -1,7 +1,7 @@ import gay.pizza.pork.buildext.AstCodegenType plugins { - id("org.jetbrains.intellij") version "1.15.0" + id("org.jetbrains.intellij") version "1.16.0" id("gay.pizza.pork.module") id("gay.pizza.pork.ast") } diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSymbolDeclaration.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSymbolDeclaration.kt index 731cc8a..33e0496 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSymbolDeclaration.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSymbolDeclaration.kt @@ -11,9 +11,8 @@ import gay.pizza.pork.idea.psi.gen.PorkElement class PorkSymbolDeclaration(val element: PorkElement) : PsiSymbolDeclaration { override fun getDeclaringElement(): PsiElement = element override fun getRangeInDeclaringElement(): TextRange { - val textRangeOfSymbol = PorkElementHelpers.symbolElementOf(element)?.psi?.textRangeInParent + return PorkElementHelpers.symbolElementOf(element)?.psi?.textRangeInParent ?: throw RuntimeException("Unable to get symbol of element: $element") - return textRangeOfSymbol } override fun getSymbol(): Symbol = PorkElementHelpers.psiSymbolFor(element) ?: