diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt index 9688c7d..e93d869 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt @@ -1,13 +1,9 @@ package gay.pizza.pork.parser import gay.pizza.pork.ast.* -import kotlin.math.exp class Parser(source: TokenSource, attribution: NodeAttribution) : ParserBase(source, attribution) { - private var storedSymbol: Symbol? = null - private var storedDefinitionModifiers: DefinitionModifiers? = null - override fun parseBlock(): Block = guarded(NodeType.Block) { expect(TokenType.LeftCurly) val items = collect(TokenType.RightCurly) { @@ -100,16 +96,15 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : while (!peek(TokenType.EndOfFile)) { if (declarationAccepted) { - val definition = maybeParseDefinition() - if (definition != null) { - declarationAccepted = false - definitions.add(definition) + val declaration = parseDeclarationMaybe() + if (declaration != null) { + declarations.add(declaration) continue + } else { + declarationAccepted = false } - declarations.add(parseDeclaration()) - } else { - definitions.add(parseDefinition()) } + definitions.add(parseDefinition()) } CompilationUnit(declarations, definitions) @@ -135,34 +130,46 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : return modifiers } - private fun maybeParseDefinition(): Definition? { - try { - storedDefinitionModifiers = parseDefinitionModifiers() - val token = peek() - return when (token.type) { - TokenType.Func -> parseFunctionDefinition() - TokenType.Let -> parseLetDefinition() - else -> null + fun peekAheadUntilNotIn(vararg types: TokenType): TokenType { + var i = 0 + while (true) { + val token = peek(i) + if (!types.contains(token)) { + return token } - } finally { - storedDefinitionModifiers = null + i++ } } - override fun parseDeclaration(): Declaration = guarded { + private fun parseDeclarationMaybe(): Declaration? { val token = peek() - return@guarded when (token.type) { + return when (token.type) { TokenType.Import -> parseImportDeclaration() - else -> throw ParseError( + else -> null + } + } + + override fun parseDeclaration(): Declaration { + val declaration = parseDeclarationMaybe() + if (declaration == null) { + val token = peek() + throw ParseError( "Failed to parse token: ${token.type} '${token.text}' as" + " declaration (index ${source.currentIndex})" ) } + return declaration } - override fun parseDefinition(): Definition = guarded { - maybeParseDefinition() ?: throw ParseError("Unable to parse definition") - } + override fun parseDefinition(): Definition = + when (val type = peekAheadUntilNotIn(*TokenType.DeclarationModifiers)) { + TokenType.Func -> parseFunctionDefinition() + TokenType.Let -> parseLetDefinition() + else -> throw ParseError( + "Failed to parse token: ${type.name} as" + + " declaration (index ${source.currentIndex})" + ) + } override fun parseDoubleLiteral(): DoubleLiteral = guarded(NodeType.DoubleLiteral) { DoubleLiteral(expect(TokenType.NumberLiteral).text.toDouble()) @@ -177,11 +184,9 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : ForIn(symbol, value, block) } - override fun parseFunctionCall(): FunctionCall = - parseFunctionCall(null) - - fun parseFunctionCall(target: Symbol?): FunctionCall = guarded(NodeType.FunctionCall) { - val symbol = target ?: parseSymbol() + override fun parseFunctionCall(): FunctionCall = guarded(NodeType.FunctionCall) { + val symbol = parseSymbol() + expect(TokenType.LeftParentheses) val arguments = collect(TokenType.RightParentheses, TokenType.Comma) { parseExpression() } @@ -190,7 +195,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : } override fun parseFunctionDefinition(): FunctionDefinition = guarded(NodeType.FunctionDefinition) { - val modifiers = storedDefinitionModifiers ?: parseDefinitionModifiers() + val modifiers = parseDefinitionModifiers() expect(TokenType.Func) val name = parseSymbol() expect(TokenType.LeftParentheses) @@ -275,7 +280,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : } override fun parseLetDefinition(): LetDefinition = guarded(NodeType.LetDefinition) { - val definitionModifiers = storedDefinitionModifiers ?: parseDefinitionModifiers() + val definitionModifiers = parseDefinitionModifiers() expect(TokenType.Let) val name = parseSymbol() expect(TokenType.Equals) @@ -322,7 +327,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : } override fun parseSetAssignment(): SetAssignment = guarded(NodeType.SetAssignment) { - val symbol = storedSymbol ?: parseSymbol() + val symbol = parseSymbol() expect(TokenType.Equals) val value = parseExpression() SetAssignment(symbol, value) @@ -343,17 +348,11 @@ class Parser(source: TokenSource, attribution: NodeAttribution) : } private fun parseSymbolCases(): Expression = guarded { - val symbol = parseSymbol() - if (next(TokenType.LeftParentheses)) { - parseFunctionCall(symbol) - } else { - val reference = SymbolReference(symbol) - if (peek(TokenType.PlusPlus, TokenType.MinusMinus)) { - expect(TokenType.PlusPlus, TokenType.MinusMinus) { - SuffixOperation(ParserHelpers.convertSuffixOperator(it), reference) - } - } else reference - } + if (peek(1, TokenType.LeftParentheses)) { + parseFunctionCall() + } else if (peek(1, TokenType.PlusPlus, TokenType.MinusMinus)) { + parseSuffixOperation() + } else parseSymbolReference() } override fun parseSymbol(): Symbol = guarded(NodeType.Symbol) { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt index bbe5e60..9ad3909 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt @@ -4,7 +4,9 @@ import gay.pizza.pork.ast.Node import gay.pizza.pork.ast.NodeParser import gay.pizza.pork.ast.NodeType -abstract class ParserBase(val source: TokenSource, val attribution: NodeAttribution) : NodeParser { +abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution) : NodeParser { + val source: TokenSource = source.ignoringParserIgnoredTypes() + @Suppress("NOTHING_TO_INLINE") protected inline fun guarded(type: NodeType? = null, noinline block: () -> T): T = attribution.guarded(type, block) @@ -64,31 +66,8 @@ abstract class ParserBase(val source: TokenSource, val attribution: NodeAttribut throw ExpectedTokenError(token, token.sourceIndex, *types) } - protected fun next(): Token { - while (true) { - val token = source.next() - if (ignoredByParser(token.type)) { - continue - } - return token - } - } - - protected fun peek(): Token { - while (true) { - val token = source.peek() - if (ignoredByParser(token.type)) { - source.next() - continue - } - return token - } - } - - private fun ignoredByParser(type: TokenType): Boolean = when (type) { - TokenType.BlockComment -> true - TokenType.LineComment -> true - TokenType.Whitespace -> true - else -> false - } + protected fun next(): Token = source.next() + protected fun peek(): Token = source.peek() + protected fun peek(ahead: Int): TokenType = source.peekTypeAhead(ahead) + protected fun peek(ahead: Int, vararg types: TokenType): Boolean = types.contains(source.peekTypeAhead(ahead)) } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt index 7b54326..75977a0 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt @@ -1,3 +1,19 @@ package gay.pizza.pork.parser -interface TokenSource : PeekableSource +interface TokenSource : PeekableSource { + fun peekTypeAhead(ahead: Int): TokenType + fun consumeAllRemainingTokens(): List { + val tokens = mutableListOf() + while (true) { + val token = next() + tokens.add(token) + if (token.type == TokenType.EndOfFile) { + break + } + } + return tokens + } + + fun ignoringParserIgnoredTypes(): TokenSource = + TokenStreamSource(TokenStream(consumeAllRemainingTokens().filter { !TokenType.ParserIgnoredTypes.contains(it.type) })) +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt index c29ca09..a6a46b4 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt @@ -2,6 +2,7 @@ package gay.pizza.pork.parser class TokenStreamSource(val stream: TokenStream) : TokenSource { private var index = 0 + override val currentIndex: Int get() = index @@ -20,4 +21,12 @@ class TokenStreamSource(val stream: TokenStream) : TokenSource { } return stream.tokens[index] } + + override fun peekTypeAhead(ahead: Int): TokenType { + val calculated = index + ahead + if (calculated >= stream.tokens.size) { + return stream.tokens.last().type + } + return stream.tokens[calculated].type + } } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt index 5a14606..73793b5 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt @@ -90,5 +90,15 @@ enum class TokenType(vararg properties: TokenTypeProperty) { val SingleChars = entries.filter { item -> item.singleChar != null } val CharConsumers = entries.filter { item -> item.charConsumer != null || item.charIndexConsumer != null } + + val ParserIgnoredTypes: Array = arrayOf( + Whitespace, + BlockComment, + LineComment + ) + + val DeclarationModifiers: Array = arrayOf( + Export + ) } } diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParser.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParser.kt index 38c7f96..f6f6129 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParser.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParser.kt @@ -17,7 +17,7 @@ class PorkParser : PsiParser { return builder.treeBuilt } - class ExitParser(val error: String? = null) : RuntimeException( + class ExitParser(error: String? = null) : RuntimeException( if (error == null) "Fast Exit" else "Exit Parser: $error" ) } diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt index fd7274c..d9b4131 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt @@ -4,6 +4,7 @@ import com.intellij.lang.PsiBuilder import gay.pizza.pork.parser.SourceIndex import gay.pizza.pork.parser.Token import gay.pizza.pork.parser.TokenSource +import gay.pizza.pork.parser.TokenType import com.intellij.psi.TokenType as PsiTokenType @Suppress("UnstableApiUsage") @@ -29,5 +30,18 @@ class PsiBuilderTokenSource(val builder: PsiBuilder) : TokenSource { return Token(tokenType, SourceIndex.indexOnly(builder.currentOffset), builder.tokenText!!) } + override fun peekTypeAhead(ahead: Int): TokenType { + if (builder.eof()) { + return TokenType.EndOfFile + } + val elementType = builder.lookAhead(ahead) + if (elementType == null || elementType == PsiTokenType.BAD_CHARACTER) { + return TokenType.EndOfFile + } + return PorkElementTypes.tokenTypeFor(elementType) ?: TokenType.EndOfFile + } + + override fun ignoringParserIgnoredTypes(): TokenSource = this + class BadCharacterError(error: String) : RuntimeException(error) }