idea: implement psi parser and the start of symbol declarations

This commit is contained in:
Alex Zenla 2023-09-11 22:43:34 -04:00
parent b64c7fb259
commit 7aa9d95221
Signed by: alex
GPG Key ID: C0780728420EBFE5
21 changed files with 340 additions and 80 deletions

View File

@ -3,8 +3,8 @@ package gay.pizza.pork.parser
import gay.pizza.pork.ast.Node
object DiscardNodeAttribution : NodeAttribution {
override fun enter() {}
override fun push(token: Token) {}
override fun <T : Node> adopt(node: T) {}
override fun <T : Node> exit(node: T): T = node
override fun <T : Node> guarded(block: () -> T): T =
block()
}

View File

@ -3,8 +3,7 @@ package gay.pizza.pork.parser
import gay.pizza.pork.ast.Node
interface NodeAttribution {
fun enter()
fun push(token: Token)
fun <T: Node> adopt(node: T)
fun <T: Node> exit(node: T): T
fun <T: Node> guarded(block: () -> T): T
}

View File

@ -0,0 +1,3 @@
package gay.pizza.pork.parser
class ParseError(val error: String) : RuntimeException(error)

View File

@ -185,7 +185,7 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
}
else -> {
throw RuntimeException(
throw ParseError(
"Failed to parse token: ${token.type} '${token.text}' as" +
" expression (index ${unsanitizedSource.currentIndex})"
)
@ -308,7 +308,7 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
return definition
}
val token = peek()
throw RuntimeException(
throw ParseError(
"Failed to parse token: ${token.type} '${token.text}' as" +
" definition (index ${unsanitizedSource.currentIndex})"
)
@ -318,7 +318,7 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
val token = peek()
return when (token.type) {
TokenType.Import -> readImportDeclaration()
else -> throw RuntimeException(
else -> throw ParseError(
"Failed to parse token: ${token.type} '${token.text}' as" +
" declaration (index ${unsanitizedSource.currentIndex})"
)
@ -343,7 +343,7 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
TokenType.GreaterEqual -> InfixOperator.GreaterEqual
TokenType.And -> InfixOperator.BooleanAnd
TokenType.Or -> InfixOperator.BooleanOr
else -> throw RuntimeException("Unknown Infix Operator")
else -> throw ParseError("Unknown Infix Operator")
}
private fun convertPrefixOperator(token: Token): PrefixOperator = when (token.type) {
@ -351,13 +351,13 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
TokenType.Plus -> PrefixOperator.UnaryPlus
TokenType.Minus -> PrefixOperator.UnaryMinus
TokenType.Tilde -> PrefixOperator.BinaryNot
else -> throw RuntimeException("Unknown Prefix Operator")
else -> throw ParseError("Unknown Prefix Operator")
}
private fun convertSuffixOperator(token: Token): SuffixOperator = when (token.type) {
TokenType.PlusPlus -> SuffixOperator.Increment
TokenType.MinusMinus -> SuffixOperator.Decrement
else -> throw RuntimeException("Unknown Suffix Operator")
else -> throw ParseError("Unknown Suffix Operator")
}
fun readCompilationUnit(): CompilationUnit = within {
@ -425,7 +425,7 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
private fun expect(vararg types: TokenType): Token {
val token = next()
if (!types.contains(token.type)) {
throw RuntimeException(
throw ParseError(
"Expected one of ${types.joinToString(", ")}" +
" but got type ${token.type} '${token.text}'"
)
@ -459,10 +459,7 @@ class Parser(source: PeekableSource<Token>, val attribution: NodeAttribution) {
}
}
private fun <T: Node> within(block: () -> T): T {
attribution.enter()
return attribution.exit(block())
}
fun <T: Node> within(block: () -> T): T = attribution.guarded(block)
private fun ignoredByParser(type: TokenType): Boolean = when (type) {
TokenType.BlockComment -> true

View File

@ -3,16 +3,10 @@ package gay.pizza.pork.parser
import gay.pizza.pork.ast.Node
import gay.pizza.pork.ast.data
class ParserNodeAttribution : NodeAttribution {
open class ParserNodeAttribution : NodeAttribution {
private val stack = mutableListOf<MutableList<Token>>()
private var current: MutableList<Token>? = null
override fun enter() {
val store = mutableListOf<Token>()
current = store
stack.add(store)
}
override fun push(token: Token) {
val store = current ?: throw RuntimeException("enter() not called!")
store.add(token)
@ -28,8 +22,12 @@ class ParserNodeAttribution : NodeAttribution {
}
}
override fun <T: Node> exit(node: T): T {
val store = stack.removeLast()
override fun <T : Node> guarded(block: () -> T): T {
var store = mutableListOf<Token>()
current = store
stack.add(store)
val node = block()
store = stack.removeLast()
current = stack.lastOrNull()
node.data = ParserAttributes(store)
return node

View File

@ -9,7 +9,9 @@ class Tokenizer(val source: CharSource) {
var endOfComment = false
while (true) {
val char = source.next()
if (char == CharSource.NullChar) throw RuntimeException("Unterminated block comment")
if (char == CharSource.NullChar) {
throw ParseError("Unterminated block comment")
}
append(char)
if (endOfComment) {
@ -48,7 +50,7 @@ class Tokenizer(val source: CharSource) {
while (true) {
val char = source.peek()
if (char == CharSource.NullChar) {
throw RuntimeException("Unterminated string.")
throw ParseError("Unterminated string.")
}
append(source.next())
if (char == '"') {
@ -107,7 +109,7 @@ class Tokenizer(val source: CharSource) {
continue
}
} else {
throw RuntimeException("Unknown Char Consumer")
throw ParseError("Unknown Char Consumer")
}
val text = buildString {
@ -134,7 +136,7 @@ class Tokenizer(val source: CharSource) {
return readStringLiteral(char)
}
throw RuntimeException("Failed to parse: (${char}) next ${source.peek()}")
throw ParseError("Failed to parse: (${char}) next ${source.peek()}")
}
return Token.endOfFile(source.currentIndex)
}

View File

@ -0,0 +1,25 @@
package gay.pizza.pork.idea
import com.intellij.lang.Commenter
class PorkCommenter : Commenter {
override fun getLineCommentPrefix(): String {
return "//"
}
override fun getBlockCommentPrefix(): String {
return "/*"
}
override fun getBlockCommentSuffix(): String {
return "*/"
}
override fun getCommentedBlockCommentPrefix(): String? {
return null
}
override fun getCommentedBlockCommentSuffix(): String? {
return null
}
}

View File

@ -0,0 +1,49 @@
package gay.pizza.pork.idea
import com.intellij.psi.tree.IElementType
import com.intellij.psi.tree.TokenSet
import gay.pizza.pork.ast.NodeType
import gay.pizza.pork.parser.TokenType
object PorkElementTypes {
private val tokenTypeToElementType = mutableMapOf<TokenType, IElementType>()
private val elementTypeToTokenType = mutableMapOf<IElementType, TokenType>()
private val nodeTypeToElementType = mutableMapOf<NodeType, IElementType>()
private val elementTypeToNodeType = mutableMapOf<IElementType, NodeType>()
init {
for (tokenType in TokenType.entries) {
val elementType = IElementType(tokenType.name, PorkLanguage)
tokenTypeToElementType[tokenType] = elementType
elementTypeToTokenType[elementType] = tokenType
}
for (nodeType in NodeType.entries) {
val elementType = IElementType(nodeType.name, PorkLanguage)
nodeTypeToElementType[nodeType] = elementType
elementTypeToNodeType[elementType] = nodeType
}
}
val CommentSet = TokenSet.create(
elementTypeFor(TokenType.BlockComment),
elementTypeFor(TokenType.LineComment)
)
val StringSet = TokenSet.create(
elementTypeFor(TokenType.StringLiteral)
)
fun tokenTypeFor(elementType: IElementType): TokenType? =
elementTypeToTokenType[elementType]
fun elementTypeFor(tokenType: TokenType): IElementType =
tokenTypeToElementType[tokenType]!!
fun nodeTypeFor(elementType: IElementType): NodeType? =
elementTypeToNodeType[elementType]
fun elementTypeFor(nodeType: NodeType): IElementType =
nodeTypeToElementType[nodeType]!!
}

View File

@ -0,0 +1,13 @@
package gay.pizza.pork.idea
import com.intellij.extapi.psi.PsiFileBase
import com.intellij.openapi.fileTypes.FileType
import com.intellij.psi.FileViewProvider
class PorkFile(viewProvider: FileViewProvider) : PsiFileBase(viewProvider, PorkLanguage) {
override fun getFileType(): FileType {
return PorkFileType
}
override fun toString(): String = "Pork File"
}

View File

@ -0,0 +1,11 @@
package gay.pizza.pork.idea
import com.intellij.model.Pointer
import com.intellij.model.Symbol
@Suppress("UnstableApiUsage")
data class PorkFunctionSymbol(val id: String) : Symbol {
override fun createPointer(): Pointer<out Symbol> {
return Pointer { this }
}
}

View File

@ -56,7 +56,7 @@ class PorkLexer : LexerBase() {
try {
val currentToken = tokenizer.next()
currentTokenType = tokenAsElement(currentToken)
currentTokenType = PorkElementTypes.elementTypeFor(currentToken.type)
internalTokenStart = currentToken.start
internalTokenEnd = currentToken.start + currentToken.text.length
} catch (e: ProcessCanceledException) {
@ -76,26 +76,6 @@ class PorkLexer : LexerBase() {
return source.endIndex
}
private fun tokenAsElement(token: Token): IElementType = when {
token.type.family == TokenFamily.KeywordFamily ->
PorkTokenTypes.Keyword
token.type.family == TokenFamily.SymbolFamily ->
PorkTokenTypes.Symbol
token.type.family == TokenFamily.OperatorFamily ->
PorkTokenTypes.Operator
token.type.family == TokenFamily.StringLiteralFamily ->
PorkTokenTypes.String
token.type.family == TokenFamily.NumericLiteralFamily ->
PorkTokenTypes.Number
token.type == TokenType.Whitespace ->
PorkTokenTypes.Whitespace
token.type == TokenType.BlockComment ->
PorkTokenTypes.BlockComment
token.type == TokenType.LineComment ->
PorkTokenTypes.LineComment
else -> PsiTokenType.CODE_FRAGMENT
}
override fun toString(): String =
"PorkLexer(start=$internalTokenStart, end=$internalTokenEnd)"
}

View File

@ -0,0 +1,6 @@
package gay.pizza.pork.idea
import com.intellij.openapi.util.Key
import gay.pizza.pork.ast.Node
object PorkNodeKey : Key<Node>("PorkAstNode")

View File

@ -0,0 +1,21 @@
package gay.pizza.pork.idea
import com.intellij.lang.ASTNode
import com.intellij.lang.PsiBuilder
import com.intellij.lang.PsiParser
import com.intellij.psi.tree.IElementType
import gay.pizza.pork.parser.Parser
class PorkParser : PsiParser {
override fun parse(root: IElementType, builder: PsiBuilder): ASTNode {
val psiBuilderMarkAttribution = PsiBuilderMarkAttribution(builder)
val source = PsiBuilderTokenSource(builder)
val parser = Parser(source, psiBuilderMarkAttribution)
try {
parser.within { parser.readCompilationUnit() }
} catch (_: ExitParser) {}
return builder.treeBuilt
}
class ExitParser(val error: String) : RuntimeException("Exit Parser: $error")
}

View File

@ -0,0 +1,45 @@
package gay.pizza.pork.idea
import com.intellij.extapi.psi.ASTWrapperPsiElement
import com.intellij.lang.ASTNode
import com.intellij.lang.ParserDefinition
import com.intellij.lang.PsiParser
import com.intellij.lexer.Lexer
import com.intellij.openapi.project.Project
import com.intellij.psi.FileViewProvider
import com.intellij.psi.PsiElement
import com.intellij.psi.PsiFile
import com.intellij.psi.tree.IFileElementType
import com.intellij.psi.tree.TokenSet
class PorkParserDefinition : ParserDefinition {
val fileElementType = IFileElementType(PorkLanguage)
override fun createLexer(project: Project?): Lexer {
return PorkLexer()
}
override fun createParser(project: Project?): PsiParser {
return PorkParser()
}
override fun getFileNodeType(): IFileElementType {
return fileElementType
}
override fun getCommentTokens(): TokenSet {
return PorkElementTypes.CommentSet
}
override fun getStringLiteralElements(): TokenSet {
return PorkElementTypes.StringSet
}
override fun createElement(node: ASTNode): PsiElement {
return ASTWrapperPsiElement(node)
}
override fun createFile(viewProvider: FileViewProvider): PsiFile {
return PorkFile(viewProvider)
}
}

View File

@ -0,0 +1,29 @@
package gay.pizza.pork.idea
import com.intellij.model.Symbol
import com.intellij.model.psi.PsiSymbolDeclaration
import com.intellij.openapi.util.TextRange
import com.intellij.psi.PsiElement
import com.intellij.psi.util.elementType
import gay.pizza.pork.ast.NodeType
import gay.pizza.pork.parser.TokenType
@Suppress("UnstableApiUsage")
class PorkSymbolDeclaration(val element: PsiElement) : PsiSymbolDeclaration {
override fun getDeclaringElement(): PsiElement = element
override fun getRangeInDeclaringElement(): TextRange {
return getSymbolElement().textRange
}
override fun getSymbol(): Symbol {
val element = getSymbolElement()
val porkNode = element.getUserData(PorkNodeKey)!!
return PorkFunctionSymbol((porkNode as gay.pizza.pork.ast.Symbol).id)
}
private fun getSymbolElement(): PsiElement {
return element.children.first {
it.elementType == PorkElementTypes.elementTypeFor(NodeType.Symbol)
}
}
}

View File

@ -0,0 +1,21 @@
package gay.pizza.pork.idea
import com.intellij.model.psi.PsiSymbolDeclaration
import com.intellij.model.psi.PsiSymbolDeclarationProvider
import com.intellij.psi.PsiElement
import com.intellij.psi.util.elementType
import gay.pizza.pork.ast.NodeType
@Suppress("UnstableApiUsage")
class PorkSymbolDeclarationProvider : PsiSymbolDeclarationProvider {
override fun getDeclarations(
element: PsiElement,
offsetInElement: Int
): MutableCollection<out PsiSymbolDeclaration> {
val symbolDeclarations = mutableListOf<PsiSymbolDeclaration>()
if (element.elementType == PorkElementTypes.elementTypeFor(NodeType.FunctionDefinition)) {
symbolDeclarations.add(PorkSymbolDeclaration(element))
}
return symbolDeclarations
}
}

View File

@ -6,6 +6,8 @@ import com.intellij.openapi.editor.colors.TextAttributesKey
import com.intellij.openapi.fileTypes.SyntaxHighlighter
import com.intellij.openapi.fileTypes.SyntaxHighlighterBase
import com.intellij.psi.tree.IElementType
import gay.pizza.pork.parser.TokenFamily
import gay.pizza.pork.parser.TokenType
object PorkSyntaxHighlighter : SyntaxHighlighter {
override fun getHighlightingLexer(): Lexer {
@ -14,42 +16,45 @@ object PorkSyntaxHighlighter : SyntaxHighlighter {
override fun getTokenHighlights(tokenType: IElementType?): Array<TextAttributesKey> {
if (tokenType == null) return emptyArray()
val attributes = when (tokenType) {
PorkTokenTypes.Keyword ->
val ourTokenType = PorkElementTypes.tokenTypeFor(tokenType) ?: return emptyArray()
val attributes = when (ourTokenType.family) {
TokenFamily.KeywordFamily ->
TextAttributesKey.createTextAttributesKey(
"PORK.KEYWORD",
DefaultLanguageHighlighterColors.KEYWORD
)
PorkTokenTypes.Symbol ->
TokenFamily.SymbolFamily ->
TextAttributesKey.createTextAttributesKey(
"PORK.SYMBOL",
DefaultLanguageHighlighterColors.LOCAL_VARIABLE
)
PorkTokenTypes.Operator ->
TokenFamily.OperatorFamily ->
TextAttributesKey.createTextAttributesKey(
"PORK.OPERATOR",
DefaultLanguageHighlighterColors.OPERATION_SIGN
)
PorkTokenTypes.String ->
TokenFamily.StringLiteralFamily ->
TextAttributesKey.createTextAttributesKey(
"PORK.STRING",
DefaultLanguageHighlighterColors.STRING
)
PorkTokenTypes.Number ->
TokenFamily.NumericLiteralFamily ->
TextAttributesKey.createTextAttributesKey(
"PORK.NUMBER",
DefaultLanguageHighlighterColors.NUMBER
)
PorkTokenTypes.BlockComment ->
TextAttributesKey.createTextAttributesKey(
"PORK.COMMENT.BLOCK",
DefaultLanguageHighlighterColors.BLOCK_COMMENT
)
PorkTokenTypes.LineComment ->
TextAttributesKey.createTextAttributesKey(
"PORK.COMMENT.LINE",
DefaultLanguageHighlighterColors.LINE_COMMENT
)
TokenFamily.CommentFamily ->
when (ourTokenType) {
TokenType.LineComment -> TextAttributesKey.createTextAttributesKey(
"PORK.COMMENT.LINE",
DefaultLanguageHighlighterColors.LINE_COMMENT
)
else -> TextAttributesKey.createTextAttributesKey(
"PORK.COMMENT.BLOCK",
DefaultLanguageHighlighterColors.BLOCK_COMMENT
)
}
else -> null
}
return if (attributes == null)

View File

@ -1,15 +0,0 @@
package gay.pizza.pork.idea
import com.intellij.psi.TokenType
import com.intellij.psi.tree.IElementType
object PorkTokenTypes {
val Whitespace = TokenType.WHITE_SPACE
val Keyword = IElementType("keyword", PorkLanguage)
val Symbol = IElementType("symbol", PorkLanguage)
val Operator = IElementType("operator", PorkLanguage)
val String = IElementType("string", PorkLanguage)
val Number = IElementType("number", PorkLanguage)
val BlockComment = IElementType("lineComment", PorkLanguage)
val LineComment = IElementType("blockComment", PorkLanguage)
}

View File

@ -0,0 +1,32 @@
package gay.pizza.pork.idea
import com.intellij.lang.PsiBuilder
import gay.pizza.pork.ast.Node
import gay.pizza.pork.parser.ParseError
import gay.pizza.pork.parser.ParserNodeAttribution
class PsiBuilderMarkAttribution(val builder: PsiBuilder) : ParserNodeAttribution() {
override fun <T : Node> guarded(block: () -> T): T {
val marker = builder.mark()
try {
val item = super.guarded(block)
marker.done(PorkElementTypes.elementTypeFor(item.type))
return item
} catch (e: PsiBuilderTokenSource.BadCharacterError) {
marker.error("Bad character.")
while (!builder.eof()) {
builder.advanceLexer()
}
throw PorkParser.ExitParser(e.error)
} catch (e: ParseError) {
while (!builder.eof()) {
builder.advanceLexer()
}
marker.error(e.error)
throw PorkParser.ExitParser(e.error)
} catch (e: PorkParser.ExitParser) {
marker.error(e.error)
throw e
}
}
}

View File

@ -0,0 +1,32 @@
package gay.pizza.pork.idea
import com.intellij.lang.PsiBuilder
import gay.pizza.pork.parser.Token
import gay.pizza.pork.parser.TokenSource
import com.intellij.psi.TokenType as PsiTokenType
@Suppress("UnstableApiUsage")
class PsiBuilderTokenSource(val builder: PsiBuilder) : TokenSource {
override val currentIndex: Int = 0
override fun next(): Token {
val token = peek()
builder.advanceLexer()
return token
}
override fun peek(): Token {
if (builder.eof()) {
return Token.endOfFile(builder.currentOffset)
}
val elementType = builder.tokenType!!
if (elementType == PsiTokenType.BAD_CHARACTER) {
throw BadCharacterError("Invalid character.")
}
val tokenType = PorkElementTypes.tokenTypeFor(elementType) ?:
throw RuntimeException("Lexing failure: ${elementType.debugName}")
return Token(tokenType, builder.currentOffset, builder.tokenText!!)
}
class BadCharacterError(val error: String) : RuntimeException(error)
}

View File

@ -11,6 +11,13 @@
<lang.syntaxHighlighterFactory
language="Pork"
implementationClass="gay.pizza.pork.idea.PorkSyntaxHighlighterFactory"/>
<lang.parserDefinition
language="Pork"
implementationClass="gay.pizza.pork.idea.PorkParserDefinition"/>
<lang.commenter
language="Pork"
implementationClass="gay.pizza.pork.idea.PorkCommenter"/>
<psi.declarationProvider implementation="gay.pizza.pork.idea.PorkSymbolDeclarationProvider"/>
</extensions>
<applicationListeners>