parser: ensure that the parse functions are always directly called by reworking whitespace

parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt

@@ -1,13 +1,9 @@
 package gay.pizza.pork.parser
 
 import gay.pizza.pork.ast.*
-import kotlin.math.exp
 
 class Parser(source: TokenSource, attribution: NodeAttribution) :
   ParserBase(source, attribution) {
-  private var storedSymbol: Symbol? = null
-  private var storedDefinitionModifiers: DefinitionModifiers? = null
-
   override fun parseBlock(): Block = guarded(NodeType.Block) {
     expect(TokenType.LeftCurly)
     val items = collect(TokenType.RightCurly) {
@@ -100,16 +96,15 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
 
     while (!peek(TokenType.EndOfFile)) {
       if (declarationAccepted) {
-        val definition = maybeParseDefinition()
+        val declaration = parseDeclarationMaybe()
-        if (definition != null) {
+        if (declaration != null) {
-          declarationAccepted = false
+          declarations.add(declaration)
-          definitions.add(definition)
           continue
+        } else {
+          declarationAccepted = false
         }
-        declarations.add(parseDeclaration())
-      } else {
-        definitions.add(parseDefinition())
       }
+      definitions.add(parseDefinition())
     }
 
     CompilationUnit(declarations, definitions)
@@ -135,34 +130,46 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     return modifiers
   }
 
-  private fun maybeParseDefinition(): Definition? {
+  fun peekAheadUntilNotIn(vararg types: TokenType): TokenType {
-    try {
+    var i = 0
-      storedDefinitionModifiers = parseDefinitionModifiers()
+    while (true) {
-      val token = peek()
+      val token = peek(i)
-      return when (token.type) {
+      if (!types.contains(token)) {
-        TokenType.Func -> parseFunctionDefinition()
+        return token
-        TokenType.Let -> parseLetDefinition()
-        else -> null
       }
-    } finally {
+      i++
-      storedDefinitionModifiers = null
     }
   }
 
-  override fun parseDeclaration(): Declaration = guarded {
+  private fun parseDeclarationMaybe(): Declaration? {
     val token = peek()
-    return@guarded when (token.type) {
+    return when (token.type) {
       TokenType.Import -> parseImportDeclaration()
-      else -> throw ParseError(
+      else -> null
+    }
+  }
+
+  override fun parseDeclaration(): Declaration {
+    val declaration = parseDeclarationMaybe()
+    if (declaration == null) {
+      val token = peek()
+      throw ParseError(
         "Failed to parse token: ${token.type} '${token.text}' as" +
           " declaration (index ${source.currentIndex})"
       )
     }
+    return declaration
   }
 
-  override fun parseDefinition(): Definition = guarded {
+  override fun parseDefinition(): Definition =
-    maybeParseDefinition() ?: throw ParseError("Unable to parse definition")
+    when (val type = peekAheadUntilNotIn(*TokenType.DeclarationModifiers)) {
-  }
+      TokenType.Func -> parseFunctionDefinition()
+      TokenType.Let -> parseLetDefinition()
+      else -> throw ParseError(
+        "Failed to parse token: ${type.name} as" +
+          " declaration (index ${source.currentIndex})"
+      )
+    }
 
   override fun parseDoubleLiteral(): DoubleLiteral = guarded(NodeType.DoubleLiteral) {
     DoubleLiteral(expect(TokenType.NumberLiteral).text.toDouble())
@@ -177,11 +184,9 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     ForIn(symbol, value, block)
   }
 
-  override fun parseFunctionCall(): FunctionCall =
+  override fun parseFunctionCall(): FunctionCall = guarded(NodeType.FunctionCall) {
-    parseFunctionCall(null)
+    val symbol = parseSymbol()
-
+    expect(TokenType.LeftParentheses)
-  fun parseFunctionCall(target: Symbol?): FunctionCall = guarded(NodeType.FunctionCall) {
-    val symbol = target ?: parseSymbol()
     val arguments = collect(TokenType.RightParentheses, TokenType.Comma) {
       parseExpression()
     }
@@ -190,7 +195,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
   }
 
   override fun parseFunctionDefinition(): FunctionDefinition = guarded(NodeType.FunctionDefinition) {
-    val modifiers = storedDefinitionModifiers ?: parseDefinitionModifiers()
+    val modifiers = parseDefinitionModifiers()
     expect(TokenType.Func)
     val name = parseSymbol()
     expect(TokenType.LeftParentheses)
@@ -275,7 +280,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
   }
 
   override fun parseLetDefinition(): LetDefinition = guarded(NodeType.LetDefinition) {
-    val definitionModifiers = storedDefinitionModifiers ?: parseDefinitionModifiers()
+    val definitionModifiers = parseDefinitionModifiers()
     expect(TokenType.Let)
     val name = parseSymbol()
     expect(TokenType.Equals)
@@ -322,7 +327,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
   }
 
   override fun parseSetAssignment(): SetAssignment = guarded(NodeType.SetAssignment) {
-    val symbol = storedSymbol ?: parseSymbol()
+    val symbol = parseSymbol()
     expect(TokenType.Equals)
     val value = parseExpression()
     SetAssignment(symbol, value)
@@ -343,17 +348,11 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
   }
 
   private fun parseSymbolCases(): Expression = guarded {
-    val symbol = parseSymbol()
+    if (peek(1, TokenType.LeftParentheses)) {
-    if (next(TokenType.LeftParentheses)) {
+      parseFunctionCall()
-      parseFunctionCall(symbol)
+    } else if (peek(1, TokenType.PlusPlus, TokenType.MinusMinus)) {
-    } else {
+      parseSuffixOperation()
-      val reference = SymbolReference(symbol)
+    } else parseSymbolReference()
-      if (peek(TokenType.PlusPlus, TokenType.MinusMinus)) {
-        expect(TokenType.PlusPlus, TokenType.MinusMinus) {
-          SuffixOperation(ParserHelpers.convertSuffixOperator(it), reference)
-        }
-      } else reference
-    }
   }
 
   override fun parseSymbol(): Symbol = guarded(NodeType.Symbol) {

parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt

@@ -4,7 +4,9 @@ import gay.pizza.pork.ast.Node
 import gay.pizza.pork.ast.NodeParser
 import gay.pizza.pork.ast.NodeType
 
-abstract class ParserBase(val source: TokenSource, val attribution: NodeAttribution) : NodeParser {
+abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution) : NodeParser {
+  val source: TokenSource = source.ignoringParserIgnoredTypes()
+
   @Suppress("NOTHING_TO_INLINE")
   protected inline fun <T: Node> guarded(type: NodeType? = null, noinline block: () -> T): T =
     attribution.guarded(type, block)
@@ -64,31 +66,8 @@ abstract class ParserBase(val source: TokenSource, val attribution: NodeAttribut
     throw ExpectedTokenError(token, token.sourceIndex, *types)
   }
 
-  protected fun next(): Token {
+  protected fun next(): Token = source.next()
-    while (true) {
+  protected fun peek(): Token = source.peek()
-      val token = source.next()
+  protected fun peek(ahead: Int): TokenType = source.peekTypeAhead(ahead)
-      if (ignoredByParser(token.type)) {
+  protected fun peek(ahead: Int, vararg types: TokenType): Boolean = types.contains(source.peekTypeAhead(ahead))
-        continue
-      }
-      return token
-    }
-  }
-
-  protected fun peek(): Token {
-    while (true) {
-      val token = source.peek()
-      if (ignoredByParser(token.type)) {
-        source.next()
-        continue
-      }
-      return token
-    }
-  }
-
-  private fun ignoredByParser(type: TokenType): Boolean = when (type) {
-    TokenType.BlockComment -> true
-    TokenType.LineComment -> true
-    TokenType.Whitespace -> true
-    else -> false
-  }
 }

parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt

@@ -1,3 +1,19 @@
 package gay.pizza.pork.parser
 
-interface TokenSource : PeekableSource<Token>
+interface TokenSource : PeekableSource<Token> {
+  fun peekTypeAhead(ahead: Int): TokenType
+  fun consumeAllRemainingTokens(): List<Token> {
+    val tokens = mutableListOf<Token>()
+    while (true) {
+      val token = next()
+      tokens.add(token)
+      if (token.type == TokenType.EndOfFile) {
+        break
+      }
+    }
+    return tokens
+  }
+
+  fun ignoringParserIgnoredTypes(): TokenSource =
+    TokenStreamSource(TokenStream(consumeAllRemainingTokens().filter { !TokenType.ParserIgnoredTypes.contains(it.type) }))
+}

parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt

@@ -2,6 +2,7 @@ package gay.pizza.pork.parser
 
 class TokenStreamSource(val stream: TokenStream) : TokenSource {
   private var index = 0
+
   override val currentIndex: Int
     get() = index
 
@@ -20,4 +21,12 @@ class TokenStreamSource(val stream: TokenStream) : TokenSource {
     }
     return stream.tokens[index]
   }
+
+  override fun peekTypeAhead(ahead: Int): TokenType {
+    val calculated = index + ahead
+    if (calculated >= stream.tokens.size) {
+      return stream.tokens.last().type
+    }
+    return stream.tokens[calculated].type
+  }
 }

parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt

@@ -90,5 +90,15 @@ enum class TokenType(vararg properties: TokenTypeProperty) {
     val SingleChars = entries.filter { item -> item.singleChar != null }
     val CharConsumers = entries.filter { item ->
       item.charConsumer != null || item.charIndexConsumer != null }
+
+    val ParserIgnoredTypes: Array<TokenType> = arrayOf(
+      Whitespace,
+      BlockComment,
+      LineComment
+    )
+
+    val DeclarationModifiers: Array<TokenType> = arrayOf(
+      Export
+    )
   }
 }

support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParser.kt

@@ -17,7 +17,7 @@ class PorkParser : PsiParser {
     return builder.treeBuilt
   }
 
-  class ExitParser(val error: String? = null) : RuntimeException(
+  class ExitParser(error: String? = null) : RuntimeException(
     if (error == null) "Fast Exit" else "Exit Parser: $error"
   )
 }

support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt

@@ -4,6 +4,7 @@ import com.intellij.lang.PsiBuilder
 import gay.pizza.pork.parser.SourceIndex
 import gay.pizza.pork.parser.Token
 import gay.pizza.pork.parser.TokenSource
+import gay.pizza.pork.parser.TokenType
 import com.intellij.psi.TokenType as PsiTokenType
 
 @Suppress("UnstableApiUsage")
@@ -29,5 +30,18 @@ class PsiBuilderTokenSource(val builder: PsiBuilder) : TokenSource {
     return Token(tokenType, SourceIndex.indexOnly(builder.currentOffset), builder.tokenText!!)
   }
 
+  override fun peekTypeAhead(ahead: Int): TokenType {
+    if (builder.eof()) {
+      return TokenType.EndOfFile
+    }
+    val elementType = builder.lookAhead(ahead)
+    if (elementType == null || elementType == PsiTokenType.BAD_CHARACTER) {
+      return TokenType.EndOfFile
+    }
+    return PorkElementTypes.tokenTypeFor(elementType) ?: TokenType.EndOfFile
+  }
+
+  override fun ignoringParserIgnoredTypes(): TokenSource = this
+
   class BadCharacterError(error: String) : RuntimeException(error)
 }