strips/pork
2
0
Fork 0
mirror of https://github.com/GayPizzaSpecifications/pork.git synced 2025-11-03 17:39:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

build: move parser and tokenizer into separate modules

Browse Source
This commit is contained in:
Alex Zenla
2023-10-16 21:52:21 -07:00
parent 9338b01b48
commit 15f5f313cc
57 changed files with 92 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ffi/src/main/kotlin/gay/pizza/pork/ffi/JavaAutogenContentSource.kt
View File

@ -2,9 +2,9 @@ package gay.pizza.pork.ffi
import gay.pizza.pork.ast.gen.visit import gay.pizza.pork.ast.gen.visit
import gay.pizza.pork.frontend.ContentSource import gay.pizza.pork.frontend.ContentSource
import gay.pizza.pork.parser.CharSource import gay.pizza.pork.tokenizer.CharSource
import gay.pizza.pork.parser.Printer import gay.pizza.pork.parser.Printer
import gay.pizza.pork.parser.StringCharSource import gay.pizza.pork.tokenizer.StringCharSource
object JavaAutogenContentSource : ContentSource { object JavaAutogenContentSource : ContentSource {
override fun loadAsCharSource(path: String): CharSource { override fun loadAsCharSource(path: String): CharSource {

2
frontend/src/main/kotlin/gay/pizza/pork/frontend/ContentSource.kt
View File

@ -1,6 +1,6 @@
package gay.pizza.pork.frontend package gay.pizza.pork.frontend
import gay.pizza.pork.parser.CharSource import gay.pizza.pork.tokenizer.CharSource
interface ContentSource { interface ContentSource {
fun loadAsCharSource(path: String): CharSource fun loadAsCharSource(path: String): CharSource

4
frontend/src/main/kotlin/gay/pizza/pork/frontend/FsContentSource.kt
View File

@ -3,8 +3,8 @@ package gay.pizza.pork.frontend
import gay.pizza.dough.fs.FsPath import gay.pizza.dough.fs.FsPath
import gay.pizza.dough.fs.PlatformFsProvider import gay.pizza.dough.fs.PlatformFsProvider
import gay.pizza.dough.fs.readString import gay.pizza.dough.fs.readString
import gay.pizza.pork.parser.CharSource import gay.pizza.pork.tokenizer.CharSource
import gay.pizza.pork.parser.StringCharSource import gay.pizza.pork.tokenizer.StringCharSource
class FsContentSource(val root: FsPath) : ContentSource { class FsContentSource(val root: FsPath) : ContentSource {
override fun loadAsCharSource(path: String): CharSource = override fun loadAsCharSource(path: String): CharSource =

2
frontend/src/main/kotlin/gay/pizza/pork/frontend/World.kt
View File

@ -4,7 +4,7 @@ import gay.pizza.pork.ast.gen.CompilationUnit
import gay.pizza.pork.ast.gen.ImportDeclaration import gay.pizza.pork.ast.gen.ImportDeclaration
import gay.pizza.pork.parser.DiscardNodeAttribution import gay.pizza.pork.parser.DiscardNodeAttribution
import gay.pizza.pork.parser.Parser import gay.pizza.pork.parser.Parser
import gay.pizza.pork.parser.Tokenizer import gay.pizza.pork.tokenizer.Tokenizer
class World(val importSource: ImportSource) { class World(val importSource: ImportSource) {
private val internalUnits = mutableMapOf<String, CompilationUnit>() private val internalUnits = mutableMapOf<String, CompilationUnit>()

4
minimal/src/main/kotlin/gay/pizza/pork/minimal/FileTool.kt
View File

@ -4,8 +4,8 @@ import gay.pizza.dough.fs.FsPath
import gay.pizza.dough.fs.readString import gay.pizza.dough.fs.readString
import gay.pizza.pork.frontend.ContentSource import gay.pizza.pork.frontend.ContentSource
import gay.pizza.pork.frontend.FsContentSource import gay.pizza.pork.frontend.FsContentSource
import gay.pizza.pork.parser.CharSource import gay.pizza.pork.tokenizer.CharSource
import gay.pizza.pork.parser.StringCharSource import gay.pizza.pork.tokenizer.StringCharSource
class FileTool(val path: FsPath) : Tool() { class FileTool(val path: FsPath) : Tool() {
override fun createCharSource(): CharSource = override fun createCharSource(): CharSource =

1
minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt
View File

@ -13,6 +13,7 @@ import gay.pizza.pork.frontend.DynamicImportSource
import gay.pizza.pork.frontend.World import gay.pizza.pork.frontend.World
import gay.pizza.pork.parser.* import gay.pizza.pork.parser.*
import gay.pizza.pork.stdlib.PorkStdlib import gay.pizza.pork.stdlib.PorkStdlib
import gay.pizza.pork.tokenizer.*
abstract class Tool { abstract class Tool {
abstract fun createCharSource(): CharSource abstract fun createCharSource(): CharSource

1
parser/build.gradle.kts
View File

@ -4,6 +4,7 @@ plugins {
dependencies { dependencies {
api(project(":ast")) api(project(":ast"))
api(project(":tokenizer"))
implementation(project(":common")) implementation(project(":common"))
} }

5
parser/src/main/kotlin/gay/pizza/pork/parser/BadCharacterError.kt
View File

@ -1,5 +0,0 @@
package gay.pizza.pork.parser
class BadCharacterError(val char: Char, sourceIndex: SourceIndex, state: TokenizerState) : ParseError(
"Failed to produce token for '${char}' at $sourceIndex in state $state"
)

1
parser/src/main/kotlin/gay/pizza/pork/parser/DiscardNodeAttribution.kt
View File

@ -2,6 +2,7 @@ package gay.pizza.pork.parser
import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.Node
import gay.pizza.pork.ast.gen.NodeType import gay.pizza.pork.ast.gen.NodeType
import gay.pizza.pork.tokenizer.Token
object DiscardNodeAttribution : NodeAttribution { object DiscardNodeAttribution : NodeAttribution {
override fun push(token: Token) {} override fun push(token: Token) {}

4
parser/src/main/kotlin/gay/pizza/pork/parser/LazySkippingTokenSource.kt
View File

@ -1,5 +1,9 @@
package gay.pizza.pork.parser package gay.pizza.pork.parser
import gay.pizza.pork.tokenizer.Token
import gay.pizza.pork.tokenizer.TokenSource
import gay.pizza.pork.tokenizer.TokenType
class LazySkippingTokenSource(val source: TokenSource, val skipping: Set<TokenType>) : ParserAwareTokenSource { class LazySkippingTokenSource(val source: TokenSource, val skipping: Set<TokenType>) : ParserAwareTokenSource {
private var index = 0 private var index = 0
override val currentIndex: Int override val currentIndex: Int

1
parser/src/main/kotlin/gay/pizza/pork/parser/NodeAttribution.kt
View File

@ -2,6 +2,7 @@ package gay.pizza.pork.parser
import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.Node
import gay.pizza.pork.ast.gen.NodeType import gay.pizza.pork.ast.gen.NodeType
import gay.pizza.pork.tokenizer.Token
interface NodeAttribution { interface NodeAttribution {
fun push(token: Token) fun push(token: Token)

2
parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt
View File

@ -1,6 +1,8 @@
package gay.pizza.pork.parser package gay.pizza.pork.parser
import gay.pizza.pork.ast.gen.* import gay.pizza.pork.ast.gen.*
import gay.pizza.pork.tokenizer.TokenSource
import gay.pizza.pork.tokenizer.TokenType
class Parser(source: TokenSource, attribution: NodeAttribution) : class Parser(source: TokenSource, attribution: NodeAttribution) :
ParserBase(source, attribution) { ParserBase(source, attribution) {

1
parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt
View File

@ -4,6 +4,7 @@ import gay.pizza.pork.ast.gen.Node
import gay.pizza.pork.ast.gen.NodeCoalescer import gay.pizza.pork.ast.gen.NodeCoalescer
import gay.pizza.pork.ast.gen.data import gay.pizza.pork.ast.gen.data
import gay.pizza.pork.ast.gen.visit import gay.pizza.pork.ast.gen.visit
import gay.pizza.pork.tokenizer.Token
data class ParserAttributes(val tokens: List<Token>) { data class ParserAttributes(val tokens: List<Token>) {
companion object { companion object {

2
parser/src/main/kotlin/gay/pizza/pork/parser/ParserAwareTokenSource.kt
View File

@ -1,3 +1,5 @@
package gay.pizza.pork.parser package gay.pizza.pork.parser
import gay.pizza.pork.tokenizer.TokenSource
interface ParserAwareTokenSource : TokenSource interface ParserAwareTokenSource : TokenSource

1
parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt
View File

@ -3,6 +3,7 @@ package gay.pizza.pork.parser
import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.Node
import gay.pizza.pork.ast.gen.NodeParser import gay.pizza.pork.ast.gen.NodeParser
import gay.pizza.pork.ast.gen.NodeType import gay.pizza.pork.ast.gen.NodeType
import gay.pizza.pork.tokenizer.*
abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution) : NodeParser { abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution) : NodeParser {
val source: TokenSource = if (source is ParserAwareTokenSource) { val source: TokenSource = if (source is ParserAwareTokenSource) {

2
parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt
View File

@ -3,6 +3,8 @@ package gay.pizza.pork.parser
import gay.pizza.pork.ast.gen.InfixOperator import gay.pizza.pork.ast.gen.InfixOperator
import gay.pizza.pork.ast.gen.PrefixOperator import gay.pizza.pork.ast.gen.PrefixOperator
import gay.pizza.pork.ast.gen.SuffixOperator import gay.pizza.pork.ast.gen.SuffixOperator
import gay.pizza.pork.tokenizer.Token
import gay.pizza.pork.tokenizer.TokenType
internal object ParserHelpers { internal object ParserHelpers {
fun convertInfixOperator(token: Token): InfixOperator = when (token.type) { fun convertInfixOperator(token: Token): InfixOperator = when (token.type) {

1
parser/src/main/kotlin/gay/pizza/pork/parser/ParserNodeAttribution.kt
View File

@ -3,6 +3,7 @@ package gay.pizza.pork.parser
import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.Node
import gay.pizza.pork.ast.gen.NodeType import gay.pizza.pork.ast.gen.NodeType
import gay.pizza.pork.ast.gen.data import gay.pizza.pork.ast.gen.data
import gay.pizza.pork.tokenizer.Token
open class ParserNodeAttribution : NodeAttribution { open class ParserNodeAttribution : NodeAttribution {
private val stack = mutableListOf<MutableList<Token>>() private val stack = mutableListOf<MutableList<Token>>()

1
settings.gradle.kts
View File

@ -4,6 +4,7 @@ includeBuild("buildext")
include( include(
":common", ":common",
":tokenizer",
":ast", ":ast",
":parser", ":parser",
":frontend", ":frontend",

4
stdlib/src/main/kotlin/gay/pizza/pork/stdlib/PorkStdlib.kt
View File

@ -1,8 +1,8 @@
package gay.pizza.pork.stdlib package gay.pizza.pork.stdlib
import gay.pizza.pork.frontend.ContentSource import gay.pizza.pork.frontend.ContentSource
import gay.pizza.pork.parser.CharSource import gay.pizza.pork.tokenizer.CharSource
import gay.pizza.pork.parser.StringCharSource import gay.pizza.pork.tokenizer.StringCharSource
object PorkStdlib : ContentSource { object PorkStdlib : ContentSource {
private val stdlibClass = PorkStdlib::class.java private val stdlibClass = PorkStdlib::class.java

2
support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkBraceMatcher.kt
View File

@ -4,7 +4,7 @@ import com.intellij.lang.BracePair
import com.intellij.lang.PairedBraceMatcher import com.intellij.lang.PairedBraceMatcher
import com.intellij.psi.PsiFile import com.intellij.psi.PsiFile
import com.intellij.psi.tree.IElementType import com.intellij.psi.tree.IElementType
import gay.pizza.pork.parser.TokenType import gay.pizza.pork.tokenizer.TokenType
class PorkBraceMatcher : PairedBraceMatcher { class PorkBraceMatcher : PairedBraceMatcher {
override fun getPairs(): Array<BracePair> = arrayOf( override fun getPairs(): Array<BracePair> = arrayOf(

2
support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkElementTypes.kt
View File

@ -3,7 +3,7 @@ package gay.pizza.pork.idea
import com.intellij.psi.tree.IElementType import com.intellij.psi.tree.IElementType
import com.intellij.psi.tree.TokenSet import com.intellij.psi.tree.TokenSet
import gay.pizza.pork.ast.gen.NodeType import gay.pizza.pork.ast.gen.NodeType
import gay.pizza.pork.parser.TokenType import gay.pizza.pork.tokenizer.TokenType
object PorkElementTypes { object PorkElementTypes {
private val tokenTypeToElementType = mutableMapOf<TokenType, IElementType>() private val tokenTypeToElementType = mutableMapOf<TokenType, IElementType>()

8
support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt
View File

@ -3,10 +3,10 @@ package gay.pizza.pork.idea
import com.intellij.lexer.LexerBase import com.intellij.lexer.LexerBase
import com.intellij.openapi.progress.ProcessCanceledException import com.intellij.openapi.progress.ProcessCanceledException
import com.intellij.psi.tree.IElementType import com.intellij.psi.tree.IElementType
import gay.pizza.pork.parser.BadCharacterError import gay.pizza.pork.tokenizer.BadCharacterError
import gay.pizza.pork.parser.StringCharSource import gay.pizza.pork.tokenizer.StringCharSource
import gay.pizza.pork.parser.Tokenizer import gay.pizza.pork.tokenizer.Tokenizer
import gay.pizza.pork.parser.UnterminatedTokenError import gay.pizza.pork.tokenizer.UnterminatedTokenError
import com.intellij.psi.TokenType as PsiTokenType import com.intellij.psi.TokenType as PsiTokenType
class PorkLexer : LexerBase() { class PorkLexer : LexerBase() {

2
support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParameterInfoHandler.kt
View File

@ -11,7 +11,7 @@ import com.intellij.psi.util.elementsAtOffsetUp
import gay.pizza.pork.idea.psi.gen.ArgumentSpecElement import gay.pizza.pork.idea.psi.gen.ArgumentSpecElement
import gay.pizza.pork.idea.psi.gen.FunctionCallElement import gay.pizza.pork.idea.psi.gen.FunctionCallElement
import gay.pizza.pork.idea.psi.gen.FunctionDefinitionElement import gay.pizza.pork.idea.psi.gen.FunctionDefinitionElement
import gay.pizza.pork.parser.TokenType import gay.pizza.pork.tokenizer.TokenType
@Suppress("UnstableApiUsage") @Suppress("UnstableApiUsage")
class PorkParameterInfoHandler : ParameterInfoHandler<FunctionCallElement, FunctionDefinitionElement> { class PorkParameterInfoHandler : ParameterInfoHandler<FunctionCallElement, FunctionDefinitionElement> {

2
support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParserDefinition.kt
View File

@ -11,7 +11,7 @@ import com.intellij.psi.PsiFile
import com.intellij.psi.tree.IFileElementType import com.intellij.psi.tree.IFileElementType
import com.intellij.psi.tree.TokenSet import com.intellij.psi.tree.TokenSet
import gay.pizza.pork.idea.psi.gen.PorkElementFactory import gay.pizza.pork.idea.psi.gen.PorkElementFactory
import gay.pizza.pork.parser.TokenType import gay.pizza.pork.tokenizer.TokenType
class PorkParserDefinition : ParserDefinition { class PorkParserDefinition : ParserDefinition {
val fileElementType = IFileElementType(PorkLanguage) val fileElementType = IFileElementType(PorkLanguage)

4
support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSyntaxHighlighter.kt
View File

@ -6,8 +6,8 @@ import com.intellij.openapi.editor.colors.TextAttributesKey
import com.intellij.openapi.fileTypes.SyntaxHighlighter import com.intellij.openapi.fileTypes.SyntaxHighlighter
import com.intellij.openapi.fileTypes.SyntaxHighlighterBase import com.intellij.openapi.fileTypes.SyntaxHighlighterBase
import com.intellij.psi.tree.IElementType import com.intellij.psi.tree.IElementType
import gay.pizza.pork.parser.TokenFamily import gay.pizza.pork.tokenizer.TokenFamily
import gay.pizza.pork.parser.TokenType import gay.pizza.pork.tokenizer.TokenType
object PorkSyntaxHighlighter : SyntaxHighlighter { object PorkSyntaxHighlighter : SyntaxHighlighter {
override fun getHighlightingLexer(): Lexer { override fun getHighlightingLexer(): Lexer {

5
support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt
View File

@ -1,7 +1,10 @@
package gay.pizza.pork.idea package gay.pizza.pork.idea
import com.intellij.lang.PsiBuilder import com.intellij.lang.PsiBuilder
import gay.pizza.pork.parser.* import gay.pizza.pork.parser.ParserAwareTokenSource
import gay.pizza.pork.tokenizer.SourceIndex
import gay.pizza.pork.tokenizer.Token
import gay.pizza.pork.tokenizer.TokenType
import com.intellij.psi.TokenType as PsiTokenType import com.intellij.psi.TokenType as PsiTokenType
@Suppress("UnstableApiUsage") @Suppress("UnstableApiUsage")

7
tokenizer/build.gradle.kts Normal file
View File

@ -0,0 +1,7 @@
plugins {
id("gay.pizza.pork.module")
}
dependencies {
implementation(project(":common"))
}

2
parser/src/main/kotlin/gay/pizza/pork/parser/AnsiHighlightScheme.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/AnsiHighlightScheme.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
open class AnsiHighlightScheme : HighlightScheme { open class AnsiHighlightScheme : HighlightScheme {
override fun highlight(token: Token): Highlight { override fun highlight(token: Token): Highlight {

5
tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/BadCharacterError.kt Normal file
View File

@ -0,0 +1,5 @@
package gay.pizza.pork.tokenizer
class BadCharacterError(char: Char, sourceIndex: SourceIndex, state: TokenizerState) : TokenizeError(
"Failed to produce token for '${char}' at $sourceIndex in state $state"
)

2
parser/src/main/kotlin/gay/pizza/pork/parser/CharConsumer.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharConsumer.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
interface CharConsumer { interface CharConsumer {
fun consume(type: TokenType, tokenizer: Tokenizer): String? fun consume(type: TokenType, tokenizer: Tokenizer): String?

2
parser/src/main/kotlin/gay/pizza/pork/parser/CharMatcher.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharMatcher.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
fun interface CharMatcher { fun interface CharMatcher {
fun valid(char: Char, index: Int): Boolean fun valid(char: Char, index: Int): Boolean

2
parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSource.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
interface CharSource : PeekableSource<Char> { interface CharSource : PeekableSource<Char> {
fun peek(index: Int): Char fun peek(index: Int): Char

2
parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSourceExtensions.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
fun CharSource.readToString(): String = buildString { fun CharSource.readToString(): String = buildString {
while (peek() != CharSource.EndOfFile) { while (peek() != CharSource.EndOfFile) {

4
parser/src/main/kotlin/gay/pizza/pork/parser/ExpectedTokenError.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ExpectedTokenError.kt
View File

@ -1,6 +1,6 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class ExpectedTokenError(got: Token, sourceIndex: SourceIndex, vararg expectedTypes: TokenType) : ParseError( class ExpectedTokenError(got: Token, sourceIndex: SourceIndex, vararg expectedTypes: TokenType) : TokenizeError(
message(got, sourceIndex, expectedTypes) message(got, sourceIndex, expectedTypes)
) { ) {
companion object { companion object {

2
parser/src/main/kotlin/gay/pizza/pork/parser/Highlight.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlight.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class Highlight(val token: Token, val text: String? = null) { class Highlight(val token: Token, val text: String? = null) {
override fun toString(): String = text ?: token.text override fun toString(): String = text ?: token.text

2
parser/src/main/kotlin/gay/pizza/pork/parser/HighlightScheme.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/HighlightScheme.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
interface HighlightScheme { interface HighlightScheme {
fun highlight(token: Token): Highlight fun highlight(token: Token): Highlight

2
parser/src/main/kotlin/gay/pizza/pork/parser/Highlighter.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class Highlighter(val scheme: HighlightScheme) { class Highlighter(val scheme: HighlightScheme) {
fun highlight(stream: TokenStream): List<Highlight> = fun highlight(stream: TokenStream): List<Highlight> =

2
parser/src/main/kotlin/gay/pizza/pork/parser/MatchedCharConsumer.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/MatchedCharConsumer.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
@Suppress("CanBeParameter") @Suppress("CanBeParameter")
class MatchedCharConsumer( class MatchedCharConsumer(

2
parser/src/main/kotlin/gay/pizza/pork/parser/PeekableSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/PeekableSource.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
interface PeekableSource<T> { interface PeekableSource<T> {
val currentIndex: Int val currentIndex: Int

2
parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndex.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndex.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
data class SourceIndex(val index: Int, val line: Int, val column: Int, val locationReliable: Boolean = true) { data class SourceIndex(val index: Int, val line: Int, val column: Int, val locationReliable: Boolean = true) {
companion object { companion object {

2
parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndexCharSource.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class SourceIndexCharSource(val delegate: CharSource) : CharSource by delegate { class SourceIndexCharSource(val delegate: CharSource) : CharSource by delegate {
private var currentLineIndex = 1 private var currentLineIndex = 1

2
parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharConsumer.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
object StringCharConsumer : CharConsumer { object StringCharConsumer : CharConsumer {
override fun consume(type: TokenType, tokenizer: Tokenizer): String { override fun consume(type: TokenType, tokenizer: Tokenizer): String {

2
parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharSource.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class StringCharSource( class StringCharSource(
val input: CharSequence, val input: CharSequence,

2
parser/src/main/kotlin/gay/pizza/pork/parser/Token.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Token.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class Token(val type: TokenType, val sourceIndex: SourceIndex, val text: String) { class Token(val type: TokenType, val sourceIndex: SourceIndex, val text: String) {
override fun toString(): String = override fun toString(): String =

2
parser/src/main/kotlin/gay/pizza/pork/parser/TokenFamily.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenFamily.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
enum class TokenFamily : TokenTypeProperty { enum class TokenFamily : TokenTypeProperty {
OperatorFamily, OperatorFamily,

2
parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
interface TokenSource : PeekableSource<Token> { interface TokenSource : PeekableSource<Token> {
fun peekTypeAhead(ahead: Int): TokenType fun peekTypeAhead(ahead: Int): TokenType

2
parser/src/main/kotlin/gay/pizza/pork/parser/TokenStream.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class TokenStream(val tokens: List<Token>) { class TokenStream(val tokens: List<Token>) {
override fun toString(): String = tokens.toString() override fun toString(): String = tokens.toString()

2
parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class TokenStreamSource(val stream: TokenStream) : TokenSource { class TokenStreamSource(val stream: TokenStream) : TokenSource {
private var index = 0 private var index = 0

12
parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenType.kt
View File

@ -1,10 +1,10 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
import gay.pizza.pork.parser.CharMatcher.* import gay.pizza.pork.tokenizer.CharMatcher.*
import gay.pizza.pork.parser.MatchedCharConsumer.Options.AllowEofTermination import gay.pizza.pork.tokenizer.MatchedCharConsumer.Options.AllowEofTermination
import gay.pizza.pork.parser.TokenTypeProperty.* import gay.pizza.pork.tokenizer.TokenTypeProperty.*
import gay.pizza.pork.parser.TokenFamily.* import gay.pizza.pork.tokenizer.TokenFamily.*
import gay.pizza.pork.parser.TokenTypeProperty.AnyOf import gay.pizza.pork.tokenizer.TokenTypeProperty.AnyOf
enum class TokenType(vararg val properties: TokenTypeProperty) { enum class TokenType(vararg val properties: TokenTypeProperty) {
NumberLiteral(NumericLiteralFamily, CharMatch(CharMatcher.AnyOf( NumberLiteral(NumericLiteralFamily, CharMatch(CharMatcher.AnyOf(

2
parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenTypeProperty.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
interface TokenTypeProperty { interface TokenTypeProperty {
class SingleChar(val char: Char) : TokenTypeProperty class SingleChar(val char: Char) : TokenTypeProperty

3
tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenizeError.kt Normal file
View File

@ -0,0 +1,3 @@
package gay.pizza.pork.tokenizer
abstract class TokenizeError(message: String) : RuntimeException(message)

2
parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Tokenizer.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class Tokenizer(source: CharSource) : TokenSource { class Tokenizer(source: CharSource) : TokenSource {
internal val source = SourceIndexCharSource(source) internal val source = SourceIndexCharSource(source)

2
parser/src/main/kotlin/gay/pizza/pork/parser/TokenizerState.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenizerState.kt
View File

@ -1,4 +1,4 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
enum class TokenizerState(vararg val transitions: Transition) { enum class TokenizerState(vararg val transitions: Transition) {
Normal(Transition({ TokenType.Quote }) { StringLiteralStart }), Normal(Transition({ TokenType.Quote }) { StringLiteralStart }),

4
parser/src/main/kotlin/gay/pizza/pork/parser/UnterminatedTokenError.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/UnterminatedTokenError.kt
View File

@ -1,5 +1,5 @@
package gay.pizza.pork.parser package gay.pizza.pork.tokenizer
class UnterminatedTokenError(what: String, sourceIndex: SourceIndex) : ParseError( class UnterminatedTokenError(what: String, sourceIndex: SourceIndex) : TokenizeError(
"Unterminated $what at $sourceIndex" "Unterminated $what at $sourceIndex"
) )

2
tool/src/main/kotlin/gay/pizza/pork/tool/CopyStdlibCommand.kt
View File

@ -7,7 +7,7 @@ import gay.pizza.dough.fs.PlatformFsProvider
import gay.pizza.dough.fs.createDirectories import gay.pizza.dough.fs.createDirectories
import gay.pizza.dough.fs.exists import gay.pizza.dough.fs.exists
import gay.pizza.dough.fs.writeString import gay.pizza.dough.fs.writeString
import gay.pizza.pork.parser.readToString import gay.pizza.pork.tokenizer.readToString
import gay.pizza.pork.stdlib.PorkStdlib import gay.pizza.pork.stdlib.PorkStdlib
class CopyStdlibCommand : CliktCommand(help = "Copy Stdlib", name = "copy-stdlib") { class CopyStdlibCommand : CliktCommand(help = "Copy Stdlib", name = "copy-stdlib") {

2
tool/src/main/kotlin/gay/pizza/pork/tool/HighlightCommand.kt
View File

@ -4,7 +4,7 @@ import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.argument
import gay.pizza.dough.fs.PlatformFsProvider import gay.pizza.dough.fs.PlatformFsProvider
import gay.pizza.pork.minimal.FileTool import gay.pizza.pork.minimal.FileTool
import gay.pizza.pork.parser.AnsiHighlightScheme import gay.pizza.pork.tokenizer.AnsiHighlightScheme
class HighlightCommand : CliktCommand(help = "Syntax Highlighter", name = "highlight") { class HighlightCommand : CliktCommand(help = "Syntax Highlighter", name = "highlight") {
val path by argument("file") val path by argument("file")

2
tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt
View File

@ -4,7 +4,7 @@ import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.argument
import gay.pizza.dough.fs.PlatformFsProvider import gay.pizza.dough.fs.PlatformFsProvider
import gay.pizza.pork.minimal.FileTool import gay.pizza.pork.minimal.FileTool
import gay.pizza.pork.parser.TokenType import gay.pizza.pork.tokenizer.TokenType
class TokenizeCommand : CliktCommand(help = "Tokenize Compilation Unit", name = "tokenize") { class TokenizeCommand : CliktCommand(help = "Tokenize Compilation Unit", name = "tokenize") {
val path by argument("file") val path by argument("file")