diff --git a/ffi/src/main/kotlin/gay/pizza/pork/ffi/JavaAutogenContentSource.kt b/ffi/src/main/kotlin/gay/pizza/pork/ffi/JavaAutogenContentSource.kt index 31888a4..3596595 100644 --- a/ffi/src/main/kotlin/gay/pizza/pork/ffi/JavaAutogenContentSource.kt +++ b/ffi/src/main/kotlin/gay/pizza/pork/ffi/JavaAutogenContentSource.kt @@ -2,9 +2,9 @@ package gay.pizza.pork.ffi import gay.pizza.pork.ast.gen.visit import gay.pizza.pork.frontend.ContentSource -import gay.pizza.pork.parser.CharSource +import gay.pizza.pork.tokenizer.CharSource import gay.pizza.pork.parser.Printer -import gay.pizza.pork.parser.StringCharSource +import gay.pizza.pork.tokenizer.StringCharSource object JavaAutogenContentSource : ContentSource { override fun loadAsCharSource(path: String): CharSource { diff --git a/frontend/src/main/kotlin/gay/pizza/pork/frontend/ContentSource.kt b/frontend/src/main/kotlin/gay/pizza/pork/frontend/ContentSource.kt index c4ccc17..0d134fb 100644 --- a/frontend/src/main/kotlin/gay/pizza/pork/frontend/ContentSource.kt +++ b/frontend/src/main/kotlin/gay/pizza/pork/frontend/ContentSource.kt @@ -1,6 +1,6 @@ package gay.pizza.pork.frontend -import gay.pizza.pork.parser.CharSource +import gay.pizza.pork.tokenizer.CharSource interface ContentSource { fun loadAsCharSource(path: String): CharSource diff --git a/frontend/src/main/kotlin/gay/pizza/pork/frontend/FsContentSource.kt b/frontend/src/main/kotlin/gay/pizza/pork/frontend/FsContentSource.kt index 04b8046..5257f60 100644 --- a/frontend/src/main/kotlin/gay/pizza/pork/frontend/FsContentSource.kt +++ b/frontend/src/main/kotlin/gay/pizza/pork/frontend/FsContentSource.kt @@ -3,8 +3,8 @@ package gay.pizza.pork.frontend import gay.pizza.dough.fs.FsPath import gay.pizza.dough.fs.PlatformFsProvider import gay.pizza.dough.fs.readString -import gay.pizza.pork.parser.CharSource -import gay.pizza.pork.parser.StringCharSource +import gay.pizza.pork.tokenizer.CharSource +import gay.pizza.pork.tokenizer.StringCharSource class FsContentSource(val root: FsPath) : ContentSource { override fun loadAsCharSource(path: String): CharSource = diff --git a/frontend/src/main/kotlin/gay/pizza/pork/frontend/World.kt b/frontend/src/main/kotlin/gay/pizza/pork/frontend/World.kt index 0aba1ea..7bfaf56 100644 --- a/frontend/src/main/kotlin/gay/pizza/pork/frontend/World.kt +++ b/frontend/src/main/kotlin/gay/pizza/pork/frontend/World.kt @@ -4,7 +4,7 @@ import gay.pizza.pork.ast.gen.CompilationUnit import gay.pizza.pork.ast.gen.ImportDeclaration import gay.pizza.pork.parser.DiscardNodeAttribution import gay.pizza.pork.parser.Parser -import gay.pizza.pork.parser.Tokenizer +import gay.pizza.pork.tokenizer.Tokenizer class World(val importSource: ImportSource) { private val internalUnits = mutableMapOf() diff --git a/minimal/src/main/kotlin/gay/pizza/pork/minimal/FileTool.kt b/minimal/src/main/kotlin/gay/pizza/pork/minimal/FileTool.kt index 70fd898..fe1ec3a 100644 --- a/minimal/src/main/kotlin/gay/pizza/pork/minimal/FileTool.kt +++ b/minimal/src/main/kotlin/gay/pizza/pork/minimal/FileTool.kt @@ -4,8 +4,8 @@ import gay.pizza.dough.fs.FsPath import gay.pizza.dough.fs.readString import gay.pizza.pork.frontend.ContentSource import gay.pizza.pork.frontend.FsContentSource -import gay.pizza.pork.parser.CharSource -import gay.pizza.pork.parser.StringCharSource +import gay.pizza.pork.tokenizer.CharSource +import gay.pizza.pork.tokenizer.StringCharSource class FileTool(val path: FsPath) : Tool() { override fun createCharSource(): CharSource = diff --git a/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt b/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt index 8228733..d4e70d0 100644 --- a/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt +++ b/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt @@ -13,6 +13,7 @@ import gay.pizza.pork.frontend.DynamicImportSource import gay.pizza.pork.frontend.World import gay.pizza.pork.parser.* import gay.pizza.pork.stdlib.PorkStdlib +import gay.pizza.pork.tokenizer.* abstract class Tool { abstract fun createCharSource(): CharSource diff --git a/parser/build.gradle.kts b/parser/build.gradle.kts index 4394faa..36f358f 100644 --- a/parser/build.gradle.kts +++ b/parser/build.gradle.kts @@ -4,6 +4,7 @@ plugins { dependencies { api(project(":ast")) + api(project(":tokenizer")) implementation(project(":common")) } diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/BadCharacterError.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/BadCharacterError.kt deleted file mode 100644 index d254c2c..0000000 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/BadCharacterError.kt +++ /dev/null @@ -1,5 +0,0 @@ -package gay.pizza.pork.parser - -class BadCharacterError(val char: Char, sourceIndex: SourceIndex, state: TokenizerState) : ParseError( - "Failed to produce token for '${char}' at $sourceIndex in state $state" -) diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/DiscardNodeAttribution.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/DiscardNodeAttribution.kt index a6b6d00..02bd1f4 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/DiscardNodeAttribution.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/DiscardNodeAttribution.kt @@ -2,6 +2,7 @@ package gay.pizza.pork.parser import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.NodeType +import gay.pizza.pork.tokenizer.Token object DiscardNodeAttribution : NodeAttribution { override fun push(token: Token) {} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/LazySkippingTokenSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/LazySkippingTokenSource.kt index fec0cf3..061c784 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/LazySkippingTokenSource.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/LazySkippingTokenSource.kt @@ -1,5 +1,9 @@ package gay.pizza.pork.parser +import gay.pizza.pork.tokenizer.Token +import gay.pizza.pork.tokenizer.TokenSource +import gay.pizza.pork.tokenizer.TokenType + class LazySkippingTokenSource(val source: TokenSource, val skipping: Set) : ParserAwareTokenSource { private var index = 0 override val currentIndex: Int diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/NodeAttribution.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/NodeAttribution.kt index 63bd49a..4cfdcc1 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/NodeAttribution.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/NodeAttribution.kt @@ -2,6 +2,7 @@ package gay.pizza.pork.parser import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.NodeType +import gay.pizza.pork.tokenizer.Token interface NodeAttribution { fun push(token: Token) diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt index f69d1c7..56cdc20 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt @@ -1,6 +1,8 @@ package gay.pizza.pork.parser import gay.pizza.pork.ast.gen.* +import gay.pizza.pork.tokenizer.TokenSource +import gay.pizza.pork.tokenizer.TokenType class Parser(source: TokenSource, attribution: NodeAttribution) : ParserBase(source, attribution) { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt index e50c021..ae44bc6 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt @@ -4,6 +4,7 @@ import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.NodeCoalescer import gay.pizza.pork.ast.gen.data import gay.pizza.pork.ast.gen.visit +import gay.pizza.pork.tokenizer.Token data class ParserAttributes(val tokens: List) { companion object { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAwareTokenSource.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAwareTokenSource.kt index 60456db..f06ca34 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAwareTokenSource.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserAwareTokenSource.kt @@ -1,3 +1,5 @@ package gay.pizza.pork.parser +import gay.pizza.pork.tokenizer.TokenSource + interface ParserAwareTokenSource : TokenSource diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt index c2e293e..032661c 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt @@ -3,6 +3,7 @@ package gay.pizza.pork.parser import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.NodeParser import gay.pizza.pork.ast.gen.NodeType +import gay.pizza.pork.tokenizer.* abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution) : NodeParser { val source: TokenSource = if (source is ParserAwareTokenSource) { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt index 4fa5423..5ec6802 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt @@ -3,6 +3,8 @@ package gay.pizza.pork.parser import gay.pizza.pork.ast.gen.InfixOperator import gay.pizza.pork.ast.gen.PrefixOperator import gay.pizza.pork.ast.gen.SuffixOperator +import gay.pizza.pork.tokenizer.Token +import gay.pizza.pork.tokenizer.TokenType internal object ParserHelpers { fun convertInfixOperator(token: Token): InfixOperator = when (token.type) { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserNodeAttribution.kt b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserNodeAttribution.kt index fd946d3..a41f822 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ParserNodeAttribution.kt +++ b/parser/src/main/kotlin/gay/pizza/pork/parser/ParserNodeAttribution.kt @@ -3,6 +3,7 @@ package gay.pizza.pork.parser import gay.pizza.pork.ast.gen.Node import gay.pizza.pork.ast.gen.NodeType import gay.pizza.pork.ast.gen.data +import gay.pizza.pork.tokenizer.Token open class ParserNodeAttribution : NodeAttribution { private val stack = mutableListOf>() diff --git a/settings.gradle.kts b/settings.gradle.kts index 5d5b50f..4191bba 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -4,6 +4,7 @@ includeBuild("buildext") include( ":common", + ":tokenizer", ":ast", ":parser", ":frontend", diff --git a/stdlib/src/main/kotlin/gay/pizza/pork/stdlib/PorkStdlib.kt b/stdlib/src/main/kotlin/gay/pizza/pork/stdlib/PorkStdlib.kt index 240aa80..6ae7c61 100644 --- a/stdlib/src/main/kotlin/gay/pizza/pork/stdlib/PorkStdlib.kt +++ b/stdlib/src/main/kotlin/gay/pizza/pork/stdlib/PorkStdlib.kt @@ -1,8 +1,8 @@ package gay.pizza.pork.stdlib import gay.pizza.pork.frontend.ContentSource -import gay.pizza.pork.parser.CharSource -import gay.pizza.pork.parser.StringCharSource +import gay.pizza.pork.tokenizer.CharSource +import gay.pizza.pork.tokenizer.StringCharSource object PorkStdlib : ContentSource { private val stdlibClass = PorkStdlib::class.java diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkBraceMatcher.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkBraceMatcher.kt index bc5ee42..559db08 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkBraceMatcher.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkBraceMatcher.kt @@ -4,7 +4,7 @@ import com.intellij.lang.BracePair import com.intellij.lang.PairedBraceMatcher import com.intellij.psi.PsiFile import com.intellij.psi.tree.IElementType -import gay.pizza.pork.parser.TokenType +import gay.pizza.pork.tokenizer.TokenType class PorkBraceMatcher : PairedBraceMatcher { override fun getPairs(): Array = arrayOf( diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkElementTypes.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkElementTypes.kt index 9703188..e1bf8c4 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkElementTypes.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkElementTypes.kt @@ -3,7 +3,7 @@ package gay.pizza.pork.idea import com.intellij.psi.tree.IElementType import com.intellij.psi.tree.TokenSet import gay.pizza.pork.ast.gen.NodeType -import gay.pizza.pork.parser.TokenType +import gay.pizza.pork.tokenizer.TokenType object PorkElementTypes { private val tokenTypeToElementType = mutableMapOf() diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt index ed1b2fa..a68180e 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt @@ -3,10 +3,10 @@ package gay.pizza.pork.idea import com.intellij.lexer.LexerBase import com.intellij.openapi.progress.ProcessCanceledException import com.intellij.psi.tree.IElementType -import gay.pizza.pork.parser.BadCharacterError -import gay.pizza.pork.parser.StringCharSource -import gay.pizza.pork.parser.Tokenizer -import gay.pizza.pork.parser.UnterminatedTokenError +import gay.pizza.pork.tokenizer.BadCharacterError +import gay.pizza.pork.tokenizer.StringCharSource +import gay.pizza.pork.tokenizer.Tokenizer +import gay.pizza.pork.tokenizer.UnterminatedTokenError import com.intellij.psi.TokenType as PsiTokenType class PorkLexer : LexerBase() { diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParameterInfoHandler.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParameterInfoHandler.kt index 3da7526..c0e4e74 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParameterInfoHandler.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParameterInfoHandler.kt @@ -11,7 +11,7 @@ import com.intellij.psi.util.elementsAtOffsetUp import gay.pizza.pork.idea.psi.gen.ArgumentSpecElement import gay.pizza.pork.idea.psi.gen.FunctionCallElement import gay.pizza.pork.idea.psi.gen.FunctionDefinitionElement -import gay.pizza.pork.parser.TokenType +import gay.pizza.pork.tokenizer.TokenType @Suppress("UnstableApiUsage") class PorkParameterInfoHandler : ParameterInfoHandler { diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParserDefinition.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParserDefinition.kt index 25df9f2..5484e72 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParserDefinition.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParserDefinition.kt @@ -11,7 +11,7 @@ import com.intellij.psi.PsiFile import com.intellij.psi.tree.IFileElementType import com.intellij.psi.tree.TokenSet import gay.pizza.pork.idea.psi.gen.PorkElementFactory -import gay.pizza.pork.parser.TokenType +import gay.pizza.pork.tokenizer.TokenType class PorkParserDefinition : ParserDefinition { val fileElementType = IFileElementType(PorkLanguage) diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSyntaxHighlighter.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSyntaxHighlighter.kt index 3550b50..fcb5588 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSyntaxHighlighter.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSyntaxHighlighter.kt @@ -6,8 +6,8 @@ import com.intellij.openapi.editor.colors.TextAttributesKey import com.intellij.openapi.fileTypes.SyntaxHighlighter import com.intellij.openapi.fileTypes.SyntaxHighlighterBase import com.intellij.psi.tree.IElementType -import gay.pizza.pork.parser.TokenFamily -import gay.pizza.pork.parser.TokenType +import gay.pizza.pork.tokenizer.TokenFamily +import gay.pizza.pork.tokenizer.TokenType object PorkSyntaxHighlighter : SyntaxHighlighter { override fun getHighlightingLexer(): Lexer { diff --git a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt index 9cad63b..aed8933 100644 --- a/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt +++ b/support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt @@ -1,7 +1,10 @@ package gay.pizza.pork.idea import com.intellij.lang.PsiBuilder -import gay.pizza.pork.parser.* +import gay.pizza.pork.parser.ParserAwareTokenSource +import gay.pizza.pork.tokenizer.SourceIndex +import gay.pizza.pork.tokenizer.Token +import gay.pizza.pork.tokenizer.TokenType import com.intellij.psi.TokenType as PsiTokenType @Suppress("UnstableApiUsage") diff --git a/tokenizer/build.gradle.kts b/tokenizer/build.gradle.kts new file mode 100644 index 0000000..55f1ba8 --- /dev/null +++ b/tokenizer/build.gradle.kts @@ -0,0 +1,7 @@ +plugins { + id("gay.pizza.pork.module") +} + +dependencies { + implementation(project(":common")) +} diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/AnsiHighlightScheme.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/AnsiHighlightScheme.kt similarity index 96% rename from parser/src/main/kotlin/gay/pizza/pork/parser/AnsiHighlightScheme.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/AnsiHighlightScheme.kt index 4767a56..f70170f 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/AnsiHighlightScheme.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/AnsiHighlightScheme.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer open class AnsiHighlightScheme : HighlightScheme { override fun highlight(token: Token): Highlight { diff --git a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/BadCharacterError.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/BadCharacterError.kt new file mode 100644 index 0000000..f82e33b --- /dev/null +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/BadCharacterError.kt @@ -0,0 +1,5 @@ +package gay.pizza.pork.tokenizer + +class BadCharacterError(char: Char, sourceIndex: SourceIndex, state: TokenizerState) : TokenizeError( + "Failed to produce token for '${char}' at $sourceIndex in state $state" +) diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/CharConsumer.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharConsumer.kt similarity index 73% rename from parser/src/main/kotlin/gay/pizza/pork/parser/CharConsumer.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharConsumer.kt index da59bb6..2a60604 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/CharConsumer.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharConsumer.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer interface CharConsumer { fun consume(type: TokenType, tokenizer: Tokenizer): String? diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/CharMatcher.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharMatcher.kt similarity index 95% rename from parser/src/main/kotlin/gay/pizza/pork/parser/CharMatcher.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharMatcher.kt index 2044812..2d252f7 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/CharMatcher.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharMatcher.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer fun interface CharMatcher { fun valid(char: Char, index: Int): Boolean diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSource.kt similarity index 84% rename from parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSource.kt index c095dea..dd489c9 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSource.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer interface CharSource : PeekableSource { fun peek(index: Int): Char diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSourceExtensions.kt similarity index 78% rename from parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSourceExtensions.kt index 307a9de..ff0f709 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSourceExtensions.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer fun CharSource.readToString(): String = buildString { while (peek() != CharSource.EndOfFile) { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/ExpectedTokenError.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ExpectedTokenError.kt similarity index 91% rename from parser/src/main/kotlin/gay/pizza/pork/parser/ExpectedTokenError.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ExpectedTokenError.kt index 4be4f1e..bf327d5 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/ExpectedTokenError.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ExpectedTokenError.kt @@ -1,6 +1,6 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer -class ExpectedTokenError(got: Token, sourceIndex: SourceIndex, vararg expectedTypes: TokenType) : ParseError( +class ExpectedTokenError(got: Token, sourceIndex: SourceIndex, vararg expectedTypes: TokenType) : TokenizeError( message(got, sourceIndex, expectedTypes) ) { companion object { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Highlight.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlight.kt similarity index 78% rename from parser/src/main/kotlin/gay/pizza/pork/parser/Highlight.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlight.kt index 303310d..362fd97 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Highlight.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlight.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer class Highlight(val token: Token, val text: String? = null) { override fun toString(): String = text ?: token.text diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/HighlightScheme.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/HighlightScheme.kt similarity index 68% rename from parser/src/main/kotlin/gay/pizza/pork/parser/HighlightScheme.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/HighlightScheme.kt index 30e1a14..370dd2a 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/HighlightScheme.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/HighlightScheme.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer interface HighlightScheme { fun highlight(token: Token): Highlight diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Highlighter.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt similarity index 82% rename from parser/src/main/kotlin/gay/pizza/pork/parser/Highlighter.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt index 18dd782..a2a51bf 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Highlighter.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer class Highlighter(val scheme: HighlightScheme) { fun highlight(stream: TokenStream): List = diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/MatchedCharConsumer.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/MatchedCharConsumer.kt similarity index 97% rename from parser/src/main/kotlin/gay/pizza/pork/parser/MatchedCharConsumer.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/MatchedCharConsumer.kt index 5f55127..156034b 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/MatchedCharConsumer.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/MatchedCharConsumer.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer @Suppress("CanBeParameter") class MatchedCharConsumer( diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/PeekableSource.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/PeekableSource.kt similarity index 72% rename from parser/src/main/kotlin/gay/pizza/pork/parser/PeekableSource.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/PeekableSource.kt index fda5bc5..3d57023 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/PeekableSource.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/PeekableSource.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer interface PeekableSource { val currentIndex: Int diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndex.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndex.kt similarity index 91% rename from parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndex.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndex.kt index 6e94c00..efc53a3 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndex.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndex.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer data class SourceIndex(val index: Int, val line: Int, val column: Int, val locationReliable: Boolean = true) { companion object { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndexCharSource.kt similarity index 93% rename from parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndexCharSource.kt index 709601a..f8a8646 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndexCharSource.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer class SourceIndexCharSource(val delegate: CharSource) : CharSource by delegate { private var currentLineIndex = 1 diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharConsumer.kt similarity index 94% rename from parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharConsumer.kt index d6016ed..7b61473 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharConsumer.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer object StringCharConsumer : CharConsumer { override fun consume(type: TokenType, tokenizer: Tokenizer): String { diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharSource.kt similarity index 95% rename from parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharSource.kt index a7188b9..997f768 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharSource.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer class StringCharSource( val input: CharSequence, diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Token.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Token.kt similarity index 92% rename from parser/src/main/kotlin/gay/pizza/pork/parser/Token.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Token.kt index 514ac1f..2e2902f 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Token.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Token.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer class Token(val type: TokenType, val sourceIndex: SourceIndex, val text: String) { override fun toString(): String = diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenFamily.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenFamily.kt similarity index 84% rename from parser/src/main/kotlin/gay/pizza/pork/parser/TokenFamily.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenFamily.kt index 8c1795d..b613fd0 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenFamily.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenFamily.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer enum class TokenFamily : TokenTypeProperty { OperatorFamily, diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt similarity index 92% rename from parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt index 0814f74..783d71d 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer interface TokenSource : PeekableSource { fun peekTypeAhead(ahead: Int): TokenType diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStream.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt similarity index 75% rename from parser/src/main/kotlin/gay/pizza/pork/parser/TokenStream.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt index 4ee6df4..5cad828 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStream.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer class TokenStream(val tokens: List) { override fun toString(): String = tokens.toString() diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt similarity index 95% rename from parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt index a6a46b4..4407bc0 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer class TokenStreamSource(val stream: TokenStream) : TokenSource { private var index = 0 diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenType.kt similarity index 93% rename from parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenType.kt index 2a045c2..32bd769 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenType.kt @@ -1,10 +1,10 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer -import gay.pizza.pork.parser.CharMatcher.* -import gay.pizza.pork.parser.MatchedCharConsumer.Options.AllowEofTermination -import gay.pizza.pork.parser.TokenTypeProperty.* -import gay.pizza.pork.parser.TokenFamily.* -import gay.pizza.pork.parser.TokenTypeProperty.AnyOf +import gay.pizza.pork.tokenizer.CharMatcher.* +import gay.pizza.pork.tokenizer.MatchedCharConsumer.Options.AllowEofTermination +import gay.pizza.pork.tokenizer.TokenTypeProperty.* +import gay.pizza.pork.tokenizer.TokenFamily.* +import gay.pizza.pork.tokenizer.TokenTypeProperty.AnyOf enum class TokenType(vararg val properties: TokenTypeProperty) { NumberLiteral(NumericLiteralFamily, CharMatch(CharMatcher.AnyOf( diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenTypeProperty.kt similarity index 97% rename from parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenTypeProperty.kt index 878f79d..e89ee01 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenTypeProperty.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer interface TokenTypeProperty { class SingleChar(val char: Char) : TokenTypeProperty diff --git a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenizeError.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenizeError.kt new file mode 100644 index 0000000..acdf1a8 --- /dev/null +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenizeError.kt @@ -0,0 +1,3 @@ +package gay.pizza.pork.tokenizer + +abstract class TokenizeError(message: String) : RuntimeException(message) diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Tokenizer.kt similarity index 99% rename from parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Tokenizer.kt index 69a26ed..3bb045a 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/Tokenizer.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Tokenizer.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer class Tokenizer(source: CharSource) : TokenSource { internal val source = SourceIndexCharSource(source) diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenizerState.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenizerState.kt similarity index 93% rename from parser/src/main/kotlin/gay/pizza/pork/parser/TokenizerState.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenizerState.kt index 3f0f08a..873f92b 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/TokenizerState.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenizerState.kt @@ -1,4 +1,4 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer enum class TokenizerState(vararg val transitions: Transition) { Normal(Transition({ TokenType.Quote }) { StringLiteralStart }), diff --git a/parser/src/main/kotlin/gay/pizza/pork/parser/UnterminatedTokenError.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/UnterminatedTokenError.kt similarity index 65% rename from parser/src/main/kotlin/gay/pizza/pork/parser/UnterminatedTokenError.kt rename to tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/UnterminatedTokenError.kt index 65fdba3..eb8f3db 100644 --- a/parser/src/main/kotlin/gay/pizza/pork/parser/UnterminatedTokenError.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/UnterminatedTokenError.kt @@ -1,5 +1,5 @@ -package gay.pizza.pork.parser +package gay.pizza.pork.tokenizer -class UnterminatedTokenError(what: String, sourceIndex: SourceIndex) : ParseError( +class UnterminatedTokenError(what: String, sourceIndex: SourceIndex) : TokenizeError( "Unterminated $what at $sourceIndex" ) diff --git a/tool/src/main/kotlin/gay/pizza/pork/tool/CopyStdlibCommand.kt b/tool/src/main/kotlin/gay/pizza/pork/tool/CopyStdlibCommand.kt index 3682cdc..849cd1f 100644 --- a/tool/src/main/kotlin/gay/pizza/pork/tool/CopyStdlibCommand.kt +++ b/tool/src/main/kotlin/gay/pizza/pork/tool/CopyStdlibCommand.kt @@ -7,7 +7,7 @@ import gay.pizza.dough.fs.PlatformFsProvider import gay.pizza.dough.fs.createDirectories import gay.pizza.dough.fs.exists import gay.pizza.dough.fs.writeString -import gay.pizza.pork.parser.readToString +import gay.pizza.pork.tokenizer.readToString import gay.pizza.pork.stdlib.PorkStdlib class CopyStdlibCommand : CliktCommand(help = "Copy Stdlib", name = "copy-stdlib") { diff --git a/tool/src/main/kotlin/gay/pizza/pork/tool/HighlightCommand.kt b/tool/src/main/kotlin/gay/pizza/pork/tool/HighlightCommand.kt index 97754a9..7494313 100644 --- a/tool/src/main/kotlin/gay/pizza/pork/tool/HighlightCommand.kt +++ b/tool/src/main/kotlin/gay/pizza/pork/tool/HighlightCommand.kt @@ -4,7 +4,7 @@ import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import gay.pizza.dough.fs.PlatformFsProvider import gay.pizza.pork.minimal.FileTool -import gay.pizza.pork.parser.AnsiHighlightScheme +import gay.pizza.pork.tokenizer.AnsiHighlightScheme class HighlightCommand : CliktCommand(help = "Syntax Highlighter", name = "highlight") { val path by argument("file") diff --git a/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt b/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt index c91e791..d304bf5 100644 --- a/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt +++ b/tool/src/main/kotlin/gay/pizza/pork/tool/TokenizeCommand.kt @@ -4,7 +4,7 @@ import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import gay.pizza.dough.fs.PlatformFsProvider import gay.pizza.pork.minimal.FileTool -import gay.pizza.pork.parser.TokenType +import gay.pizza.pork.tokenizer.TokenType class TokenizeCommand : CliktCommand(help = "Tokenize Compilation Unit", name = "tokenize") { val path by argument("file")