mirror of
https://github.com/GayPizzaSpecifications/pork.git
synced 2025-08-02 21:00:56 +00:00
Add AnyOf to the tokenizer
This commit is contained in:
parent
7b931327ff
commit
e56e815f06
@ -230,7 +230,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
|
|||||||
}
|
}
|
||||||
|
|
||||||
override fun parseImportDeclaration(): ImportDeclaration = guarded {
|
override fun parseImportDeclaration(): ImportDeclaration = guarded {
|
||||||
expect(TokenType.Import, TokenType.Impork, TokenType.PorkLoad)
|
expect(TokenType.Import)
|
||||||
val form = parseSymbol()
|
val form = parseSymbol()
|
||||||
val components = oneAndContinuedBy(TokenType.Dot) {
|
val components = oneAndContinuedBy(TokenType.Dot) {
|
||||||
parseSymbol()
|
parseSymbol()
|
||||||
|
@ -55,9 +55,7 @@ enum class TokenType(vararg properties: TokenTypeProperty) {
|
|||||||
In(ManyChars("in"), KeywordFamily),
|
In(ManyChars("in"), KeywordFamily),
|
||||||
Continue(ManyChars("continue"), KeywordFamily),
|
Continue(ManyChars("continue"), KeywordFamily),
|
||||||
Break(ManyChars("break"), KeywordFamily),
|
Break(ManyChars("break"), KeywordFamily),
|
||||||
Import(ManyChars("import"), KeywordFamily),
|
Import(AnyOf("import", "impork", "porkload"), KeywordFamily),
|
||||||
Impork(ManyChars("impork"), KeywordFamily),
|
|
||||||
PorkLoad(ManyChars("porkload"), KeywordFamily),
|
|
||||||
Export(ManyChars("export"), KeywordFamily),
|
Export(ManyChars("export"), KeywordFamily),
|
||||||
Func(ManyChars("func"), KeywordFamily),
|
Func(ManyChars("func"), KeywordFamily),
|
||||||
Native(ManyChars("native"), KeywordFamily),
|
Native(ManyChars("native"), KeywordFamily),
|
||||||
@ -72,6 +70,8 @@ enum class TokenType(vararg properties: TokenTypeProperty) {
|
|||||||
properties.filterIsInstance<Promotion>()
|
properties.filterIsInstance<Promotion>()
|
||||||
val manyChars: ManyChars? =
|
val manyChars: ManyChars? =
|
||||||
properties.filterIsInstance<ManyChars>().singleOrNull()
|
properties.filterIsInstance<ManyChars>().singleOrNull()
|
||||||
|
val anyOf: AnyOf? =
|
||||||
|
properties.filterIsInstance<AnyOf>().singleOrNull()
|
||||||
val singleChar: SingleChar? =
|
val singleChar: SingleChar? =
|
||||||
properties.filterIsInstance<SingleChar>().singleOrNull()
|
properties.filterIsInstance<SingleChar>().singleOrNull()
|
||||||
val family: TokenFamily =
|
val family: TokenFamily =
|
||||||
@ -83,6 +83,7 @@ enum class TokenType(vararg properties: TokenTypeProperty) {
|
|||||||
properties.filterIsInstance<TokenUpgrader>().singleOrNull()
|
properties.filterIsInstance<TokenUpgrader>().singleOrNull()
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
|
val AnyOf = entries.filter { item -> item.anyOf != null }
|
||||||
val ManyChars = entries.filter { item -> item.manyChars != null }
|
val ManyChars = entries.filter { item -> item.manyChars != null }
|
||||||
val SingleChars = entries.filter { item -> item.singleChar != null }
|
val SingleChars = entries.filter { item -> item.singleChar != null }
|
||||||
val CharConsumers = entries.filter { item ->
|
val CharConsumers = entries.filter { item ->
|
||||||
|
@ -4,6 +4,7 @@ interface TokenTypeProperty {
|
|||||||
class SingleChar(val char: Char) : TokenTypeProperty
|
class SingleChar(val char: Char) : TokenTypeProperty
|
||||||
class Promotion(val nextChar: Char, val type: TokenType) : TokenTypeProperty
|
class Promotion(val nextChar: Char, val type: TokenType) : TokenTypeProperty
|
||||||
class ManyChars(val text: String) : TokenTypeProperty
|
class ManyChars(val text: String) : TokenTypeProperty
|
||||||
|
class AnyOf(vararg val strings: String): TokenTypeProperty
|
||||||
class CharConsumer(val isValid: (Char) -> Boolean) : TokenTypeProperty
|
class CharConsumer(val isValid: (Char) -> Boolean) : TokenTypeProperty
|
||||||
class CharIndexConsumer(val isValid: (Char, Int) -> Boolean) : TokenTypeProperty
|
class CharIndexConsumer(val isValid: (Char, Int) -> Boolean) : TokenTypeProperty
|
||||||
open class TokenUpgrader(val maybeUpgrade: (Token) -> Token?) : TokenTypeProperty
|
open class TokenUpgrader(val maybeUpgrade: (Token) -> Token?) : TokenTypeProperty
|
||||||
@ -16,6 +17,14 @@ interface TokenTypeProperty {
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (upgraded == null) {
|
||||||
|
for(item in TokenType.AnyOf) {
|
||||||
|
if(item.anyOf != null && item.anyOf.strings.contains(token.text)) {
|
||||||
|
upgraded = Token(item, token.start, token.text)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
upgraded
|
upgraded
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user