diff --git a/src/main/kotlin/gay/pizza/pork/parse/Highlighter.kt b/src/main/kotlin/gay/pizza/pork/parse/Highlighter.kt new file mode 100644 index 0000000..1f5370d --- /dev/null +++ b/src/main/kotlin/gay/pizza/pork/parse/Highlighter.kt @@ -0,0 +1,19 @@ +package gay.pizza.pork.parse + +abstract class Highlighter(source: TokenSource) : TokenProcessor(source) { + override fun process(token: Token) { + when { + token.type.keyword != null -> { + keyword(token) + } + token.type == TokenType.Symbol -> { + symbol(token) + } + else -> other(token) + } + } + + abstract fun keyword(token: Token) + abstract fun symbol(token: Token) + abstract fun other(token: Token) +} \ No newline at end of file diff --git a/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt b/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt index e656eac..8dc871e 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt @@ -1,6 +1,6 @@ package gay.pizza.pork.parse -class PorkTokenizer(val source: CharSource) { +class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = false) { private var tokenStart: Int = 0 private fun isSymbol(c: Char): Boolean = @@ -40,10 +40,14 @@ class PorkTokenizer(val source: CharSource) { return Token(TokenType.IntLiteral, number) } - private fun skipWhitespace() { - while (isWhitespace(source.peek())) { - source.next() + private fun readWhitespace(): Token { + val whitespace = buildString { + while (isWhitespace(source.peek())) { + val char = source.next() + append(char) + } } + return Token(TokenType.Whitespace, whitespace) } fun next(): Token { @@ -57,7 +61,10 @@ class PorkTokenizer(val source: CharSource) { } if (isWhitespace(char)) { - skipWhitespace() + val whitespace = readWhitespace() + if (preserveWhitespace) { + return whitespace + } continue } diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt new file mode 100644 index 0000000..c9a62e6 --- /dev/null +++ b/src/main/kotlin/gay/pizza/pork/parse/TokenProcessor.kt @@ -0,0 +1,15 @@ +package gay.pizza.pork.parse + +abstract class TokenProcessor(val source: TokenSource) { + fun processAll() { + while (true) { + val token = source.next() + process(token) + if (token.type == TokenType.EndOfFile) { + break + } + } + } + + abstract fun process(token: Token) +} \ No newline at end of file diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt index 83d7c75..80a600d 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt @@ -21,6 +21,7 @@ enum class TokenType(val char: Char? = null, val keyword: String? = null) { If(keyword = "if"), Then(keyword = "then"), Else(keyword = "else"), + Whitespace, EndOfFile; companion object {