diff --git a/src/main/kotlin/gay/pizza/pork/main.kt b/src/main/kotlin/gay/pizza/pork/main.kt index b2d0ae8..d78b8d5 100644 --- a/src/main/kotlin/gay/pizza/pork/main.kt +++ b/src/main/kotlin/gay/pizza/pork/main.kt @@ -4,29 +4,32 @@ import gay.pizza.pork.ast.Program import gay.pizza.pork.eval.Arguments import gay.pizza.pork.eval.PorkEvaluator import gay.pizza.pork.eval.Scope -import gay.pizza.pork.parse.PorkParser -import gay.pizza.pork.parse.PorkTokenizer -import gay.pizza.pork.parse.StringCharSource -import gay.pizza.pork.parse.TokenStreamSource +import gay.pizza.pork.parse.* import kotlin.io.path.Path import kotlin.io.path.readText -fun main(args: Array) { - fun eval(ast: Program) { - val scope = Scope() - val evaluator = PorkEvaluator(scope) - evaluator.visit(ast) - println("> ${scope.call("main", Arguments.Zero)}") - } +fun eval(ast: Program) { + val scope = Scope() + val evaluator = PorkEvaluator(scope) + evaluator.visit(ast) + println("> ${scope.call("main", Arguments.Zero)}") +} +fun main(args: Array) { val code = Path(args[0]).readText() - val stream = PorkTokenizer(StringCharSource(code)).tokenize() + val stream = tokenize(code).excludeAllWhitespace() println(stream.tokens.joinToString("\n")) - val parser = PorkParser(TokenStreamSource(stream)) - val program = parser.readProgram() + val program = parse(stream) eval(program) - val exactStream = PorkTokenizer(StringCharSource(code), preserveWhitespace = true).tokenize() + val exactStream = tokenize(code) val exactCode = exactStream.tokens.joinToString("") { it.text } println(exactCode) + println(code == exactCode) } + +fun tokenize(input: String): TokenStream = + PorkTokenizer(StringCharSource(input)).tokenize() + +fun parse(stream: TokenStream): Program = + PorkParser(TokenStreamSource(stream)).readProgram() diff --git a/src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt b/src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt index 98fcce6..ff3fc24 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt @@ -2,7 +2,9 @@ package gay.pizza.pork.parse import gay.pizza.pork.ast.* -class PorkParser(val source: PeekableSource) { +class PorkParser(source: PeekableSource) { + private val whitespaceIncludedSource = source + private fun readIntLiteral(): IntLiteral { val token = expect(TokenType.IntLiteral) return IntLiteral(token.text.toInt()) @@ -61,7 +63,7 @@ class PorkParser(val source: PeekableSource) { } fun readExpression(): Expression { - val token = source.peek() + val token = peek() val expression = when (token.type) { TokenType.IntLiteral -> { readIntLiteral() @@ -110,8 +112,13 @@ class PorkParser(val source: PeekableSource) { } } - if (peekType(TokenType.Plus, TokenType.Minus, TokenType.Multiply, TokenType.Divide, TokenType.Equality)) { - val infixToken = source.next() + if (peekType( + TokenType.Plus, + TokenType.Minus, + TokenType.Multiply, + TokenType.Divide, + TokenType.Equality)) { + val infixToken = next() val infixOperator = convertInfixOperator(infixToken) return InfixOperation(expression, infixOperator, readExpression()) } @@ -155,15 +162,36 @@ class PorkParser(val source: PeekableSource) { } private fun peekType(vararg types: TokenType): Boolean { - val token = source.peek() + val token = peek() return types.contains(token.type) } private fun expect(type: TokenType): Token { - val token = source.next() + val token = next() if (token.type != type) { throw RuntimeException("Expected token type '${type}' but got type ${token.type} '${token.text}'") } return token } + + private fun next(): Token { + while (true) { + val token = whitespaceIncludedSource.next() + if (token.type == TokenType.Whitespace) { + continue + } + return token + } + } + + private fun peek(): Token { + while (true) { + val token = whitespaceIncludedSource.peek() + if (token.type == TokenType.Whitespace) { + whitespaceIncludedSource.next() + continue + } + return token + } + } } diff --git a/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt b/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt index 304417b..2badaad 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt @@ -1,6 +1,6 @@ package gay.pizza.pork.parse -class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = false) { +class PorkTokenizer(val source: CharSource) { private var tokenStart: Int = 0 private fun isSymbol(c: Char): Boolean = @@ -22,12 +22,12 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa var type = TokenType.Symbol for (keyword in TokenType.Keywords) { - if (symbol == keyword.keyword) { + if (symbol == keyword.keyword?.text) { type = keyword } } - return Token(type, symbol) + return Token(type, tokenStart, symbol) } private fun readIntLiteral(firstChar: Char): Token { @@ -37,7 +37,7 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa append(source.next()) } } - return Token(TokenType.IntLiteral, number) + return Token(TokenType.IntLiteral, tokenStart, number) } private fun readWhitespace(firstChar: Char): Token { @@ -48,7 +48,7 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa append(char) } } - return Token(TokenType.Whitespace, whitespace) + return Token(TokenType.Whitespace, tokenStart, whitespace) } fun next(): Token { @@ -57,12 +57,13 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa val char = source.next() for (item in TokenType.SingleChars) { - if (item.char != char) { + val itemChar = item.singleChar!!.char + if (itemChar != char) { continue } var type = item - var text = item.char.toString() + var text = itemChar.toString() for (promotion in item.promotions) { if (source.peek() != promotion.nextChar) { continue @@ -71,15 +72,11 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa type = promotion.type text += nextChar } - return Token(type, text) + return Token(type, tokenStart, text) } if (isWhitespace(char)) { - val whitespace = readWhitespace(char) - if (preserveWhitespace) { - return whitespace - } - continue + return readWhitespace(char) } if (isDigit(char)) { @@ -91,7 +88,7 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa } throw RuntimeException("Failed to parse: (${char}) next ${source.peek()}") } - return TokenSource.EndOfFile + return Token.endOfFile(source.currentIndex) } fun tokenize(): TokenStream { diff --git a/src/main/kotlin/gay/pizza/pork/parse/Token.kt b/src/main/kotlin/gay/pizza/pork/parse/Token.kt index ba87ee9..184151a 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/Token.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/Token.kt @@ -1,5 +1,10 @@ package gay.pizza.pork.parse -class Token(val type: TokenType, val text: String) { +class Token(val type: TokenType, val start: Int, val text: String) { override fun toString(): String = "${type.name} $text" + + companion object { + fun endOfFile(size: Int): Token = + Token(TokenType.EndOfFile, size, "") + } } diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenPromotion.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenPromotion.kt deleted file mode 100644 index 4fb743c..0000000 --- a/src/main/kotlin/gay/pizza/pork/parse/TokenPromotion.kt +++ /dev/null @@ -1,3 +0,0 @@ -package gay.pizza.pork.parse - -class TokenPromotion(val nextChar: Char, val type: TokenType) diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenSource.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenSource.kt index eebf28e..069e219 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/TokenSource.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/TokenSource.kt @@ -1,7 +1,3 @@ package gay.pizza.pork.parse -interface TokenSource : PeekableSource { - companion object { - val EndOfFile = Token(TokenType.EndOfFile, "") - } -} +interface TokenSource : PeekableSource diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenStream.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenStream.kt index b55fd5b..e415d66 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/TokenStream.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/TokenStream.kt @@ -1,5 +1,8 @@ package gay.pizza.pork.parse class TokenStream(val tokens: List) { + fun excludeAllWhitespace(): TokenStream = + TokenStream(tokens.filter { it.type != TokenType.Whitespace }) + override fun toString(): String = tokens.toString() } diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt index 3719415..85940cc 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt @@ -6,7 +6,7 @@ class TokenStreamSource(val stream: TokenStream) : TokenSource { override fun next(): Token { if (index == stream.tokens.size) { - return TokenSource.EndOfFile + return Token.endOfFile(stream.tokens.size) } val char = stream.tokens[index] index++ @@ -15,7 +15,7 @@ class TokenStreamSource(val stream: TokenStream) : TokenSource { override fun peek(): Token { if (index == stream.tokens.size) { - return TokenSource.EndOfFile + return Token.endOfFile(stream.tokens.size) } return stream.tokens[index] } diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt index a3ace9d..d41ecab 100644 --- a/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt +++ b/src/main/kotlin/gay/pizza/pork/parse/TokenType.kt @@ -1,33 +1,39 @@ package gay.pizza.pork.parse -enum class TokenType(val char: Char? = null, val keyword: String? = null, val promotions: List = emptyList()) { +import gay.pizza.pork.parse.TokenTypeProperty.* + +enum class TokenType(vararg properties: TokenTypeProperty) { Symbol, IntLiteral, Equality, - Equals(char = '=', promotions = listOf(TokenPromotion('=', Equality))), - Plus(char = '+'), - Minus(char = '-'), - Multiply(char = '*'), - Divide(char = '/'), - LeftCurly(char = '{'), - RightCurly(char = '}'), - LeftBracket(char = '['), - RightBracket(char = ']'), - LeftParentheses(char = '('), - RightParentheses(char = ')'), - Negation(char = '!'), - Comma(char = ','), - False(keyword = "false"), - True(keyword = "true"), - In(keyword = "in"), - If(keyword = "if"), - Then(keyword = "then"), - Else(keyword = "else"), + Equals(SingleChar('='), Promotion('=', Equality)), + Plus(SingleChar('+')), + Minus(SingleChar('-')), + Multiply(SingleChar('*')), + Divide(SingleChar('/')), + LeftCurly(SingleChar('{')), + RightCurly(SingleChar('}')), + LeftBracket(SingleChar('[')), + RightBracket(SingleChar(']')), + LeftParentheses(SingleChar('(')), + RightParentheses(SingleChar(')')), + Negation(SingleChar('!')), + Comma(SingleChar(',')), + False(Keyword("false")), + True(Keyword("true")), + In(Keyword("in")), + If(Keyword("if")), + Then(Keyword("then")), + Else(Keyword("else")), Whitespace, EndOfFile; + val promotions: List = properties.filterIsInstance() + val keyword: Keyword? = properties.filterIsInstance().singleOrNull() + val singleChar: SingleChar? = properties.filterIsInstance().singleOrNull() + companion object { - val Keywords = entries.filter { it.keyword != null } - val SingleChars = entries.filter { it.char != null } + val Keywords = entries.filter { item -> item.keyword != null } + val SingleChars = entries.filter { item -> item.singleChar != null } } } diff --git a/src/main/kotlin/gay/pizza/pork/parse/TokenTypeProperty.kt b/src/main/kotlin/gay/pizza/pork/parse/TokenTypeProperty.kt new file mode 100644 index 0000000..f166143 --- /dev/null +++ b/src/main/kotlin/gay/pizza/pork/parse/TokenTypeProperty.kt @@ -0,0 +1,7 @@ +package gay.pizza.pork.parse + +sealed class TokenTypeProperty { + class SingleChar(val char: Char) : TokenTypeProperty() + class Promotion(val nextChar: Char, val type: TokenType) : TokenTypeProperty() + class Keyword(val text: String) : TokenTypeProperty() +}