strips/pork
2
0
Fork 0
mirror of https://github.com/GayPizzaSpecifications/pork.git synced 2025-08-03 05:10:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ensure that whitespace preservation in the tokenizer produces exact results.

Browse Source
This commit is contained in:
Alex Zenla 2023-08-20 00:54:28 -07:00
parent 72163a2aa6
commit c5d0881893
Signed by: alex
GPG Key ID: C0780728420EBFE5
7 changed files with 15 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/kotlin/gay/pizza/pork/main.kt
View File

@ -1,11 +1,13 @@
package gay.pizza.pork package gay.pizza.pork
import gay.pizza.pork.ast.* import gay.pizza.pork.ast.Program
import gay.pizza.pork.compiler.KotlinCompiler
import gay.pizza.pork.eval.Arguments import gay.pizza.pork.eval.Arguments
import gay.pizza.pork.eval.Scope
import gay.pizza.pork.eval.PorkEvaluator import gay.pizza.pork.eval.PorkEvaluator
import gay.pizza.pork.parse.* import gay.pizza.pork.eval.Scope
import gay.pizza.pork.parse.PorkParser
import gay.pizza.pork.parse.PorkTokenizer
import gay.pizza.pork.parse.StringCharSource
import gay.pizza.pork.parse.TokenStreamSource
import kotlin.io.path.Path import kotlin.io.path.Path
import kotlin.io.path.readText import kotlin.io.path.readText
@ -18,13 +20,13 @@ fun main(args: Array<String>) {
} }
val code = Path(args[0]).readText() val code = Path(args[0]).readText()
val tokenizer = PorkTokenizer(StringCharSource(code)) val stream = PorkTokenizer(StringCharSource(code)).tokenize()
val stream = tokenizer.tokenize()
println(stream.tokens.joinToString("\n")) println(stream.tokens.joinToString("\n"))
val parser = PorkParser(TokenStreamSource(stream)) val parser = PorkParser(TokenStreamSource(stream))
val program = parser.readProgram() val program = parser.readProgram()
eval(program) eval(program)
val kotlinCompiler = KotlinCompiler()
val kotlinCode = kotlinCompiler.visit(program) val exactStream = PorkTokenizer(StringCharSource(code), preserveWhitespace = true).tokenize()
println(kotlinCode) val exactCode = exactStream.tokens.joinToString("") { it.text }
println(exactCode)
} }

1
src/main/kotlin/gay/pizza/pork/parse/PeekableSource.kt
View File

@ -2,7 +2,6 @@ package gay.pizza.pork.parse
interface PeekableSource<T> { interface PeekableSource<T> {
val currentIndex: Int val currentIndex: Int
fun back()
fun next(): T fun next(): T
fun peek(): T fun peek(): T
} }

4
src/main/kotlin/gay/pizza/pork/parse/PorkParser.kt
View File

@ -35,10 +35,6 @@ class PorkParser(val source: PeekableSource<Token>) {
FunctionCall(symbol, arguments) FunctionCall(symbol, arguments)
} else if (peekType(TokenType.Equals)) { } else if (peekType(TokenType.Equals)) {
expect(TokenType.Equals) expect(TokenType.Equals)
if (peekType(TokenType.Equals)) {
source.back()
return SymbolReference(symbol)
}
Define(symbol, readExpression()) Define(symbol, readExpression())
} else { } else {
SymbolReference(symbol) SymbolReference(symbol)

5
src/main/kotlin/gay/pizza/pork/parse/PorkTokenizer.kt
View File

@ -40,8 +40,9 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa
return Token(TokenType.IntLiteral, number) return Token(TokenType.IntLiteral, number)
} }
private fun readWhitespace(): Token { private fun readWhitespace(firstChar: Char): Token {
val whitespace = buildString { val whitespace = buildString {
append(firstChar)
while (isWhitespace(source.peek())) { while (isWhitespace(source.peek())) {
val char = source.next() val char = source.next()
append(char) append(char)
@ -74,7 +75,7 @@ class PorkTokenizer(val source: CharSource, val preserveWhitespace: Boolean = fa
} }
if (isWhitespace(char)) { if (isWhitespace(char)) {
val whitespace = readWhitespace() val whitespace = readWhitespace(char)
if (preserveWhitespace) { if (preserveWhitespace) {
return whitespace return whitespace
} }

3
src/main/kotlin/gay/pizza/pork/parse/StringCharSource.kt
View File

@ -3,9 +3,6 @@ package gay.pizza.pork.parse
class StringCharSource(val input: String) : CharSource { class StringCharSource(val input: String) : CharSource {
private var index = 0 private var index = 0
override val currentIndex: Int = index override val currentIndex: Int = index
override fun back() {
index--
}
override fun next(): Char { override fun next(): Char {
if (index == input.length) { if (index == input.length) {

3
src/main/kotlin/gay/pizza/pork/parse/TokenStreamSource.kt
View File

@ -3,9 +3,6 @@ package gay.pizza.pork.parse
class TokenStreamSource(val stream: TokenStream) : TokenSource { class TokenStreamSource(val stream: TokenStream) : TokenSource {
private var index = 0 private var index = 0
override val currentIndex: Int = index override val currentIndex: Int = index
override fun back() {
index--
}
override fun next(): Token { override fun next(): Token {
if (index == stream.tokens.size) { if (index == stream.tokens.size) {