Heimdall/Gjallarhorn: Chunk Export Improvements and Chunk Export Renderer

This commit is contained in:
Kenneth Endfinger 2022-02-17 21:37:38 -05:00
parent ac2e99052d
commit 86800e59f4
No known key found for this signature in database
GPG Key ID: C4E68E5647420E10
18 changed files with 272 additions and 54 deletions

View File

@ -8,7 +8,6 @@ import org.bukkit.Server
import org.bukkit.World
import org.bukkit.plugin.Plugin
import java.io.File
import java.util.concurrent.atomic.AtomicBoolean
import java.util.zip.GZIPOutputStream
class ChunkExporter(private val plugin: Plugin, private val server: Server, val world: World) {
@ -21,29 +20,14 @@ class ChunkExporter(private val plugin: Plugin, private val server: Server, val
}
private fun exportChunkListAsync(chunks: List<Chunk>) {
val listOfChunks = chunks.toMutableList()
doExportChunkList(listOfChunks, AtomicBoolean(false))
}
private fun doExportChunkList(chunks: MutableList<Chunk>, check: AtomicBoolean) {
check.set(false)
val chunk = chunks.removeFirstOrNull()
if (chunk == null) {
plugin.slF4JLogger.info("Chunk Export Complete")
return
}
val snapshot = chunk.chunkSnapshot
server.scheduler.runTaskAsynchronously(plugin) { ->
saveChunkSnapshotAndScheduleNext(snapshot, chunks, check)
}
}
private fun saveChunkSnapshotAndScheduleNext(snapshot: ChunkSnapshot, chunks: MutableList<Chunk>, check: AtomicBoolean) {
exportChunkSnapshot(snapshot)
if (!check.getAndSet(true)) {
plugin.server.scheduler.runTask(plugin) { -> doExportChunkList(chunks, check) }
}
plugin.slF4JLogger.info("Exporting ${chunks.size} Chunks")
val snapshots = chunks.map { it.chunkSnapshot }
Thread {
for (snapshot in snapshots) {
exportChunkSnapshot(snapshot)
}
plugin.slF4JLogger.info("Exported ${chunks.size} Chunks")
}.start()
}
private fun exportChunkSnapshot(snapshot: ChunkSnapshot) {

View File

@ -39,7 +39,7 @@ class BlockChangeTimelapseCommand : CliktCommand("Block Change Timelapse", name
help = "Timelapse Change Speed Minimum Interval Seconds"
).int()
private val render by option("--render", help = "Render Top Down Image").enum<RenderType> { it.id }.required()
private val render by option("--render", help = "Render Top Down Image").enum<ImageRenderType> { it.id }.required()
private val considerAirBlocks by option("--consider-air-blocks", help = "Enable Air Block Consideration").flag()
@ -144,16 +144,6 @@ class BlockChangeTimelapseCommand : CliktCommand("Block Change Timelapse", name
return fromBlock to toBlock
}
@Suppress("unused")
enum class RenderType(
val id: String,
val create: (BlockExpanse, Database) -> BlockImageRenderer
) {
BlockDiversity("block-diversity", { expanse, _ -> BlockDiversityRenderer(expanse) }),
HeightMap("height-map", { expanse, _ -> BlockHeightMapRenderer(expanse) }),
PlayerPosition("player-position", { expanse, db -> PlayerLocationShareRenderer(expanse, db) })
}
@Suppress("unused")
enum class TimelapseMode(val id: String, val interval: Duration) {
ByHour("hours", Duration.ofHours(1)),

View File

@ -0,0 +1,36 @@
package cloud.kubelet.foundation.gjallarhorn.commands
import cloud.kubelet.foundation.gjallarhorn.export.ChunkExportLoader
import cloud.kubelet.foundation.gjallarhorn.state.BlockExpanse
import cloud.kubelet.foundation.gjallarhorn.state.ChangelogSlice
import cloud.kubelet.foundation.gjallarhorn.state.SparseBlockStateMap
import cloud.kubelet.foundation.gjallarhorn.util.savePngFile
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.core.requireObject
import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.enum
import com.github.ajalt.clikt.parameters.types.path
import org.jetbrains.exposed.sql.Database
class ChunkExportLoaderCommand : CliktCommand("Chunk Export Loader", name = "chunk-export-loader") {
private val db by requireObject<Database>()
private val exportDirectoryPath by argument("export-directory-path").path()
private val world by argument("world")
private val render by option("--render", help = "Render Top Down Image").enum<ImageRenderType> { it.id }
override fun run() {
val map = SparseBlockStateMap()
val loader = ChunkExportLoader(map)
loader.loadAllChunksForWorld(exportDirectoryPath, world, fast = true)
if (render != null) {
val expanse = BlockExpanse.offsetAndMax(map.calculateZeroBlockOffset(), map.calculateMaxBlock())
map.applyCoordinateOffset(expanse.offset)
val renderer = render!!.create(expanse, db)
val image = renderer.render(ChangelogSlice.none, map)
image.savePngFile("full.png")
}
}
}

View File

@ -0,0 +1,18 @@
package cloud.kubelet.foundation.gjallarhorn.commands
import cloud.kubelet.foundation.gjallarhorn.render.BlockDiversityRenderer
import cloud.kubelet.foundation.gjallarhorn.render.BlockHeightMapRenderer
import cloud.kubelet.foundation.gjallarhorn.render.BlockImageRenderer
import cloud.kubelet.foundation.gjallarhorn.render.PlayerLocationShareRenderer
import cloud.kubelet.foundation.gjallarhorn.state.BlockExpanse
import org.jetbrains.exposed.sql.Database
@Suppress("unused")
enum class ImageRenderType(
val id: String,
val create: (BlockExpanse, Database) -> BlockImageRenderer
) {
BlockDiversity("block-diversity", { expanse, _ -> BlockDiversityRenderer(expanse) }),
HeightMap("height-map", { expanse, _ -> BlockHeightMapRenderer(expanse) }),
PlayerPosition("player-position", { expanse, db -> PlayerLocationShareRenderer(expanse, db) })
}

View File

@ -0,0 +1,53 @@
package cloud.kubelet.foundation.gjallarhorn.export
import cloud.kubelet.foundation.gjallarhorn.state.BlockCoordinate
import cloud.kubelet.foundation.gjallarhorn.state.BlockState
import cloud.kubelet.foundation.gjallarhorn.state.SparseBlockStateMap
import cloud.kubelet.foundation.heimdall.export.ExportedChunk
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
import org.slf4j.LoggerFactory
import java.nio.file.Path
import java.util.zip.GZIPInputStream
import kotlin.io.path.inputStream
import kotlin.io.path.listDirectoryEntries
class ChunkExportLoader(val map: SparseBlockStateMap) {
fun loadAllChunksForWorld(path: Path, world: String, fast: Boolean = false) {
val chunkFiles = path.listDirectoryEntries("${world}_chunk_*.json.gz")
if (fast) {
chunkFiles.parallelStream().forEach { loadChunkFile(it, id = chunkFiles.indexOf(it)) }
} else {
for (filePath in chunkFiles) {
loadChunkFile(filePath, id = chunkFiles.indexOf(filePath))
}
}
}
fun loadChunkFile(path: Path, id: Int = 0) {
val fileInputStream = path.inputStream()
val gzipInputStream = GZIPInputStream(fileInputStream)
val chunk = Json.decodeFromStream(ExportedChunk.serializer(), gzipInputStream)
var blockCount = 0L
for (section in chunk.sections) {
val x = (chunk.x * 16) + section.x
val z = (chunk.z * 16) + section.z
for ((y, block) in section.blocks.withIndex()) {
if (block.type == "minecraft:air") {
continue
}
val coordinate = BlockCoordinate(x.toLong(), y.toLong(), z.toLong())
val state = BlockState.cached(block.type)
map.put(coordinate, state)
blockCount++
}
}
logger.info("($id) Chunk X=${chunk.x} Z=${chunk.z} had $blockCount blocks")
}
companion object {
private val logger = LoggerFactory.getLogger(ChunkExportLoader::class.java)
}
}

View File

@ -1,6 +1,7 @@
package cloud.kubelet.foundation.gjallarhorn
import cloud.kubelet.foundation.gjallarhorn.commands.BlockChangeTimelapseCommand
import cloud.kubelet.foundation.gjallarhorn.commands.ChunkExportLoaderCommand
import cloud.kubelet.foundation.gjallarhorn.commands.PlayerPositionExport
import cloud.kubelet.foundation.gjallarhorn.commands.PlayerSessionExport
import com.github.ajalt.clikt.core.subcommands
@ -8,5 +9,6 @@ import com.github.ajalt.clikt.core.subcommands
fun main(args: Array<String>) = GjallarhornCommand().subcommands(
BlockChangeTimelapseCommand(),
PlayerSessionExport(),
PlayerPositionExport()
PlayerPositionExport(),
ChunkExportLoaderCommand()
).main(args)

View File

@ -2,6 +2,7 @@ package cloud.kubelet.foundation.gjallarhorn.render
import cloud.kubelet.foundation.gjallarhorn.state.BlockExpanse
import cloud.kubelet.foundation.gjallarhorn.state.BlockStateMap
import cloud.kubelet.foundation.gjallarhorn.state.SparseBlockStateMap
import cloud.kubelet.foundation.gjallarhorn.state.ChangelogSlice
import cloud.kubelet.foundation.gjallarhorn.util.BlockColorKey
import cloud.kubelet.foundation.gjallarhorn.util.defaultBlockColorMap
@ -13,7 +14,7 @@ class BlockDiversityRenderer(val expanse: BlockExpanse, quadPixelSize: Int = def
private val blockColorKey = BlockColorKey(defaultBlockColorMap)
override fun render(slice: ChangelogSlice, map: BlockStateMap): BufferedImage = buildPixelQuadImage(expanse) { graphics, x, z ->
val maybeYBlocks = map.blocks[x]?.get(z)
val maybeYBlocks = map.getVerticalSection(x, z)
if (maybeYBlocks == null) {
setPixelQuad(graphics, x, z, Color.white)
return@buildPixelQuadImage

View File

@ -2,6 +2,7 @@ package cloud.kubelet.foundation.gjallarhorn.render
import cloud.kubelet.foundation.gjallarhorn.state.BlockExpanse
import cloud.kubelet.foundation.gjallarhorn.state.BlockStateMap
import cloud.kubelet.foundation.gjallarhorn.state.SparseBlockStateMap
import cloud.kubelet.foundation.gjallarhorn.state.ChangelogSlice
import cloud.kubelet.foundation.gjallarhorn.util.FloatClamp
import java.awt.image.BufferedImage
@ -9,10 +10,11 @@ import java.awt.image.BufferedImage
class BlockHeightMapRenderer(val expanse: BlockExpanse, quadPixelSize: Int = defaultQuadPixelSize) :
BlockHeatMapRenderer(quadPixelSize) {
override fun render(slice: ChangelogSlice, map: BlockStateMap): BufferedImage {
val yMin = map.blocks.minOf { xSection -> xSection.value.minOf { zSection -> zSection.value.minOf { it.key } } }
val yMax = map.blocks.maxOf { xSection -> xSection.value.maxOf { zSection -> zSection.value.maxOf { it.key } } }
val blockMap = map as SparseBlockStateMap
val yMin = blockMap.blocks.minOf { xSection -> xSection.value.minOf { zSection -> zSection.value.minOf { it.key } } }
val yMax = blockMap.blocks.maxOf { xSection -> xSection.value.maxOf { zSection -> zSection.value.maxOf { it.key } } }
val clamp = FloatClamp(yMin, yMax)
return buildHeatMapImage(expanse, clamp) { x, z -> map.blocks[x]?.get(z)?.maxOf { it.key } }
return buildHeatMapImage(expanse, clamp) { x, z -> blockMap.blocks[x]?.get(z)?.maxOf { it.key } }
}
}

View File

@ -0,0 +1,20 @@
package cloud.kubelet.foundation.gjallarhorn.render
import cloud.kubelet.foundation.gjallarhorn.state.BlockExpanse
import cloud.kubelet.foundation.gjallarhorn.state.BlockStateMap
import cloud.kubelet.foundation.gjallarhorn.state.SparseBlockStateMap
import cloud.kubelet.foundation.gjallarhorn.state.ChangelogSlice
import cloud.kubelet.foundation.gjallarhorn.util.FloatClamp
import java.awt.image.BufferedImage
class BlockVerticalFillMapRenderer(val expanse: BlockExpanse, quadPixelSize: Int = defaultQuadPixelSize) :
BlockHeatMapRenderer(quadPixelSize) {
override fun render(slice: ChangelogSlice, map: BlockStateMap): BufferedImage {
val blockMap = map as SparseBlockStateMap
val yMin = blockMap.blocks.minOf { xSection -> xSection.value.minOf { zSection -> zSection.value.size } }
val yMax = blockMap.blocks.maxOf { xSection -> xSection.value.maxOf { zSection -> zSection.value.size } }
val clamp = FloatClamp(yMin.toLong(), yMax.toLong())
return buildHeatMapImage(expanse, clamp) { x, z -> blockMap.blocks[x]?.get(z)?.maxOf { it.key } }
}
}

View File

@ -1,23 +1,29 @@
package cloud.kubelet.foundation.gjallarhorn.state
import cloud.kubelet.foundation.gjallarhorn.util.maxOfAll
import cloud.kubelet.foundation.gjallarhorn.util.minOfAll
import java.util.*
import kotlin.math.absoluteValue
open class BlockCoordinateSparseMap<T> {
val blocks = TreeMap<Long, TreeMap<Long, TreeMap<Long, T>>>()
open class BlockCoordinateSparseMap<T> : BlockCoordinateStore<T> {
private var internalBlocks = TreeMap<Long, TreeMap<Long, TreeMap<Long, T>>>()
fun get(position: BlockCoordinate): T? = blocks[position.x]?.get(position.z)?.get(position.z)
fun getVerticalSection(x: Long, z: Long): Map<Long, T>? = blocks[x]?.get(z)
fun getXSection(x: Long): Map<Long, Map<Long, T>>? = blocks[x]
val blocks: TreeMap<Long, TreeMap<Long, TreeMap<Long, T>>>
get() = internalBlocks
fun put(position: BlockCoordinate, value: T) {
blocks.getOrPut(position.x) {
override fun get(position: BlockCoordinate): T? = internalBlocks[position.x]?.get(position.z)?.get(position.z)
override fun getVerticalSection(x: Long, z: Long): Map<Long, T>? = internalBlocks[x]?.get(z)
override fun getXSection(x: Long): Map<Long, Map<Long, T>>? = internalBlocks[x]
override fun put(position: BlockCoordinate, value: T) {
internalBlocks.getOrPut(position.x) {
TreeMap()
}.getOrPut(position.z) {
TreeMap()
}[position.y] = value
}
fun createOrModify(position: BlockCoordinate, create: () -> T, modify: (T) -> Unit) {
override fun createOrModify(position: BlockCoordinate, create: () -> T, modify: (T) -> Unit) {
val existing = get(position)
if (existing == null) {
put(position, create())
@ -25,4 +31,36 @@ open class BlockCoordinateSparseMap<T> {
modify(existing)
}
}
fun coordinateSequence(): Sequence<BlockCoordinate> = internalBlocks.asSequence().flatMap { x ->
x.value.asSequence().flatMap { z ->
z.value.asSequence().map { y -> BlockCoordinate(x.key, z.key, y.key) }
}
}
fun calculateZeroBlockOffset(): BlockCoordinate {
val (x, y, z) = coordinateSequence().minOfAll(3) { listOf(it.x, it.y, it.z) }
val xOffset = if (x < 0) x.absoluteValue else 0
val yOffset = if (y < 0) y.absoluteValue else 0
val zOffset = if (z < 0) z.absoluteValue else 0
return BlockCoordinate(xOffset, yOffset, zOffset)
}
fun calculateMaxBlock(): BlockCoordinate {
val (x, y, z) = coordinateSequence().maxOfAll(3) { listOf(it.x, it.y, it.z) }
return BlockCoordinate(x, y, z)
}
fun applyCoordinateOffset(offset: BlockCoordinate) {
val root = TreeMap<Long, TreeMap<Long, TreeMap<Long, T>>>()
internalBlocks = internalBlocks.map { xSection ->
val zSectionMap = TreeMap<Long, TreeMap<Long, T>>()
(xSection.key + offset.x) to xSection.value.map { zSection ->
val ySectionMap = TreeMap<Long, T>()
(zSection.key + offset.z) to zSection.value.map { ySection ->
(ySection.key + offset.y) to ySection.value
}.toMap(ySectionMap)
}.toMap(zSectionMap)
}.toMap(root)
}
}

View File

@ -0,0 +1,9 @@
package cloud.kubelet.foundation.gjallarhorn.state
interface BlockCoordinateStore<T> {
fun get(position: BlockCoordinate): T?
fun getVerticalSection(x: Long, z: Long): Map<Long, T>?
fun getXSection(x: Long): Map<Long, Map<Long, T>>?
fun put(position: BlockCoordinate, value: T)
fun createOrModify(position: BlockCoordinate, create: () -> T, modify: (T) -> Unit)
}

View File

@ -1,9 +1,10 @@
package cloud.kubelet.foundation.gjallarhorn.state
import java.util.concurrent.ConcurrentHashMap
import kotlin.math.absoluteValue
class BlockLogTracker(private val mode: BlockTrackMode = BlockTrackMode.RemoveOnDelete) {
val blocks = HashMap<BlockCoordinate, BlockState>()
class BlockLogTracker(private val mode: BlockTrackMode = BlockTrackMode.RemoveOnDelete, isConcurrent: Boolean = false) {
internal val blocks: MutableMap<BlockCoordinate, BlockState> = if (isConcurrent) ConcurrentHashMap() else mutableMapOf()
fun place(position: BlockCoordinate, state: BlockState) {
blocks[position] = state
@ -39,8 +40,8 @@ class BlockLogTracker(private val mode: BlockTrackMode = BlockTrackMode.RemoveOn
fun isEmpty() = blocks.isEmpty()
fun isNotEmpty() = !isEmpty()
fun buildBlockMap(offset: BlockCoordinate = BlockCoordinate.zero): BlockStateMap {
val map = BlockStateMap()
fun buildBlockMap(offset: BlockCoordinate = BlockCoordinate.zero): SparseBlockStateMap {
val map = SparseBlockStateMap()
blocks.forEach { (position, state) ->
val realPosition = offset.applyAsOffset(position)
map.put(realPosition, state)
@ -55,4 +56,6 @@ class BlockLogTracker(private val mode: BlockTrackMode = BlockTrackMode.RemoveOn
place(change.location, change.to)
}
}
fun get(position: BlockCoordinate): BlockState? = blocks[position]
}

View File

@ -0,0 +1,21 @@
package cloud.kubelet.foundation.gjallarhorn.state
class BlockLogTrackerStateMap(val tracker: BlockLogTracker) : BlockStateMap {
override fun get(position: BlockCoordinate): BlockState? = tracker.get(position)
override fun getVerticalSection(x: Long, z: Long): Map<Long, BlockState> {
return tracker.blocks.filter { it.key.x == x && it.key.z == z }.mapKeys { it.key.y }
}
override fun getXSection(x: Long): Map<Long, Map<Long, BlockState>>? {
throw RuntimeException("X section not supported.")
}
override fun put(position: BlockCoordinate, value: BlockState) {
throw RuntimeException("Modification not supported.")
}
override fun createOrModify(position: BlockCoordinate, create: () -> BlockState, modify: (BlockState) -> Unit) {
throw RuntimeException("Modification not supported.")
}
}

View File

@ -1,3 +1,3 @@
package cloud.kubelet.foundation.gjallarhorn.state
class BlockStateMap : BlockCoordinateSparseMap<BlockState>()
typealias BlockStateMap = BlockCoordinateStore<BlockState>

View File

@ -22,4 +22,8 @@ data class ChangelogSlice(val rootStartTime: Instant, val sliceEndTime: Instant,
ChangelogSlice(rootStartTime, sliceEndTime, half)
)
}
companion object {
val none = ChangelogSlice(Instant.MIN, Instant.MIN, Duration.ZERO)
}
}

View File

@ -0,0 +1,3 @@
package cloud.kubelet.foundation.gjallarhorn.state
class SparseBlockStateMap : BlockCoordinateSparseMap<BlockState>()

View File

@ -13,5 +13,6 @@ val defaultBlockColorMap = mapOf<String, Color>(
"minecraft:stone_brick_stairs" to Color.decode("#b8a18c"),
"minecraft:dirt_path" to Color.decode("#8f743d"),
"minecraft:deepslate_tiles" to Color.decode("#49494b"),
"minecraft:spruce_planks" to Color.decode("#60492d")
"minecraft:spruce_planks" to Color.decode("#60492d"),
"minecraft:water" to Color.decode("#1f54ff")
)

View File

@ -0,0 +1,33 @@
package cloud.kubelet.foundation.gjallarhorn.util
fun <T> Sequence<T>.minOfAll(fieldCount: Int, block: (value: T) -> List<Long>): List<Long> {
val fieldRange = 0 until fieldCount
val results = fieldRange.map { Long.MAX_VALUE }.toMutableList()
for (item in this) {
val numerics = block(item)
for (field in fieldRange) {
val current = results[field]
val number = numerics[field]
if (number < current) {
results[field] = number
}
}
}
return results
}
fun <T> Sequence<T>.maxOfAll(fieldCount: Int, block: (value: T) -> List<Long>): List<Long> {
val fieldRange = 0 until fieldCount
val results = fieldRange.map { Long.MIN_VALUE }.toMutableList()
for (item in this) {
val numerics = block(item)
for (field in fieldRange) {
val current = results[field]
val number = numerics[field]
if (number > current) {
results[field] = number
}
}
}
return results
}