Gjallarhorn: Implement combined chunk format for storing many chunks in one map.

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/commands/ChunkExportLoaderCommand.kt

@@ -1,17 +1,23 @@
 package cloud.kubelet.foundation.gjallarhorn.commands
 
 import cloud.kubelet.foundation.gjallarhorn.export.ChunkExportLoader
+import cloud.kubelet.foundation.gjallarhorn.export.CombinedChunkFormat
 import cloud.kubelet.foundation.gjallarhorn.state.BlockExpanse
 import cloud.kubelet.foundation.gjallarhorn.state.BlockLogTracker
 import cloud.kubelet.foundation.gjallarhorn.state.ChangelogSlice
+import cloud.kubelet.foundation.gjallarhorn.state.SparseBlockStateMap
 import cloud.kubelet.foundation.gjallarhorn.util.savePngFile
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.core.requireObject
 import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.options.flag
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.types.enum
 import com.github.ajalt.clikt.parameters.types.int
 import com.github.ajalt.clikt.parameters.types.path
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.decodeFromStream
+import kotlinx.serialization.json.encodeToStream
 import org.jetbrains.exposed.sql.Database
 
 class ChunkExportLoaderCommand : CliktCommand("Chunk Export Loader", name = "chunk-export-loader") {
@@ -21,17 +27,35 @@ class ChunkExportLoaderCommand : CliktCommand("Chunk Export Loader", name = "chu
   private val world by argument("world")
   private val chunkLoadLimit by option("--chunk-limit", help = "Chunk Limit").int()
   private val render by option("--render", help = "Render Top Down Image").enum<ImageRenderType> { it.id }
+  private val loadCombinedFormat by option("--load-combined-format").flag()
+  private val saveCombinedFormat by option("--save-combined-format").flag()
 
   override fun run() {
-    val tracker = BlockLogTracker(isConcurrent = true)
+    val format: CombinedChunkFormat
-    val loader = ChunkExportLoader(tracker = tracker)
+
-    loader.loadAllChunksForWorld(exportDirectoryPath, world, fast = true, limit = chunkLoadLimit)
+    val combinedFormatFile = exportDirectoryPath.resolve("combined.json").toFile()
-    if (render != null) {
+    format = if (loadCombinedFormat) {
+      Json.decodeFromStream(CombinedChunkFormat.serializer(), combinedFormatFile.inputStream())
+    } else {
+      val tracker = BlockLogTracker(isConcurrent = true)
+      val loader = ChunkExportLoader(tracker = tracker)
+      loader.loadAllChunksForWorld(exportDirectoryPath, world, fast = true, limit = chunkLoadLimit)
       val expanse = BlockExpanse.zeroOffsetAndMax(tracker.calculateZeroBlockOffset(), tracker.calculateMaxBlock())
       val map = tracker.buildBlockMap(expanse.offset)
-      val renderer = render!!.createNewRenderer(expanse, db)
+      CombinedChunkFormat(expanse, map)
-      val image = renderer.render(ChangelogSlice.none, map)
+    }
+
+    if (render != null) {
+      val renderer = render!!.createNewRenderer(format.expanse, db)
+      val image = renderer.render(ChangelogSlice.none, format.map)
       image.savePngFile("full.png")
     }
+
+    if (saveCombinedFormat) {
+      if (combinedFormatFile.exists()) {
+        combinedFormatFile.delete()
+      }
+      Json.encodeToStream(CombinedChunkFormat.serializer(), format, combinedFormatFile.outputStream())
+    }
   }
 }

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/export/ChunkExportLoader.kt

@@ -1,9 +1,6 @@
 package cloud.kubelet.foundation.gjallarhorn.export
 
-import cloud.kubelet.foundation.gjallarhorn.state.BlockCoordinate
+import cloud.kubelet.foundation.gjallarhorn.state.*
-import cloud.kubelet.foundation.gjallarhorn.state.BlockLogTracker
-import cloud.kubelet.foundation.gjallarhorn.state.BlockState
-import cloud.kubelet.foundation.gjallarhorn.state.SparseBlockStateMap
 import cloud.kubelet.foundation.heimdall.export.ExportedChunk
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.decodeFromStream

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/export/CombinedChunkFormat.kt

@@ -0,0 +1,10 @@
+package cloud.kubelet.foundation.gjallarhorn.export
+
+import cloud.kubelet.foundation.gjallarhorn.state.BlockExpanse
+import cloud.kubelet.foundation.gjallarhorn.state.SparseBlockStateMap
+
+@kotlinx.serialization.Serializable
+class CombinedChunkFormat(
+  val expanse: BlockExpanse,
+  val map: SparseBlockStateMap
+)

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockCoordinate.kt

@@ -1,7 +1,9 @@
 package cloud.kubelet.foundation.gjallarhorn.state
 
 import java.util.*
+import kotlinx.serialization.Serializable
 
+@Serializable
 data class BlockCoordinate(
   val x: Long,
   val y: Long,

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockCoordinateSparseMap.kt

@@ -5,10 +5,10 @@ import cloud.kubelet.foundation.gjallarhorn.util.minOfAll
 import java.util.*
 import kotlin.math.absoluteValue
 
-open class BlockCoordinateSparseMap<T> : BlockCoordinateStore<T> {
+open class BlockCoordinateSparseMap<T>(blocks: Map<Long, Map<Long, Map<Long, T>>> = mutableMapOf()) : BlockCoordinateStore<T> {
-  private var internalBlocks = TreeMap<Long, TreeMap<Long, TreeMap<Long, T>>>()
+  private var internalBlocks = blocks
 
-  val blocks: TreeMap<Long, TreeMap<Long, TreeMap<Long, T>>>
+  val blocks: Map<Long, Map<Long, Map<Long, T>>>
     get() = internalBlocks
 
   override fun get(position: BlockCoordinate): T? = internalBlocks[position.x]?.get(position.z)?.get(position.z)
@@ -16,11 +16,11 @@ open class BlockCoordinateSparseMap<T> : BlockCoordinateStore<T> {
   override fun getXSection(x: Long): Map<Long, Map<Long, T>>? = internalBlocks[x]
 
   override fun put(position: BlockCoordinate, value: T) {
-    internalBlocks.getOrPut(position.x) {
+    (((internalBlocks as MutableMap).getOrPut(position.x) {
-      TreeMap()
+      mutableMapOf()
-    }.getOrPut(position.z) {
+    } as MutableMap).getOrPut(position.z) {
-      TreeMap()
+      mutableMapOf()
-    }[position.y] = value
+    } as MutableMap)[position.y] = value
   }
 
   override fun createOrModify(position: BlockCoordinate, create: () -> T, modify: (T) -> Unit) {
@@ -52,11 +52,11 @@ open class BlockCoordinateSparseMap<T> : BlockCoordinateStore<T> {
   }
 
   fun applyCoordinateOffset(offset: BlockCoordinate) {
-    val root = TreeMap<Long, TreeMap<Long, TreeMap<Long, T>>>()
+    val root = mutableMapOf<Long, MutableMap<Long, MutableMap<Long, T>>>()
     internalBlocks = internalBlocks.map { xSection ->
-      val zSectionMap = TreeMap<Long, TreeMap<Long, T>>()
+      val zSectionMap = mutableMapOf<Long, MutableMap<Long, T>>()
       (xSection.key + offset.x) to xSection.value.map { zSection ->
-        val ySectionMap = TreeMap<Long, T>()
+        val ySectionMap = mutableMapOf<Long, T>()
         (zSection.key + offset.z) to zSection.value.mapKeys {
           (it.key + offset.y)
         }.toMap(ySectionMap)

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockExpanse.kt

@@ -1,6 +1,9 @@
 package cloud.kubelet.foundation.gjallarhorn.state
 
-class BlockExpanse(
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class BlockExpanse(
   val offset: BlockCoordinate,
   val size: BlockCoordinate
 ) {

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockState.kt

@@ -1,8 +1,10 @@
 package cloud.kubelet.foundation.gjallarhorn.state
 
+import kotlinx.serialization.Serializable
 import java.util.concurrent.ConcurrentHashMap
 
-class BlockState(val type: String) {
+@Serializable(BlockStateSerializer::class)
+data class BlockState(val type: String) {
   companion object {
     private val cache = ConcurrentHashMap<String, BlockState>()
 

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockStateSerializer.kt

@@ -0,0 +1,20 @@
+package cloud.kubelet.foundation.gjallarhorn.state
+
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+
+class BlockStateSerializer : KSerializer<BlockState> {
+  override val descriptor: SerialDescriptor
+    get() = String.serializer().descriptor
+
+  override fun deserialize(decoder: Decoder): BlockState {
+    return BlockState.cached(decoder.decodeString())
+  }
+
+  override fun serialize(encoder: Encoder, value: BlockState) {
+    encoder.encodeString(value.type)
+  }
+}

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/SparseBlockStateMap.kt

@@ -1,3 +1,7 @@
 package cloud.kubelet.foundation.gjallarhorn.state
 
-class SparseBlockStateMap : BlockCoordinateSparseMap<BlockState>()
+import kotlinx.serialization.Serializable
+
+@Serializable(SparseBlockStateMapSerializer::class)
+class SparseBlockStateMap(blocks: Map<Long, Map<Long, Map<Long, BlockState>>> = mutableMapOf()) :
+  BlockCoordinateSparseMap<BlockState>(blocks)

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/SparseBlockStateMapSerializer.kt

@@ -0,0 +1,23 @@
+package cloud.kubelet.foundation.gjallarhorn.state
+
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.builtins.MapSerializer
+import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+
+class SparseBlockStateMapSerializer : KSerializer<SparseBlockStateMap> {
+  private val internal = MapSerializer(Long.serializer(), MapSerializer(Long.serializer(), MapSerializer(Long.serializer(), BlockState.serializer())))
+  override val descriptor: SerialDescriptor
+    get() = internal.descriptor
+
+  override fun deserialize(decoder: Decoder): SparseBlockStateMap {
+    val data = internal.deserialize(decoder)
+    return SparseBlockStateMap(data)
+  }
+
+  override fun serialize(encoder: Encoder, value: SparseBlockStateMap) {
+    internal.serialize(encoder, value.blocks)
+  }
+}