Gjallarhorn: Introduce concept of block changelogs, which makes timelapse rendering more efficient.

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/commands/BlockLogReplay.kt

@@ -14,17 +14,15 @@ import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.options.required
 import com.github.ajalt.clikt.parameters.types.enum
 import com.github.ajalt.clikt.parameters.types.int
-import jetbrains.exodus.kotlin.notNull
+import org.jetbrains.exposed.sql.Database
-import org.jetbrains.exposed.sql.*
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.lessEq
-import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.and
 import org.slf4j.LoggerFactory
 import java.time.Duration
 import java.time.Instant
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.ScheduledThreadPoolExecutor
 import java.util.concurrent.TimeUnit
-import java.util.concurrent.atomic.AtomicLong
 
 class BlockLogReplay : CliktCommand("Replay Block Logs", name = "replay-block-log") {
   private val db by requireObject<Database>()
@@ -42,13 +40,8 @@ class BlockLogReplay : CliktCommand("Replay Block Logs", name = "replay-block-lo
 
   override fun run() {
     if (timelapseMode != null) {
-      val (start, end) = transaction(db) {
+      val changelog = BlockChangelog.query(db)
-        val minTimeColumn = BlockChangeView.time.min().notNull
+      val (start, end) = changelog.changeTimeRange
-        val maxTimeColumn = BlockChangeView.time.max().notNull
-        val row = BlockChangeView.slice(minTimeColumn, maxTimeColumn).selectAll().single()
-        row[minTimeColumn]!! to row[maxTimeColumn]!!
-      }
-
       var intervals = mutableListOf<Instant>()
       var current = start
       while (!current.isAfter(end)) {
@@ -65,7 +58,8 @@ class BlockLogReplay : CliktCommand("Replay Block Logs", name = "replay-block-lo
       for (time in intervals) {
         trackerPool.submit {
           val index = intervals.indexOf(time) + 1
-          val tracker = buildTrackerState(time, "Timelapse-${index}")
+          val tracker =
+            buildTrackerState(changelog.slice(time.minus(timelapseMode!!.interval) to time), "Timelapse-${index}")
           if (tracker.isEmpty()) {
             return@submit
           }
@@ -102,51 +96,33 @@ class BlockLogReplay : CliktCommand("Replay Block Logs", name = "replay-block-lo
       logger.info("Rendering Completed")
     } else {
       val time = if (exactTimeAsString != null) Instant.parse(exactTimeAsString) else null
-      val tracker = buildTrackerState(time, "Single-Time")
+      val filter = compose(
+        combine = { a, b -> a and b },
+        { time != null } to { BlockChangeView.time lessEq time!! }
+      )
+      val changelog = BlockChangelog.query(db, filter)
+      val tracker = buildTrackerState(changelog, "Single-Time")
       val expanse = BlockExpanse.offsetAndMax(tracker.calculateZeroBlockOffset(), tracker.calculateMaxBlock())
       saveRenderImage(render.create(expanse), tracker, expanse)
     }
   }
 
-  fun saveRenderImage(renderer: BlockImageRenderer, tracker: BlockLogTracker, expanse: BlockExpanse, suffix: String = "") {
+  fun saveRenderImage(
+    renderer: BlockImageRenderer,
+    tracker: BlockLogTracker,
+    expanse: BlockExpanse,
+    suffix: String = ""
+  ) {
     val map = tracker.buildBlockMap(expanse.offset)
     val image = renderer.render(map)
     image.savePngFile("${render.id}${suffix}.png")
   }
 
-  fun buildTrackerState(time: Instant?, job: String): BlockLogTracker {
+  fun buildTrackerState(changelog: BlockChangelog, job: String): BlockLogTracker {
-    val filter = compose(
-      combine = { a, b -> a and b },
-      { time != null } to { BlockChangeView.time lessEq time!! }
-    )
-
     val tracker =
       BlockLogTracker(if (considerAirBlocks) BlockTrackMode.AirOnDelete else BlockTrackMode.RemoveOnDelete)
-
+    tracker.replay(changelog)
-    val blockChangeCounter = AtomicLong()
+    logger.info("Job $job Total Block Changes... ${changelog.changes.size}")
-    transaction(db) {
-      BlockChangeView.select(filter).orderBy(BlockChangeView.time).forEach { row ->
-        val changeIsBreak = row[BlockChangeView.isBreak]
-        val x = row[BlockChangeView.x]
-        val y = row[BlockChangeView.y]
-        val z = row[BlockChangeView.z]
-        val block = row[BlockChangeView.block]
-
-        val location = BlockCoordinate(x.toLong(), y.toLong(), z.toLong())
-        if (changeIsBreak) {
-          tracker.delete(location)
-        } else {
-          tracker.place(location, BlockState(block))
-        }
-
-        val count = blockChangeCounter.addAndGet(1)
-        if (count % 1000L == 0L) {
-          logger.info("Job $job Calculating Block Changes... $count")
-        }
-      }
-    }
-    logger.info("Job $job Total Block Changes... ${blockChangeCounter.get()}")
-
     val uniqueBlockPositions = tracker.blocks.size
     logger.info("Job $job Unique Block Positions... $uniqueBlockPositions")
     maybeTrimState(tracker)

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/render/BlockDiversityRenderer.kt

@@ -6,7 +6,8 @@ import cloud.kubelet.foundation.gjallarhorn.util.RandomColorKey
 import java.awt.Color
 import java.awt.image.BufferedImage
 
-class BlockDiversityRenderer(val expanse: BlockExpanse, quadPixelSize: Int = defaultQuadPixelSize) : BlockGridRenderer(quadPixelSize) {
+class BlockDiversityRenderer(val expanse: BlockExpanse, quadPixelSize: Int = defaultQuadPixelSize) :
+  BlockGridRenderer(quadPixelSize) {
   private val randomColorKey = RandomColorKey()
 
   override fun render(map: BlockMap): BufferedImage = buildPixelQuadImage(expanse) { x, z ->

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/render/BlockHeatMapRenderer.kt

@@ -7,7 +7,11 @@ import java.awt.Color
 import java.awt.image.BufferedImage
 
 abstract class BlockHeatMapRenderer(quadPixelSize: Int = defaultQuadPixelSize) : BlockGridRenderer(quadPixelSize) {
-  protected fun buildHeatMapImage(expanse: BlockExpanse, clamp: FloatClamp, calculate: (Long, Long) -> Long?): BufferedImage =
+  protected fun buildHeatMapImage(
+    expanse: BlockExpanse,
+    clamp: FloatClamp,
+    calculate: (Long, Long) -> Long?
+  ): BufferedImage =
     buildPixelQuadImage(expanse) { x, z ->
       val value = calculate(x, z)
       val color = if (value != null) {

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockChange.kt

@@ -0,0 +1,11 @@
+package cloud.kubelet.foundation.gjallarhorn.state
+
+import java.time.Instant
+
+data class BlockChange(
+  val time: Instant,
+  val type: BlockChangeType,
+  val location: BlockCoordinate,
+  val from: BlockState,
+  val to: BlockState
+)

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockChangeType.kt

@@ -0,0 +1,6 @@
+package cloud.kubelet.foundation.gjallarhorn.state
+
+enum class BlockChangeType {
+  Place,
+  Break
+}

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockChangelog.kt

@@ -0,0 +1,54 @@
+package cloud.kubelet.foundation.gjallarhorn.state
+
+import cloud.kubelet.foundation.heimdall.view.BlockChangeView
+import org.jetbrains.exposed.sql.Database
+import org.jetbrains.exposed.sql.Op
+import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.transactions.transaction
+import java.time.Instant
+
+class BlockChangelog(
+  val changes: List<BlockChange>
+) {
+  fun slice(range: Pair<Instant, Instant>): BlockChangelog = BlockChangelog(changes.filter {
+    it.time >= range.first &&
+        it.time <= range.second
+  })
+
+  val changeTimeRange: Pair<Instant, Instant>
+    get() = changes.minOf { it.time } to changes.maxOf { it.time }
+
+  companion object {
+    fun query(db: Database, filter: Op<Boolean> = Op.TRUE): BlockChangelog = transaction(db) {
+      BlockChangelog(BlockChangeView.select(filter).orderBy(BlockChangeView.time).map { row ->
+        val time = row[BlockChangeView.time]
+        val changeIsBreak = row[BlockChangeView.isBreak]
+        val x = row[BlockChangeView.x]
+        val y = row[BlockChangeView.y]
+        val z = row[BlockChangeView.z]
+        val block = row[BlockChangeView.block]
+        val location = BlockCoordinate(x.toLong(), y.toLong(), z.toLong())
+
+        val fromBlock = if (changeIsBreak) {
+          BlockState(block)
+        } else {
+          BlockState.AirBlock
+        }
+
+        val toBlock = if (changeIsBreak) {
+          BlockState.AirBlock
+        } else {
+          BlockState(block)
+        }
+
+        BlockChange(
+          time,
+          if (changeIsBreak) BlockChangeType.Break else BlockChangeType.Place,
+          location,
+          fromBlock,
+          toBlock
+        )
+      })
+    }
+  }
+}

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockLogTracker.kt

@@ -57,4 +57,12 @@ class BlockLogTracker(private val mode: BlockTrackMode = BlockTrackMode.RemoveOn
     }
     return map
   }
+
+  fun replay(changelog: BlockChangelog) = changelog.changes.forEach { change ->
+    if (change.type == BlockChangeType.Break) {
+      delete(change.location)
+    } else {
+      place(change.location, change.to)
+    }
+  }
 }

tool-gjallarhorn/src/main/kotlin/cloud/kubelet/foundation/gjallarhorn/state/BlockState.kt

@@ -3,4 +3,8 @@ package cloud.kubelet.foundation.gjallarhorn.state
 import kotlinx.serialization.Serializable
 
 @Serializable
-data class BlockState(val type: String)
+data class BlockState(val type: String) {
+  companion object {
+    val AirBlock = BlockState("minecraft:air")
+  }
+}