feat: oci packer can now use mksquashfs if available (#70)

* feat: oci packer can now use mksquashfs if available

* fix: use nproc in kernel build script for default jobs, and fix DEV.md guide

* feat: working erofs backend

crates/oci/Cargo.toml

@@ -14,11 +14,13 @@ async-compression = { workspace = true, features = ["tokio", "gzip", "zstd"] }
 async-trait = { workspace = true }
 backhand = { workspace = true }
 bytes = { workspace = true }
+indexmap = { workspace = true }
 krata-tokio-tar = { workspace = true }
 log = { workspace = true }
 oci-spec = { workspace = true }
 path-clean = { workspace = true }
 reqwest = { workspace = true }
+scopeguard = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 sha256 = { workspace = true }

crates/oci/examples/squashify.rs

@@ -3,7 +3,11 @@ use std::{env::args, path::PathBuf};
 use anyhow::Result;
 use env_logger::Env;
 use krataoci::{
-    cache::ImageCache, compiler::ImageCompiler, name::ImageName, progress::OciProgressContext,
+    cache::ImageCache,
+    compiler::OciImageCompiler,
+    name::ImageName,
+    packer::OciPackerFormat,
+    progress::{OciProgress, OciProgressContext},
 };
 use tokio::{fs, sync::broadcast};
 
@@ -21,21 +25,30 @@ async fn main() -> Result<()> {
 
     let cache = ImageCache::new(&cache_dir)?;
 
-    let (sender, mut receiver) = broadcast::channel(1000);
+    let (sender, mut receiver) = broadcast::channel::<OciProgress>(1000);
     tokio::task::spawn(async move {
         loop {
-            let Some(_) = receiver.recv().await.ok() else {
+            let Some(progress) = receiver.recv().await.ok() else {
                 break;
             };
+            println!("phase {:?}", progress.phase);
+            for (id, layer) in progress.layers {
+                println!(
+                    "{} {:?} {} of {}",
+                    id, layer.phase, layer.value, layer.total
+                )
+            }
         }
     });
     let context = OciProgressContext::new(sender);
-    let compiler = ImageCompiler::new(&cache, seed, context)?;
-    let info = compiler.compile(&image.to_string(), &image).await?;
+    let compiler = OciImageCompiler::new(&cache, seed, context)?;
+    let info = compiler
+        .compile(&image.to_string(), &image, OciPackerFormat::Squashfs)
+        .await?;
     println!(
         "generated squashfs of {} to {}",
         image,
-        info.image_squashfs.to_string_lossy()
+        info.image.to_string_lossy()
     );
     Ok(())
 }

crates/oci/src/cache.rs

@@ -1,3 +1,5 @@
+use crate::packer::OciPackerFormat;
+
 use super::compiler::ImageInfo;
 use anyhow::Result;
 use log::debug;
@@ -17,19 +19,19 @@ impl ImageCache {
         })
     }
 
-    pub async fn recall(&self, digest: &str) -> Result<Option<ImageInfo>> {
-        let mut squashfs_path = self.cache_dir.clone();
+    pub async fn recall(&self, digest: &str, format: OciPackerFormat) -> Result<Option<ImageInfo>> {
+        let mut fs_path = self.cache_dir.clone();
         let mut config_path = self.cache_dir.clone();
         let mut manifest_path = self.cache_dir.clone();
-        squashfs_path.push(format!("{}.squashfs", digest));
+        fs_path.push(format!("{}.{}", digest, format.extension()));
         manifest_path.push(format!("{}.manifest.json", digest));
         config_path.push(format!("{}.config.json", digest));
         Ok(
-            if squashfs_path.exists() && manifest_path.exists() && config_path.exists() {
-                let squashfs_metadata = fs::metadata(&squashfs_path).await?;
+            if fs_path.exists() && manifest_path.exists() && config_path.exists() {
+                let image_metadata = fs::metadata(&fs_path).await?;
                 let manifest_metadata = fs::metadata(&manifest_path).await?;
                 let config_metadata = fs::metadata(&config_path).await?;
-                if squashfs_metadata.is_file()
+                if image_metadata.is_file()
                     && manifest_metadata.is_file()
                     && config_metadata.is_file()
                 {
@@ -38,7 +40,7 @@ impl ImageCache {
                     let config_text = fs::read_to_string(&config_path).await?;
                     let config: ImageConfiguration = serde_json::from_str(&config_text)?;
                     debug!("cache hit digest={}", digest);
-                    Some(ImageInfo::new(squashfs_path.clone(), manifest, config)?)
+                    Some(ImageInfo::new(fs_path.clone(), manifest, config)?)
                 } else {
                     None
                 }
@@ -49,23 +51,24 @@ impl ImageCache {
         )
     }
 
-    pub async fn store(&self, digest: &str, info: &ImageInfo) -> Result<ImageInfo> {
+    pub async fn store(
+        &self,
+        digest: &str,
+        info: &ImageInfo,
+        format: OciPackerFormat,
+    ) -> Result<ImageInfo> {
         debug!("cache store digest={}", digest);
-        let mut squashfs_path = self.cache_dir.clone();
+        let mut fs_path = self.cache_dir.clone();
         let mut manifest_path = self.cache_dir.clone();
         let mut config_path = self.cache_dir.clone();
-        squashfs_path.push(format!("{}.squashfs", digest));
+        fs_path.push(format!("{}.{}", digest, format.extension()));
         manifest_path.push(format!("{}.manifest.json", digest));
         config_path.push(format!("{}.config.json", digest));
-        fs::copy(&info.image_squashfs, &squashfs_path).await?;
+        fs::copy(&info.image, &fs_path).await?;
         let manifest_text = serde_json::to_string_pretty(&info.manifest)?;
         fs::write(&manifest_path, manifest_text).await?;
         let config_text = serde_json::to_string_pretty(&info.config)?;
         fs::write(&config_path, config_text).await?;
-        ImageInfo::new(
-            squashfs_path.clone(),
-            info.manifest.clone(),
-            info.config.clone(),
-        )
+        ImageInfo::new(fs_path.clone(), info.manifest.clone(), info.config.clone())
     }
 }

crates/oci/src/compiler.rs

@@ -1,18 +1,14 @@
 use crate::cache::ImageCache;
 use crate::fetch::{OciImageDownloader, OciImageLayer};
 use crate::name::ImageName;
+use crate::packer::OciPackerFormat;
 use crate::progress::{OciProgress, OciProgressContext, OciProgressPhase};
 use crate::registry::OciRegistryPlatform;
 use anyhow::{anyhow, Result};
-use backhand::compression::Compressor;
-use backhand::{FilesystemCompressor, FilesystemWriter, NodeHeader};
-use log::{debug, trace, warn};
+use indexmap::IndexMap;
+use log::{debug, trace};
 use oci_spec::image::{ImageConfiguration, ImageManifest};
 use std::borrow::Cow;
-use std::collections::BTreeMap;
-use std::fs::File;
-use std::io::{BufWriter, ErrorKind, Read};
-use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
 use std::path::{Path, PathBuf};
 use std::pin::Pin;
 use tokio::fs;
@@ -20,51 +16,55 @@ use tokio::io::AsyncRead;
 use tokio_stream::StreamExt;
 use tokio_tar::{Archive, Entry};
 use uuid::Uuid;
-use walkdir::WalkDir;
 
-pub const IMAGE_SQUASHFS_VERSION: u64 = 2;
+pub const IMAGE_PACKER_VERSION: u64 = 2;
 
 pub struct ImageInfo {
-    pub image_squashfs: PathBuf,
+    pub image: PathBuf,
     pub manifest: ImageManifest,
     pub config: ImageConfiguration,
 }
 
 impl ImageInfo {
     pub fn new(
-        squashfs: PathBuf,
+        image: PathBuf,
         manifest: ImageManifest,
         config: ImageConfiguration,
     ) -> Result<ImageInfo> {
         Ok(ImageInfo {
-            image_squashfs: squashfs,
+            image,
             manifest,
             config,
         })
     }
 }
 
-pub struct ImageCompiler<'a> {
+pub struct OciImageCompiler<'a> {
     cache: &'a ImageCache,
     seed: Option<PathBuf>,
     progress: OciProgressContext,
 }
 
-impl ImageCompiler<'_> {
+impl OciImageCompiler<'_> {
     pub fn new(
         cache: &ImageCache,
         seed: Option<PathBuf>,
         progress: OciProgressContext,
-    ) -> Result<ImageCompiler> {
-        Ok(ImageCompiler {
+    ) -> Result<OciImageCompiler> {
+        Ok(OciImageCompiler {
             cache,
             seed,
             progress,
         })
     }
 
-    pub async fn compile(&self, id: &str, image: &ImageName) -> Result<ImageInfo> {
-        debug!("compile image={image}");
+    pub async fn compile(
+        &self,
+        id: &str,
+        image: &ImageName,
+        format: OciPackerFormat,
+    ) -> Result<ImageInfo> {
+        debug!("compile image={image} format={:?}", format);
         let mut tmp_dir = std::env::temp_dir().clone();
         tmp_dir.push(format!("krata-compile-{}", Uuid::new_v4()));
 
@@ -76,12 +76,17 @@ impl ImageCompiler<'_> {
         layer_dir.push("layer");
         fs::create_dir_all(&layer_dir).await?;
 
-        let mut squash_file = tmp_dir.clone();
-        squash_file.push("image.squashfs");
+        let mut packed_file = tmp_dir.clone();
+        packed_file.push("image.packed");
+
+        let _guard = scopeguard::guard(tmp_dir.to_path_buf(), |delete| {
+            tokio::task::spawn(async move {
+                let _ = fs::remove_dir_all(delete).await;
+            });
+        });
         let info = self
-            .download_and_compile(id, image, &layer_dir, &image_dir, &squash_file)
+            .download_and_compile(id, image, &layer_dir, &image_dir, &packed_file, format)
             .await?;
-        fs::remove_dir_all(&tmp_dir).await?;
         Ok(info)
     }
 
@@ -91,12 +96,13 @@ impl ImageCompiler<'_> {
         image: &ImageName,
         layer_dir: &Path,
         image_dir: &Path,
-        squash_file: &Path,
+        packed_file: &Path,
+        format: OciPackerFormat,
     ) -> Result<ImageInfo> {
         let mut progress = OciProgress {
             id: id.to_string(),
             phase: OciProgressPhase::Resolving,
-            layers: BTreeMap::new(),
+            layers: IndexMap::new(),
             value: 0,
             total: 0,
         };
@@ -109,14 +115,14 @@ impl ImageCompiler<'_> {
         );
         let resolved = downloader.resolve(image.clone()).await?;
         let cache_key = format!(
-            "manifest={}:squashfs-version={}\n",
-            resolved.digest, IMAGE_SQUASHFS_VERSION
+            "manifest={}:version={}:format={}\n",
+            resolved.digest,
+            IMAGE_PACKER_VERSION,
+            format.id(),
         );
         let cache_digest = sha256::digest(cache_key);
 
-        progress.phase = OciProgressPhase::Complete;
-        self.progress.update(&progress);
-        if let Some(cached) = self.cache.recall(&cache_digest).await? {
+        if let Some(cached) = self.cache.recall(&cache_digest, format).await? {
             return Ok(cached);
         }
 
@@ -132,7 +138,7 @@ impl ImageCompiler<'_> {
                 "process layer digest={} compression={:?}",
                 &layer.digest, layer.compression,
             );
-            progress.extracting_layer(&layer.digest, 0, 0);
+            progress.extracting_layer(&layer.digest, 0, 1);
             self.progress.update(&progress);
             let (whiteouts, count) = self.process_layer_whiteout(layer, image_dir).await?;
             progress.extracting_layer(&layer.digest, 0, count);
@@ -149,7 +155,9 @@ impl ImageCompiler<'_> {
                 let path = entry.path()?;
                 let mut maybe_whiteout_path_str =
                     path.to_str().map(|x| x.to_string()).unwrap_or_default();
-                progress.extracting_layer(&layer.digest, completed, count);
+                if (completed % 10) == 0 {
+                    progress.extracting_layer(&layer.digest, completed, count);
+                }
                 completed += 1;
                 self.progress.update(&progress);
                 if whiteouts.contains(&maybe_whiteout_path_str) {
@@ -183,26 +191,28 @@ impl ImageCompiler<'_> {
             }
         }
 
-        let image_dir_squash = image_dir.to_path_buf();
-        let squash_file_squash = squash_file.to_path_buf();
-        let progress_squash = progress.clone();
+        let image_dir_pack = image_dir.to_path_buf();
+        let packed_file_pack = packed_file.to_path_buf();
+        let progress_pack = progress.clone();
         let progress_context = self.progress.clone();
+        let format_pack = format;
         progress = tokio::task::spawn_blocking(move || {
-            ImageCompiler::squash(
-                &image_dir_squash,
-                &squash_file_squash,
-                progress_squash,
+            OciImageCompiler::pack(
+                format_pack,
+                &image_dir_pack,
+                &packed_file_pack,
+                progress_pack,
                 progress_context,
             )
         })
         .await??;
 
         let info = ImageInfo::new(
-            squash_file.to_path_buf(),
+            packed_file.to_path_buf(),
             local.image.manifest,
             local.config,
         )?;
-        let info = self.cache.store(&cache_digest, &info).await?;
+        let info = self.cache.store(&cache_digest, &info, format).await?;
         progress.phase = OciProgressPhase::Complete;
         progress.value = 0;
         progress.total = 0;
@@ -359,128 +369,20 @@ impl ImageCompiler<'_> {
         Ok(())
     }
 
-    fn squash(
+    fn pack(
+        format: OciPackerFormat,
         image_dir: &Path,
-        squash_file: &Path,
+        packed_file: &Path,
         mut progress: OciProgress,
         progress_context: OciProgressContext,
     ) -> Result<OciProgress> {
-        progress.phase = OciProgressPhase::Packing;
-        progress.total = 2;
-        progress.value = 0;
-        progress_context.update(&progress);
-        let mut writer = FilesystemWriter::default();
-        writer.set_compressor(FilesystemCompressor::new(Compressor::Gzip, None)?);
-        let walk = WalkDir::new(image_dir).follow_links(false);
-        for entry in walk {
-            let entry = entry?;
-            let rel = entry
-                .path()
-                .strip_prefix(image_dir)?
-                .to_str()
-                .ok_or_else(|| anyhow!("failed to strip prefix of tmpdir"))?;
-            let rel = format!("/{}", rel);
-            trace!("squash write {}", rel);
-            let typ = entry.file_type();
-            let metadata = std::fs::symlink_metadata(entry.path())?;
-            let uid = metadata.uid();
-            let gid = metadata.gid();
-            let mode = metadata.permissions().mode();
-            let mtime = metadata.mtime();
-
-            if rel == "/" {
-                writer.set_root_uid(uid);
-                writer.set_root_gid(gid);
-                writer.set_root_mode(mode as u16);
-                continue;
-            }
-
-            let header = NodeHeader {
-                permissions: mode as u16,
-                uid,
-                gid,
-                mtime: mtime as u32,
-            };
-            if typ.is_symlink() {
-                let symlink = std::fs::read_link(entry.path())?;
-                let symlink = symlink
-                    .to_str()
-                    .ok_or_else(|| anyhow!("failed to read symlink"))?;
-                writer.push_symlink(symlink, rel, header)?;
-            } else if typ.is_dir() {
-                writer.push_dir(rel, header)?;
-            } else if typ.is_file() {
-                writer.push_file(ConsumingFileReader::new(entry.path()), rel, header)?;
-            } else if typ.is_block_device() {
-                let device = metadata.dev();
-                writer.push_block_device(device as u32, rel, header)?;
-            } else if typ.is_char_device() {
-                let device = metadata.dev();
-                writer.push_char_device(device as u32, rel, header)?;
-            } else if typ.is_fifo() {
-                writer.push_fifo(rel, header)?;
-            } else if typ.is_socket() {
-                writer.push_socket(rel, header)?;
-            } else {
-                return Err(anyhow!("invalid file type"));
-            }
-        }
-
-        progress.phase = OciProgressPhase::Packing;
-        progress.value = 1;
-        progress_context.update(&progress);
-
-        let squash_file_path = squash_file
-            .to_str()
-            .ok_or_else(|| anyhow!("failed to convert squashfs string"))?;
-
-        let file = File::create(squash_file)?;
-        let mut bufwrite = BufWriter::new(file);
-        trace!("squash generate: {}", squash_file_path);
-        writer.write(&mut bufwrite)?;
+        let backend = format.detect_best_backend();
+        let backend = backend.create();
+        backend.pack(&mut progress, &progress_context, image_dir, packed_file)?;
         std::fs::remove_dir_all(image_dir)?;
         progress.phase = OciProgressPhase::Packing;
-        progress.value = 2;
+        progress.value = progress.total;
         progress_context.update(&progress);
         Ok(progress)
     }
 }
-
-struct ConsumingFileReader {
-    path: PathBuf,
-    file: Option<File>,
-}
-
-impl ConsumingFileReader {
-    fn new(path: &Path) -> ConsumingFileReader {
-        ConsumingFileReader {
-            path: path.to_path_buf(),
-            file: None,
-        }
-    }
-}
-
-impl Read for ConsumingFileReader {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        if self.file.is_none() {
-            self.file = Some(File::open(&self.path)?);
-        }
-        let Some(ref mut file) = self.file else {
-            return Err(std::io::Error::new(
-                ErrorKind::NotFound,
-                "file was not opened",
-            ));
-        };
-        file.read(buf)
-    }
-}
-
-impl Drop for ConsumingFileReader {
-    fn drop(&mut self) {
-        let file = self.file.take();
-        drop(file);
-        if let Err(error) = std::fs::remove_file(&self.path) {
-            warn!("failed to delete consuming file {:?}: {}", self.path, error);
-        }
-    }
-}

crates/oci/src/lib.rs

@@ -2,5 +2,6 @@ pub mod cache;
 pub mod compiler;
 pub mod fetch;
 pub mod name;
+pub mod packer;
 pub mod progress;
 pub mod registry;

crates/oci/src/packer.rs

@@ -0,0 +1,307 @@
+use std::{
+    fs::File,
+    io::{BufWriter, ErrorKind, Read},
+    os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt},
+    path::{Path, PathBuf},
+    process::{Command, Stdio},
+};
+
+use anyhow::{anyhow, Result};
+use backhand::{compression::Compressor, FilesystemCompressor, FilesystemWriter, NodeHeader};
+use log::{trace, warn};
+use walkdir::WalkDir;
+
+use crate::progress::{OciProgress, OciProgressContext, OciProgressPhase};
+
+#[derive(Debug, Default, Clone, Copy)]
+pub enum OciPackerFormat {
+    #[default]
+    Squashfs,
+    Erofs,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum OciPackerBackendType {
+    Backhand,
+    MkSquashfs,
+    MkfsErofs,
+}
+
+impl OciPackerFormat {
+    pub fn id(&self) -> u8 {
+        match self {
+            OciPackerFormat::Squashfs => 0,
+            OciPackerFormat::Erofs => 1,
+        }
+    }
+
+    pub fn extension(&self) -> &str {
+        match self {
+            OciPackerFormat::Squashfs => "erofs",
+            OciPackerFormat::Erofs => "erofs",
+        }
+    }
+
+    pub fn detect_best_backend(&self) -> OciPackerBackendType {
+        match self {
+            OciPackerFormat::Squashfs => {
+                let status = Command::new("mksquashfs")
+                    .arg("-version")
+                    .stdin(Stdio::null())
+                    .stderr(Stdio::null())
+                    .stdout(Stdio::null())
+                    .status()
+                    .ok();
+
+                let Some(code) = status.and_then(|x| x.code()) else {
+                    return OciPackerBackendType::Backhand;
+                };
+
+                if code == 0 {
+                    OciPackerBackendType::MkSquashfs
+                } else {
+                    OciPackerBackendType::Backhand
+                }
+            }
+            OciPackerFormat::Erofs => OciPackerBackendType::MkfsErofs,
+        }
+    }
+}
+
+impl OciPackerBackendType {
+    pub fn format(&self) -> OciPackerFormat {
+        match self {
+            OciPackerBackendType::Backhand => OciPackerFormat::Squashfs,
+            OciPackerBackendType::MkSquashfs => OciPackerFormat::Squashfs,
+            OciPackerBackendType::MkfsErofs => OciPackerFormat::Erofs,
+        }
+    }
+
+    pub fn create(&self) -> Box<dyn OciPackerBackend> {
+        match self {
+            OciPackerBackendType::Backhand => {
+                Box::new(OciPackerBackhand {}) as Box<dyn OciPackerBackend>
+            }
+            OciPackerBackendType::MkSquashfs => {
+                Box::new(OciPackerMkSquashfs {}) as Box<dyn OciPackerBackend>
+            }
+            OciPackerBackendType::MkfsErofs => {
+                Box::new(OciPackerMkfsErofs {}) as Box<dyn OciPackerBackend>
+            }
+        }
+    }
+}
+
+pub trait OciPackerBackend {
+    fn pack(
+        &self,
+        progress: &mut OciProgress,
+        progress_context: &OciProgressContext,
+        directory: &Path,
+        file: &Path,
+    ) -> Result<()>;
+}
+
+pub struct OciPackerBackhand {}
+
+impl OciPackerBackend for OciPackerBackhand {
+    fn pack(
+        &self,
+        progress: &mut OciProgress,
+        progress_context: &OciProgressContext,
+        directory: &Path,
+        file: &Path,
+    ) -> Result<()> {
+        progress.phase = OciProgressPhase::Packing;
+        progress.total = 1;
+        progress.value = 0;
+        progress_context.update(progress);
+        let mut writer = FilesystemWriter::default();
+        writer.set_compressor(FilesystemCompressor::new(Compressor::Gzip, None)?);
+        let walk = WalkDir::new(directory).follow_links(false);
+        for entry in walk {
+            let entry = entry?;
+            let rel = entry
+                .path()
+                .strip_prefix(directory)?
+                .to_str()
+                .ok_or_else(|| anyhow!("failed to strip prefix of tmpdir"))?;
+            let rel = format!("/{}", rel);
+            trace!("squash write {}", rel);
+            let typ = entry.file_type();
+            let metadata = std::fs::symlink_metadata(entry.path())?;
+            let uid = metadata.uid();
+            let gid = metadata.gid();
+            let mode = metadata.permissions().mode();
+            let mtime = metadata.mtime();
+
+            if rel == "/" {
+                writer.set_root_uid(uid);
+                writer.set_root_gid(gid);
+                writer.set_root_mode(mode as u16);
+                continue;
+            }
+
+            let header = NodeHeader {
+                permissions: mode as u16,
+                uid,
+                gid,
+                mtime: mtime as u32,
+            };
+            if typ.is_symlink() {
+                let symlink = std::fs::read_link(entry.path())?;
+                let symlink = symlink
+                    .to_str()
+                    .ok_or_else(|| anyhow!("failed to read symlink"))?;
+                writer.push_symlink(symlink, rel, header)?;
+            } else if typ.is_dir() {
+                writer.push_dir(rel, header)?;
+            } else if typ.is_file() {
+                writer.push_file(ConsumingFileReader::new(entry.path()), rel, header)?;
+            } else if typ.is_block_device() {
+                let device = metadata.dev();
+                writer.push_block_device(device as u32, rel, header)?;
+            } else if typ.is_char_device() {
+                let device = metadata.dev();
+                writer.push_char_device(device as u32, rel, header)?;
+            } else if typ.is_fifo() {
+                writer.push_fifo(rel, header)?;
+            } else if typ.is_socket() {
+                writer.push_socket(rel, header)?;
+            } else {
+                return Err(anyhow!("invalid file type"));
+            }
+        }
+
+        progress.phase = OciProgressPhase::Packing;
+        progress.value = 1;
+        progress_context.update(progress);
+
+        let squash_file_path = file
+            .to_str()
+            .ok_or_else(|| anyhow!("failed to convert squashfs string"))?;
+
+        let file = File::create(file)?;
+        let mut bufwrite = BufWriter::new(file);
+        trace!("squash generate: {}", squash_file_path);
+        writer.write(&mut bufwrite)?;
+        Ok(())
+    }
+}
+
+struct ConsumingFileReader {
+    path: PathBuf,
+    file: Option<File>,
+}
+
+impl ConsumingFileReader {
+    fn new(path: &Path) -> ConsumingFileReader {
+        ConsumingFileReader {
+            path: path.to_path_buf(),
+            file: None,
+        }
+    }
+}
+
+impl Read for ConsumingFileReader {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        if self.file.is_none() {
+            self.file = Some(File::open(&self.path)?);
+        }
+        let Some(ref mut file) = self.file else {
+            return Err(std::io::Error::new(
+                ErrorKind::NotFound,
+                "file was not opened",
+            ));
+        };
+        file.read(buf)
+    }
+}
+
+impl Drop for ConsumingFileReader {
+    fn drop(&mut self) {
+        let file = self.file.take();
+        drop(file);
+        if let Err(error) = std::fs::remove_file(&self.path) {
+            warn!("failed to delete consuming file {:?}: {}", self.path, error);
+        }
+    }
+}
+
+pub struct OciPackerMkSquashfs {}
+
+impl OciPackerBackend for OciPackerMkSquashfs {
+    fn pack(
+        &self,
+        progress: &mut OciProgress,
+        progress_context: &OciProgressContext,
+        directory: &Path,
+        file: &Path,
+    ) -> Result<()> {
+        progress.phase = OciProgressPhase::Packing;
+        progress.total = 1;
+        progress.value = 0;
+        progress_context.update(progress);
+        let mut child = Command::new("mksquashfs")
+            .arg(directory)
+            .arg(file)
+            .arg("-comp")
+            .arg("gzip")
+            .stdin(Stdio::null())
+            .stderr(Stdio::null())
+            .stdout(Stdio::null())
+            .spawn()?;
+        let status = child.wait()?;
+        if !status.success() {
+            Err(anyhow!(
+                "mksquashfs failed with exit code: {}",
+                status.code().unwrap()
+            ))
+        } else {
+            progress.phase = OciProgressPhase::Packing;
+            progress.total = 1;
+            progress.value = 1;
+            progress_context.update(progress);
+            Ok(())
+        }
+    }
+}
+
+pub struct OciPackerMkfsErofs {}
+
+impl OciPackerBackend for OciPackerMkfsErofs {
+    fn pack(
+        &self,
+        progress: &mut OciProgress,
+        progress_context: &OciProgressContext,
+        directory: &Path,
+        file: &Path,
+    ) -> Result<()> {
+        progress.phase = OciProgressPhase::Packing;
+        progress.total = 1;
+        progress.value = 0;
+        progress_context.update(progress);
+        let mut child = Command::new("mkfs.erofs")
+            .arg("-L")
+            .arg("root")
+            .arg(file)
+            .arg(directory)
+            .stdin(Stdio::null())
+            .stderr(Stdio::null())
+            .stdout(Stdio::null())
+            .spawn()?;
+        let status = child.wait()?;
+        if !status.success() {
+            Err(anyhow!(
+                "mkfs.erofs failed with exit code: {}",
+                status.code().unwrap()
+            ))
+        } else {
+            progress.phase = OciProgressPhase::Packing;
+            progress.total = 1;
+            progress.value = 1;
+            progress_context.update(progress);
+            Ok(())
+        }
+    }
+}

crates/oci/src/progress.rs

@@ -1,12 +1,11 @@
-use std::collections::BTreeMap;
-
+use indexmap::IndexMap;
 use tokio::sync::broadcast::Sender;
 
 #[derive(Clone, Debug)]
 pub struct OciProgress {
     pub id: String,
     pub phase: OciProgressPhase,
-    pub layers: BTreeMap<String, OciProgressLayer>,
+    pub layers: IndexMap<String, OciProgressLayer>,
     pub value: u64,
     pub total: u64,
 }