feat: oci compliance work (#85)

* chore: rework oci crate to be more composable

* feat: image pull is now internally explicit

* feat: utilize vfs for assembling oci images

* feat: rework oci to preserve permissions via a vfs

crates/oci/src/packer/backend.rs

@@ -0,0 +1,201 @@
+use std::{path::Path, process::Stdio, sync::Arc};
+
+use super::OciPackedFormat;
+use crate::{
+    progress::{OciBoundProgress, OciProgressPhase},
+    vfs::VfsTree,
+};
+use anyhow::{anyhow, Result};
+use log::warn;
+use tokio::{pin, process::Command, select};
+
+#[derive(Debug, Clone, Copy)]
+pub enum OciPackerBackendType {
+    MkSquashfs,
+    MkfsErofs,
+}
+
+impl OciPackerBackendType {
+    pub fn format(&self) -> OciPackedFormat {
+        match self {
+            OciPackerBackendType::MkSquashfs => OciPackedFormat::Squashfs,
+            OciPackerBackendType::MkfsErofs => OciPackedFormat::Erofs,
+        }
+    }
+
+    pub fn create(&self) -> Box<dyn OciPackerBackend> {
+        match self {
+            OciPackerBackendType::MkSquashfs => {
+                Box::new(OciPackerMkSquashfs {}) as Box<dyn OciPackerBackend>
+            }
+            OciPackerBackendType::MkfsErofs => {
+                Box::new(OciPackerMkfsErofs {}) as Box<dyn OciPackerBackend>
+            }
+        }
+    }
+}
+
+#[async_trait::async_trait]
+pub trait OciPackerBackend: Send + Sync {
+    async fn pack(&self, progress: OciBoundProgress, vfs: Arc<VfsTree>, file: &Path) -> Result<()>;
+}
+
+pub struct OciPackerMkSquashfs {}
+
+#[async_trait::async_trait]
+impl OciPackerBackend for OciPackerMkSquashfs {
+    async fn pack(&self, progress: OciBoundProgress, vfs: Arc<VfsTree>, file: &Path) -> Result<()> {
+        progress
+            .update(|progress| {
+                progress.phase = OciProgressPhase::Packing;
+                progress.total = 1;
+                progress.value = 0;
+            })
+            .await;
+
+        let mut child = Command::new("mksquashfs")
+            .arg("-")
+            .arg(file)
+            .arg("-comp")
+            .arg("gzip")
+            .arg("-tar")
+            .stdin(Stdio::piped())
+            .stderr(Stdio::null())
+            .stdout(Stdio::null())
+            .spawn()?;
+        let stdin = child
+            .stdin
+            .take()
+            .ok_or(anyhow!("unable to acquire stdin stream"))?;
+        let mut writer = Some(tokio::task::spawn(async move {
+            if let Err(error) = vfs.write_to_tar(stdin).await {
+                warn!("failed to write tar: {}", error);
+                return Err(error);
+            }
+            Ok(())
+        }));
+        let wait = child.wait();
+        pin!(wait);
+        let status_result = loop {
+            if let Some(inner) = writer.as_mut() {
+                select! {
+                    x = inner => {
+                        writer = None;
+                        match x {
+                            Ok(_) => {},
+                            Err(error) => {
+                                return Err(error.into());
+                            }
+                        }
+                    },
+                    status = &mut wait => {
+                        break status;
+                    }
+                };
+            } else {
+                select! {
+                    status = &mut wait => {
+                        break status;
+                    }
+                };
+            }
+        };
+        if let Some(writer) = writer {
+            writer.await??;
+        }
+        let status = status_result?;
+        if !status.success() {
+            Err(anyhow!(
+                "mksquashfs failed with exit code: {}",
+                status.code().unwrap()
+            ))
+        } else {
+            progress
+                .update(|progress| {
+                    progress.phase = OciProgressPhase::Packing;
+                    progress.total = 1;
+                    progress.value = 1;
+                })
+                .await;
+            Ok(())
+        }
+    }
+}
+
+pub struct OciPackerMkfsErofs {}
+
+#[async_trait::async_trait]
+impl OciPackerBackend for OciPackerMkfsErofs {
+    async fn pack(&self, progress: OciBoundProgress, vfs: Arc<VfsTree>, path: &Path) -> Result<()> {
+        progress
+            .update(|progress| {
+                progress.phase = OciProgressPhase::Packing;
+                progress.total = 1;
+                progress.value = 0;
+            })
+            .await;
+
+        let mut child = Command::new("mkfs.erofs")
+            .arg("-L")
+            .arg("root")
+            .arg("--tar=-")
+            .arg(path)
+            .stdin(Stdio::piped())
+            .stderr(Stdio::null())
+            .stdout(Stdio::null())
+            .spawn()?;
+        let stdin = child
+            .stdin
+            .take()
+            .ok_or(anyhow!("unable to acquire stdin stream"))?;
+        let mut writer = Some(tokio::task::spawn(
+            async move { vfs.write_to_tar(stdin).await },
+        ));
+        let wait = child.wait();
+        pin!(wait);
+        let status_result = loop {
+            if let Some(inner) = writer.as_mut() {
+                select! {
+                    x = inner => {
+                        match x {
+                            Ok(_) => {
+                                writer = None;
+                            },
+                            Err(error) => {
+                                return Err(error.into());
+                            }
+                        }
+                    },
+                    status = &mut wait => {
+                        break status;
+                    }
+                };
+            } else {
+                select! {
+                    status = &mut wait => {
+                        break status;
+                    }
+                };
+            }
+        };
+        if let Some(writer) = writer {
+            writer.await??;
+        }
+        let status = status_result?;
+        if !status.success() {
+            Err(anyhow!(
+                "mkfs.erofs failed with exit code: {}",
+                status.code().unwrap()
+            ))
+        } else {
+            progress
+                .update(|progress| {
+                    progress.phase = OciProgressPhase::Packing;
+                    progress.total = 1;
+                    progress.value = 1;
+                })
+                .await;
+            Ok(())
+        }
+    }
+}

crates/oci/src/packer/cache.rs

@@ -0,0 +1,84 @@
+use crate::packer::{OciImagePacked, OciPackedFormat};
+
+use anyhow::Result;
+use log::debug;
+use oci_spec::image::{ImageConfiguration, ImageManifest};
+use std::path::{Path, PathBuf};
+use tokio::fs;
+
+#[derive(Clone)]
+pub struct OciPackerCache {
+    cache_dir: PathBuf,
+}
+
+impl OciPackerCache {
+    pub fn new(cache_dir: &Path) -> Result<OciPackerCache> {
+        Ok(OciPackerCache {
+            cache_dir: cache_dir.to_path_buf(),
+        })
+    }
+
+    pub async fn recall(
+        &self,
+        digest: &str,
+        format: OciPackedFormat,
+    ) -> Result<Option<OciImagePacked>> {
+        let mut fs_path = self.cache_dir.clone();
+        let mut config_path = self.cache_dir.clone();
+        let mut manifest_path = self.cache_dir.clone();
+        fs_path.push(format!("{}.{}", digest, format.extension()));
+        manifest_path.push(format!("{}.manifest.json", digest));
+        config_path.push(format!("{}.config.json", digest));
+        Ok(
+            if fs_path.exists() && manifest_path.exists() && config_path.exists() {
+                let image_metadata = fs::metadata(&fs_path).await?;
+                let manifest_metadata = fs::metadata(&manifest_path).await?;
+                let config_metadata = fs::metadata(&config_path).await?;
+                if image_metadata.is_file()
+                    && manifest_metadata.is_file()
+                    && config_metadata.is_file()
+                {
+                    let manifest_text = fs::read_to_string(&manifest_path).await?;
+                    let manifest: ImageManifest = serde_json::from_str(&manifest_text)?;
+                    let config_text = fs::read_to_string(&config_path).await?;
+                    let config: ImageConfiguration = serde_json::from_str(&config_text)?;
+                    debug!("cache hit digest={}", digest);
+                    Some(OciImagePacked::new(
+                        digest.to_string(),
+                        fs_path.clone(),
+                        format,
+                        config,
+                        manifest,
+                    ))
+                } else {
+                    None
+                }
+            } else {
+                debug!("cache miss digest={}", digest);
+                None
+            },
+        )
+    }
+
+    pub async fn store(&self, packed: OciImagePacked) -> Result<OciImagePacked> {
+        debug!("cache store digest={}", packed.digest);
+        let mut fs_path = self.cache_dir.clone();
+        let mut manifest_path = self.cache_dir.clone();
+        let mut config_path = self.cache_dir.clone();
+        fs_path.push(format!("{}.{}", packed.digest, packed.format.extension()));
+        manifest_path.push(format!("{}.manifest.json", packed.digest));
+        config_path.push(format!("{}.config.json", packed.digest));
+        fs::copy(&packed.path, &fs_path).await?;
+        let manifest_text = serde_json::to_string_pretty(&packed.manifest)?;
+        fs::write(&manifest_path, manifest_text).await?;
+        let config_text = serde_json::to_string_pretty(&packed.config)?;
+        fs::write(&config_path, config_text).await?;
+        Ok(OciImagePacked::new(
+            packed.digest,
+            fs_path.clone(),
+            packed.format,
+            packed.config,
+            packed.manifest,
+        ))
+    }
+}

crates/oci/src/packer/mod.rs

@@ -0,0 +1,58 @@
+use std::path::PathBuf;
+
+use self::backend::OciPackerBackendType;
+use oci_spec::image::{ImageConfiguration, ImageManifest};
+
+pub mod backend;
+pub mod cache;
+pub mod service;
+
+#[derive(Debug, Default, Clone, Copy)]
+pub enum OciPackedFormat {
+    #[default]
+    Squashfs,
+    Erofs,
+}
+
+impl OciPackedFormat {
+    pub fn extension(&self) -> &str {
+        match self {
+            OciPackedFormat::Squashfs => "squashfs",
+            OciPackedFormat::Erofs => "erofs",
+        }
+    }
+
+    pub fn backend(&self) -> OciPackerBackendType {
+        match self {
+            OciPackedFormat::Squashfs => OciPackerBackendType::MkSquashfs,
+            OciPackedFormat::Erofs => OciPackerBackendType::MkfsErofs,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct OciImagePacked {
+    pub digest: String,
+    pub path: PathBuf,
+    pub format: OciPackedFormat,
+    pub config: ImageConfiguration,
+    pub manifest: ImageManifest,
+}
+
+impl OciImagePacked {
+    pub fn new(
+        digest: String,
+        path: PathBuf,
+        format: OciPackedFormat,
+        config: ImageConfiguration,
+        manifest: ImageManifest,
+    ) -> OciImagePacked {
+        OciImagePacked {
+            digest,
+            path,
+            format,
+            config,
+            manifest,
+        }
+    }
+}

crates/oci/src/packer/service.rs

@@ -0,0 +1,81 @@
+use std::path::{Path, PathBuf};
+
+use anyhow::{anyhow, Result};
+
+use crate::{
+    assemble::OciImageAssembler,
+    fetch::OciImageFetcher,
+    name::ImageName,
+    progress::{OciBoundProgress, OciProgress, OciProgressContext},
+    registry::OciPlatform,
+};
+
+use super::{cache::OciPackerCache, OciImagePacked, OciPackedFormat};
+
+#[derive(Clone)]
+pub struct OciPackerService {
+    seed: Option<PathBuf>,
+    platform: OciPlatform,
+    cache: OciPackerCache,
+}
+
+impl OciPackerService {
+    pub fn new(
+        seed: Option<PathBuf>,
+        cache_dir: &Path,
+        platform: OciPlatform,
+    ) -> Result<OciPackerService> {
+        Ok(OciPackerService {
+            seed,
+            cache: OciPackerCache::new(cache_dir)?,
+            platform,
+        })
+    }
+
+    pub async fn recall(
+        &self,
+        digest: &str,
+        format: OciPackedFormat,
+    ) -> Result<Option<OciImagePacked>> {
+        self.cache.recall(digest, format).await
+    }
+
+    pub async fn request(
+        &self,
+        name: ImageName,
+        format: OciPackedFormat,
+        progress_context: OciProgressContext,
+    ) -> Result<OciImagePacked> {
+        let progress = OciProgress::new();
+        let progress = OciBoundProgress::new(progress_context.clone(), progress);
+        let fetcher =
+            OciImageFetcher::new(self.seed.clone(), self.platform.clone(), progress.clone());
+        let resolved = fetcher.resolve(name).await?;
+        if let Some(cached) = self.cache.recall(&resolved.digest, format).await? {
+            return Ok(cached);
+        }
+        let assembler =
+            OciImageAssembler::new(fetcher, resolved, progress.clone(), None, None).await?;
+        let assembled = assembler.assemble().await?;
+        let mut file = assembled
+            .tmp_dir
+            .clone()
+            .ok_or(anyhow!("tmp_dir was missing when packing image"))?;
+        file.push("image.pack");
+        let target = file.clone();
+        let packer = format.backend().create();
+        packer
+            .pack(progress, assembled.vfs.clone(), &target)
+            .await?;
+
+        let packed = OciImagePacked::new(
+            assembled.digest.clone(),
+            file,
+            format,
+            assembled.config.clone(),
+            assembled.manifest.clone(),
+        );
+        let packed = self.cache.store(packed).await?;
+        Ok(packed)
+    }
+}