oci: retain bit-perfect copies of manifest and config on disk

crates/oci/src/assemble.rs

@@ -1,5 +1,6 @@
 use crate::fetch::{OciImageFetcher, OciImageLayer, OciResolvedImage};
 use crate::progress::OciBoundProgress;
+use crate::schema::OciSchema;
 use crate::vfs::{VfsNode, VfsTree};
 use anyhow::{anyhow, Result};
 use log::{debug, trace, warn};
@@ -15,8 +16,8 @@ use uuid::Uuid;
 
 pub struct OciImageAssembled {
     pub digest: String,
-    pub manifest: ImageManifest,
+    pub manifest: OciSchema<ImageManifest>,
-    pub config: ImageConfiguration,
+    pub config: OciSchema<ImageConfiguration>,
     pub vfs: Arc<VfsTree>,
     pub tmp_dir: Option<PathBuf>,
 }

crates/oci/src/fetch.rs

@@ -1,4 +1,7 @@
-use crate::progress::{OciBoundProgress, OciProgressPhase};
+use crate::{
+    progress::{OciBoundProgress, OciProgressPhase},
+    schema::OciSchema,
+};
 
 use super::{
     name::ImageName,
@@ -6,6 +9,7 @@ use super::{
 };
 
 use std::{
+    fmt::Debug,
     path::{Path, PathBuf},
     pin::Pin,
 };
@@ -66,13 +70,13 @@ impl OciImageLayer {
 pub struct OciResolvedImage {
     pub name: ImageName,
     pub digest: String,
-    pub manifest: ImageManifest,
+    pub manifest: OciSchema<ImageManifest>,
 }
 
 #[derive(Clone, Debug)]
 pub struct OciLocalImage {
     pub image: OciResolvedImage,
-    pub config: ImageConfiguration,
+    pub config: OciSchema<ImageConfiguration>,
     pub layers: Vec<OciImageLayer>,
 }
 
@@ -89,10 +93,10 @@ impl OciImageFetcher {
         }
     }
 
-    async fn load_seed_json_blob<T: DeserializeOwned>(
+    async fn load_seed_json_blob<T: Clone + Debug + DeserializeOwned>(
         &self,
         descriptor: &Descriptor,
-    ) -> Result<Option<T>> {
+    ) -> Result<Option<OciSchema<T>>> {
         let digest = descriptor.digest();
         let Some((digest_type, digest_content)) = digest.split_once(':') else {
             return Err(anyhow!("digest content was not properly formatted"));
@@ -101,7 +105,10 @@ impl OciImageFetcher {
         self.load_seed_json(&want).await
     }
 
-    async fn load_seed_json<T: DeserializeOwned>(&self, want: &str) -> Result<Option<T>> {
+    async fn load_seed_json<T: Clone + Debug + DeserializeOwned>(
+        &self,
+        want: &str,
+    ) -> Result<Option<OciSchema<T>>> {
         let Some(ref seed) = self.seed else {
             return Ok(None);
         };
@@ -113,10 +120,10 @@ impl OciImageFetcher {
             let mut entry = entry?;
             let path = String::from_utf8(entry.path_bytes().to_vec())?;
             if path == want {
-                let mut content = String::new();
+                let mut content = Vec::new();
-                entry.read_to_string(&mut content).await?;
+                entry.read_to_end(&mut content).await?;
-                let data = serde_json::from_str::<T>(&content)?;
+                let item = serde_json::from_slice::<T>(&content)?;
-                return Ok(Some(data));
+                return Ok(Some(OciSchema::new(content, item)));
             }
         }
         Ok(None)
@@ -154,7 +161,7 @@ impl OciImageFetcher {
 
         if let Some(index) = self.load_seed_json::<ImageIndex>("index.json").await? {
             let mut found: Option<&Descriptor> = None;
-            for manifest in index.manifests() {
+            for manifest in index.item().manifests() {
                 let Some(annotations) = manifest.annotations() else {
                     continue;
                 };
@@ -215,7 +222,7 @@ impl OciImageFetcher {
         image: OciResolvedImage,
         layer_dir: &Path,
     ) -> Result<OciLocalImage> {
-        let config: ImageConfiguration;
+        let config: OciSchema<ImageConfiguration>;
         self.progress
             .update(|progress| {
                 progress.phase = OciProgressPhase::ConfigAcquire;
@@ -223,27 +230,30 @@ impl OciImageFetcher {
             .await;
         let mut client = OciRegistryClient::new(image.name.registry_url()?, self.platform.clone())?;
         if let Some(seeded) = self
-            .load_seed_json_blob::<ImageConfiguration>(image.manifest.config())
+            .load_seed_json_blob::<ImageConfiguration>(image.manifest.item().config())
             .await?
         {
             config = seeded;
         } else {
             let config_bytes = client
-                .get_blob(&image.name.name, image.manifest.config())
+                .get_blob(&image.name.name, image.manifest.item().config())
                 .await?;
-            config = serde_json::from_slice(&config_bytes)?;
+            config = OciSchema::new(
+                config_bytes.to_vec(),
+                serde_json::from_slice(&config_bytes)?,
+            );
         }
         self.progress
             .update(|progress| {
                 progress.phase = OciProgressPhase::LayerAcquire;
 
-                for layer in image.manifest.layers() {
+                for layer in image.manifest.item().layers() {
                     progress.add_layer(layer.digest(), layer.size() as usize);
                 }
             })
             .await;
         let mut layers = Vec::new();
-        for layer in image.manifest.layers() {
+        for layer in image.manifest.item().layers() {
             self.progress
                 .update(|progress| {
                     progress.downloading_layer(layer.digest(), 0, layer.size() as usize);

crates/oci/src/lib.rs

@@ -4,4 +4,5 @@ pub mod name;
 pub mod packer;
 pub mod progress;
 pub mod registry;
+pub mod schema;
 pub mod vfs;

crates/oci/src/packer/cache.rs

@@ -1,4 +1,7 @@
-use crate::packer::{OciImagePacked, OciPackedFormat};
+use crate::{
+    packer::{OciImagePacked, OciPackedFormat},
+    schema::OciSchema,
+};
 
 use anyhow::Result;
 use log::debug;
@@ -38,17 +41,17 @@ impl OciPackerCache {
                     && manifest_metadata.is_file()
                     && config_metadata.is_file()
                 {
-                    let manifest_text = fs::read_to_string(&manifest_path).await?;
+                    let manifest_bytes = fs::read(&manifest_path).await?;
-                    let manifest: ImageManifest = serde_json::from_str(&manifest_text)?;
+                    let manifest: ImageManifest = serde_json::from_slice(&manifest_bytes)?;
-                    let config_text = fs::read_to_string(&config_path).await?;
+                    let config_bytes = fs::read(&config_path).await?;
-                    let config: ImageConfiguration = serde_json::from_str(&config_text)?;
+                    let config: ImageConfiguration = serde_json::from_slice(&config_bytes)?;
                     debug!("cache hit digest={}", digest);
                     Some(OciImagePacked::new(
                         digest.to_string(),
                         fs_path.clone(),
                         format,
-                        config,
+                        OciSchema::new(config_bytes, config),
-                        manifest,
+                        OciSchema::new(manifest_bytes, manifest),
                     ))
                 } else {
                     None
@@ -68,11 +71,9 @@ impl OciPackerCache {
         fs_path.push(format!("{}.{}", packed.digest, packed.format.extension()));
         manifest_path.push(format!("{}.manifest.json", packed.digest));
         config_path.push(format!("{}.config.json", packed.digest));
-        fs::copy(&packed.path, &fs_path).await?;
+        fs::rename(&packed.path, &fs_path).await?;
-        let manifest_text = serde_json::to_string_pretty(&packed.manifest)?;
+        fs::write(&config_path, packed.config.raw()).await?;
-        fs::write(&manifest_path, manifest_text).await?;
+        fs::write(&manifest_path, packed.manifest.raw()).await?;
-        let config_text = serde_json::to_string_pretty(&packed.config)?;
-        fs::write(&config_path, config_text).await?;
         Ok(OciImagePacked::new(
             packed.digest,
             fs_path.clone(),

crates/oci/src/packer/mod.rs

@@ -1,5 +1,7 @@
 use std::path::PathBuf;
 
+use crate::schema::OciSchema;
+
 use self::backend::OciPackerBackendType;
 use oci_spec::image::{ImageConfiguration, ImageManifest};
 
@@ -35,8 +37,8 @@ pub struct OciImagePacked {
     pub digest: String,
     pub path: PathBuf,
     pub format: OciPackedFormat,
-    pub config: ImageConfiguration,
+    pub config: OciSchema<ImageConfiguration>,
-    pub manifest: ImageManifest,
+    pub manifest: OciSchema<ImageManifest>,
 }
 
 impl OciImagePacked {
@@ -44,8 +46,8 @@ impl OciImagePacked {
         digest: String,
         path: PathBuf,
         format: OciPackedFormat,
-        config: ImageConfiguration,
+        config: OciSchema<ImageConfiguration>,
-        manifest: ImageManifest,
+        manifest: OciSchema<ImageManifest>,
     ) -> OciImagePacked {
         OciImagePacked {
             digest,

crates/oci/src/packer/service.rs

@@ -67,7 +67,6 @@ impl OciPackerService {
         packer
             .pack(progress, assembled.vfs.clone(), &target)
             .await?;
-
         let packed = OciImagePacked::new(
             assembled.digest.clone(),
             file,

crates/oci/src/registry.rs

@@ -7,7 +7,7 @@ use reqwest::{Client, RequestBuilder, Response, StatusCode};
 use tokio::{fs::File, io::AsyncWriteExt};
 use url::Url;
 
-use crate::progress::OciBoundProgress;
+use crate::{progress::OciBoundProgress, schema::OciSchema};
 
 #[derive(Clone, Debug)]
 pub struct OciPlatform {
@@ -176,7 +176,7 @@ impl OciRegistryClient {
         &mut self,
         name: N,
         reference: R,
-    ) -> Result<(ImageManifest, String)> {
+    ) -> Result<(OciSchema<ImageManifest>, String)> {
         let url = self.url.join(&format!(
             "/v2/{}/manifests/{}",
             name.as_ref(),
@@ -198,15 +198,16 @@ impl OciRegistryClient {
             .ok_or_else(|| anyhow!("fetching manifest did not yield a content digest"))?
             .to_str()?
             .to_string();
-        let manifest = serde_json::from_str(&response.text().await?)?;
+        let bytes = response.bytes().await?;
-        Ok((manifest, digest))
+        let manifest = serde_json::from_slice(&bytes)?;
+        Ok((OciSchema::new(bytes.to_vec(), manifest), digest))
     }
 
     pub async fn get_manifest_with_digest<N: AsRef<str>, R: AsRef<str>>(
         &mut self,
         name: N,
         reference: R,
-    ) -> Result<(ImageManifest, String)> {
+    ) -> Result<(OciSchema<ImageManifest>, String)> {
         let url = self.url.join(&format!(
             "/v2/{}/manifests/{}",
             name.as_ref(),
@@ -244,8 +245,9 @@ impl OciRegistryClient {
             .ok_or_else(|| anyhow!("fetching manifest did not yield a content digest"))?
             .to_str()?
             .to_string();
-        let manifest = serde_json::from_str(&response.text().await?)?;
+        let bytes = response.bytes().await?;
-        Ok((manifest, digest))
+        let manifest = serde_json::from_slice(&bytes)?;
+        Ok((OciSchema::new(bytes.to_vec(), manifest), digest))
     }
 
     fn pick_manifest(&mut self, index: ImageIndex) -> Option<Descriptor> {

crates/oci/src/schema.rs

@@ -0,0 +1,29 @@
+use std::fmt::Debug;
+
+#[derive(Clone, Debug)]
+pub struct OciSchema<T: Clone + Debug> {
+    raw: Vec<u8>,
+    item: T,
+}
+
+impl<T: Clone + Debug> OciSchema<T> {
+    pub fn new(raw: Vec<u8>, item: T) -> OciSchema<T> {
+        OciSchema { raw, item }
+    }
+
+    pub fn raw(&self) -> &[u8] {
+        &self.raw
+    }
+
+    pub fn item(&self) -> &T {
+        &self.item
+    }
+
+    pub fn into_raw(self) -> Vec<u8> {
+        self.raw
+    }
+
+    pub fn into_item(self) -> T {
+        self.item
+    }
+}

crates/runtime/src/cfgblk.rs

@@ -26,7 +26,7 @@ impl ConfigBlock<'_> {
 
     pub fn build(&self, launch_config: &LaunchInfo) -> Result<()> {
         trace!("build launch_config={:?}", launch_config);
-        let manifest = self.image.config.to_string()?;
+        let config = self.image.config.raw();
         let launch = serde_json::to_string(launch_config)?;
         let mut writer = FilesystemWriter::default();
         writer.push_dir(
@@ -39,7 +39,7 @@ impl ConfigBlock<'_> {
             },
         )?;
         writer.push_file(
-            manifest.as_bytes(),
+            config,
             "/image/config.json",
             NodeHeader {
                 permissions: 384,