From e88fadd3bc8106ae1e36fed544d684d197ccd3ef Mon Sep 17 00:00:00 2001 From: Alex Zenla Date: Tue, 16 Apr 2024 05:22:58 +0000 Subject: [PATCH] oci: retain bit-perfect copies of manifest and config on disk --- crates/oci/src/assemble.rs | 5 ++-- crates/oci/src/fetch.rs | 44 ++++++++++++++++++++------------ crates/oci/src/lib.rs | 1 + crates/oci/src/packer/cache.rs | 25 +++++++++--------- crates/oci/src/packer/mod.rs | 10 +++++--- crates/oci/src/packer/service.rs | 1 - crates/oci/src/registry.rs | 16 +++++++----- crates/oci/src/schema.rs | 29 +++++++++++++++++++++ crates/runtime/src/cfgblk.rs | 4 +-- 9 files changed, 90 insertions(+), 45 deletions(-) create mode 100644 crates/oci/src/schema.rs diff --git a/crates/oci/src/assemble.rs b/crates/oci/src/assemble.rs index 5f13ae3..2db1d99 100644 --- a/crates/oci/src/assemble.rs +++ b/crates/oci/src/assemble.rs @@ -1,5 +1,6 @@ use crate::fetch::{OciImageFetcher, OciImageLayer, OciResolvedImage}; use crate::progress::OciBoundProgress; +use crate::schema::OciSchema; use crate::vfs::{VfsNode, VfsTree}; use anyhow::{anyhow, Result}; use log::{debug, trace, warn}; @@ -15,8 +16,8 @@ use uuid::Uuid; pub struct OciImageAssembled { pub digest: String, - pub manifest: ImageManifest, - pub config: ImageConfiguration, + pub manifest: OciSchema, + pub config: OciSchema, pub vfs: Arc, pub tmp_dir: Option, } diff --git a/crates/oci/src/fetch.rs b/crates/oci/src/fetch.rs index 9260afe..3e89f10 100644 --- a/crates/oci/src/fetch.rs +++ b/crates/oci/src/fetch.rs @@ -1,4 +1,7 @@ -use crate::progress::{OciBoundProgress, OciProgressPhase}; +use crate::{ + progress::{OciBoundProgress, OciProgressPhase}, + schema::OciSchema, +}; use super::{ name::ImageName, @@ -6,6 +9,7 @@ use super::{ }; use std::{ + fmt::Debug, path::{Path, PathBuf}, pin::Pin, }; @@ -66,13 +70,13 @@ impl OciImageLayer { pub struct OciResolvedImage { pub name: ImageName, pub digest: String, - pub manifest: ImageManifest, + pub manifest: OciSchema, } #[derive(Clone, Debug)] pub struct OciLocalImage { pub image: OciResolvedImage, - pub config: ImageConfiguration, + pub config: OciSchema, pub layers: Vec, } @@ -89,10 +93,10 @@ impl OciImageFetcher { } } - async fn load_seed_json_blob( + async fn load_seed_json_blob( &self, descriptor: &Descriptor, - ) -> Result> { + ) -> Result>> { let digest = descriptor.digest(); let Some((digest_type, digest_content)) = digest.split_once(':') else { return Err(anyhow!("digest content was not properly formatted")); @@ -101,7 +105,10 @@ impl OciImageFetcher { self.load_seed_json(&want).await } - async fn load_seed_json(&self, want: &str) -> Result> { + async fn load_seed_json( + &self, + want: &str, + ) -> Result>> { let Some(ref seed) = self.seed else { return Ok(None); }; @@ -113,10 +120,10 @@ impl OciImageFetcher { let mut entry = entry?; let path = String::from_utf8(entry.path_bytes().to_vec())?; if path == want { - let mut content = String::new(); - entry.read_to_string(&mut content).await?; - let data = serde_json::from_str::(&content)?; - return Ok(Some(data)); + let mut content = Vec::new(); + entry.read_to_end(&mut content).await?; + let item = serde_json::from_slice::(&content)?; + return Ok(Some(OciSchema::new(content, item))); } } Ok(None) @@ -154,7 +161,7 @@ impl OciImageFetcher { if let Some(index) = self.load_seed_json::("index.json").await? { let mut found: Option<&Descriptor> = None; - for manifest in index.manifests() { + for manifest in index.item().manifests() { let Some(annotations) = manifest.annotations() else { continue; }; @@ -215,7 +222,7 @@ impl OciImageFetcher { image: OciResolvedImage, layer_dir: &Path, ) -> Result { - let config: ImageConfiguration; + let config: OciSchema; self.progress .update(|progress| { progress.phase = OciProgressPhase::ConfigAcquire; @@ -223,27 +230,30 @@ impl OciImageFetcher { .await; let mut client = OciRegistryClient::new(image.name.registry_url()?, self.platform.clone())?; if let Some(seeded) = self - .load_seed_json_blob::(image.manifest.config()) + .load_seed_json_blob::(image.manifest.item().config()) .await? { config = seeded; } else { let config_bytes = client - .get_blob(&image.name.name, image.manifest.config()) + .get_blob(&image.name.name, image.manifest.item().config()) .await?; - config = serde_json::from_slice(&config_bytes)?; + config = OciSchema::new( + config_bytes.to_vec(), + serde_json::from_slice(&config_bytes)?, + ); } self.progress .update(|progress| { progress.phase = OciProgressPhase::LayerAcquire; - for layer in image.manifest.layers() { + for layer in image.manifest.item().layers() { progress.add_layer(layer.digest(), layer.size() as usize); } }) .await; let mut layers = Vec::new(); - for layer in image.manifest.layers() { + for layer in image.manifest.item().layers() { self.progress .update(|progress| { progress.downloading_layer(layer.digest(), 0, layer.size() as usize); diff --git a/crates/oci/src/lib.rs b/crates/oci/src/lib.rs index 4372fd3..de3f17d 100644 --- a/crates/oci/src/lib.rs +++ b/crates/oci/src/lib.rs @@ -4,4 +4,5 @@ pub mod name; pub mod packer; pub mod progress; pub mod registry; +pub mod schema; pub mod vfs; diff --git a/crates/oci/src/packer/cache.rs b/crates/oci/src/packer/cache.rs index 460520a..f872e64 100644 --- a/crates/oci/src/packer/cache.rs +++ b/crates/oci/src/packer/cache.rs @@ -1,4 +1,7 @@ -use crate::packer::{OciImagePacked, OciPackedFormat}; +use crate::{ + packer::{OciImagePacked, OciPackedFormat}, + schema::OciSchema, +}; use anyhow::Result; use log::debug; @@ -38,17 +41,17 @@ impl OciPackerCache { && manifest_metadata.is_file() && config_metadata.is_file() { - let manifest_text = fs::read_to_string(&manifest_path).await?; - let manifest: ImageManifest = serde_json::from_str(&manifest_text)?; - let config_text = fs::read_to_string(&config_path).await?; - let config: ImageConfiguration = serde_json::from_str(&config_text)?; + let manifest_bytes = fs::read(&manifest_path).await?; + let manifest: ImageManifest = serde_json::from_slice(&manifest_bytes)?; + let config_bytes = fs::read(&config_path).await?; + let config: ImageConfiguration = serde_json::from_slice(&config_bytes)?; debug!("cache hit digest={}", digest); Some(OciImagePacked::new( digest.to_string(), fs_path.clone(), format, - config, - manifest, + OciSchema::new(config_bytes, config), + OciSchema::new(manifest_bytes, manifest), )) } else { None @@ -68,11 +71,9 @@ impl OciPackerCache { fs_path.push(format!("{}.{}", packed.digest, packed.format.extension())); manifest_path.push(format!("{}.manifest.json", packed.digest)); config_path.push(format!("{}.config.json", packed.digest)); - fs::copy(&packed.path, &fs_path).await?; - let manifest_text = serde_json::to_string_pretty(&packed.manifest)?; - fs::write(&manifest_path, manifest_text).await?; - let config_text = serde_json::to_string_pretty(&packed.config)?; - fs::write(&config_path, config_text).await?; + fs::rename(&packed.path, &fs_path).await?; + fs::write(&config_path, packed.config.raw()).await?; + fs::write(&manifest_path, packed.manifest.raw()).await?; Ok(OciImagePacked::new( packed.digest, fs_path.clone(), diff --git a/crates/oci/src/packer/mod.rs b/crates/oci/src/packer/mod.rs index da1c4b2..7ddbc93 100644 --- a/crates/oci/src/packer/mod.rs +++ b/crates/oci/src/packer/mod.rs @@ -1,5 +1,7 @@ use std::path::PathBuf; +use crate::schema::OciSchema; + use self::backend::OciPackerBackendType; use oci_spec::image::{ImageConfiguration, ImageManifest}; @@ -35,8 +37,8 @@ pub struct OciImagePacked { pub digest: String, pub path: PathBuf, pub format: OciPackedFormat, - pub config: ImageConfiguration, - pub manifest: ImageManifest, + pub config: OciSchema, + pub manifest: OciSchema, } impl OciImagePacked { @@ -44,8 +46,8 @@ impl OciImagePacked { digest: String, path: PathBuf, format: OciPackedFormat, - config: ImageConfiguration, - manifest: ImageManifest, + config: OciSchema, + manifest: OciSchema, ) -> OciImagePacked { OciImagePacked { digest, diff --git a/crates/oci/src/packer/service.rs b/crates/oci/src/packer/service.rs index 54cf444..1d47ee0 100644 --- a/crates/oci/src/packer/service.rs +++ b/crates/oci/src/packer/service.rs @@ -67,7 +67,6 @@ impl OciPackerService { packer .pack(progress, assembled.vfs.clone(), &target) .await?; - let packed = OciImagePacked::new( assembled.digest.clone(), file, diff --git a/crates/oci/src/registry.rs b/crates/oci/src/registry.rs index 1b93e51..86597f6 100644 --- a/crates/oci/src/registry.rs +++ b/crates/oci/src/registry.rs @@ -7,7 +7,7 @@ use reqwest::{Client, RequestBuilder, Response, StatusCode}; use tokio::{fs::File, io::AsyncWriteExt}; use url::Url; -use crate::progress::OciBoundProgress; +use crate::{progress::OciBoundProgress, schema::OciSchema}; #[derive(Clone, Debug)] pub struct OciPlatform { @@ -176,7 +176,7 @@ impl OciRegistryClient { &mut self, name: N, reference: R, - ) -> Result<(ImageManifest, String)> { + ) -> Result<(OciSchema, String)> { let url = self.url.join(&format!( "/v2/{}/manifests/{}", name.as_ref(), @@ -198,15 +198,16 @@ impl OciRegistryClient { .ok_or_else(|| anyhow!("fetching manifest did not yield a content digest"))? .to_str()? .to_string(); - let manifest = serde_json::from_str(&response.text().await?)?; - Ok((manifest, digest)) + let bytes = response.bytes().await?; + let manifest = serde_json::from_slice(&bytes)?; + Ok((OciSchema::new(bytes.to_vec(), manifest), digest)) } pub async fn get_manifest_with_digest, R: AsRef>( &mut self, name: N, reference: R, - ) -> Result<(ImageManifest, String)> { + ) -> Result<(OciSchema, String)> { let url = self.url.join(&format!( "/v2/{}/manifests/{}", name.as_ref(), @@ -244,8 +245,9 @@ impl OciRegistryClient { .ok_or_else(|| anyhow!("fetching manifest did not yield a content digest"))? .to_str()? .to_string(); - let manifest = serde_json::from_str(&response.text().await?)?; - Ok((manifest, digest)) + let bytes = response.bytes().await?; + let manifest = serde_json::from_slice(&bytes)?; + Ok((OciSchema::new(bytes.to_vec(), manifest), digest)) } fn pick_manifest(&mut self, index: ImageIndex) -> Option { diff --git a/crates/oci/src/schema.rs b/crates/oci/src/schema.rs new file mode 100644 index 0000000..04bc203 --- /dev/null +++ b/crates/oci/src/schema.rs @@ -0,0 +1,29 @@ +use std::fmt::Debug; + +#[derive(Clone, Debug)] +pub struct OciSchema { + raw: Vec, + item: T, +} + +impl OciSchema { + pub fn new(raw: Vec, item: T) -> OciSchema { + OciSchema { raw, item } + } + + pub fn raw(&self) -> &[u8] { + &self.raw + } + + pub fn item(&self) -> &T { + &self.item + } + + pub fn into_raw(self) -> Vec { + self.raw + } + + pub fn into_item(self) -> T { + self.item + } +} diff --git a/crates/runtime/src/cfgblk.rs b/crates/runtime/src/cfgblk.rs index 0ae491f..fbfe563 100644 --- a/crates/runtime/src/cfgblk.rs +++ b/crates/runtime/src/cfgblk.rs @@ -26,7 +26,7 @@ impl ConfigBlock<'_> { pub fn build(&self, launch_config: &LaunchInfo) -> Result<()> { trace!("build launch_config={:?}", launch_config); - let manifest = self.image.config.to_string()?; + let config = self.image.config.raw(); let launch = serde_json::to_string(launch_config)?; let mut writer = FilesystemWriter::default(); writer.push_dir( @@ -39,7 +39,7 @@ impl ConfigBlock<'_> { }, )?; writer.push_file( - manifest.as_bytes(), + config, "/image/config.json", NodeHeader { permissions: 384,