mirrors/krata
1
0
Fork 0
mirror of https://github.com/edera-dev/krata.git synced 2025-08-05 06:01:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

oci: retain bit-perfect copies of manifest and config on disk

Browse Source
This commit is contained in:
Alex Zenla
2024-04-16 05:22:58 +00:00
parent 218f848170
commit e88fadd3bc
9 changed files with 90 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crates/oci/src/assemble.rs
View File

@ -1,5 +1,6 @@
use crate::fetch::{OciImageFetcher, OciImageLayer, OciResolvedImage};
use crate::progress::OciBoundProgress;
use crate::schema::OciSchema;
use crate::vfs::{VfsNode, VfsTree};
use anyhow::{anyhow, Result};
use log::{debug, trace, warn};
@ -15,8 +16,8 @@ use uuid::Uuid;
pub struct OciImageAssembled {
pub digest: String,
pub manifest: ImageManifest,
pub config: ImageConfiguration,
pub manifest: OciSchema<ImageManifest>,
pub config: OciSchema<ImageConfiguration>,
pub vfs: Arc<VfsTree>,
pub tmp_dir: Option<PathBuf>,
}

44
crates/oci/src/fetch.rs
View File

@ -1,4 +1,7 @@
use crate::progress::{OciBoundProgress, OciProgressPhase};
use crate::{
progress::{OciBoundProgress, OciProgressPhase},
schema::OciSchema,
};
use super::{
name::ImageName,
@ -6,6 +9,7 @@ use super::{
};
use std::{
fmt::Debug,
path::{Path, PathBuf},
pin::Pin,
};
@ -66,13 +70,13 @@ impl OciImageLayer {
pub struct OciResolvedImage {
pub name: ImageName,
pub digest: String,
pub manifest: ImageManifest,
pub manifest: OciSchema<ImageManifest>,
}
#[derive(Clone, Debug)]
pub struct OciLocalImage {
pub image: OciResolvedImage,
pub config: ImageConfiguration,
pub config: OciSchema<ImageConfiguration>,
pub layers: Vec<OciImageLayer>,
}
@ -89,10 +93,10 @@ impl OciImageFetcher {
}
}
async fn load_seed_json_blob<T: DeserializeOwned>(
async fn load_seed_json_blob<T: Clone + Debug + DeserializeOwned>(
&self,
descriptor: &Descriptor,
) -> Result<Option<T>> {
) -> Result<Option<OciSchema<T>>> {
let digest = descriptor.digest();
let Some((digest_type, digest_content)) = digest.split_once(':') else {
return Err(anyhow!("digest content was not properly formatted"));
@ -101,7 +105,10 @@ impl OciImageFetcher {
self.load_seed_json(&want).await
}
async fn load_seed_json<T: DeserializeOwned>(&self, want: &str) -> Result<Option<T>> {
async fn load_seed_json<T: Clone + Debug + DeserializeOwned>(
&self,
want: &str,
) -> Result<Option<OciSchema<T>>> {
let Some(ref seed) = self.seed else {
return Ok(None);
};
@ -113,10 +120,10 @@ impl OciImageFetcher {
let mut entry = entry?;
let path = String::from_utf8(entry.path_bytes().to_vec())?;
if path == want {
let mut content = String::new();
entry.read_to_string(&mut content).await?;
let data = serde_json::from_str::<T>(&content)?;
return Ok(Some(data));
let mut content = Vec::new();
entry.read_to_end(&mut content).await?;
let item = serde_json::from_slice::<T>(&content)?;
return Ok(Some(OciSchema::new(content, item)));
}
}
Ok(None)
@ -154,7 +161,7 @@ impl OciImageFetcher {
if let Some(index) = self.load_seed_json::<ImageIndex>("index.json").await? {
let mut found: Option<&Descriptor> = None;
for manifest in index.manifests() {
for manifest in index.item().manifests() {
let Some(annotations) = manifest.annotations() else {
continue;
};
@ -215,7 +222,7 @@ impl OciImageFetcher {
image: OciResolvedImage,
layer_dir: &Path,
) -> Result<OciLocalImage> {
let config: ImageConfiguration;
let config: OciSchema<ImageConfiguration>;
self.progress
.update(|progress| {
progress.phase = OciProgressPhase::ConfigAcquire;
@ -223,27 +230,30 @@ impl OciImageFetcher {
.await;
let mut client = OciRegistryClient::new(image.name.registry_url()?, self.platform.clone())?;
if let Some(seeded) = self
.load_seed_json_blob::<ImageConfiguration>(image.manifest.config())
.load_seed_json_blob::<ImageConfiguration>(image.manifest.item().config())
.await?
{
config = seeded;
} else {
let config_bytes = client
.get_blob(&image.name.name, image.manifest.config())
.get_blob(&image.name.name, image.manifest.item().config())
.await?;
config = serde_json::from_slice(&config_bytes)?;
config = OciSchema::new(
config_bytes.to_vec(),
serde_json::from_slice(&config_bytes)?,
);
}
self.progress
.update(|progress| {
progress.phase = OciProgressPhase::LayerAcquire;
for layer in image.manifest.layers() {
for layer in image.manifest.item().layers() {
progress.add_layer(layer.digest(), layer.size() as usize);
}
})
.await;
let mut layers = Vec::new();
for layer in image.manifest.layers() {
for layer in image.manifest.item().layers() {
self.progress
.update(|progress| {
progress.downloading_layer(layer.digest(), 0, layer.size() as usize);

1
crates/oci/src/lib.rs
View File

@ -4,4 +4,5 @@ pub mod name;
pub mod packer;
pub mod progress;
pub mod registry;
pub mod schema;
pub mod vfs;

25
crates/oci/src/packer/cache.rs
View File

@ -1,4 +1,7 @@
use crate::packer::{OciImagePacked, OciPackedFormat};
use crate::{
packer::{OciImagePacked, OciPackedFormat},
schema::OciSchema,
};
use anyhow::Result;
use log::debug;
@ -38,17 +41,17 @@ impl OciPackerCache {
&& manifest_metadata.is_file()
&& config_metadata.is_file()
{
let manifest_text = fs::read_to_string(&manifest_path).await?;
let manifest: ImageManifest = serde_json::from_str(&manifest_text)?;
let config_text = fs::read_to_string(&config_path).await?;
let config: ImageConfiguration = serde_json::from_str(&config_text)?;
let manifest_bytes = fs::read(&manifest_path).await?;
let manifest: ImageManifest = serde_json::from_slice(&manifest_bytes)?;
let config_bytes = fs::read(&config_path).await?;
let config: ImageConfiguration = serde_json::from_slice(&config_bytes)?;
debug!("cache hit digest={}", digest);
Some(OciImagePacked::new(
digest.to_string(),
fs_path.clone(),
format,
config,
manifest,
OciSchema::new(config_bytes, config),
OciSchema::new(manifest_bytes, manifest),
))
} else {
None
@ -68,11 +71,9 @@ impl OciPackerCache {
fs_path.push(format!("{}.{}", packed.digest, packed.format.extension()));
manifest_path.push(format!("{}.manifest.json", packed.digest));
config_path.push(format!("{}.config.json", packed.digest));
fs::copy(&packed.path, &fs_path).await?;
let manifest_text = serde_json::to_string_pretty(&packed.manifest)?;
fs::write(&manifest_path, manifest_text).await?;
let config_text = serde_json::to_string_pretty(&packed.config)?;
fs::write(&config_path, config_text).await?;
fs::rename(&packed.path, &fs_path).await?;
fs::write(&config_path, packed.config.raw()).await?;
fs::write(&manifest_path, packed.manifest.raw()).await?;
Ok(OciImagePacked::new(
packed.digest,
fs_path.clone(),

10
crates/oci/src/packer/mod.rs
View File

@ -1,5 +1,7 @@
use std::path::PathBuf;
use crate::schema::OciSchema;
use self::backend::OciPackerBackendType;
use oci_spec::image::{ImageConfiguration, ImageManifest};
@ -35,8 +37,8 @@ pub struct OciImagePacked {
pub digest: String,
pub path: PathBuf,
pub format: OciPackedFormat,
pub config: ImageConfiguration,
pub manifest: ImageManifest,
pub config: OciSchema<ImageConfiguration>,
pub manifest: OciSchema<ImageManifest>,
}
impl OciImagePacked {
@ -44,8 +46,8 @@ impl OciImagePacked {
digest: String,
path: PathBuf,
format: OciPackedFormat,
config: ImageConfiguration,
manifest: ImageManifest,
config: OciSchema<ImageConfiguration>,
manifest: OciSchema<ImageManifest>,
) -> OciImagePacked {
OciImagePacked {
digest,

1
crates/oci/src/packer/service.rs
View File

@ -67,7 +67,6 @@ impl OciPackerService {
packer
.pack(progress, assembled.vfs.clone(), &target)
.await?;
let packed = OciImagePacked::new(
assembled.digest.clone(),
file,

16
crates/oci/src/registry.rs
View File

@ -7,7 +7,7 @@ use reqwest::{Client, RequestBuilder, Response, StatusCode};
use tokio::{fs::File, io::AsyncWriteExt};
use url::Url;
use crate::progress::OciBoundProgress;
use crate::{progress::OciBoundProgress, schema::OciSchema};
#[derive(Clone, Debug)]
pub struct OciPlatform {
@ -176,7 +176,7 @@ impl OciRegistryClient {
&mut self,
name: N,
reference: R,
) -> Result<(ImageManifest, String)> {
) -> Result<(OciSchema<ImageManifest>, String)> {
let url = self.url.join(&format!(
"/v2/{}/manifests/{}",
name.as_ref(),
@ -198,15 +198,16 @@ impl OciRegistryClient {
.ok_or_else(|| anyhow!("fetching manifest did not yield a content digest"))?
.to_str()?
.to_string();
let manifest = serde_json::from_str(&response.text().await?)?;
Ok((manifest, digest))
let bytes = response.bytes().await?;
let manifest = serde_json::from_slice(&bytes)?;
Ok((OciSchema::new(bytes.to_vec(), manifest), digest))
}
pub async fn get_manifest_with_digest<N: AsRef<str>, R: AsRef<str>>(
&mut self,
name: N,
reference: R,
) -> Result<(ImageManifest, String)> {
) -> Result<(OciSchema<ImageManifest>, String)> {
let url = self.url.join(&format!(
"/v2/{}/manifests/{}",
name.as_ref(),
@ -244,8 +245,9 @@ impl OciRegistryClient {
.ok_or_else(|| anyhow!("fetching manifest did not yield a content digest"))?
.to_str()?
.to_string();
let manifest = serde_json::from_str(&response.text().await?)?;
Ok((manifest, digest))
let bytes = response.bytes().await?;
let manifest = serde_json::from_slice(&bytes)?;
Ok((OciSchema::new(bytes.to_vec(), manifest), digest))
}
fn pick_manifest(&mut self, index: ImageIndex) -> Option<Descriptor> {

29
crates/oci/src/schema.rs Normal file
View File

@ -0,0 +1,29 @@
use std::fmt::Debug;
#[derive(Clone, Debug)]
pub struct OciSchema<T: Clone + Debug> {
raw: Vec<u8>,
item: T,
}
impl<T: Clone + Debug> OciSchema<T> {
pub fn new(raw: Vec<u8>, item: T) -> OciSchema<T> {
OciSchema { raw, item }
}
pub fn raw(&self) -> &[u8] {
&self.raw
}
pub fn item(&self) -> &T {
&self.item
}
pub fn into_raw(self) -> Vec<u8> {
self.raw
}
pub fn into_item(self) -> T {
self.item
}
}

4
crates/runtime/src/cfgblk.rs
View File

@ -26,7 +26,7 @@ impl ConfigBlock<'_> {
pub fn build(&self, launch_config: &LaunchInfo) -> Result<()> {
trace!("build launch_config={:?}", launch_config);
let manifest = self.image.config.to_string()?;
let config = self.image.config.raw();
let launch = serde_json::to_string(launch_config)?;
let mut writer = FilesystemWriter::default();
writer.push_dir(
@ -39,7 +39,7 @@ impl ConfigBlock<'_> {
},
)?;
writer.push_file(
manifest.as_bytes(),
config,
"/image/config.json",
NodeHeader {
permissions: 384,