feat: oci concurrency improvements (#95)

* feat: implement improved and detailed oci progress indication

* feat: implement on-disk indexes of images

* oci: utilize rw-lock for increased cache performance

crates/oci/src/fetch.rs

@@ -10,6 +10,8 @@ use super::{
 
 use std::{
     fmt::Debug,
+    io::SeekFrom,
+    os::unix::fs::MetadataExt,
     path::{Path, PathBuf},
     pin::Pin,
 };
@@ -18,12 +20,13 @@ use anyhow::{anyhow, Result};
 use async_compression::tokio::bufread::{GzipDecoder, ZstdDecoder};
 use log::debug;
 use oci_spec::image::{
-    Descriptor, ImageConfiguration, ImageIndex, ImageManifest, MediaType, ToDockerV2S2,
+    Descriptor, DescriptorBuilder, ImageConfiguration, ImageIndex, ImageManifest, MediaType,
+    ToDockerV2S2,
 };
 use serde::de::DeserializeOwned;
 use tokio::{
-    fs::File,
-    io::{AsyncRead, AsyncReadExt, BufReader, BufWriter},
+    fs::{self, File},
+    io::{AsyncRead, AsyncReadExt, AsyncSeekExt, BufReader, BufWriter},
 };
 use tokio_stream::StreamExt;
 use tokio_tar::Archive;
@@ -43,16 +46,43 @@ pub enum OciImageLayerCompression {
 
 #[derive(Clone, Debug)]
 pub struct OciImageLayer {
+    pub metadata: Descriptor,
     pub path: PathBuf,
     pub digest: String,
     pub compression: OciImageLayerCompression,
 }
 
+#[async_trait::async_trait]
+pub trait OciImageLayerReader: AsyncRead + Sync {
+    async fn position(&mut self) -> Result<u64>;
+}
+
+#[async_trait::async_trait]
+impl OciImageLayerReader for BufReader<File> {
+    async fn position(&mut self) -> Result<u64> {
+        Ok(self.seek(SeekFrom::Current(0)).await?)
+    }
+}
+
+#[async_trait::async_trait]
+impl OciImageLayerReader for GzipDecoder<BufReader<File>> {
+    async fn position(&mut self) -> Result<u64> {
+        self.get_mut().position().await
+    }
+}
+
+#[async_trait::async_trait]
+impl OciImageLayerReader for ZstdDecoder<BufReader<File>> {
+    async fn position(&mut self) -> Result<u64> {
+        self.get_mut().position().await
+    }
+}
+
 impl OciImageLayer {
-    pub async fn decompress(&self) -> Result<Pin<Box<dyn AsyncRead + Send>>> {
+    pub async fn decompress(&self) -> Result<Pin<Box<dyn OciImageLayerReader + Send>>> {
         let file = File::open(&self.path).await?;
         let reader = BufReader::new(file);
-        let reader: Pin<Box<dyn AsyncRead + Send>> = match self.compression {
+        let reader: Pin<Box<dyn OciImageLayerReader + Send>> = match self.compression {
             OciImageLayerCompression::None => Box::pin(reader),
             OciImageLayerCompression::Gzip => Box::pin(GzipDecoder::new(reader)),
             OciImageLayerCompression::Zstd => Box::pin(ZstdDecoder::new(reader)),
@@ -60,7 +90,7 @@ impl OciImageLayer {
         Ok(reader)
     }
 
-    pub async fn archive(&self) -> Result<Archive<Pin<Box<dyn AsyncRead + Send>>>> {
+    pub async fn archive(&self) -> Result<Archive<Pin<Box<dyn OciImageLayerReader + Send>>>> {
         let decompress = self.decompress().await?;
         Ok(Archive::new(decompress))
     }
@@ -70,6 +100,7 @@ impl OciImageLayer {
 pub struct OciResolvedImage {
     pub name: ImageName,
     pub digest: String,
+    pub descriptor: Descriptor,
     pub manifest: OciSchema<ImageManifest>,
 }
 
@@ -199,6 +230,7 @@ impl OciImageFetcher {
                     );
                     return Ok(OciResolvedImage {
                         name: image,
+                        descriptor: found.clone(),
                         digest: found.digest().clone(),
                         manifest,
                     });
@@ -207,11 +239,20 @@ impl OciImageFetcher {
         }
 
         let mut client = OciRegistryClient::new(image.registry_url()?, self.platform.clone())?;
-        let (manifest, digest) = client
+        let (manifest, descriptor, digest) = client
             .get_manifest_with_digest(&image.name, &image.reference)
             .await?;
+        let descriptor = descriptor.unwrap_or_else(|| {
+            DescriptorBuilder::default()
+                .media_type(MediaType::ImageManifest)
+                .size(manifest.raw().len() as i64)
+                .digest(digest.clone())
+                .build()
+                .unwrap()
+        });
         Ok(OciResolvedImage {
             name: image,
+            descriptor,
             digest,
             manifest,
         })
@@ -225,7 +266,7 @@ impl OciImageFetcher {
         let config: OciSchema<ImageConfiguration>;
         self.progress
             .update(|progress| {
-                progress.phase = OciProgressPhase::ConfigAcquire;
+                progress.phase = OciProgressPhase::ConfigDownload;
             })
             .await;
         let mut client = OciRegistryClient::new(image.name.registry_url()?, self.platform.clone())?;
@@ -245,10 +286,10 @@ impl OciImageFetcher {
         }
         self.progress
             .update(|progress| {
-                progress.phase = OciProgressPhase::LayerAcquire;
+                progress.phase = OciProgressPhase::LayerDownload;
 
                 for layer in image.manifest.item().layers() {
-                    progress.add_layer(layer.digest(), layer.size() as usize);
+                    progress.add_layer(layer.digest());
                 }
             })
             .await;
@@ -256,7 +297,7 @@ impl OciImageFetcher {
         for layer in image.manifest.item().layers() {
             self.progress
                 .update(|progress| {
-                    progress.downloading_layer(layer.digest(), 0, layer.size() as usize);
+                    progress.downloading_layer(layer.digest(), 0, layer.size() as u64);
                 })
                 .await;
             layers.push(
@@ -265,7 +306,7 @@ impl OciImageFetcher {
             );
             self.progress
                 .update(|progress| {
-                    progress.downloaded_layer(layer.digest());
+                    progress.downloaded_layer(layer.digest(), layer.size() as u64);
                 })
                 .await;
         }
@@ -304,6 +345,12 @@ impl OciImageFetcher {
             }
         }
 
+        let metadata = fs::metadata(&layer_path).await?;
+
+        if layer.size() as u64 != metadata.size() {
+            return Err(anyhow!("layer size differs from size in manifest",));
+        }
+
         let mut media_type = layer.media_type().clone();
 
         // docker layer compatibility
@@ -318,6 +365,7 @@ impl OciImageFetcher {
             other => return Err(anyhow!("found layer with unknown media type: {}", other)),
         };
         Ok(OciImageLayer {
+            metadata: layer.clone(),
             path: layer_path,
             digest: layer.digest().clone(),
             compression,