From 0bf3eb27f4b23f0585b45d5c1202918289fae34d Mon Sep 17 00:00:00 2001 From: Alex Zenla Date: Thu, 18 Jan 2024 10:16:59 -0800 Subject: [PATCH] hypha: implement custom image fetcher --- hypha/Cargo.toml | 7 ++-- hypha/src/ctl/mod.rs | 2 +- hypha/src/error.rs | 38 ++++++++++++++++++---- hypha/src/image/fetch.rs | 69 ++++++++++++++++++++++++++++++++++++++++ hypha/src/image/mod.rs | 20 ++++++------ hypha/src/image/name.rs | 67 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 181 insertions(+), 22 deletions(-) create mode 100644 hypha/src/image/fetch.rs create mode 100644 hypha/src/image/name.rs diff --git a/hypha/Cargo.toml b/hypha/Cargo.toml index 11f905d..d76813c 100644 --- a/hypha/Cargo.toml +++ b/hypha/Cargo.toml @@ -17,16 +17,15 @@ walkdir = "2" serde = "1.0.195" serde_json = "1.0.111" sha256 = "1.5.0" +url = "2.5.0" +ureq = "2.9.1" [dependencies.clap] version = "4.4.18" features = ["derive"] -[dependencies.ocipkg] -version = "0.2.8" - [dependencies.oci-spec] -version = "0.5.8" +version = "0.6.4" [dependencies.backhand] version = "0.14.2" diff --git a/hypha/src/ctl/mod.rs b/hypha/src/ctl/mod.rs index 83aa99c..3004a9e 100644 --- a/hypha/src/ctl/mod.rs +++ b/hypha/src/ctl/mod.rs @@ -1,7 +1,7 @@ use crate::error::{HyphaError, Result}; use crate::image::cache::ImageCache; +use crate::image::name::ImageName; use crate::image::{ImageCompiler, ImageInfo}; -use ocipkg::ImageName; use std::fs; use std::path::PathBuf; use uuid::Uuid; diff --git a/hypha/src/error.rs b/hypha/src/error.rs index ea29cfa..e1948ba 100644 --- a/hypha/src/error.rs +++ b/hypha/src/error.rs @@ -1,6 +1,8 @@ use backhand::BackhandError; +use oci_spec::OciSpecError; use std::error::Error; use std::fmt::{Display, Formatter}; +use std::num::ParseIntError; use std::path::StripPrefixError; use xenclient::XenClientError; @@ -43,12 +45,6 @@ impl From for HyphaError { } } -impl From for HyphaError { - fn from(value: ocipkg::error::Error) -> Self { - HyphaError::new(value.to_string().as_str()) - } -} - impl From for HyphaError { fn from(value: walkdir::Error) -> Self { HyphaError::new(value.to_string().as_str()) @@ -72,3 +68,33 @@ impl From for HyphaError { HyphaError::new(value.to_string().as_str()) } } + +impl From for HyphaError { + fn from(value: ureq::Error) -> Self { + HyphaError::new(value.to_string().as_str()) + } +} + +impl From for HyphaError { + fn from(value: ParseIntError) -> Self { + HyphaError::new(value.to_string().as_str()) + } +} + +impl From for HyphaError { + fn from(value: OciSpecError) -> Self { + HyphaError::new(value.to_string().as_str()) + } +} + +impl From for HyphaError { + fn from(value: url::ParseError) -> Self { + HyphaError::new(value.to_string().as_str()) + } +} + +impl From for HyphaError { + fn from(value: std::fmt::Error) -> Self { + HyphaError::new(value.to_string().as_str()) + } +} diff --git a/hypha/src/image/fetch.rs b/hypha/src/image/fetch.rs new file mode 100644 index 0000000..76c1ae8 --- /dev/null +++ b/hypha/src/image/fetch.rs @@ -0,0 +1,69 @@ +use crate::error::{HyphaError, Result}; +use oci_spec::image::{Arch, Descriptor, ImageIndex, ImageManifest, MediaType, Os, ToDockerV2S2}; +use std::io::Read; +use ureq::{Agent, Request, Response}; +use url::Url; + +pub struct RegistryClient { + agent: Agent, + url: Url, +} + +impl RegistryClient { + pub fn new(url: Url) -> Result { + Ok(RegistryClient { + agent: Agent::new(), + url, + }) + } + + fn call(&mut self, req: Request) -> Result { + Ok(req.call()?) + } + + pub fn get_blob(&mut self, name: &str, descriptor: &Descriptor) -> Result> { + let url = self + .url + .join(&format!("/v2/{}/blobs/{}", name, descriptor.digest()))?; + let response = self.call(self.agent.get(url.as_str()))?; + let mut buffer: Vec = Vec::new(); + response.into_reader().read_to_end(&mut buffer)?; + Ok(buffer) + } + + pub fn get_manifest(&mut self, name: &str, reference: &str) -> Result { + let url = self + .url + .join(&format!("/v2/{}/manifests/{}", name, reference))?; + let accept = format!( + "{}, {}, {}", + MediaType::ImageManifest.to_docker_v2s2()?, + MediaType::ImageManifest, + MediaType::ImageIndex, + ); + let response = self.call(self.agent.get(url.as_str()).set("Accept", &accept))?; + let content_type = response.header("Content-Type").ok_or_else(|| { + HyphaError::new("registry response did not have a Content-Type header") + })?; + if content_type == MediaType::ImageIndex.to_string() { + let index = ImageIndex::from_reader(response.into_reader())?; + let descriptor = self + .pick_manifest(index) + .ok_or_else(|| HyphaError::new("unable to pick manifest from index"))?; + return self.get_manifest(name, descriptor.digest()); + } + let manifest = ImageManifest::from_reader(response.into_reader())?; + Ok(manifest) + } + + fn pick_manifest(&mut self, index: ImageIndex) -> Option { + for item in index.manifests() { + if let Some(platform) = item.platform() { + if *platform.os() == Os::Linux && *platform.architecture() == Arch::Amd64 { + return Some(item.clone()); + } + } + } + None + } +} diff --git a/hypha/src/image/mod.rs b/hypha/src/image/mod.rs index 74ea9a1..d324883 100644 --- a/hypha/src/image/mod.rs +++ b/hypha/src/image/mod.rs @@ -1,13 +1,14 @@ pub mod cache; +pub mod fetch; +pub mod name; use crate::error::{HyphaError, Result}; use crate::image::cache::ImageCache; +use crate::image::fetch::RegistryClient; +use crate::image::name::ImageName; use backhand::{FilesystemWriter, NodeHeader}; use log::{debug, trace}; use oci_spec::image::{ImageConfiguration, ImageManifest, MediaType}; -use ocipkg::distribution::Client; -use ocipkg::error::Error; -use ocipkg::{Digest, ImageName}; use std::fs; use std::fs::File; use std::io::BufReader; @@ -71,11 +72,8 @@ impl ImageCompiler<'_> { "ImageCompiler download image={image}, image_dir={}", image_dir.to_str().unwrap() ); - let ImageName { - name, reference, .. - } = image; - let mut client = Client::new(image.registry_url()?, name.clone())?; - let manifest = client.get_manifest(reference)?; + let mut client = RegistryClient::new(image.registry_url()?)?; + let manifest = client.get_manifest(&image.name, &image.reference)?; let manifest_serialized = serde_json::to_string(&manifest)?; let cache_key = format!( "manifest\n{}squashfs-version\n{}\n", @@ -87,7 +85,7 @@ impl ImageCompiler<'_> { return Ok(cached); } - let config_bytes = client.get_blob(&Digest::new(manifest.config().digest())?)?; + let config_bytes = client.get_blob(&image.name, manifest.config())?; let config: ImageConfiguration = serde_json::from_slice(&config_bytes)?; for layer in manifest.layers() { @@ -97,7 +95,7 @@ impl ImageCompiler<'_> { layer.size() ); - let blob = client.get_blob(&Digest::new(layer.digest())?)?; + let blob = client.get_blob(&image.name, layer)?; match layer.media_type() { MediaType::ImageLayerGzip => {} MediaType::Other(ty) => { @@ -123,7 +121,7 @@ impl ImageCompiler<'_> { let info = ImageInfo::new(squash_file.clone(), manifest.clone(), config)?; return self.cache.store(&cache_digest, &info); } - Err(Error::MissingLayer.into()) + Err(HyphaError::new("unable to find image layer")) } fn squash(&self, image_dir: &PathBuf, squash_file: &PathBuf) -> Result<()> { diff --git a/hypha/src/image/name.rs b/hypha/src/image/name.rs new file mode 100644 index 0000000..e145575 --- /dev/null +++ b/hypha/src/image/name.rs @@ -0,0 +1,67 @@ +use crate::error::Result; +use std::fmt; +use url::Url; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ImageName { + pub hostname: String, + pub port: Option, + pub name: String, + pub reference: String, +} + +impl fmt::Display for ImageName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(port) = self.port { + write!( + f, + "{}:{}/{}:{}", + self.hostname, port, self.name, self.reference + ) + } else { + write!(f, "{}/{}:{}", self.hostname, self.name, self.reference) + } + } +} + +impl Default for ImageName { + fn default() -> Self { + Self::parse(&format!("{}", uuid::Uuid::new_v4().as_hyphenated())) + .expect("UUID hyphenated must be valid name") + } +} + +impl ImageName { + pub fn parse(name: &str) -> Result { + let (hostname, name) = name + .split_once('/') + .unwrap_or(("registry-1.docker.io", name)); + let (hostname, port) = if let Some((hostname, port)) = hostname.split_once(':') { + (hostname, Some(str::parse(port)?)) + } else { + (hostname, None) + }; + let (name, reference) = name.split_once(':').unwrap_or((name, "latest")); + Ok(ImageName { + hostname: hostname.to_string(), + port, + name: name.to_string(), + reference: reference.to_string(), + }) + } + + /// URL for OCI distribution API endpoint + pub fn registry_url(&self) -> Result { + let hostname = if let Some(port) = self.port { + format!("{}:{}", self.hostname, port) + } else { + self.hostname.clone() + }; + let url = if self.hostname.starts_with("localhost") { + format!("http://{}", hostname) + } else { + format!("https://{}", hostname) + }; + Ok(Url::parse(&url)?) + } +}