From 0b5a9616c78ed3e0b927fa11791d66e4c2c98767 Mon Sep 17 00:00:00 2001 From: Ariel Miculas-Trif Date: Wed, 25 Sep 2024 08:59:36 +0300 Subject: [PATCH 1/4] Support symlinks for blobs/sha256, useful for sharing blobs across OCI repos Use open_with_external_blobs which adds support for external blobs, added in ocidir-rs [1]. We'll use the latest version of ocidir-rs from github and we'll switch to version 0.4.0 when it gets released. Closes #131 [1] https://github.com/containers/ocidir-rs/pull/22 Signed-off-by: Ariel Miculas-Trif --- Cargo.lock | 7 +++---- puzzlefs-lib/Cargo.toml | 2 +- puzzlefs-lib/src/builder.rs | 22 ++++++++++------------ puzzlefs-lib/src/oci.rs | 34 +++++++++++++++++----------------- 4 files changed, 31 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a65273b..0d0e297 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "adler2" @@ -1018,9 +1018,8 @@ dependencies = [ [[package]] name = "ocidir" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1123c697592d4240224b7e09d50375c7da3f17320ed741c922f54b5377b79eb0" +version = "0.3.1" +source = "git+https://github.com/containers/ocidir-rs#a943d18a54806b7b6a5425bc9cacddd4d7f326d1" dependencies = [ "camino", "cap-std-ext", diff --git a/puzzlefs-lib/Cargo.toml b/puzzlefs-lib/Cargo.toml index e361200..bf61611 100644 --- a/puzzlefs-lib/Cargo.toml +++ b/puzzlefs-lib/Cargo.toml @@ -40,7 +40,7 @@ os_pipe = "1.1.2" tempfile = "3.10" openat = "0.1.21" zstd-seekable = "0.1.23" -ocidir = "0.3.0" +ocidir = {git="https://github.com/containers/ocidir-rs"} cap-std = "3.2.0" diff --git a/puzzlefs-lib/src/builder.rs b/puzzlefs-lib/src/builder.rs index d2eea81..bce01a5 100644 --- a/puzzlefs-lib/src/builder.rs +++ b/puzzlefs-lib/src/builder.rs @@ -502,14 +502,12 @@ pub fn enable_fs_verity(oci: Image, tag: &str, manifest_root_hash: &str) -> Resu .find_manifest_with_tag(tag)? .ok_or_else(|| WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture()))?; let config_digest = manifest.config().digest().digest(); - let config_digest_path = oci.blob_path().join(config_digest); - enable_verity_for_file(&oci.0.dir.open(config_digest_path)?)?; + let config_digest_path = Image::blob_path().join(config_digest); + enable_verity_for_file(&oci.0.dir().open(config_digest_path)?)?; for (content_addressed_file, verity_hash) in rootfs.get_verity_data()? { - let file_path = oci - .blob_path() - .join(Digest::new(&content_addressed_file).to_string()); - let fd = oci.0.dir.open(&file_path)?; + let file_path = Image::blob_path().join(Digest::new(&content_addressed_file).to_string()); + let fd = oci.0.dir().open(&file_path)?; if let Err(e) = fsverity_enable( fd.as_raw_fd(), FS_VERITY_BLOCK_SIZE_DEFAULT, @@ -564,8 +562,8 @@ pub mod tests { let md = image .0 - .dir - .symlink_metadata(image.blob_path().join(FILE_DIGEST)) + .dir() + .symlink_metadata(Image::blob_path().join(FILE_DIGEST)) .unwrap(); assert!(md.is_file()); @@ -662,8 +660,8 @@ pub mod tests { matching == a.len() } - fn get_image_blobs(image: &Image) -> Vec { - WalkDir::new(image.blob_path()) + fn get_image_blobs() -> Vec { + WalkDir::new(Image::blob_path()) .contents_first(false) .follow_links(false) .same_file_system(true) @@ -685,7 +683,7 @@ pub mod tests { for (i, image) in images.iter().enumerate() { build_test_fs(path, image, "test").unwrap(); - let ents = get_image_blobs(image); + let ents = get_image_blobs(); sha_suite.push(ents); if i != 0 && !do_vecs_match(&sha_suite[i - 1], &sha_suite[i]) { @@ -708,7 +706,7 @@ pub mod tests { for (i, image) in images.iter().enumerate() { build_test_fs(&path[i], image, "test").unwrap(); - let ents = get_image_blobs(image); + let ents = get_image_blobs(); sha_suite.push(ents); if i != 0 && !do_vecs_match(&sha_suite[i - 1], &sha_suite[i]) { diff --git a/puzzlefs-lib/src/oci.rs b/puzzlefs-lib/src/oci.rs index abc01df..6c82c3a 100644 --- a/puzzlefs-lib/src/oci.rs +++ b/puzzlefs-lib/src/oci.rs @@ -20,7 +20,6 @@ pub use ocidir::oci_spec::image::Descriptor; use ocidir::oci_spec::image::{ DescriptorBuilder, ImageIndex, ImageManifest, ImageManifestBuilder, MediaType, Sha256Digest, }; -use ocidir::oci_spec::OciSpecError; use ocidir::OciDir; use std::collections::HashMap; use std::str::FromStr; @@ -36,18 +35,22 @@ impl Image { pub fn new(oci_dir: &Path) -> Result { fs::create_dir_all(oci_dir)?; let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?; - let oci_dir = OciDir::ensure(&d)?; + let oci_dir = OciDir::ensure(d)?; Ok(Self(oci_dir)) } pub fn open(oci_dir: &Path) -> Result { let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?; - let oci_dir = OciDir::open(&d)?; + let blobs_dir = cap_std::fs::Dir::open_ambient_dir( + oci_dir.join(Self::blob_path()), + cap_std::ambient_authority(), + )?; + let oci_dir = OciDir::open_with_external_blobs(d, blobs_dir)?; Ok(Self(oci_dir)) } - pub fn blob_path(&self) -> PathBuf { + pub fn blob_path() -> PathBuf { // TODO: use BLOBDIR constant from ocidir after making it public PathBuf::from("blobs/sha256") } @@ -103,12 +106,12 @@ impl Image { ); descriptor.set_annotations(Some(annotations)); } - let path = self.blob_path().join(descriptor.digest().digest()); + let path = Self::blob_path().join(descriptor.digest().digest()); // avoid replacing the data blob so we don't drop fsverity data - if self.0.dir.exists(&path) { + if self.0.dir().exists(&path) { let mut hasher = Sha256::new(); - let mut file = self.0.dir.open(&path)?; + let mut file = self.0.dir().open(&path)?; io::copy(&mut file, &mut hasher)?; let existing_digest = hasher.finalize(); if existing_digest != digest { @@ -120,7 +123,7 @@ impl Image { .into()); } } else { - self.0.dir.write(&path, final_data)?; + self.0.dir().write(&path, final_data)?; } // Let's make the PuzzleFS image rootfs the first layer so it's easy to find @@ -136,7 +139,7 @@ impl Image { } fn open_raw_blob(&self, digest: &str, verity: Option<&[u8]>) -> io::Result { - let file = self.0.dir.open(self.blob_path().join(digest))?; + let file = self.0.blobs_dir().open(digest)?; if let Some(verity) = verity { check_fs_verity(&file, verity).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; } @@ -270,10 +273,7 @@ impl Image { } pub fn get_index(&self) -> Result { - Ok(self - .0 - .read_index()? - .ok_or_else(|| OciSpecError::Other("missing OCI index".to_string()))?) + Ok(self.0.read_index()?) } pub fn get_empty_manifest(&self) -> Result { @@ -287,8 +287,8 @@ impl Image { .data("e30=") .build()?; - if !self.0.dir.exists( - self.blob_path() + if !self.0.dir().exists( + Self::blob_path() .join("44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"), ) { let mut blob = self.0.create_blob()?; @@ -330,8 +330,8 @@ mod tests { let md = image .0 - .dir - .symlink_metadata(image.blob_path().join(DIGEST))?; + .dir() + .symlink_metadata(Image::blob_path().join(DIGEST))?; assert!(md.is_file()); Ok(()) } From 0e4e62760b731f5ef40c0a00d7fad784d1b86e7f Mon Sep 17 00:00:00 2001 From: Ariel Miculas-Trif Date: Thu, 3 Oct 2024 13:34:34 +0300 Subject: [PATCH 2/4] Use find_manifest_descriptor_with_tag from ocidir-rs Now that [1] was merged, we can use find_manifest_descriptor_with_tag and remove copy-pasted code from ocidir-rs. [1] https://github.com/containers/ocidir-rs/pull/28 Signed-off-by: Ariel Miculas-Trif --- puzzlefs-lib/src/oci.rs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/puzzlefs-lib/src/oci.rs b/puzzlefs-lib/src/oci.rs index 6c82c3a..9ddc55f 100644 --- a/puzzlefs-lib/src/oci.rs +++ b/puzzlefs-lib/src/oci.rs @@ -27,7 +27,6 @@ use std::str::FromStr; use std::io::Cursor; pub mod media_types; -const OCI_TAG_ANNOTATION: &str = "org.opencontainers.image.ref.name"; pub struct Image(pub OciDir); @@ -204,21 +203,10 @@ impl Image { Ok(file) } - // TODO: export this function from ocidr / find another way to avoid code duplication - fn descriptor_is_tagged(d: &Descriptor, tag: &str) -> bool { - d.annotations() - .as_ref() - .and_then(|annos| annos.get(OCI_TAG_ANNOTATION)) - .filter(|tagval| tagval.as_str() == tag) - .is_some() - } - pub fn get_image_manifest_fd(&self, tag: &str) -> Result { - let index = self.get_index()?; - let image_manifest = index - .manifests() - .iter() - .find(|desc| Self::descriptor_is_tagged(desc, tag)) + let image_manifest = self + .0 + .find_manifest_descriptor_with_tag(tag)? .ok_or_else(|| { WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture()) })?; From 79ee4d567ccc1004d2517e2c4918eb0664167705 Mon Sep 17 00:00:00 2001 From: Ariel Miculas-Trif Date: Mon, 14 Oct 2024 11:25:14 +0300 Subject: [PATCH 3/4] Use new_empty_manifest from ocidir-rs Now that [1] was merged into ocidir-rs, we can use the new_empty_manifest function and we'll get a valid OCI image with an empty config descriptor. [1] https://github.com/containers/ocidir-rs/pull/31 Signed-off-by: Ariel Miculas-Trif --- puzzlefs-lib/src/oci.rs | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/puzzlefs-lib/src/oci.rs b/puzzlefs-lib/src/oci.rs index 9ddc55f..dcfd213 100644 --- a/puzzlefs-lib/src/oci.rs +++ b/puzzlefs-lib/src/oci.rs @@ -3,7 +3,6 @@ use std::any::Any; use std::backtrace::Backtrace; use std::fs; use std::io; -use std::io::Write; use std::io::{Read, Seek}; use std::path::{Path, PathBuf}; @@ -17,9 +16,7 @@ pub use crate::format::Digest; use crate::oci::media_types::{PuzzleFSMediaType, PUZZLEFS_ROOTFS, VERITY_ROOT_HASH_ANNOTATION}; use ocidir::oci_spec::image; pub use ocidir::oci_spec::image::Descriptor; -use ocidir::oci_spec::image::{ - DescriptorBuilder, ImageIndex, ImageManifest, ImageManifestBuilder, MediaType, Sha256Digest, -}; +use ocidir::oci_spec::image::{ImageIndex, ImageManifest, MediaType}; use ocidir::OciDir; use std::collections::HashMap; use std::str::FromStr; @@ -265,32 +262,7 @@ impl Image { } pub fn get_empty_manifest(&self) -> Result { - // see https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor - let config = DescriptorBuilder::default() - .media_type(MediaType::EmptyJSON) - .size(2_u32) - .digest(Sha256Digest::from_str( - "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", - )?) - .data("e30=") - .build()?; - - if !self.0.dir().exists( - Self::blob_path() - .join("44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"), - ) { - let mut blob = self.0.create_blob()?; - blob.write_all("{}".as_bytes())?; - // TODO: blob.complete_verified_as(&config)? once https://github.com/containers/ocidir-rs/pull/18 is merged - blob.complete()?; - } - - let image_manifest = ImageManifestBuilder::default() - .schema_version(2_u32) - .config(config) - .layers(Vec::new()) - .build()?; - Ok(image_manifest) + Ok(self.0.new_empty_manifest()?.build()?) } } From 37b83f0af21f984a7ac9f8dea1e9b53ea60cd846 Mon Sep 17 00:00:00 2001 From: Ariel Miculas-Trif Date: Mon, 14 Oct 2024 12:12:17 +0300 Subject: [PATCH 4/4] Store the right size of the blobs in the OCI descriptors The final size of the blobs should be stored in the descriptors instead of the uncompressed size. Make sure we have a valid oci image by calling the fsck function from ocidir-rs. Closes #55 Signed-off-by: Ariel Miculas-Trif --- puzzlefs-lib/src/builder.rs | 10 +++++++--- puzzlefs-lib/src/oci.rs | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/puzzlefs-lib/src/builder.rs b/puzzlefs-lib/src/builder.rs index bce01a5..5e3e5aa 100644 --- a/puzzlefs-lib/src/builder.rs +++ b/puzzlefs-lib/src/builder.rs @@ -601,14 +601,15 @@ pub mod tests { chunks[0].len, decompressor.get_uncompressed_length().unwrap() ); - Ok(()) } else { panic!("bad inode mode: {:?}", inodes[1].mode); - } + }; + image.0.fsck()?; + Ok::<(), anyhow::Error>(()) } #[test] - fn test_delta_generation() { + fn test_delta_generation() -> anyhow::Result<()> { let dir = tempdir().unwrap(); let image = Image::new(dir.path()).unwrap(); let tag = "test"; @@ -621,6 +622,7 @@ pub mod tests { delta_dir.join("SekienAkashita.jpg"), ) .unwrap(); + image.0.fsck()?; let new_tag = "test2"; let (_desc, image) = @@ -629,6 +631,7 @@ pub mod tests { assert_eq!(delta.metadatas.len(), 2); let image = Image::new(dir.path()).unwrap(); + image.0.fsck()?; let mut pfs = PuzzleFS::open(image, new_tag, None).unwrap(); assert_eq!(pfs.max_inode().unwrap(), 3); let mut walker = WalkPuzzleFS::walk(&mut pfs).unwrap(); @@ -649,6 +652,7 @@ pub mod tests { assert_eq!(foo_dir.inode.dir_entries().unwrap().len(), 0); assert!(walker.next().is_none()); + Ok(()) } fn do_vecs_match(a: &[T], b: &[T]) -> bool { diff --git a/puzzlefs-lib/src/oci.rs b/puzzlefs-lib/src/oci.rs index dcfd213..158bef4 100644 --- a/puzzlefs-lib/src/oci.rs +++ b/puzzlefs-lib/src/oci.rs @@ -71,6 +71,7 @@ impl Image { let uncompressed_size = io::copy(&mut <&[u8]>::clone(&buf), &mut compressed)?; compressed.end()?; let compressed_size = compressed_data.get_ref().len() as u64; + let final_size = std::cmp::min(compressed_size, uncompressed_size); // store the uncompressed blob if the compressed version has bigger size let final_data = if compressed_blob && compressed_size >= uncompressed_size { @@ -89,7 +90,7 @@ impl Image { let fs_verity_digest = get_fs_verity_digest(&compressed_data.get_ref()[..])?; let mut descriptor = Descriptor::new( MediaType::Other(media_type_with_extension), - uncompressed_size, + final_size, image::Digest::from_str(&digest_string)?, ); // We need to store the PuzzleFS Rootfs verity digest as an annotation (obviously we cannot