Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support a shared cache for the OCI blobs, used in the LXC OCI template #134

Merged
merged 4 commits into from
Oct 15, 2024
Merged

Support a shared cache for the OCI blobs, used in the LXC OCI template #134

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0b5a961
Support symlinks for blobs/sha256, useful for sharing blobs across OC…
ariel-miculas Sep 25, 2024
0e4e627
Use find_manifest_descriptor_with_tag from ocidir-rs
ariel-miculas Oct 3, 2024
79ee4d5
Use new_empty_manifest from ocidir-rs
ariel-miculas Oct 14, 2024
37b83f0
Store the right size of the blobs in the OCI descriptors
ariel-miculas Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion puzzlefs-lib/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ os_pipe = "1.1.2"
tempfile = "3.10"
openat = "0.1.21"
zstd-seekable = "0.1.23"
ocidir = "0.3.0"
ocidir = {git="https://github.com/containers/ocidir-rs"}
cap-std = "3.2.0"


Expand Down
32 changes: 17 additions & 15 deletions puzzlefs-lib/src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -502,14 +502,12 @@ pub fn enable_fs_verity(oci: Image, tag: &str, manifest_root_hash: &str) -> Resu
.find_manifest_with_tag(tag)?
.ok_or_else(|| WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture()))?;
let config_digest = manifest.config().digest().digest();
let config_digest_path = oci.blob_path().join(config_digest);
enable_verity_for_file(&oci.0.dir.open(config_digest_path)?)?;
let config_digest_path = Image::blob_path().join(config_digest);
enable_verity_for_file(&oci.0.dir().open(config_digest_path)?)?;

for (content_addressed_file, verity_hash) in rootfs.get_verity_data()? {
let file_path = oci
.blob_path()
.join(Digest::new(&content_addressed_file).to_string());
let fd = oci.0.dir.open(&file_path)?;
let file_path = Image::blob_path().join(Digest::new(&content_addressed_file).to_string());
let fd = oci.0.dir().open(&file_path)?;
if let Err(e) = fsverity_enable(
fd.as_raw_fd(),
FS_VERITY_BLOCK_SIZE_DEFAULT,
Expand Down Expand Up @@ -564,8 +562,8 @@ pub mod tests {

let md = image
.0
.dir
.symlink_metadata(image.blob_path().join(FILE_DIGEST))
.dir()
.symlink_metadata(Image::blob_path().join(FILE_DIGEST))
.unwrap();
assert!(md.is_file());

Expand Down Expand Up @@ -603,14 +601,15 @@ pub mod tests {
chunks[0].len,
decompressor.get_uncompressed_length().unwrap()
);
Ok(())
} else {
panic!("bad inode mode: {:?}", inodes[1].mode);
}
};
image.0.fsck()?;
Ok::<(), anyhow::Error>(())
}

#[test]
fn test_delta_generation() {
fn test_delta_generation() -> anyhow::Result<()> {
let dir = tempdir().unwrap();
let image = Image::new(dir.path()).unwrap();
let tag = "test";
Expand All @@ -623,6 +622,7 @@ pub mod tests {
delta_dir.join("SekienAkashita.jpg"),
)
.unwrap();
image.0.fsck()?;

let new_tag = "test2";
let (_desc, image) =
Expand All @@ -631,6 +631,7 @@ pub mod tests {
assert_eq!(delta.metadatas.len(), 2);

let image = Image::new(dir.path()).unwrap();
image.0.fsck()?;
let mut pfs = PuzzleFS::open(image, new_tag, None).unwrap();
assert_eq!(pfs.max_inode().unwrap(), 3);
let mut walker = WalkPuzzleFS::walk(&mut pfs).unwrap();
Expand All @@ -651,6 +652,7 @@ pub mod tests {
assert_eq!(foo_dir.inode.dir_entries().unwrap().len(), 0);

assert!(walker.next().is_none());
Ok(())
}

fn do_vecs_match<T: PartialEq>(a: &[T], b: &[T]) -> bool {
Expand All @@ -662,8 +664,8 @@ pub mod tests {
matching == a.len()
}

fn get_image_blobs(image: &Image) -> Vec<OsString> {
WalkDir::new(image.blob_path())
fn get_image_blobs() -> Vec<OsString> {
WalkDir::new(Image::blob_path())
.contents_first(false)
.follow_links(false)
.same_file_system(true)
Expand All @@ -685,7 +687,7 @@ pub mod tests {

for (i, image) in images.iter().enumerate() {
build_test_fs(path, image, "test").unwrap();
let ents = get_image_blobs(image);
let ents = get_image_blobs();
sha_suite.push(ents);

if i != 0 && !do_vecs_match(&sha_suite[i - 1], &sha_suite[i]) {
Expand All @@ -708,7 +710,7 @@ pub mod tests {

for (i, image) in images.iter().enumerate() {
build_test_fs(&path[i], image, "test").unwrap();
let ents = get_image_blobs(image);
let ents = get_image_blobs();
sha_suite.push(ents);

if i != 0 && !do_vecs_match(&sha_suite[i - 1], &sha_suite[i]) {
Expand Down
83 changes: 22 additions & 61 deletions puzzlefs-lib/src/oci.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ use std::any::Any;
use std::backtrace::Backtrace;
use std::fs;
use std::io;
use std::io::Write;
use std::io::{Read, Seek};
use std::path::{Path, PathBuf};

Expand All @@ -17,37 +16,37 @@ pub use crate::format::Digest;
use crate::oci::media_types::{PuzzleFSMediaType, PUZZLEFS_ROOTFS, VERITY_ROOT_HASH_ANNOTATION};
use ocidir::oci_spec::image;
pub use ocidir::oci_spec::image::Descriptor;
use ocidir::oci_spec::image::{
DescriptorBuilder, ImageIndex, ImageManifest, ImageManifestBuilder, MediaType, Sha256Digest,
};
use ocidir::oci_spec::OciSpecError;
use ocidir::oci_spec::image::{ImageIndex, ImageManifest, MediaType};
use ocidir::OciDir;
use std::collections::HashMap;
use std::str::FromStr;

use std::io::Cursor;

pub mod media_types;
const OCI_TAG_ANNOTATION: &str = "org.opencontainers.image.ref.name";

pub struct Image(pub OciDir);

impl Image {
pub fn new(oci_dir: &Path) -> Result<Self> {
fs::create_dir_all(oci_dir)?;
let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?;
let oci_dir = OciDir::ensure(&d)?;
let oci_dir = OciDir::ensure(d)?;

Ok(Self(oci_dir))
}

pub fn open(oci_dir: &Path) -> Result<Self> {
let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?;
let oci_dir = OciDir::open(&d)?;
let blobs_dir = cap_std::fs::Dir::open_ambient_dir(
oci_dir.join(Self::blob_path()),
cap_std::ambient_authority(),
)?;
let oci_dir = OciDir::open_with_external_blobs(d, blobs_dir)?;
Ok(Self(oci_dir))
}

pub fn blob_path(&self) -> PathBuf {
pub fn blob_path() -> PathBuf {
// TODO: use BLOBDIR constant from ocidir after making it public
PathBuf::from("blobs/sha256")
}
Expand All @@ -72,6 +71,7 @@ impl Image {
let uncompressed_size = io::copy(&mut <&[u8]>::clone(&buf), &mut compressed)?;
compressed.end()?;
let compressed_size = compressed_data.get_ref().len() as u64;
let final_size = std::cmp::min(compressed_size, uncompressed_size);

// store the uncompressed blob if the compressed version has bigger size
let final_data = if compressed_blob && compressed_size >= uncompressed_size {
Expand All @@ -90,7 +90,7 @@ impl Image {
let fs_verity_digest = get_fs_verity_digest(&compressed_data.get_ref()[..])?;
let mut descriptor = Descriptor::new(
MediaType::Other(media_type_with_extension),
uncompressed_size,
final_size,
image::Digest::from_str(&digest_string)?,
);
// We need to store the PuzzleFS Rootfs verity digest as an annotation (obviously we cannot
Expand All @@ -103,12 +103,12 @@ impl Image {
);
descriptor.set_annotations(Some(annotations));
}
let path = self.blob_path().join(descriptor.digest().digest());
let path = Self::blob_path().join(descriptor.digest().digest());

// avoid replacing the data blob so we don't drop fsverity data
if self.0.dir.exists(&path) {
if self.0.dir().exists(&path) {
let mut hasher = Sha256::new();
let mut file = self.0.dir.open(&path)?;
let mut file = self.0.dir().open(&path)?;
io::copy(&mut file, &mut hasher)?;
let existing_digest = hasher.finalize();
if existing_digest != digest {
Expand All @@ -120,7 +120,7 @@ impl Image {
.into());
}
} else {
self.0.dir.write(&path, final_data)?;
self.0.dir().write(&path, final_data)?;
}

// Let's make the PuzzleFS image rootfs the first layer so it's easy to find
Expand All @@ -136,7 +136,7 @@ impl Image {
}

fn open_raw_blob(&self, digest: &str, verity: Option<&[u8]>) -> io::Result<cap_std::fs::File> {
let file = self.0.dir.open(self.blob_path().join(digest))?;
let file = self.0.blobs_dir().open(digest)?;
if let Some(verity) = verity {
check_fs_verity(&file, verity).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
}
Expand Down Expand Up @@ -201,21 +201,10 @@ impl Image {
Ok(file)
}

// TODO: export this function from ocidr / find another way to avoid code duplication
fn descriptor_is_tagged(d: &Descriptor, tag: &str) -> bool {
d.annotations()
.as_ref()
.and_then(|annos| annos.get(OCI_TAG_ANNOTATION))
.filter(|tagval| tagval.as_str() == tag)
.is_some()
}

pub fn get_image_manifest_fd(&self, tag: &str) -> Result<cap_std::fs::File> {
let index = self.get_index()?;
let image_manifest = index
.manifests()
.iter()
.find(|desc| Self::descriptor_is_tagged(desc, tag))
let image_manifest = self
.0
.find_manifest_descriptor_with_tag(tag)?
.ok_or_else(|| {
WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture())
})?;
Expand Down Expand Up @@ -270,39 +259,11 @@ impl Image {
}

pub fn get_index(&self) -> Result<ImageIndex> {
Ok(self
.0
.read_index()?
.ok_or_else(|| OciSpecError::Other("missing OCI index".to_string()))?)
Ok(self.0.read_index()?)
}

pub fn get_empty_manifest(&self) -> Result<ImageManifest> {
// see https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor
let config = DescriptorBuilder::default()
.media_type(MediaType::EmptyJSON)
.size(2_u32)
.digest(Sha256Digest::from_str(
"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
)?)
.data("e30=")
.build()?;

if !self.0.dir.exists(
self.blob_path()
.join("44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"),
) {
let mut blob = self.0.create_blob()?;
blob.write_all("{}".as_bytes())?;
// TODO: blob.complete_verified_as(&config)? once https://github.com/containers/ocidir-rs/pull/18 is merged
blob.complete()?;
}

let image_manifest = ImageManifestBuilder::default()
.schema_version(2_u32)
.config(config)
.layers(Vec::new())
.build()?;
Ok(image_manifest)
Ok(self.0.new_empty_manifest()?.build()?)
}
}

Expand Down Expand Up @@ -330,8 +291,8 @@ mod tests {

let md = image
.0
.dir
.symlink_metadata(image.blob_path().join(DIGEST))?;
.dir()
.symlink_metadata(Image::blob_path().join(DIGEST))?;
assert!(md.is_file());
Ok(())
}
Expand Down
Loading