From 4f4866d5267606ee8514cf2a014b965a3a698cd4 Mon Sep 17 00:00:00 2001 From: Ariel Miculas-Trif Date: Thu, 12 Sep 2024 00:10:46 +0300 Subject: [PATCH] Integrate the PuzzleFS image into the OCI image specification Previously, the OCI Image Index contained a list of manifests which were referencing the PuzzleFS rootfs image, i.e. the metadata of the PuzzleFS image in Capnproto format. Now the Image Index [1] references an Image Manifest [2] and the PuzzleFS image (the PuzzleFS rootfs image together with the file chunks) is embedded into the layers field of the Image Manifest. Where PuzzleFS diverges from the Image Manifest spec is in the layers definition: our layers are not self contained images and thus they do not stack. Instead, we have a rootfs layer which stores the PuzzleFS image rootfs and multiple file chunks which contain the actual data of the filesystem. No extraction step is performed. Instead, when mounting a PuzzleFS image, the filesystem is reconstructed from the PuzzleFS metadata and the file chunks, not unlike how squashfs/erofs archives are mounted directly. See the "Inspecting a puzzlefs image" section from the README for more details about the format. The image config is an empty descriptor [3] for now, but we don't store it in blobs/sha256, which causes `skopeo copy` to fail because it doesn't find the blob referenced by the empty descriptor in the data store. This will be addressed in a subsequent commit. See #55 for more context. [1] https://github.com/opencontainers/image-spec/blob/main/image-index.md [2] https://github.com/opencontainers/image-spec/blob/main/manifest.md [3] https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor Signed-off-by: Ariel Miculas-Trif --- Cargo.lock | 999 +++++++++++++++++++++++---- README.md | 175 +++-- exe/src/main.rs | 14 +- exe/tests/verity.rs | 2 +- puzzlefs-lib/Cargo.toml | 2 + puzzlefs-lib/src/builder.rs | 102 ++- puzzlefs-lib/src/extractor.rs | 15 +- puzzlefs-lib/src/format/error.rs | 15 +- puzzlefs-lib/src/format/types.rs | 2 +- puzzlefs-lib/src/fsverity_helpers.rs | 3 +- puzzlefs-lib/src/oci.rs | 351 ++++++---- puzzlefs-lib/src/oci/descriptor.rs | 38 - puzzlefs-lib/src/oci/index.rs | 76 -- puzzlefs-lib/src/oci/media_types.rs | 19 +- puzzlefs-lib/src/reader/fuse.rs | 3 +- puzzlefs-lib/src/reader/puzzlefs.rs | 6 +- puzzlefs-lib/src/reader/walk.rs | 6 +- 17 files changed, 1298 insertions(+), 530 deletions(-) delete mode 100644 puzzlefs-lib/src/oci/descriptor.rs delete mode 100644 puzzlefs-lib/src/oci/index.rs diff --git a/Cargo.lock b/Cargo.lock index fa64b7c..3b96daf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -11,69 +17,92 @@ dependencies = [ "memchr", ] +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" -version = "0.6.13" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "assert_cmd" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8" +checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d" dependencies = [ "anstyle", "bstr", "doc-comment", + "libc", "predicates", "predicates-core", "predicates-tree", @@ -91,6 +120,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + [[package]] name = "bincode" version = "1.3.3" @@ -102,9 +137,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "block-buffer" @@ -123,26 +158,90 @@ checksum = "5988cb1d626264ac94100be357308f29ff7cbdd3b36bda27f450a4ee3f713426" [[package]] name = "bstr" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" dependencies = [ "memchr", "regex-automata", "serde", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" + +[[package]] +name = "cap-primitives" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d00bd8d26c4270d950eaaa837387964a2089a1c3c349a690a1fa03221d29531" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes", + "ipnet", + "maybe-owned", + "rustix", + "windows-sys 0.52.0", + "winx", +] + +[[package]] +name = "cap-std" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19eb8e3d71996828751c1ed3908a439639752ac6bdc874e41469ef7fc15fbd7f" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes", + "rustix", +] + +[[package]] +name = "cap-std-ext" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0279cf1f7b6cbeeb98e6946e8fea58136f691d4d0aa8c775f4439a05030a481" +dependencies = [ + "cap-primitives", + "cap-tempfile", + "rustix", +] + +[[package]] +name = "cap-tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53880047c3f37cd64947775f0526795498d614182603a718c792616b762ce777" +dependencies = [ + "cap-std", + "rand", + "rustix", + "uuid", +] + [[package]] name = "capnp" -version = "0.19.3" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b11832e6fb7a695c4a63cc42bd97bd2cda7165cd850caf5aff9a3d0e617720ed" +checksum = "de71387912cac7dd3cb7c219e09628411620a18061bba58c71453c26ae7bf66a" dependencies = [ "embedded-io", ] @@ -158,13 +257,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.95" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +checksum = "e9d013ecb737093c0e86b151a7b837993cf9ec6c502946cfb44bedc392421e0b" dependencies = [ "jobserver", "libc", - "once_cell", + "shlex", ] [[package]] @@ -175,15 +274,29 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" -version = "0.1.1" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] [[package]] name = "clap" -version = "4.5.4" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -191,9 +304,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstream", "anstyle", @@ -203,9 +316,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck", "proc-macro2", @@ -215,25 +328,40 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -246,12 +374,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.4" +version = "3.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" +checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3" dependencies = [ - "nix 0.28.0", - "windows-sys", + "nix 0.29.0", + "windows-sys 0.59.0", ] [[package]] @@ -264,6 +392,41 @@ dependencies = [ "libc", ] +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "deranged" version = "0.3.11" @@ -273,6 +436,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd33f37ee6a119146a1781d3356a7c26028f83d779b2e04ecd45fdc75c76877b" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7431fa049613920234f22c47fdc33e6cf3ee83067091ea4277a3f8c4587aae38" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "difflib" version = "0.4.0" @@ -331,12 +525,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -356,9 +550,64 @@ checksum = "c47726595a8a071d7d8045a837d1179b1964633e256300675aa50c31284a23e2" [[package]] name = "fastrand" -version = "2.0.2" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + +[[package]] +name = "flate2" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +dependencies = [ + "crc32fast", + "libz-sys", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fs-set-times" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033b337d725b97690d86893f9de22b67b80dcc4e9ad815f348254c38119db8fb" +dependencies = [ + "io-lifetimes", + "rustix", + "windows-sys 0.52.0", +] [[package]] name = "fs-verity" @@ -397,11 +646,34 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getset" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f636605b743120a8d32ed92fc27b6cde1a769f8f936c065151eb66f88ded513c" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "heck" @@ -447,16 +719,73 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "indexmap" -version = "2.2.6" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown", ] +[[package]] +name = "io-extras" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9f046b9af244f13b3bd939f55d16830ac3a201e8a9ba9661bfcb03e2be72b9b" +dependencies = [ + "io-lifetimes", + "windows-sys 0.52.0", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a611371471e98973dbcab4e0ec66c31a10bc356eeb4d54a0e05eac8158fe38c" + +[[package]] +name = "ipnet" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itoa" version = "1.0.11" @@ -465,30 +794,61 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "libc" -version = "0.2.153" +version = "0.2.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" + +[[package]] +name = "libredox" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags", + "libc", + "redox_syscall", +] + +[[package]] +name = "libz-sys" +version = "1.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "match_cfg" @@ -496,11 +856,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" @@ -511,6 +877,15 @@ dependencies = [ "libc", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "nix" version = "0.27.1" @@ -524,9 +899,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags", "cfg-if", @@ -540,6 +915,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -580,6 +964,53 @@ dependencies = [ "libc", ] +[[package]] +name = "oci-spec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cee185ce7cf1cce45e194e34cd87c0bad7ff0aa2e8917009a2da4f7b31fb363" +dependencies = [ + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror", +] + +[[package]] +name = "ocidir" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1123c697592d4240224b7e09d50375c7da3f17320ed741c922f54b5377b79eb0" +dependencies = [ + "camino", + "cap-std-ext", + "chrono", + "flate2", + "hex", + "oci-spec", + "olpc-cjson", + "openssl", + "serde", + "serde_json", + "tar", + "thiserror", +] + +[[package]] +name = "olpc-cjson" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d637c9c15b639ccff597da8f4fa968300651ad2f1e968aefc3b4927a6fb2027a" +dependencies = [ + "serde", + "serde_json", + "unicode-normalization", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -595,14 +1026,52 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-sys" +version = "0.9.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_pipe" -version = "1.1.5" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57119c3b893986491ec9aa85056780d3a0f3cf4da7cc09dd3650dbd6c6738fb9" +checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -653,11 +1122,20 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + [[package]] name = "predicates" -version = "3.1.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8" +checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97" dependencies = [ "anstyle", "difflib", @@ -666,15 +1144,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" +checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931" [[package]] name = "predicates-tree" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13" dependencies = [ "predicates-core", "termtree", @@ -690,11 +1168,33 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" -version = "1.0.81" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -725,6 +1225,7 @@ name = "puzzlefs-lib" version = "0.1.0" dependencies = [ "anyhow", + "cap-std", "capnp", "capnpc", "fastcdc", @@ -734,6 +1235,7 @@ dependencies = [ "log", "memmap2", "nix 0.27.1", + "ocidir", "openat", "os_pipe", "serde", @@ -749,34 +1251,73 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" -version = "1.10.4" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", ] [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", ] [[package]] @@ -787,28 +1328,36 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" dependencies = [ "bitflags", "errno", + "itoa", "libc", "linux-raw-sys", - "windows-sys", + "once_cell", + "windows-sys 0.52.0", ] [[package]] -name = "ryu" +name = "rustversion" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "same-file" @@ -821,18 +1370,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.198" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.198" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", "quote", @@ -841,11 +1390,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -861,6 +1411,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "smallvec" version = "1.13.2" @@ -896,11 +1452,30 @@ dependencies = [ "syn", ] +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "syn" -version = "2.0.60" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -920,16 +1495,28 @@ dependencies = [ "time", ] +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -949,18 +1536,18 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.59" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.59" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", @@ -1009,11 +1596,26 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" @@ -1038,17 +1640,41 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "getrandom", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wait-timeout" @@ -1069,6 +1695,67 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + [[package]] name = "winapi" version = "0.3.9" @@ -1087,11 +1774,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -1100,6 +1787,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -1109,11 +1805,20 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -1127,51 +1832,51 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" @@ -1182,6 +1887,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winx" +version = "0.36.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9643b83820c0cd246ecabe5fa454dd04ba4fa67996369466d0747472d337346" +dependencies = [ + "bitflags", + "windows-sys 0.52.0", +] + [[package]] name = "xattr" version = "1.3.1" @@ -1195,9 +1910,9 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", "zerocopy-derive", @@ -1205,9 +1920,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", @@ -1216,18 +1931,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] @@ -1250,9 +1965,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/README.md b/README.md index 8e3825a..5405a6d 100644 --- a/README.md +++ b/README.md @@ -272,80 +272,127 @@ Otherwise, run `fusermount -u /tmp/mounted-image`. You will need to have `fuse` ### Inspecting a puzzlefs image ``` $ cd /tmp/puzzlefs-image -$ cat index.json | jq . +$ cat index.json | jq { - "schemaVersion": -1, "manifests": [ { - "digest": "sha256:0efa2a4b490abb02a5b9b5f2d43c8262643dba48c67f14b236df0a6f1ea745d8", - "size": 272, - "media_type": "application/vnd.puzzlefs.image.rootfs.v1", "annotations": { "org.opencontainers.image.ref.name": "puzzlefs_example" - } + }, + "digest": "sha256:c9106994f5e18833e45164e2028431e9c822b4697172f8a997a0d9a3b0d26c9e", + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "platform": { + "architecture": "amd64", + "os": "linux" + }, + "size": 619 } ], - "annotations": {} + "schemaVersion": 2 } ``` -The `digest` specifies the puzzlefs image manifest, which needs to be decoded using the `capnp tool` and the manifest schema -(assuming you've cloned puzzlefs in `~/puzzlefs`): -``` -$ capnp convert binary:json ~/puzzlefs/format/manifest.capnp Rootfs < blobs/sha256/0efa2a4b490abb02a5b9b5f2d43c8262643dba48c67f14b236df0a6f1ea745d8 - -{ "metadatas": [{ "digest": [102, 197, 227, 96, 136, 156, 147, 144, 139, 154, 248, 228, 29, 161, 252, 228, 118, 222, 21, 44, 132, 0, 214, 164, 80, 74, 121, 156, 26, 85, 123, 57], - "offset": "0", - "compressed": false }], - "fsVerityData": [ - { "digest": [102, 197, 227, 96, 136, 156, 147, 144, 139, 154, 248, 228, 29, 161, 252, 228, 118, 222, 21, 44, 132, 0, 214, 164, 80, 74, 121, 156, 26, 85, 123, 57], - "verity": [224, 180, 63, 193, 142, 198, 24, 175, 78, 42, 126, 227, 253, 187, 102, 162, 31, 77, 85, 252, 205, 137, 198, 216, 26, 213, 113, 238, 144, 79, 93, 244] }, - { "digest": [239, 32, 68, 39, 210, 105, 37, 83, 131, 158, 224, 24, 162, 25, 96, 90, 140, 95, 158, 194, 97, 2, 153, 175, 54, 197, 216, 193, 115, 121, 62, 22], - "verity": [196, 54, 71, 79, 3, 104, 3, 253, 163, 243, 85, 213, 67, 235, 144, 210, 20, 206, 160, 209, 75, 164, 93, 22, 79, 84, 41, 119, 20, 84, 64, 164] } ], - "manifestVersion": "1" } -``` -`metadatas` contains a list of layers (in this case only one) which can be further decoded (the sha of the blob is obtained by a decimal to hexadecimal conversion): -``` -$ capnp convert binary:json ~/puzzlefs/format/metadata.capnp InodeVector < blobs/sha256/66c5e360889c93908b9af8e41da1fce476de152c8400d6a4504a799c1a557b39 - -{"inodes": [ - { "ino": "1", - "mode": {"dir": { - "entries": [ - { "ino": "2", - "name": [97, 108, 103, 111, 114, 105, 116, 104, 109, 115] }, - { "ino": "3", - "name": [108, 111, 114, 101, 109, 95, 105, 112, 115, 117, 109, 46, 116, 120, 116] } ], - "lookBelow": false }}, - "uid": 1000, - "gid": 1000, - "permissions": 493 }, - { "ino": "2", - "mode": {"dir": { - "entries": [{ "ino": "4", - "name": [98, 105, 110, 97, 114, 121, 45, 115, 101, 97, 114, 99, 104, 46, 116, 120, 116] }], - "lookBelow": false }}, - "uid": 1000, - "gid": 1000, - "permissions": 493 }, - { "ino": "3", - "mode": {"file": {"chunks": [{ "blob": { - "digest": [239, 32, 68, 39, 210, 105, 37, 83, 131, 158, 224, 24, 162, 25, 96, 90, 140, 95, 158, 194, 97, 2, 153, 175, 54, 197, 216, 193, 115, 121, 62, 22], - "offset": "0", - "compressed": false }, - "len": "865" }]}}, - "uid": 1000, - "gid": 1000, - "permissions": 420 }, - { "ino": "4", - "mode": {"file": {"chunks": [{ "blob": { - "digest": [239, 32, 68, 39, 210, 105, 37, 83, 131, 158, 224, 24, 162, 25, 96, 90, 140, 95, 158, 194, 97, 2, 153, 175, 54, 197, 216, 193, 115, 121, 62, 22], - "offset": "865", - "compressed": false }, - "len": "278" }]}}, - "uid": 1000, - "gid": 1000, - "permissions": 420 } ]} +`index.json` follows the [OCI Image Index Specification](https://github.com/opencontainers/image-spec/blob/main/image-index.md). + +The digest tagged with the `puzzlefs_example` tag is an [OCI Image +Manifest](https://github.com/opencontainers/image-spec/blob/main/manifest.md) +with the caveat that `layers` are not applied in the usual way (i.e. by +stacking each one on top of one another). See below for details about the +PuzzleFS `layer` descriptors. + +The Image Manifest looks like this: ``` +$ cat blobs/sha256/c9106994f5e18833e45164e2028431e9c822b4697172f8a997a0d9a3b0d26c9e | jq +{ + "config": { + "data": "e30=", + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "mediaType": "application/vnd.oci.empty.v1+json", + "size": 2 + }, + "layers": [ + { + "digest": "sha256:b7f1ee9373416a49835747455ec4d287bcccc5a4bf8c38156483d46b35ce4dbd", + "mediaType": "application/vnd.puzzlefs.image.filedata.v1", + "size": 27 + }, + { + "annotations": { + "io.puzzlefsoci.puzzlefs.puzzlefs_verity_root_hash": "7b22d0210c16134159be75d8239d100817b451591d39af2031d94ae84ac4f8c7" + }, + "digest": "sha256:9e2edc6917b65606b1112ac8663665dfd2d945cfea960ca595accf790922b910", + "mediaType": "application/vnd.puzzlefs.image.rootfs.v1", + "size": 552 + } + ], + "schemaVersion": 2 +} +``` + +There are two types of layer descriptors: +* `application/vnd.puzzlefs.image.rootfs.v1`: the PuzzleFS image rootfs which + contains metadata in Capnproto format and must appear only once in the + `layers` array +* `application/vnd.puzzlefs.image.filedata.v1`: a PuzzleFS data chunk generated + by the FastCDC algorithm; usually there are multiple chunks in an image and + they contain all the filesystem data + +There is no extraction step for these layers, PuzzleFS mounts the filesystem by +reading the PuzzleFS image rootfs and using this metadata to combine the data +chunks back into the original files. In fact, the data chunks are part of the +OCI Image Manifest so that the other tools copy the image correctly. For +example, with skopeo: +``` +$ skopeo --version +skopeo version 1.15.2 +$ skopeo copy oci:/tmp/puzzlefs-image:puzzlefs_example oci:/tmp/copy-puzzlefs-image:puzzlefs_example +``` +The information about the data chunks is also stored in the PuzzleFS image rootfs, +so that PuzzleFS could mount the filesystem efficiently and that the PuzzleFS +image could also be decoded in the kernel. + +The `digest` of the PuzzleFS iamge rootfs contains the filesystem metadata and +it can be decoded using the `capnp tool` and the capnp metadata schema (the +following snippet assumes that you've cloned puzzlefs in `~/puzzlefs`): +``` +$ capnp convert binary:json ~/puzzlefs/puzzlefs-lib/src/format/metadata.capnp Rootfs < blobs/sha256/9e2edc6917b65606b1112ac8663665dfd2d945cfea960ca595accf790922b910 +{ "metadatas": [{"inodes": [ + { "ino": "1", + "mode": {"dir": { + "entries": [ + { "ino": "2", + "name": [97, 108, 103, 111, 114, 105, 116, 104, 109, 115] }, + { "ino": "3", + "name": [108, 111, 114, 101, 109, 95, 105, 112, 115, 117, 109, 46, 116, 120, 116] } ], + "lookBelow": false }}, + "uid": 1000, + "gid": 1000, + "permissions": 493 }, + { "ino": "2", + "mode": {"dir": { + "entries": [{ "ino": "4", + "name": [98, 105, 110, 97, 114, 121, 45, 115, 101, 97, 114, 99, 104, 46, 116, 120, 116] }], + "lookBelow": false }}, + "uid": 1000, + "gid": 1000, + "permissions": 509 }, + { "ino": "3", + "mode": {"file": [{ "blob": { + "digest": [183, 241, 238, 147, 115, 65, 106, 73, 131, 87, 71, 69, 94, 196, 210, 135, 188, 204, 197, 164, 191, 140, 56, 21, 100, 131, 212, 107, 53, 206, 77, 189], + "offset": "0", + "compressed": false }, + "len": "27" }]}, + "uid": 1000, + "gid": 1000, + "permissions": 436 }, + {"ino": "4", "mode": {"file": []}, "uid": 1000, "gid": 1000, "permissions": 436} ]}], + "fsVerityData": [{ "digest": [183, 241, 238, 147, 115, 65, 106, 73, 131, 87, 71, 69, 94, 196, 210, 135, 188, 204, 197, 164, 191, 140, 56, 21, 100, 131, 212, 107, 53, 206, 77, 189], + "verity": [91, 20, 52, 173, 44, 8, 31, 244, 53, 178, 16, 121, 46, 144, 14, 39, 2, 30, 196, 43, 104, 230, 143, 98, 219, 173, 82, 223, 224, 201, 247, 164] }], + "manifestVersion": "3" } +``` + +`metadatas` contains a list of PuzzleFS layers, each layer consisting of a +vector of Inodes. See the [capnp +schema](./puzzlefs-lib/src/format/metadata.capnp) for details. ## Implementation diff --git a/exe/src/main.rs b/exe/src/main.rs index 63f6fdf..6164130 100644 --- a/exe/src/main.rs +++ b/exe/src/main.rs @@ -157,21 +157,19 @@ fn main() -> anyhow::Result<()> { let image = Image::new(oci_dir)?; let new_image = match b.base_layer { Some(base_layer) => { - let (desc, image) = if b.compression { - add_rootfs_delta::(rootfs, image, &base_layer)? + let (_desc, image) = if b.compression { + add_rootfs_delta::(rootfs, image, &b.tag, &base_layer)? } else { - add_rootfs_delta::(rootfs, image, &base_layer)? + add_rootfs_delta::(rootfs, image, &b.tag, &base_layer)? }; - image.add_tag(&b.tag, desc)?; image } None => { - let desc = if b.compression { - build_initial_rootfs::(rootfs, &image)? + if b.compression { + build_initial_rootfs::(rootfs, &image, &b.tag)? } else { - build_initial_rootfs::(rootfs, &image)? + build_initial_rootfs::(rootfs, &image, &b.tag)? }; - image.add_tag(&b.tag, desc)?; Arc::new(image) } }; diff --git a/exe/tests/verity.rs b/exe/tests/verity.rs index a8c7552..ef00322 100644 --- a/exe/tests/verity.rs +++ b/exe/tests/verity.rs @@ -126,7 +126,7 @@ fn test_fs_verity() -> anyhow::Result<()> { assert!(mount_output .unwrap_err() .to_string() - .contains("Error: fs error: invalid fs_verity data: fsverity mismatch")); + .contains("invalid fs_verity data: fsverity mismatch")); // test that we can mount with the right digest puzzlefs([ diff --git a/puzzlefs-lib/Cargo.toml b/puzzlefs-lib/Cargo.toml index 5e35e3d..92be84c 100644 --- a/puzzlefs-lib/Cargo.toml +++ b/puzzlefs-lib/Cargo.toml @@ -40,6 +40,8 @@ os_pipe = "1.1.2" tempfile = "3.10" openat = "0.1.21" zstd-seekable = "0.1.23" +ocidir = "0.3.0" +cap-std = "3.2.0" [dev-dependencies] diff --git a/puzzlefs-lib/src/builder.rs b/puzzlefs-lib/src/builder.rs index 7278022..e4d381b 100644 --- a/puzzlefs-lib/src/builder.rs +++ b/puzzlefs-lib/src/builder.rs @@ -26,6 +26,7 @@ use crate::metadata_capnp; use crate::oci::media_types; use crate::oci::{Descriptor, Image}; use crate::reader::{PuzzleFS, PUZZLEFS_IMAGE_MANIFEST_VERSION}; +use ocidir::oci_spec::image::{ImageManifest, Platform}; use nix::errno::Errno; @@ -92,6 +93,7 @@ fn process_chunks( mut chunker: StreamCDC, files: &mut [File], verity_data: &mut VerityData, + image_manifest: &mut ImageManifest, ) -> Result<()> { let mut file_iter = files.iter_mut(); let mut file_used = 0; @@ -108,10 +110,11 @@ fn process_chunks( let mut chunk_used: u64 = 0; let (desc, fs_verity_digest, compressed) = - oci.put_blob::(&chunk.data)?; + oci.put_blob::(&chunk.data, image_manifest, media_types::Chunk {})?; + let digest = Digest::try_from(desc.digest().digest())?.underlying(); let verity_hash = fs_verity_digest; - verity_data.insert(desc.digest.underlying(), verity_hash); + verity_data.insert(digest, verity_hash); while chunk_used < chunk.length as u64 { let room = min( @@ -121,7 +124,7 @@ fn process_chunks( let blob = BlobRef { offset: chunk_used, - digest: desc.digest.underlying(), + digest, compressed, }; @@ -164,6 +167,7 @@ fn build_delta( oci: &Image, mut existing: Option, verity_data: &mut VerityData, + image_manifest: &mut ImageManifest, ) -> Result> { let mut dirs = HashMap::::new(); let mut files = Vec::::new(); @@ -349,7 +353,7 @@ fn build_delta( AVG_CHUNK_SIZE, MAX_CHUNK_SIZE, ); - process_chunks::(oci, fcdc, &mut files, verity_data)?; + process_chunks::(oci, fcdc, &mut files, verity_data, image_manifest)?; // TODO: not render this whole thing in memory, stick it all in the same blob, etc. let mut sorted_dirs = dirs.into_values().collect::>(); @@ -392,9 +396,11 @@ fn build_delta( pub fn build_initial_rootfs( rootfs: &Path, oci: &Image, + tag: &str, ) -> Result { let mut verity_data: VerityData = BTreeMap::new(); - let inodes = build_delta::(rootfs, oci, None, &mut verity_data)?; + let mut image_manifest = Image::get_empty_manifest()?; + let inodes = build_delta::(rootfs, oci, None, &mut verity_data, &mut image_manifest)?; let rootfs_buf = serialize_metadata(Rootfs { metadatas: vec![inodes], @@ -402,9 +408,17 @@ pub fn build_initial_rootfs( manifest_version: PUZZLEFS_IMAGE_MANIFEST_VERSION, })?; - Ok(oci - .put_blob::(rootfs_buf.as_slice())? - .0) + let rootfs_descriptor = oci + .put_blob::( + rootfs_buf.as_slice(), + &mut image_manifest, + media_types::Rootfs {}, + )? + .0; + oci.0 + .insert_manifest(image_manifest, Some(tag), Platform::default())?; + + Ok(rootfs_descriptor) } // add_rootfs_delta adds whatever the delta between the current rootfs and the puzzlefs @@ -413,13 +427,22 @@ pub fn add_rootfs_delta( rootfs_path: &Path, oci: Image, tag: &str, + base_layer: &str, ) -> Result<(Descriptor, Arc)> { let mut verity_data: VerityData = BTreeMap::new(); - let pfs = PuzzleFS::open(oci, tag, None)?; + let mut image_manifest = Image::get_empty_manifest()?; + + let pfs = PuzzleFS::open(oci, base_layer, None)?; let oci = Arc::clone(&pfs.oci); - let mut rootfs = Rootfs::try_from(oci.open_rootfs_blob(tag, None)?)?; + let mut rootfs = Rootfs::try_from(oci.open_rootfs_blob(base_layer, None)?)?; - let inodes = build_delta::(rootfs_path, &oci, Some(pfs), &mut verity_data)?; + let inodes = build_delta::( + rootfs_path, + &oci, + Some(pfs), + &mut verity_data, + &mut image_manifest, + )?; if !rootfs.metadatas.iter().any(|x| *x == inodes) { rootfs.metadatas.insert(0, inodes); @@ -427,11 +450,16 @@ pub fn add_rootfs_delta( rootfs.fs_verity_data.extend(verity_data); let rootfs_buf = serialize_metadata(rootfs)?; - Ok(( - oci.put_blob::(rootfs_buf.as_slice())? - .0, - oci, - )) + let rootfs_descriptor = oci + .put_blob::( + rootfs_buf.as_slice(), + &mut image_manifest, + media_types::Rootfs {}, + )? + .0; + oci.0 + .insert_manifest(image_manifest, Some(tag), Platform::default())?; + Ok((rootfs_descriptor, oci)) } pub fn enable_fs_verity(oci: Image, tag: &str, manifest_root_hash: &str) -> Result<()> { @@ -454,11 +482,26 @@ pub fn enable_fs_verity(oci: Image, tag: &str, manifest_root_hash: &str) -> Resu let oci = Arc::clone(&pfs.oci); let rootfs = oci.open_rootfs_blob(tag, None)?; + let rootfs_fd = oci.get_pfs_rootfs(tag, None)?; + if let Err(e) = fsverity_enable( + rootfs_fd.as_raw_fd(), + FS_VERITY_BLOCK_SIZE_DEFAULT, + InnerHashAlgorithm::Sha256, + &[], + ) { + // if fsverity is enabled, ignore the error + if e.kind() != std::io::ErrorKind::AlreadyExists { + return Err(WireFormatError::from(e)); + } + } + let rootfs_verity = oci.get_pfs_rootfs_verity(tag)?; + check_fs_verity(&rootfs_fd, &rootfs_verity[..])?; + for (content_addressed_file, verity_hash) in rootfs.get_verity_data()? { let file_path = oci .blob_path() .join(Digest::new(&content_addressed_file).to_string()); - let fd = std::fs::File::open(file_path)?; + let fd = oci.0.dir.open(&file_path)?; if let Err(e) = fsverity_enable( fd.as_raw_fd(), FS_VERITY_BLOCK_SIZE_DEFAULT, @@ -477,8 +520,8 @@ pub fn enable_fs_verity(oci: Image, tag: &str, manifest_root_hash: &str) -> Resu } // TODO: figure out how to guard this with #[cfg(test)] -pub fn build_test_fs(path: &Path, image: &Image) -> Result { - build_initial_rootfs::(path, image) +pub fn build_test_fs(path: &Path, image: &Image, tag: &str) -> Result { + build_initial_rootfs::(path, image, tag) } #[cfg(test)] @@ -488,6 +531,7 @@ pub mod tests { use tempfile::tempdir; use crate::reader::WalkPuzzleFS; + use cap_std::fs::MetadataExt; use std::path::PathBuf; use tempfile::TempDir; @@ -502,8 +546,7 @@ pub mod tests { // but once all that's stabalized, we should verify the metadata hash too. let dir = tempdir().unwrap(); let image = Image::new(dir.path()).unwrap(); - let rootfs_desc = build_test_fs(Path::new("src/builder/test/test-1"), &image).unwrap(); - image.add_tag("test-tag", rootfs_desc)?; + build_test_fs(Path::new("src/builder/test/test-1"), &image, "test-tag").unwrap(); let rootfs = image.open_rootfs_blob("test-tag", None).unwrap(); // there should be a blob that matches the hash of the test data, since it all gets input @@ -511,7 +554,11 @@ pub mod tests { const FILE_DIGEST: &str = "3eee1082ab3babf6c1595f1069d11ebc2a60135890a11e402e017ddd831a220d"; - let md = fs::symlink_metadata(image.blob_path().join(FILE_DIGEST)).unwrap(); + let md = image + .0 + .dir + .symlink_metadata(image.blob_path().join(FILE_DIGEST)) + .unwrap(); assert!(md.is_file()); let mut decompressor = image @@ -558,9 +605,8 @@ pub mod tests { fn test_delta_generation() { let dir = tempdir().unwrap(); let image = Image::new(dir.path()).unwrap(); - let rootfs_desc = build_test_fs(Path::new("src/builder/test/test-1"), &image).unwrap(); let tag = "test"; - image.add_tag(tag, rootfs_desc).unwrap(); + build_test_fs(Path::new("src/builder/test/test-1"), &image, tag).unwrap(); let delta_dir = dir.path().join(Path::new("delta")); fs::create_dir_all(delta_dir.join(Path::new("foo"))).unwrap(); @@ -570,9 +616,9 @@ pub mod tests { ) .unwrap(); - let (desc, image) = add_rootfs_delta::(&delta_dir, image, tag).unwrap(); let new_tag = "test2"; - image.add_tag(new_tag, desc).unwrap(); + let (_desc, image) = + add_rootfs_delta::(&delta_dir, image, new_tag, tag).unwrap(); let delta = Rootfs::try_from(image.open_rootfs_blob(new_tag, None).unwrap()).unwrap(); assert_eq!(delta.metadatas.len(), 2); @@ -630,7 +676,7 @@ pub mod tests { .collect::>(); for (i, image) in images.iter().enumerate() { - build_test_fs(path, image).unwrap(); + build_test_fs(path, image, "test").unwrap(); let ents = get_image_blobs(image); sha_suite.push(ents); @@ -653,7 +699,7 @@ pub mod tests { .collect::>(); for (i, image) in images.iter().enumerate() { - build_test_fs(&path[i], image).unwrap(); + build_test_fs(&path[i], image, "test").unwrap(); let ents = get_image_blobs(image); sha_suite.push(ents); diff --git a/puzzlefs-lib/src/extractor.rs b/puzzlefs-lib/src/extractor.rs index 54cfe40..dfc8772 100644 --- a/puzzlefs-lib/src/extractor.rs +++ b/puzzlefs-lib/src/extractor.rs @@ -187,9 +187,7 @@ mod tests { } } - let rootfs_desc = build_test_fs(&rootfs, &image).unwrap(); - - image.add_tag("test", rootfs_desc).unwrap(); + build_test_fs(&rootfs, &image, "test").unwrap(); extract_rootfs( oci_dir.to_str().unwrap(), @@ -238,9 +236,7 @@ mod tests { std::fs::set_permissions(foo, Permissions::from_mode(TESTED_PERMISSION)).unwrap(); - let rootfs_desc = build_test_fs(&rootfs, &image).unwrap(); - - image.add_tag("test", rootfs_desc).unwrap(); + build_test_fs(&rootfs, &image, "test").unwrap(); extract_rootfs( oci_dir.to_str().unwrap(), @@ -277,9 +273,7 @@ mod tests { fs::metadata(&bar).unwrap().ino() ); - let rootfs_desc = build_test_fs(&rootfs, &image).unwrap(); - - image.add_tag("test", rootfs_desc).unwrap(); + build_test_fs(&rootfs, &image, "test").unwrap(); extract_rootfs( oci_dir.to_str().unwrap(), @@ -309,8 +303,7 @@ mod tests { fs::create_dir_all(&rootfs).unwrap(); std::fs::File::create(foo).unwrap(); - let rootfs_desc = build_test_fs(&rootfs, &image).unwrap(); - image.add_tag("test", rootfs_desc).unwrap(); + build_test_fs(&rootfs, &image, "test").unwrap(); extract_rootfs( oci_dir.to_str().unwrap(), diff --git a/puzzlefs-lib/src/format/error.rs b/puzzlefs-lib/src/format/error.rs index 17e4009..aa6a563 100644 --- a/puzzlefs-lib/src/format/error.rs +++ b/puzzlefs-lib/src/format/error.rs @@ -11,8 +11,6 @@ pub enum WireFormatError { LocalRefError(Backtrace), #[error("cannot seek to other blob")] SeekOtherError(Backtrace), - #[error("no value present")] - ValueMissing(Backtrace), #[error("invalid serialized data")] InvalidSerializedData(Backtrace), #[error("invalid image schema: {0}")] @@ -21,6 +19,10 @@ pub enum WireFormatError { InvalidImageVersion(String, Backtrace), #[error("invalid fs_verity data: {0}")] InvalidFsVerityData(String, Backtrace), + #[error("missing manifest: {0}")] + MissingManifest(String, Backtrace), + #[error("missing PuzzleFS rootfs")] + MissingRootfs(Backtrace), #[error("fs error: {0}")] IOError(#[from] io::Error, Backtrace), #[error("deserialization error (capnp): {0}")] @@ -33,6 +35,10 @@ pub enum WireFormatError { FromSliceError(#[from] std::array::TryFromSliceError, Backtrace), #[error("hex error: {0}")] HexError(#[from] hex::FromHexError, Backtrace), + #[error("Oci error: {0}")] + OciError(#[from] ocidir::oci_spec::OciSpecError, Backtrace), + #[error("Oci dir error: {0}")] + OciDirError(#[from] ocidir::Error, Backtrace), } impl WireFormatError { @@ -40,11 +46,12 @@ impl WireFormatError { match self { WireFormatError::LocalRefError(..) => Errno::EINVAL as c_int, WireFormatError::SeekOtherError(..) => Errno::ESPIPE as c_int, - WireFormatError::ValueMissing(..) => Errno::ENOENT as c_int, WireFormatError::InvalidSerializedData(..) => Errno::EINVAL as c_int, WireFormatError::InvalidImageSchema(..) => Errno::EINVAL as c_int, WireFormatError::InvalidImageVersion(..) => Errno::EINVAL as c_int, WireFormatError::InvalidFsVerityData(..) => Errno::EINVAL as c_int, + WireFormatError::MissingManifest(..) => Errno::EINVAL as c_int, + WireFormatError::MissingRootfs(..) => Errno::EINVAL as c_int, WireFormatError::IOError(ioe, ..) => { ioe.raw_os_error().unwrap_or(Errno::EINVAL as i32) as c_int } @@ -53,6 +60,8 @@ impl WireFormatError { WireFormatError::HexError(..) => Errno::EINVAL as c_int, WireFormatError::FromIntError(..) => Errno::EINVAL as c_int, WireFormatError::FromSliceError(..) => Errno::EINVAL as c_int, + WireFormatError::OciError(..) => Errno::EINVAL as c_int, + WireFormatError::OciDirError(..) => Errno::EINVAL as c_int, } } diff --git a/puzzlefs-lib/src/format/types.rs b/puzzlefs-lib/src/format/types.rs index 6f54a97..385c3d8 100644 --- a/puzzlefs-lib/src/format/types.rs +++ b/puzzlefs-lib/src/format/types.rs @@ -102,7 +102,7 @@ pub struct RootfsReader { } impl RootfsReader { - pub fn open(f: fs::File) -> Result { + pub fn open(f: cap_std::fs::File) -> Result { // We know the loaded message is safe, so we're allowing unlimited reads. let unlimited_reads = message::ReaderOptions { traversal_limit_in_words: None, diff --git a/puzzlefs-lib/src/fsverity_helpers.rs b/puzzlefs-lib/src/fsverity_helpers.rs index edb3dde..ad50e25 100644 --- a/puzzlefs-lib/src/fsverity_helpers.rs +++ b/puzzlefs-lib/src/fsverity_helpers.rs @@ -1,6 +1,5 @@ use crate::format::{Result, WireFormatError, SHA256_BLOCK_SIZE}; use std::backtrace::Backtrace; -use std::fs; use std::io::Write; use std::os::unix::io::AsRawFd; @@ -19,7 +18,7 @@ pub fn get_fs_verity_digest(data: &[u8]) -> Result<[u8; SHA256_BLOCK_SIZE]> { Ok(result.into()) } -pub fn check_fs_verity(file: &fs::File, expected: &[u8]) -> Result<()> { +pub fn check_fs_verity(file: &cap_std::fs::File, expected: &[u8]) -> Result<()> { if expected.len() != SHA256_BLOCK_SIZE { return Err(WireFormatError::InvalidFsVerityData( format!( diff --git a/puzzlefs-lib/src/oci.rs b/puzzlefs-lib/src/oci.rs index e03e965..89ffe8b 100644 --- a/puzzlefs-lib/src/oci.rs +++ b/puzzlefs-lib/src/oci.rs @@ -6,85 +6,56 @@ use std::io; use std::io::{Read, Seek}; use std::path::{Path, PathBuf}; -use serde::{Deserialize, Serialize}; use sha2::{Digest as Sha2Digest, Sha256}; -use tempfile::NamedTempFile; use crate::compression::{Compression, Decompressor, Noop, Zstd}; use crate::format::{Result, RootfsReader, VerityData, WireFormatError, SHA256_BLOCK_SIZE}; -use openat::Dir; use std::io::{Error, ErrorKind}; -mod descriptor; -pub use descriptor::{Descriptor, Digest}; +pub use crate::format::Digest; +use crate::oci::media_types::{PuzzleFSMediaType, PUZZLEFS_ROOTFS, VERITY_ROOT_HASH_ANNOTATION}; +use ocidir::oci_spec::image; +pub use ocidir::oci_spec::image::Descriptor; +use ocidir::oci_spec::image::{ + DescriptorBuilder, ImageIndex, ImageManifest, ImageManifestBuilder, MediaType, Sha256Digest, +}; +use ocidir::oci_spec::OciSpecError; +use ocidir::OciDir; +use std::collections::HashMap; +use std::str::FromStr; -mod index; -pub use index::Index; use std::io::Cursor; -use std::io::Write; pub mod media_types; +const OCI_TAG_ANNOTATION: &str = "org.opencontainers.image.ref.name"; -// this is a string, probably intended to be a real version format (though the spec doesn't say -// anything) so let's just say "puzzlefs-dev" for now since the format is in flux. -const PUZZLEFS_IMAGE_LAYOUT_VERSION: &str = "puzzlefs-dev"; - -const IMAGE_LAYOUT_PATH: &str = "oci-layout"; - -#[derive(Serialize, Deserialize, Debug)] -struct OCILayout { - #[serde(rename = "imageLayoutVersion")] - version: String, -} - -pub struct Image { - oci_dir: PathBuf, - oci_dir_fd: Dir, -} +pub struct Image(pub OciDir); impl Image { pub fn new(oci_dir: &Path) -> Result { fs::create_dir_all(oci_dir)?; - let image = Image { - oci_dir: oci_dir.to_path_buf(), - oci_dir_fd: Dir::open(oci_dir)?, - }; - fs::create_dir_all(image.blob_path())?; - let layout_file = fs::File::create(oci_dir.join(IMAGE_LAYOUT_PATH))?; - let layout = OCILayout { - version: PUZZLEFS_IMAGE_LAYOUT_VERSION.to_string(), - }; - serde_json::to_writer(layout_file, &layout)?; - Ok(image) + let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?; + let oci_dir = OciDir::ensure(&d)?; + + Ok(Self(oci_dir)) } pub fn open(oci_dir: &Path) -> Result { - let layout_file = fs::File::open(oci_dir.join(IMAGE_LAYOUT_PATH))?; - let layout = serde_json::from_reader::<_, OCILayout>(layout_file)?; - if layout.version != PUZZLEFS_IMAGE_LAYOUT_VERSION { - Err(WireFormatError::InvalidImageVersion( - layout.version, - Backtrace::capture(), - )) - } else { - Ok(Image { - oci_dir: oci_dir.to_path_buf(), - oci_dir_fd: Dir::open(oci_dir)?, - }) - } + let d = cap_std::fs::Dir::open_ambient_dir(oci_dir, cap_std::ambient_authority())?; + let oci_dir = OciDir::open(&d)?; + Ok(Self(oci_dir)) } pub fn blob_path(&self) -> PathBuf { - self.oci_dir.join("blobs/sha256") - } - - pub fn blob_path_relative(&self) -> PathBuf { + // TODO: use BLOBDIR constant from ocidir after making it public PathBuf::from("blobs/sha256") } - pub fn put_blob( + pub fn put_blob( &self, buf: &[u8], + image_manifest: &mut ImageManifest, + media_type: impl PuzzleFSMediaType, ) -> Result<(Descriptor, [u8; SHA256_BLOCK_SIZE], bool)> { let mut compressed_data = Cursor::new(Vec::::new()); let mut compressed = C::compress(&mut compressed_data)?; @@ -111,15 +82,32 @@ impl Image { hasher.update(final_data); let digest = hasher.finalize(); - let media_type = C::append_extension(MT::name()); - let descriptor = Descriptor::new(digest.into(), uncompressed_size, media_type); + let media_type_with_extension = C::append_extension(media_type.name()); + let mut digest_string = "sha256:".to_string(); + digest_string.push_str(&hex::encode(digest.as_slice())); + let fs_verity_digest = get_fs_verity_digest(&compressed_data.get_ref()[..])?; - let path = self.blob_path().join(descriptor.digest.to_string()); + let mut descriptor = Descriptor::new( + MediaType::Other(media_type_with_extension), + uncompressed_size, + image::Digest::from_str(&digest_string)?, + ); + // We need to store the PuzzleFS Rootfs verity digest as an annotation (obviously we cannot + // store it in the Rootfs itself) + if media_type.name() == PUZZLEFS_ROOTFS { + let mut annotations = HashMap::new(); + annotations.insert( + VERITY_ROOT_HASH_ANNOTATION.to_string(), + hex::encode(fs_verity_digest), + ); + descriptor.set_annotations(Some(annotations)); + } + let path = self.blob_path().join(descriptor.digest().digest()); // avoid replacing the data blob so we don't drop fsverity data - if path.exists() { + if self.0.dir.exists(&path) { let mut hasher = Sha256::new(); - let mut file = fs::File::open(path)?; + let mut file = self.0.dir.open(&path)?; io::copy(&mut file, &mut hasher)?; let existing_digest = hasher.finalize(); if existing_digest != digest { @@ -131,17 +119,14 @@ impl Image { .into()); } } else { - let mut tmp = NamedTempFile::new_in(&self.oci_dir)?; - tmp.write_all(final_data)?; - tmp.persist(path).map_err(|e| e.error)?; + self.0.dir.write(&path, final_data)?; } + image_manifest.layers_mut().push(descriptor.clone()); Ok((descriptor, fs_verity_digest, compressed_blob)) } - fn open_raw_blob(&self, digest: &Digest, verity: Option<&[u8]>) -> io::Result { - let file = self - .oci_dir_fd - .open_file(&self.blob_path_relative().join(digest.to_string()))?; + fn open_raw_blob(&self, digest: &str, verity: Option<&[u8]>) -> io::Result { + let file = self.0.dir.open(self.blob_path().join(digest))?; if let Some(verity) = verity { check_fs_verity(&file, verity).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; } @@ -153,27 +138,94 @@ impl Image { digest: &Digest, verity: Option<&[u8]>, ) -> io::Result> { - let f = self.open_raw_blob(digest, verity)?; + let f = self.open_raw_blob(&digest.to_string(), verity)?; C::decompress(f) } - pub fn get_image_manifest_fd(&self, tag: &str) -> Result { + pub fn get_pfs_rootfs_verity(&self, tag: &str) -> Result<[u8; SHA256_BLOCK_SIZE]> { + let manifest = self.0.find_manifest_with_tag(tag)?.ok_or_else(|| { + WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture()) + })?; + + let rootfs_desc = manifest + .layers() + .iter() + .find(|desc| desc.media_type() == &MediaType::Other(PUZZLEFS_ROOTFS.to_string())) + .ok_or_else(|| WireFormatError::MissingRootfs(Backtrace::capture()))?; + + let rootfs_verity = rootfs_desc + .annotations() + .as_ref() + .ok_or_else(|| { + WireFormatError::InvalidFsVerityData( + "missing rootfs annotations".to_string(), + Backtrace::capture(), + ) + })? + .get(VERITY_ROOT_HASH_ANNOTATION) + .ok_or_else(|| { + WireFormatError::InvalidFsVerityData( + "missing rootfs verity annotation".to_string(), + Backtrace::capture(), + ) + })?; + let mut verity_digest: [u8; SHA256_BLOCK_SIZE] = [0; SHA256_BLOCK_SIZE]; + hex::decode_to_slice(rootfs_verity, &mut verity_digest)?; + + Ok(verity_digest) + } + + pub fn get_pfs_rootfs(&self, tag: &str, verity: Option<&[u8]>) -> Result { + let manifest = self.0.find_manifest_with_tag(tag)?.ok_or_else(|| { + WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture()) + })?; + + let rootfs_desc = manifest + .layers() + .iter() + .find(|desc| desc.media_type() == &MediaType::Other(PUZZLEFS_ROOTFS.to_string())) + .ok_or_else(|| WireFormatError::MissingRootfs(Backtrace::capture()))?; + + let rootfs_digest = rootfs_desc.digest().digest(); + let file = self.open_raw_blob(rootfs_digest, verity)?; + Ok(file) + } + + // TODO: export this function from ocidr / find another way to avoid code duplication + fn descriptor_is_tagged(d: &Descriptor, tag: &str) -> bool { + d.annotations() + .as_ref() + .and_then(|annos| annos.get(OCI_TAG_ANNOTATION)) + .filter(|tagval| tagval.as_str() == tag) + .is_some() + } + + pub fn get_image_manifest_fd(&self, tag: &str) -> Result { let index = self.get_index()?; - let desc = index - .find_tag(tag) - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("no tag {tag}")))?; - let file = self.open_raw_blob(&desc.digest, None)?; + let image_manifest = index + .manifests() + .iter() + .find(|desc| Self::descriptor_is_tagged(desc, tag)) + .ok_or_else(|| { + WireFormatError::MissingManifest(tag.to_string(), Backtrace::capture()) + })?; + let file = self.open_raw_blob(image_manifest.digest().digest(), None)?; Ok(file) } pub fn open_rootfs_blob(&self, tag: &str, verity: Option<&[u8]>) -> Result { - let index = self.get_index()?; - let desc = index - .find_tag(tag) - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, format!("no tag {tag}")))?; + let temp_verity; + let rootfs_verity = if let Some(verity) = verity { + let manifest = self.get_image_manifest_fd(tag)?; + check_fs_verity(&manifest, verity)?; + temp_verity = self.get_pfs_rootfs_verity(tag)?; + Some(&temp_verity[..]) + } else { + None + }; - let rootfs = self.open_raw_blob(&desc.digest, verity)?; - RootfsReader::open(rootfs) + let rootfs_file = self.get_pfs_rootfs(tag, rootfs_verity)?; + RootfsReader::open(rootfs_file) } pub fn fill_from_chunk( @@ -207,92 +259,115 @@ impl Image { Ok(n) } - pub fn get_index(&self) -> Result { - Index::open(&self.oci_dir.join(index::PATH)) + pub fn get_index(&self) -> Result { + Ok(self + .0 + .read_index()? + .ok_or_else(|| OciSpecError::Other("missing OCI index".to_string()))?) } - pub fn put_index(&self, i: &Index) -> Result<()> { - i.write(&self.oci_dir.join(index::PATH)) - } - - pub fn add_tag(&self, name: &str, mut desc: Descriptor) -> Result<()> { - // check that the blob exists... - self.open_raw_blob(&desc.digest, None)?; - - let mut index = self.get_index().unwrap_or_default(); - - // untag anything that has this tag - for m in index.manifests.iter_mut() { - if m.get_name() - .map(|existing_tag| existing_tag == name) - .unwrap_or(false) - { - m.remove_name() - } - } - desc.set_name(name); - - index.manifests.push(desc); - self.put_index(&index) + pub fn get_empty_manifest() -> Result { + // see https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidance-for-an-empty-descriptor + // TODO: write the empty blob to blobs/sha256, otherwise skopeo won't be able to copy the image + // the fs verity test will also need changes because currently it makes sure that verity is + // enabled for every blob in blobs/sha256; one thing we could do is enable verity but skip + // checking if the digests match or add the verity annotations for each descriptor; another + // way is to only check the verity data for the PuzzleFS rootfs and file chunks + let config = DescriptorBuilder::default() + .media_type(MediaType::EmptyJSON) + .size(2_u32) + .digest(Sha256Digest::from_str( + "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + )?) + .data("e30=") + .build()?; + let image_manifest = ImageManifestBuilder::default() + .schema_version(2_u32) + .config(config) + .layers(Vec::new()) + .build()?; + Ok(image_manifest) } } #[cfg(test)] mod tests { use super::*; + use ocidir::oci_spec::image::{ImageIndexBuilder, Platform, ANNOTATION_REF_NAME}; + use std::collections::HashMap; use tempfile::tempdir; type DefaultCompression = Zstd; #[test] - fn test_put_blob_correct_hash() { - let dir = tempdir().unwrap(); - let image: Image = Image::new(dir.path()).unwrap(); - let (desc, ..) = image - .put_blob::("meshuggah rocks".as_bytes()) - .unwrap(); + fn test_put_blob_correct_hash() -> anyhow::Result<()> { + let dir = tempdir()?; + let mut image_manifest = Image::get_empty_manifest()?; + let image: Image = Image::new(dir.path())?; + let (desc, ..) = image.put_blob::( + "meshuggah rocks".as_bytes(), + &mut image_manifest, + media_types::Chunk {}, + )?; const DIGEST: &str = "3abd5ce0f91f640d88dca1f26b37037b02415927cacec9626d87668a715ec12d"; - assert_eq!(desc.digest.to_string(), DIGEST); + assert_eq!(desc.digest().digest(), DIGEST); - let md = fs::symlink_metadata(image.blob_path().join(DIGEST)).unwrap(); + let md = image + .0 + .dir + .symlink_metadata(image.blob_path().join(DIGEST))?; assert!(md.is_file()); + Ok(()) } #[test] - fn test_open_can_open_new_image() { - let dir = tempdir().unwrap(); - Image::new(dir.path()).unwrap(); - Image::open(dir.path()).unwrap(); + fn test_open_can_open_new_image() -> anyhow::Result<()> { + let dir = tempdir()?; + Image::new(dir.path())?; + Image::open(dir.path())?; + Ok(()) } #[test] - fn test_put_get_index() { - let dir = tempdir().unwrap(); - let image = Image::new(dir.path()).unwrap(); - let (mut desc, ..) = image - .put_blob::("meshuggah rocks".as_bytes()) - .unwrap(); - desc.set_name("foo"); - let mut index = Index::default(); - // TODO: make a real API for this that checks that descriptor has a name? - index.manifests.push(desc); - image.put_index(&index).unwrap(); - - let image2 = Image::open(dir.path()).unwrap(); - let index2 = image2.get_index().unwrap(); - assert_eq!(index.manifests, index2.manifests); + fn test_put_get_index() -> anyhow::Result<()> { + let dir = tempdir()?; + let image = Image::new(dir.path())?; + let mut image_manifest = Image::get_empty_manifest()?; + let mut annotations = HashMap::new(); + annotations.insert(ANNOTATION_REF_NAME.to_string(), "foo".to_string()); + image_manifest.set_annotations(Some(annotations)); + let image_manifest_descriptor = + image + .0 + .insert_manifest(image_manifest, None, Platform::default())?; + + let index = ImageIndexBuilder::default() + .schema_version(2_u32) + .manifests(vec![image_manifest_descriptor]) + .build()?; + + let image2 = Image::open(dir.path())?; + let index2 = image2.get_index()?; + assert_eq!(index.manifests(), index2.manifests()); + Ok(()) } #[test] - fn double_put_ok() { - let dir = tempdir().unwrap(); - let image = Image::new(dir.path()).unwrap(); - let desc1 = image - .put_blob::("meshuggah rocks".as_bytes()) - .unwrap(); - let desc2 = image - .put_blob::("meshuggah rocks".as_bytes()) - .unwrap(); + fn double_put_ok() -> anyhow::Result<()> { + let dir = tempdir()?; + let mut image_manifest = Image::get_empty_manifest()?; + let image = Image::new(dir.path())?; + let desc1 = image.put_blob::( + "meshuggah rocks".as_bytes(), + &mut image_manifest, + media_types::Chunk {}, + )?; + let desc2 = image.put_blob::( + "meshuggah rocks".as_bytes(), + &mut image_manifest, + media_types::Chunk {}, + )?; assert_eq!(desc1, desc2); + Ok(()) } } diff --git a/puzzlefs-lib/src/oci/descriptor.rs b/puzzlefs-lib/src/oci/descriptor.rs deleted file mode 100644 index e9b7ae2..0000000 --- a/puzzlefs-lib/src/oci/descriptor.rs +++ /dev/null @@ -1,38 +0,0 @@ -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -pub use crate::format::Digest; - -const NAME_ANNOTATION: &str = "org.opencontainers.image.ref.name"; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct Descriptor { - pub digest: Digest, - pub size: u64, - pub media_type: String, - pub annotations: HashMap, -} - -impl Descriptor { - pub fn new(digest: [u8; 32], size: u64, media_type: String) -> Descriptor { - Descriptor { - digest: Digest::new(&digest), - size, - media_type, - annotations: HashMap::new(), - } - } - - pub fn set_name(&mut self, name: &str) { - self.annotations - .insert(NAME_ANNOTATION.to_string(), name.into()); - } - - pub fn get_name(&self) -> Option<&String> { - self.annotations.get(NAME_ANNOTATION) - } - - pub(crate) fn remove_name(&mut self) { - self.annotations.remove_entry(NAME_ANNOTATION); - } -} diff --git a/puzzlefs-lib/src/oci/index.rs b/puzzlefs-lib/src/oci/index.rs deleted file mode 100644 index 260f535..0000000 --- a/puzzlefs-lib/src/oci/index.rs +++ /dev/null @@ -1,76 +0,0 @@ -use std::backtrace::Backtrace; -use std::collections::HashMap; -use std::fs; -use std::path::Path; - -use serde::{Deserialize, Serialize}; - -use super::descriptor::Descriptor; -use crate::format::{Result, WireFormatError}; - -// the OCI spec says this must be 2 in order for older dockers to use image layouts, and that it -// will probably be removed. We could hard code it to two, but let's use -1 as an additional -// indicator that this is a "weird" image. ...why is this defined as an int and not a uint? :) -const PUZZLEFS_SCHEMA_VERSION: i32 = -1; - -// the name of the index file as defined by the OCI spec -pub const PATH: &str = "index.json"; - -#[derive(Serialize, Deserialize, Debug)] -pub struct Index { - #[serde(rename = "schemaVersion")] - version: i32, - pub manifests: Vec, - pub annotations: HashMap, -} - -impl Default for Index { - fn default() -> Self { - Index { - version: PUZZLEFS_SCHEMA_VERSION, - manifests: Vec::new(), - annotations: HashMap::new(), - } - } -} - -impl Index { - pub(crate) fn open(p: &Path) -> Result { - let index_file = fs::File::open(p)?; - let index = serde_json::from_reader::<_, Index>(index_file)?; - if index.version != PUZZLEFS_SCHEMA_VERSION { - Err(WireFormatError::InvalidImageSchema( - index.version, - Backtrace::capture(), - )) - } else { - Ok(index) - } - } - - pub(crate) fn write(&self, p: &Path) -> Result<()> { - let index_file = fs::File::create(p)?; - serde_json::to_writer(index_file, &self)?; - Ok(()) - } - - pub fn find_tag(&self, tag: &str) -> Option<&Descriptor> { - self.manifests - .iter() - .find(|d| d.get_name().map(|n| n == tag).unwrap_or(false)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn test_can_open_new_index() { - let dir = tempdir().unwrap(); - let i = Index::default(); - i.write(&dir.path().join(PATH)).unwrap(); - Index::open(&dir.path().join(PATH)).unwrap(); - } -} diff --git a/puzzlefs-lib/src/oci/media_types.rs b/puzzlefs-lib/src/oci/media_types.rs index c80bb5c..4f6ea6d 100644 --- a/puzzlefs-lib/src/oci/media_types.rs +++ b/puzzlefs-lib/src/oci/media_types.rs @@ -1,23 +1,26 @@ -pub trait MediaType { - fn name() -> &'static str; +pub trait PuzzleFSMediaType { + fn name(&self) -> &'static str; } -const PUZZLEFS_ROOTFS: &str = "application/vnd.puzzlefs.image.rootfs.v1"; +pub(crate) const PUZZLEFS_ROOTFS: &str = "application/vnd.puzzlefs.image.rootfs.v1"; pub struct Rootfs {} -impl MediaType for Rootfs { - fn name() -> &'static str { +impl PuzzleFSMediaType for Rootfs { + fn name(&self) -> &'static str { PUZZLEFS_ROOTFS } } -const PUZZLEFS_CHUNK_DATA: &str = "application/vnd.puzzlefs.image.layer.puzzlefs.v1"; +pub(crate) const PUZZLEFS_CHUNK_DATA: &str = "application/vnd.puzzlefs.image.filedata.v1"; pub struct Chunk {} -impl MediaType for Chunk { - fn name() -> &'static str { +impl PuzzleFSMediaType for Chunk { + fn name(&self) -> &'static str { PUZZLEFS_CHUNK_DATA } } + +pub(crate) const VERITY_ROOT_HASH_ANNOTATION: &str = + "io.puzzlefsoci.puzzlefs.puzzlefs_verity_root_hash"; diff --git a/puzzlefs-lib/src/reader/fuse.rs b/puzzlefs-lib/src/reader/fuse.rs index ed1a797..3b864ae 100644 --- a/puzzlefs-lib/src/reader/fuse.rs +++ b/puzzlefs-lib/src/reader/fuse.rs @@ -689,8 +689,7 @@ mod tests { fn test_fuse() { let dir = tempdir().unwrap(); let image = Image::new(dir.path()).unwrap(); - let rootfs_desc = build_test_fs(Path::new("src/builder/test/test-1"), &image).unwrap(); - image.add_tag("test", rootfs_desc).unwrap(); + build_test_fs(Path::new("src/builder/test/test-1"), &image, "test").unwrap(); let mountpoint = tempdir().unwrap(); let _bg = crate::reader::spawn_mount::<&str>( image, diff --git a/puzzlefs-lib/src/reader/puzzlefs.rs b/puzzlefs-lib/src/reader/puzzlefs.rs index 3618a6b..68b0fa6 100644 --- a/puzzlefs-lib/src/reader/puzzlefs.rs +++ b/puzzlefs-lib/src/reader/puzzlefs.rs @@ -201,8 +201,7 @@ mod tests { // make ourselves a test image let oci_dir = tempdir().unwrap(); let image = Image::new(oci_dir.path()).unwrap(); - let rootfs_desc = build_test_fs(Path::new("src/builder/test/test-1"), &image).unwrap(); - image.add_tag("test", rootfs_desc).unwrap(); + build_test_fs(Path::new("src/builder/test/test-1"), &image, "test").unwrap(); let pfs = PuzzleFS::open(image, "test", None).unwrap(); let inode = pfs.find_inode(2).unwrap(); @@ -222,8 +221,7 @@ mod tests { fn test_path_lookup() { let oci_dir = tempdir().unwrap(); let image = Image::new(oci_dir.path()).unwrap(); - let rootfs_desc = build_test_fs(Path::new("src/builder/test/test-1"), &image).unwrap(); - image.add_tag("test", rootfs_desc).unwrap(); + build_test_fs(Path::new("src/builder/test/test-1"), &image, "test").unwrap(); let pfs = PuzzleFS::open(image, "test", None).unwrap(); assert_eq!(pfs.lookup(Path::new("/")).unwrap().unwrap().ino, 1); diff --git a/puzzlefs-lib/src/reader/walk.rs b/puzzlefs-lib/src/reader/walk.rs index 57322f0..008706f 100644 --- a/puzzlefs-lib/src/reader/walk.rs +++ b/puzzlefs-lib/src/reader/walk.rs @@ -86,8 +86,7 @@ mod tests { // make ourselves a test image let oci_dir = tempdir().unwrap(); let image = Image::new(oci_dir.path()).unwrap(); - let rootfs_desc = build_test_fs(Path::new("src/builder/test/test-1"), &image).unwrap(); - image.add_tag("test", rootfs_desc).unwrap(); + build_test_fs(Path::new("src/builder/test/test-1"), &image, "test").unwrap(); let mut pfs = PuzzleFS::open(image, "test", None).unwrap(); let mut walker = WalkPuzzleFS::walk(&mut pfs).unwrap(); @@ -125,9 +124,8 @@ mod tests { xattr::set(f, "user.meshuggah", b"rocks").unwrap(); } - let rootfs_desc = build_test_fs(&rootfs, &image).unwrap(); + build_test_fs(&rootfs, &image, "test").unwrap(); - image.add_tag("test", rootfs_desc).unwrap(); let mut pfs = PuzzleFS::open(image, "test", None).unwrap(); let mut walker = WalkPuzzleFS::walk(&mut pfs).unwrap();