From 406dfc88f29ec40861b13b74b44f90f616135968 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Fri, 11 Aug 2023 11:30:36 -0400 Subject: [PATCH] feat: add http caching to reqwest --- Cargo.lock | 489 ++++++++++++++++++++++++- Cargo.toml | 5 +- crates/cli/Cargo.toml | 21 +- crates/cli/src/package.rs | 16 +- crates/cli/src/package_manager.rs | 15 +- crates/registry/Cargo.toml | 13 +- crates/registry/src/lib.rs | 4 + crates/registry/src/package.rs | 3 +- crates/registry/src/package_version.rs | 3 +- crates/tarball/Cargo.toml | 11 +- 10 files changed, 548 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6e506301..a76339981 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,6 +81,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "anyhow" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -91,6 +97,46 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite", + "log", + "parking", + "polling", + "rustix 0.37.23", + "slab", + "socket2", + "waker-fn", +] + +[[package]] +name = "async-lock" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-trait" +version = "0.1.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -127,6 +173,15 @@ version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -176,6 +231,37 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cfa25e60aea747ec7e1124f238816749faa93759c6ff5b31f1ccdda137f4479" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", +] + [[package]] name = "cast" version = "0.3.0" @@ -279,6 +365,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "concurrent-queue" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "console" version = "0.15.7" @@ -407,6 +502,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "deranged" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7684a49fb1af197853ef7b2ee694bc1f5b4179556f1e5710e1760c5db6f5e929" + [[package]] name = "digest" version = "0.10.7" @@ -465,6 +566,30 @@ dependencies = [ "libc", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.0.0" @@ -561,6 +686,21 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.28" @@ -629,6 +769,12 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "globset" version = "0.4.12" @@ -728,6 +874,62 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-cache" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf08bcc92bf30668ea5e396f2f8a2eb3e41697c677a1748bc3620f3de04b1ac" +dependencies = [ + "async-trait", + "bincode", + "http", + "http-cache-semantics", + "httpdate", + "moka", + "serde", + "url", +] + +[[package]] +name = "http-cache-reqwest" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c5ba396182e217acf42cf5df4a3d7eeacec39dd1dcc4c871e8c7ce2105569e" +dependencies = [ + "anyhow", + "async-trait", + "http", + "http-cache", + "http-cache-semantics", + "reqwest", + "reqwest-middleware", + "serde", + "task-local-extensions", + "url", +] + +[[package]] +name = "http-cache-semantics" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14246388577086faaaa56fb59f0b94e288800fecfff75918a237813297cdda17" +dependencies = [ + "http", + "http-serde", + "serde", + "time", +] + +[[package]] +name = "http-serde" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e272971f774ba29341db2f686255ff8a979365a26fb9e4277f6b6d9ec0cdd5e" +dependencies = [ + "http", + "serde", +] + [[package]] name = "httparse" version = "1.8.0" @@ -823,6 +1025,26 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "ioctl-sys" version = "0.8.0" @@ -842,7 +1064,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix", + "rustix 0.38.4", "windows-sys 0.48.0", ] @@ -894,6 +1116,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + [[package]] name = "linux-raw-sys" version = "0.4.3" @@ -916,6 +1144,15 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + [[package]] name = "matchers" version = "0.1.0" @@ -978,6 +1215,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1024,6 +1271,31 @@ dependencies = [ "tokio", ] +[[package]] +name = "moka" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa6e72583bf6830c956235bff0d5afec8cf2952f579ebad18ae7821a917d950f" +dependencies = [ + "async-io", + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "futures-util", + "once_cell", + "parking_lot", + "quanta", + "rustc_version", + "scheduled-thread-pool", + "skeptic", + "smallvec", + "tagptr", + "thiserror", + "triomphe", + "uuid", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1213,7 +1485,9 @@ version = "0.0.1" dependencies = [ "clap", "futures-util", + "http-cache-reqwest", "insta", + "moka", "node-semver", "pacquet_cafs", "pacquet_diagnostics", @@ -1225,6 +1499,7 @@ dependencies = [ "rayon", "reflink-copy", "reqwest", + "reqwest-middleware", "serde", "serde_json", "tempfile", @@ -1288,6 +1563,7 @@ dependencies = [ "pacquet_diagnostics", "pipe-trait", "reqwest", + "reqwest-middleware", "serde", "serde_json", "tempfile", @@ -1301,6 +1577,7 @@ dependencies = [ "pacquet_cafs", "pacquet_diagnostics", "reqwest", + "reqwest-middleware", "ssri", "tar", "tempfile", @@ -1308,6 +1585,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "parking" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1395,6 +1678,22 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1416,6 +1715,33 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bccbff07d5ed689c4087d20d7307a52ab6141edeedf487c3876a55b86cf63df" +[[package]] +name = "pulldown-cmark" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +dependencies = [ + "bitflags 1.3.2", + "memchr", + "unicase", +] + +[[package]] +name = "quanta" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quote" version = "1.0.32" @@ -1455,6 +1781,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "rayon" version = "1.7.0" @@ -1571,6 +1906,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "native-tls", "once_cell", "percent-encoding", @@ -1588,6 +1924,21 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest-middleware" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff44108c7925d082f2861e683a88618b68235ad9cdc60d64d9d1188efc951cdb" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest", + "serde", + "task-local-extensions", + "thiserror", +] + [[package]] name = "result" version = "1.0.0" @@ -1600,6 +1951,29 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.37.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + [[package]] name = "rustix" version = "0.38.4" @@ -1609,7 +1983,7 @@ dependencies = [ "bitflags 2.3.3", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.3", "windows-sys 0.48.0", ] @@ -1643,6 +2017,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1672,6 +2055,15 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +dependencies = [ + "serde", +] + [[package]] name = "serde" version = "1.0.183" @@ -1792,6 +2184,21 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" +[[package]] +name = "skeptic" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" +dependencies = [ + "bytecount", + "cargo_metadata", + "error-chain", + "glob", + "pulldown-cmark", + "tempfile", + "walkdir", +] + [[package]] name = "slab" version = "0.4.8" @@ -1907,6 +2314,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tar" version = "0.4.39" @@ -1918,6 +2331,15 @@ dependencies = [ "xattr", ] +[[package]] +name = "task-local-extensions" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba323866e5d033818e3240feeb9f7db2c4296674e4d9e16b97b7bf8f490434e8" +dependencies = [ + "pin-utils", +] + [[package]] name = "tempfile" version = "3.7.1" @@ -1925,9 +2347,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ "cfg-if", - "fastrand", + "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix", + "rustix 0.38.4", "windows-sys 0.48.0", ] @@ -1982,6 +2404,34 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fdd63d58b18d663fbdf70e049f00a22c8e42be082203be7f26589213cd75ea" +dependencies = [ + "deranged", + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" +dependencies = [ + "time-core", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -2130,6 +2580,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "triomphe" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee8098afad3fb0c54a9007aab6804558410503ad676d4633f9c2559a00ac0f" + [[package]] name = "try-lock" version = "0.2.4" @@ -2142,6 +2598,15 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.13" @@ -2190,6 +2655,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -2198,6 +2664,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +dependencies = [ + "getrandom", +] + [[package]] name = "valuable" version = "0.1.0" @@ -2222,6 +2697,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "walkdir" version = "2.3.3" diff --git a/Cargo.toml b/Cargo.toml index 6c45a9275..d05ce7845 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,9 +28,12 @@ clap = { version = "4", features = ["derive", "string"] } home = { version = "0.5.5" } insta = { version = "1.31.0", features = ["yaml", "glob", "walkdir"] } futures-util = { version = "0.3.28" } +moka = { version = "0.11.3" } miette = { version = "5.9.0", features = ["fancy"] } reflink-copy = { version = "0.1.5" } reqwest = { version = "0.11", default-features = false, features = ["json", "native-tls-vendored"] } +reqwest-middleware = { version = "0.2.3" } +http-cache-reqwest = { version = "0.11.1", default-features = false, features = ["manager-moka"] } node-semver = { version = "2.1.0" } pipe-trait = { version = "0.4.0" } rayon = { version = "1.7.0" } @@ -44,7 +47,7 @@ tar = { version = "0.4.38" } thiserror = { version = "1.0.44" } tracing = { version = "0.1.37" } tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } walkdir = { version = "2.3.3" } zune-inflate = { version = "0.2.54" } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 67e584c6e..cc370823a 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -23,15 +23,18 @@ pacquet_registry = { workspace = true } pacquet_tarball = { workspace = true } pacquet_diagnostics = { workspace = true } -clap = { workspace = true } -futures-util = { workspace = true } -rayon = { workspace = true } -reflink-copy = { workspace = true } -reqwest = { workspace = true } -node-semver = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -tokio = { workspace = true } +clap = { workspace = true } +http-cache-reqwest = { workspace = true } +futures-util = { workspace = true } +moka = { workspace = true } +rayon = { workspace = true } +reflink-copy = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } +node-semver = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } [dev-dependencies] insta = { workspace = true } diff --git a/crates/cli/src/package.rs b/crates/cli/src/package.rs index cbacdad3f..f6b16e9be 100644 --- a/crates/cli/src/package.rs +++ b/crates/cli/src/package.rs @@ -3,6 +3,7 @@ use crate::package_manager::PackageManagerError; use pacquet_npmrc::Npmrc; use pacquet_registry::{Package, PackageVersion}; use pacquet_tarball::download_tarball_to_store; +use reqwest_middleware::ClientWithMiddleware; use std::path::PathBuf; /// This function execute the following and returns the package @@ -15,7 +16,7 @@ use std::path::PathBuf; /// `node_modules/.pacquet/fastify@1.0.0/node_modules`. pub async fn find_package_version_from_registry>( config: &Npmrc, - http_client: &reqwest::Client, + http_client: &ClientWithMiddleware, name: &str, version: &str, symlink_path: P, @@ -28,7 +29,7 @@ pub async fn find_package_version_from_registry>( pub async fn fetch_package_version_directly>( config: &Npmrc, - http_client: &reqwest::Client, + http_client: &ClientWithMiddleware, name: &str, version: &str, symlink_path: P, @@ -67,8 +68,11 @@ async fn internal_fetch>( #[cfg(test)] mod tests { use crate::package::find_package_version_from_registry; + use http_cache_reqwest::{Cache, CacheMode, HttpCache, HttpCacheOptions, MokaManager}; use node_semver::Version; use pacquet_npmrc::Npmrc; + use reqwest::Client; + use reqwest_middleware::ClientBuilder; use std::fs; use std::path::Path; use tempfile::tempdir; @@ -103,7 +107,13 @@ mod tests { let modules_dir = tempdir().unwrap(); let virtual_store_dir = tempdir().unwrap(); let config = get_config(store_dir.path(), modules_dir.path(), virtual_store_dir.path()); - let http_client = reqwest::Client::new(); + let http_client = ClientBuilder::new(Client::new()) + .with(Cache(HttpCache { + mode: CacheMode::ForceCache, + manager: MokaManager::default(), + options: HttpCacheOptions::default(), + })) + .build(); let symlink_path = tempdir().unwrap(); let package = find_package_version_from_registry( &config, diff --git a/crates/cli/src/package_manager.rs b/crates/cli/src/package_manager.rs index 7083c9902..8a743d34a 100644 --- a/crates/cli/src/package_manager.rs +++ b/crates/cli/src/package_manager.rs @@ -1,3 +1,6 @@ +use http_cache_reqwest::{Cache, CacheMode, HttpCache, HttpCacheOptions, MokaManager}; +use reqwest::Client; +use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use std::path::PathBuf; use pacquet_diagnostics::{ @@ -30,7 +33,7 @@ pub enum PackageManagerError { pub struct PackageManager { pub config: Box, pub package_json: Box, - pub http_client: Box, + pub http_client: Box, } impl PackageManager { @@ -38,7 +41,15 @@ impl PackageManager { Ok(PackageManager { config: Box::new(get_current_npmrc()), package_json: Box::new(PackageJson::create_if_needed(package_json_path.into())?), - http_client: Box::new(reqwest::Client::new()), + http_client: Box::new( + ClientBuilder::new(Client::new()) + .with(Cache(HttpCache { + mode: CacheMode::ForceCache, + manager: MokaManager::default(), + options: HttpCacheOptions::default(), + })) + .build(), + ), }) } } diff --git a/crates/registry/Cargo.toml b/crates/registry/Cargo.toml index 3c3aac857..10a585b82 100644 --- a/crates/registry/Cargo.toml +++ b/crates/registry/Cargo.toml @@ -13,12 +13,13 @@ repository.workspace = true [dependencies] pacquet_diagnostics = { workspace = true } -reqwest = { workspace = true } -node-semver = { workspace = true } -pipe-trait = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -tokio = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } +node-semver = { workspace = true } +pipe-trait = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/registry/src/lib.rs b/crates/registry/src/lib.rs index 87cc44709..5f96265c3 100644 --- a/crates/registry/src/lib.rs +++ b/crates/registry/src/lib.rs @@ -26,6 +26,10 @@ pub enum RegistryError { #[diagnostic(code(pacquet_registry::network_error))] Network(#[from] reqwest::Error), + #[error(transparent)] + #[diagnostic(code(pacquet_registry::network_middleware_error))] + NetworkMiddleware(#[from] reqwest_middleware::Error), + #[error(transparent)] #[diagnostic(code(pacquet_registry::io_error))] Io(#[from] std::io::Error), diff --git a/crates/registry/src/package.rs b/crates/registry/src/package.rs index 168f2095a..59039eec2 100644 --- a/crates/registry/src/package.rs +++ b/crates/registry/src/package.rs @@ -4,6 +4,7 @@ use std::{ }; use pipe_trait::Pipe; +use reqwest_middleware::ClientWithMiddleware; use serde::{Deserialize, Serialize}; use crate::{package_version::PackageVersion, RegistryError}; @@ -28,7 +29,7 @@ impl PartialEq for Package { impl Package { pub async fn fetch_from_registry( name: &str, - http_client: &reqwest::Client, + http_client: &ClientWithMiddleware, registry: &str, ) -> Result { http_client diff --git a/crates/registry/src/package_version.rs b/crates/registry/src/package_version.rs index 804f7a975..d3efdba3d 100644 --- a/crates/registry/src/package_version.rs +++ b/crates/registry/src/package_version.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use pipe_trait::Pipe; +use reqwest_middleware::ClientWithMiddleware; use serde::{Deserialize, Serialize}; use crate::package_distribution::PackageDistribution; @@ -27,7 +28,7 @@ impl PackageVersion { pub async fn fetch_from_registry( name: &str, version: &str, - http_client: &reqwest::Client, + http_client: &ClientWithMiddleware, registry: &str, ) -> Result { http_client diff --git a/crates/tarball/Cargo.toml b/crates/tarball/Cargo.toml index 0a40c23ab..f86f5bfaf 100644 --- a/crates/tarball/Cargo.toml +++ b/crates/tarball/Cargo.toml @@ -14,11 +14,12 @@ repository.workspace = true pacquet_cafs = { workspace = true } pacquet_diagnostics = { workspace = true } -reqwest = { workspace = true } -ssri = { workspace = true } -tar = { workspace = true } -tokio = { workspace = true } -zune-inflate = { workspace = true } +reqwest = { workspace = true } +reqwest-middleware = { workspace = true } +ssri = { workspace = true } +tar = { workspace = true } +tokio = { workspace = true } +zune-inflate = { workspace = true } [dev-dependencies] tempfile = { workspace = true }