fix(python): reduce network usage for python precompiled manifests

jdx · Dec 14, 2024 · 7e52e45 · 7e52e45
1 parent e3586b7
commit 7e52e45
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 19 deletions.
diff --git a/src/http.rs b/src/http.rs
@@ -53,7 +53,15 @@ impl Client {
             .zstd(true)
     }
 
-    pub async fn get<U: IntoUrl>(&self, url: U) -> Result<Response> {
+    pub fn get_bytes<U: IntoUrl>(&self, url: U) -> Result<impl AsRef<[u8]>> {
+        let url = url.into_url().unwrap();
+        RUNTIME.block_on(async {
+            let resp = self.get_async(url.clone()).await?;
+            Ok(resp.bytes().await?)
+        })
+    }
+
+    pub async fn get_async<U: IntoUrl>(&self, url: U) -> Result<Response> {
         let get = |url: Url| async move {
             debug!("GET {}", &url);
             let mut req = self.reqwest.get(url.clone());
@@ -113,7 +121,7 @@ impl Client {
     pub fn get_text<U: IntoUrl>(&self, url: U) -> Result<String> {
         let mut url = url.into_url().unwrap();
         let text = RUNTIME.block_on(async {
-            let resp = self.get(url.clone()).await?;
+            let resp = self.get_async(url.clone()).await?;
             Ok::<String, eyre::Error>(resp.text().await?)
         })?;
         if text.starts_with("<!DOCTYPE html>") {
@@ -133,7 +141,7 @@ impl Client {
     {
         let url = url.into_url().unwrap();
         let (json, headers) = RUNTIME.block_on(async {
-            let resp = self.get(url).await?;
+            let resp = self.get_async(url).await?;
             let headers = resp.headers().clone();
             Ok::<(T, HeaderMap), eyre::Error>((resp.json().await?, headers))
         })?;
@@ -157,7 +165,7 @@ impl Client {
         debug!("GET Downloading {} to {}", &url, display_path(path));
 
         RUNTIME.block_on(async {
-            let mut resp = self.get(url).await?;
+            let mut resp = self.get_async(url).await?;
             if let Some(length) = resp.content_length() {
                 if let Some(pr) = pr {
                     pr.set_length(length);

diff --git a/src/plugins/core/python.rs b/src/plugins/core/python.rs
@@ -10,33 +10,27 @@ use crate::http::{HTTP, HTTP_FETCH};
 use crate::install_context::InstallContext;
 use crate::toolset::{ToolRequest, ToolVersion, Toolset};
 use crate::ui::progress_report::SingleReport;
-use crate::{cmd, file, plugins};
+use crate::{cmd, dirs, file, plugins};
 use eyre::{bail, eyre};
 use itertools::Itertools;
+use once_cell::sync::Lazy;
 use std::collections::BTreeMap;
+use std::io::{Read};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, OnceLock};
+use flate2::read::GzDecoder;
 use versions::Versioning;
 use xx::regex;
 
 #[derive(Debug)]
 pub struct PythonPlugin {
     ba: BackendArg,
-    precompiled_cache: CacheManager<Vec<(String, String, String)>>,
 }
 
 impl PythonPlugin {
     pub fn new() -> Self {
         let ba = plugins::core::new_backend_arg("python");
-        Self {
-            precompiled_cache: CacheManagerBuilder::new(
-                ba.cache_path.join("precompiled.msgpack.z"),
-            )
-            .with_fresh_duration(SETTINGS.fetch_remote_versions_cache())
-            .with_cache_key(python_precompiled_platform())
-            .build(),
-            ba,
-        }
+        Self { ba }
     }
 
     fn python_build_path(&self) -> PathBuf {
@@ -87,12 +81,25 @@ impl PythonPlugin {
     }
 
     fn fetch_precompiled_remote_versions(&self) -> eyre::Result<&Vec<(String, String, String)>> {
-        self.precompiled_cache.get_or_try_init(|| {
-            let raw = match SETTINGS.paranoid {
-                true => HTTP_FETCH.get_text("https://mise-versions.jdx.dev/python-precompiled"),
+        static PRECOMPILED_CACHE: Lazy<CacheManager<Vec<(String, String, String)>>> =
+            Lazy::new(|| {
+                CacheManagerBuilder::new(dirs::CACHE.join("python").join("precompiled.msgpack.z"))
+                    .with_fresh_duration(SETTINGS.fetch_remote_versions_cache())
+                    .with_cache_key(python_precompiled_platform())
+                    .build()
+            });
+        PRECOMPILED_CACHE.get_or_try_init(|| {
+            let arch = python_arch();
+            let os = python_os();
+            let url_path = format!("python-precompiled-{arch}-{os}.gz");
+            let rsp = match SETTINGS.paranoid {
+                true => HTTP_FETCH.get_bytes(format!("https://mise-versions.jdx.dev/{url_path}")),
                 // using http is not a security concern and enabling tls makes mise significantly slower
-                false => HTTP_FETCH.get_text("http://mise-versions.jdx.dev/python-precompiled"),
+                false => HTTP_FETCH.get_bytes(format!("http://mise-versions.jdx.dev/{url_path}")),
             }?;
+            let mut decoder = GzDecoder::new(rsp.as_ref());
+            let mut raw = String::new();
+            decoder.read_to_string(&mut raw)?;
             let platform = python_precompiled_platform();
             // order by version, whether it is a release candidate, date, and in the preferred order of install types
             let rank = |v: &str, date: &str, name: &str| {