From c1ef23580e18ffa3cbe0377245f4c6a8b6021dba Mon Sep 17 00:00:00 2001 From: jdx <216188+jdx@users.noreply.github.com> Date: Sat, 26 Oct 2024 16:39:05 -0500 Subject: [PATCH] perf: eager load plugin/backend data (#2830) --- e2e/plugins/test_plugin_link | 7 +++++++ src/backend/asdf.rs | 15 +++++++++------ src/backend/mod.rs | 25 +++++++++++++----------- src/backend/vfox.rs | 17 +++++++++------- src/cli/plugins/link.rs | 18 ----------------- src/config/config_file/mise_toml.rs | 2 +- src/eager.rs | 16 ++++++++++++--- src/plugins/asdf_plugin.rs | 30 ++++++++++++----------------- src/plugins/mod.rs | 22 ++++++++++++++++++++- src/plugins/vfox_plugin.rs | 30 ++++++++++++----------------- 10 files changed, 99 insertions(+), 83 deletions(-) create mode 100644 e2e/plugins/test_plugin_link diff --git a/e2e/plugins/test_plugin_link b/e2e/plugins/test_plugin_link new file mode 100644 index 000000000..188756cce --- /dev/null +++ b/e2e/plugins/test_plugin_link @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +assert "mise plugin link -f tiny-link $MISE_DATA_DIR/plugins/dummy" "" +assert "mise plugin ls" "dummy +tiny-link" +assert "mise plugin uninstall tiny-link" "" +assert "mise plugin ls" "dummy" diff --git a/src/backend/asdf.rs b/src/backend/asdf.rs index e3fd5b95d..b0d83bdd0 100644 --- a/src/backend/asdf.rs +++ b/src/backend/asdf.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use color_eyre::eyre::{eyre, Result, WrapErr}; use console::style; -use rayon::prelude::*; use crate::backend::external_plugin_cache::ExternalPluginCache; use crate::backend::{ABackend, Backend, BackendList}; @@ -24,7 +23,7 @@ use crate::plugins::Script::{Download, ExecEnv, Install, ParseLegacyFile}; use crate::plugins::{Plugin, PluginType, Script, ScriptManager}; use crate::toolset::{ToolRequest, ToolVersion, Toolset}; use crate::ui::progress_report::SingleReport; -use crate::{dirs, env, file}; +use crate::{dirs, env, file, plugins}; /// This represents a plugin installed to ~/.local/share/mise/plugins pub struct AsdfBackend { @@ -89,11 +88,15 @@ impl AsdfBackend { } pub fn list() -> Result { - Ok(file::dir_subdirs(&dirs::PLUGINS)? - .into_par_iter() + Ok(plugins::INSTALLED_PLUGINS + .iter() // if metadata.lua exists it's a vfox plugin (hopefully) - .filter(|name| !dirs::PLUGINS.join(name).join("metadata.lua").exists()) - .map(|name| Arc::new(Self::from_arg(name.into())) as ABackend) + .filter(|(_, pt)| matches!(pt, PluginType::Asdf)) + .map(|(d, _)| { + Arc::new(Self::from_arg( + d.file_name().unwrap().to_string_lossy().into(), + )) as ABackend + }) .collect()) } diff --git a/src/backend/mod.rs b/src/backend/mod.rs index c0a5aae0a..31684f6e0 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -10,7 +10,6 @@ use contracts::requires; use eyre::{bail, eyre, WrapErr}; use itertools::Itertools; use once_cell::sync::Lazy; -use rayon::prelude::*; use regex::Regex; use strum::IntoEnumIterator; use versions::Versioning; @@ -84,14 +83,15 @@ fn load_tools() -> BackendMap { if let Some(backends) = &*memo_tools { return backends.clone(); } - time!("load_tools: start"); - let mut tools = CORE_PLUGINS + time!("load_tools start"); + let core_tools = CORE_PLUGINS .iter() .map(|(_, p)| p.clone()) .collect::>(); + time!("load_tools core"); let mut asdf_tools = Ok(vec![]); let mut vfox_tools = Ok(vec![]); - let mut backend_tools = Ok(vec![]); + let mut backend_tools = vec![]; rayon::scope(|s| { if !SETTINGS.disable_backends.contains(&"asdf".to_string()) { s.spawn(|_| asdf_tools = asdf::AsdfBackend::list()); @@ -99,11 +99,13 @@ fn load_tools() -> BackendMap { if !SETTINGS.disable_backends.contains(&"vfox".to_string()) { s.spawn(|_| vfox_tools = vfox::VfoxBackend::list()); } - backend_tools = list_installed_backends(); + backend_tools = INSTALLED_BACKENDS.clone(); }); + time!("load_tools backends"); + let mut tools = core_tools; tools.extend(asdf_tools.expect("asdf tools failed to load")); tools.extend(vfox_tools.expect("vfox tools failed to load")); - tools.extend(backend_tools.expect("backend tools failed to load")); + tools.extend(backend_tools); tools.retain(|backend| !SETTINGS.disable_tools.contains(backend.id())); let tools: BackendMap = tools @@ -115,13 +117,14 @@ fn load_tools() -> BackendMap { tools } -fn list_installed_backends() -> eyre::Result { - Ok(file::dir_subdirs(&dirs::INSTALLS)? - .into_par_iter() +pub static INSTALLED_BACKENDS: Lazy> = Lazy::new(|| { + file::dir_subdirs(&dirs::INSTALLS) + .unwrap() + .into_iter() .map(|dir| arg_to_backend(BackendMeta::read(&dir).into())) .filter(|f| !matches!(f.fa().backend_type, BackendType::Asdf | BackendType::Vfox)) - .collect()) -} + .collect() +}); pub fn list() -> BackendList { load_tools().values().cloned().collect() diff --git a/src/backend/vfox.rs b/src/backend/vfox.rs index e1e93872e..804671408 100644 --- a/src/backend/vfox.rs +++ b/src/backend/vfox.rs @@ -1,7 +1,6 @@ -use crate::env; +use crate::{env, plugins}; use eyre::{Report, WrapErr}; use heck::ToKebabCase; -use rayon::prelude::*; use std::collections::BTreeMap; use std::fmt::Debug; use std::path::PathBuf; @@ -18,7 +17,7 @@ use crate::git::Git; use crate::install_context::InstallContext; use crate::plugins::PluginType; use crate::toolset::{ToolVersion, Toolset}; -use crate::{dirs, file, registry}; +use crate::{dirs, registry}; use vfox::Vfox; use xx::regex; @@ -116,10 +115,14 @@ fn vfox_to_url(name: &str) -> eyre::Result { impl VfoxBackend { pub fn list() -> eyre::Result { - Ok(file::dir_subdirs(&dirs::PLUGINS)? - .into_par_iter() - .filter(|name| dirs::PLUGINS.join(name).join("metadata.lua").exists()) - .map(|name| Arc::new(Self::from_arg(name.into())) as ABackend) + Ok(plugins::INSTALLED_PLUGINS + .iter() + .filter(|(_, pt)| matches!(pt, PluginType::Vfox)) + .map(|(d, _)| { + Arc::new(Self::from_arg( + d.file_name().unwrap().to_string_lossy().into(), + )) as ABackend + }) .collect()) } diff --git a/src/cli/plugins/link.rs b/src/cli/plugins/link.rs index ddc27a12d..9ffa3ff7b 100644 --- a/src/cli/plugins/link.rs +++ b/src/cli/plugins/link.rs @@ -77,21 +77,3 @@ static AFTER_LONG_HELP: &str = color_print::cstr!( $ mise plugins link ./mise-node "# ); - -#[cfg(test)] -mod tests { - use crate::test::reset; - use test_log::test; - - #[test] - fn test_plugin_link() { - reset(); - assert_cli_snapshot!("plugin", "link", "-f", "tiny-link", "../data/plugins/tiny", @""); - assert_cli_snapshot!("plugins", "ls", @r#" - dummy - tiny - tiny-link - "#); - assert_cli_snapshot!("plugin", "uninstall", "tiny-link", @""); - } -} diff --git a/src/config/config_file/mise_toml.rs b/src/config/config_file/mise_toml.rs index 8eaa73dd7..b8902aa65 100644 --- a/src/config/config_file/mise_toml.rs +++ b/src/config/config_file/mise_toml.rs @@ -99,7 +99,7 @@ impl MiseToml { for task in rf.tasks.0.values_mut() { task.config_source.clone_from(&rf.path); } - trace!("{}", rf.dump()?); + // trace!("{}", rf.dump()?); Ok(rf) } diff --git a/src/eager.rs b/src/eager.rs index b4b1ad5ef..24cee753f 100644 --- a/src/eager.rs +++ b/src/eager.rs @@ -1,20 +1,30 @@ +use crate::backend::INSTALLED_BACKENDS; use crate::cli::version::VERSION; -use crate::plugins::VERSION_REGEX; +use crate::plugins::{INSTALLED_PLUGINS, VERSION_REGEX}; use once_cell::sync::Lazy; use std::path::PathBuf; use std::sync::Mutex; /// initializes slow parts of mise eagerly in the background pub fn early_init() { + time!("early_init"); rayon::spawn(|| { let _ = &*VERSION_REGEX; }); rayon::spawn(|| { let _ = &*VERSION; - }) + }); + rayon::spawn(|| { + let _ = &*INSTALLED_PLUGINS; + }); } pub static CONFIG_FILES: Lazy>> = Lazy::new(|| Mutex::new(Vec::new())); /// run after SETTING has been loaded -pub fn post_settings() {} +pub fn post_settings() { + time!("post_settings"); + rayon::spawn(|| { + let _ = &*INSTALLED_BACKENDS; + }); +} diff --git a/src/plugins/asdf_plugin.rs b/src/plugins/asdf_plugin.rs index bef1552c9..6e3f25682 100644 --- a/src/plugins/asdf_plugin.rs +++ b/src/plugins/asdf_plugin.rs @@ -8,19 +8,18 @@ use crate::timeout::run_with_timeout; use crate::ui::multi_progress_report::MultiProgressReport; use crate::ui::progress_report::SingleReport; use crate::ui::prompt; -use crate::{dirs, exit, lock_file, registry}; +use crate::{dirs, exit, lock_file, plugins, registry}; use clap::Command; use console::style; use contracts::requires; use eyre::{bail, eyre, Context}; use itertools::Itertools; use once_cell::sync::Lazy; -use rayon::prelude::*; use std::collections::{BTreeSet, HashMap}; use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::sync::{Mutex, MutexGuard}; -use xx::{file, regex}; +use xx::regex; #[derive(Debug)] pub struct AsdfPlugin { @@ -55,21 +54,16 @@ impl AsdfPlugin { pub fn list() -> eyre::Result { let settings = Settings::get(); - match file::ls(*dirs::PLUGINS) { - Ok(dirs) => { - let plugins = dirs - .into_par_iter() - .filter(|dir| dir.is_dir()) - .map(|dir| { - let name = dir.file_name().unwrap().to_string_lossy().to_string(); - Box::new(AsdfPlugin::new(name)) as Box - }) - .filter(|p| !settings.disable_tools.contains(p.name())) - .collect(); - Ok(plugins) - } - Err(_) => Ok(PluginList::new()), - } + let plugins = plugins::INSTALLED_PLUGINS + .iter() + .filter(|(_, pt)| matches!(pt, PluginType::Asdf)) + .map(|(dir, _)| { + let name = dir.file_name().unwrap().to_string_lossy().to_string(); + Box::new(AsdfPlugin::new(name)) as Box + }) + .filter(|p| !settings.disable_tools.contains(p.name())) + .collect(); + Ok(plugins) } fn repo(&self) -> MutexGuard { diff --git a/src/plugins/mod.rs b/src/plugins/mod.rs index a510bf4e3..889db08ee 100644 --- a/src/plugins/mod.rs +++ b/src/plugins/mod.rs @@ -1,4 +1,3 @@ -use crate::backend; use crate::backend::{ABackend, BackendList, BackendType}; use crate::cli::args::BackendArg; use crate::errors::Error::PluginNotInstalled; @@ -7,6 +6,7 @@ use crate::plugins::core::CorePlugin; use crate::plugins::vfox_plugin::VFOX_PLUGIN_NAMES; use crate::ui::multi_progress_report::MultiProgressReport; use crate::ui::progress_report::SingleReport; +use crate::{backend, dirs, file}; use clap::Command; use itertools::Itertools; use once_cell::sync::Lazy; @@ -53,6 +53,26 @@ pub static PLUGIN_NAMES_TO_TYPE: Lazy> = Lazy::new( asdf.into_iter().chain(vfox).collect() }); +pub static INSTALLED_PLUGINS: Lazy> = + Lazy::new(|| match file::dir_subdirs(&dirs::PLUGINS) { + Ok(dirs) => dirs + .into_iter() + .map(|d| { + let path = dirs::PLUGINS.join(&d); + let plugin_type = if path.join("metadata.lua").exists() { + PluginType::Vfox + } else { + PluginType::Asdf + }; + (path, plugin_type) + }) + .collect(), + Err(e) => { + warn!("error reading plugin dirs: {e}"); + vec![] + } + }); + pub fn list() -> BackendList { // TODO: replace with list2 backend::list() diff --git a/src/plugins/vfox_plugin.rs b/src/plugins/vfox_plugin.rs index 19d87355f..fd2c806fa 100644 --- a/src/plugins/vfox_plugin.rs +++ b/src/plugins/vfox_plugin.rs @@ -7,16 +7,15 @@ use crate::result::Result; use crate::ui::multi_progress_report::MultiProgressReport; use crate::ui::progress_report::SingleReport; use crate::ui::prompt; -use crate::{dirs, lock_file}; +use crate::{dirs, lock_file, plugins}; use console::style; use contracts::requires; use eyre::{bail, eyre, Context}; use once_cell::sync::Lazy; -use rayon::prelude::*; use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::sync::{Mutex, MutexGuard}; -use xx::{file, regex}; +use xx::regex; #[derive(Debug)] pub struct VfoxPlugin { @@ -49,21 +48,16 @@ impl VfoxPlugin { pub fn list() -> eyre::Result { let settings = Settings::get(); - match file::ls(*dirs::PLUGINS) { - Ok(dirs) => { - let plugins = dirs - .into_par_iter() - .filter(|dir| dir.join("metadata.lua").exists()) - .map(|dir| { - let name = dir.file_name().unwrap().to_string_lossy().to_string(); - Box::new(VfoxPlugin::new(name)) as Box - }) - .filter(|p| !settings.disable_tools.contains(p.name())) - .collect(); - Ok(plugins) - } - Err(_) => Ok(PluginList::new()), - } + let plugins = plugins::INSTALLED_PLUGINS + .iter() + .filter(|(_, t)| matches!(t, PluginType::Vfox)) + .map(|(dir, _)| { + let name = dir.file_name().unwrap().to_string_lossy().to_string(); + Box::new(VfoxPlugin::new(name)) as Box + }) + .filter(|p| !settings.disable_tools.contains(p.name())) + .collect(); + Ok(plugins) } fn repo(&self) -> MutexGuard {