From 76c5cb9578bd9baf89c13fbac315008c2c5649b1 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Fri, 8 Nov 2024 16:53:58 -0800 Subject: [PATCH 1/6] spawn wrapper to log stdout/err from plugin install --- workbench/src/main/setupAddRemovePlugin.js | 116 +++++++++++++-------- 1 file changed, 75 insertions(+), 41 deletions(-) diff --git a/workbench/src/main/setupAddRemovePlugin.js b/workbench/src/main/setupAddRemovePlugin.js index 1818eb9355..9c1e05c32a 100644 --- a/workbench/src/main/setupAddRemovePlugin.js +++ b/workbench/src/main/setupAddRemovePlugin.js @@ -2,7 +2,7 @@ import upath from 'upath'; import fs from 'fs'; import { tmpdir } from 'os'; import toml from 'toml'; -import { execSync } from 'child_process'; +import { execSync, spawn } from 'child_process'; import { ipcMain } from 'electron'; import { getLogger } from './logger'; @@ -11,59 +11,93 @@ import { settingsStore } from './settingsStore'; const logger = getLogger(__filename.split('/').slice(-1)[0]); +function customSpawn(cmd, args, options) { + logger.info(cmd, args); + const cmdProcess = spawn(cmd, args, options); + if (cmdProcess.stdout) { + cmdProcess.stderr.on('data', (data) => logger.info(data.toString())); + cmdProcess.stdout.on('data', (data) => logger.info(data.toString())); + } + return new Promise((resolve, reject) => { + cmdProcess.on('close', (code) => { resolve(code); }); + }); +} + export function setupAddPlugin() { ipcMain.handle( ipcMainChannels.ADD_PLUGIN, (e, pluginURL) => { logger.info('adding plugin at', pluginURL); + const mamba = settingsStore.get('mamba'); + let envName; + let pluginID; + let pluginName; + let pluginPyName; + try { // Create a temporary directory and check out the plugin's pyproject.toml const tmpPluginDir = fs.mkdtempSync(upath.join(tmpdir(), 'natcap-invest-')); - execSync( - `git clone --depth 1 --no-checkout ${pluginURL} "${tmpPluginDir}"`, - { stdio: 'inherit', windowsHide: true } - ); - execSync('git checkout HEAD pyproject.toml', { cwd: tmpPluginDir, stdio: 'inherit', windowsHide: true }); + customSpawn( + 'git', + ['clone', '--depth', '1', '--no-checkout', pluginURL, tmpPluginDir], + { windowsHide: true } + ).then(async () => { + await customSpawn( + 'git', + ['checkout', 'HEAD', 'pyproject.toml'], + { cwd: tmpPluginDir, windowsHide: true } + ); + }).then(async () => { + // Read in the plugin's pyproject.toml, then delete it + const pyprojectTOML = toml.parse(fs.readFileSync( + upath.join(tmpPluginDir, 'pyproject.toml') + ).toString()); + fs.rmSync(tmpPluginDir, { recursive: true, force: true }); - // Read in the plugin's pyproject.toml, then delete it - const pyprojectTOML = toml.parse(fs.readFileSync( - upath.join(tmpPluginDir, 'pyproject.toml') - ).toString()); - fs.rmSync(tmpPluginDir, { recursive: true, force: true }); + // Access plugin metadata from the pyproject.toml + pluginID = pyprojectTOML.tool.natcap.invest.model_id; + pluginName = pyprojectTOML.tool.natcap.invest.model_name; + pluginPyName = pyprojectTOML.tool.natcap.invest.pyname; - // Access plugin metadata from the pyproject.toml - const pluginID = pyprojectTOML.tool.natcap.invest.model_id; - const pluginName = pyprojectTOML.tool.natcap.invest.model_name; - const pluginPyName = pyprojectTOML.tool.natcap.invest.pyname; + // Create a conda env containing the plugin and its dependencies + envName = `invest_plugin_${pluginID}`; + await customSpawn( + mamba, + ['create', '--yes', '--name', envName, '-c', 'conda-forge', '"python<3.12"', '"gdal<3.6"'], + { windowsHide: true } + ); + logger.info('created mamba env for plugin'); + }).then(async () => { + await customSpawn( + mamba, + ['run', '--verbose', '--no-capture-output', '--name', envName, 'pip', 'install', `git+${pluginURL}`], + { windowsHide: true } + ); + logger.info('installed plugin into its env'); + }) + .then(() => { + // Write plugin metadata to the workbench's config.json + const envInfo = execSync(`${mamba} info --envs`, { windowsHide: true }).toString(); + logger.info(`env info:\n${envInfo}`); - // Create a conda env containing the plugin and its dependencies - const envName = `invest_plugin_${pluginID}`; - const mamba = settingsStore.get('mamba'); - execSync(`${mamba} create --yes --name ${envName} -c conda-forge "python<3.12" "gdal<3.6"`, - { stdio: 'inherit', windowsHide: true }); - logger.info('created mamba env for plugin'); - execSync(`${mamba} run --name ${envName} pip install "git+${pluginURL}"`, - { stdio: 'inherit', windowsHide: true }); - logger.info('installed plugin into its env'); + const regex = new RegExp(String.raw`^${envName} +(.+)$`, 'm'); + const envPath = envInfo.match(regex)[1]; - // Write plugin metadata to the workbench's config.json - const envInfo = execSync(`${mamba} info --envs`, { windowsHide: true }).toString(); - logger.info(`env info:\n${envInfo}`); - const envPath = envInfo.match(`${envName}\\s+(.+)$`)[1]; - logger.info(`env path:\n${envPath}`); - logger.info('writing plugin info to settings store'); - settingsStore.set( - `plugins.${pluginID}`, - { - model_name: pluginName, - pyname: pluginPyName, - type: 'plugin', - source: pluginURL, - env: envPath, - } - ); - logger.info('successfully added plugin'); + logger.info(`env path:\n${envPath}`); + logger.info('writing plugin info to settings store'); + settingsStore.set( + `plugins.${pluginID}`, + { + model_name: pluginName, + pyname: pluginPyName, + type: 'plugin', + source: pluginURL, + env: envPath, + } + ); + logger.info('successfully added plugin'); + }); } catch (error) { return error; } From 00c1fc318fdb8d9daf6c7893d7ca32dbc4961d26 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Mon, 11 Nov 2024 10:11:37 -0800 Subject: [PATCH 2/6] logging stdout and stderr from plugin installation --- workbench/src/main/setupAddRemovePlugin.js | 22 ++++++++++++------- .../renderer/components/PluginModal/index.jsx | 7 ++---- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/workbench/src/main/setupAddRemovePlugin.js b/workbench/src/main/setupAddRemovePlugin.js index 9c1e05c32a..8e40324021 100644 --- a/workbench/src/main/setupAddRemovePlugin.js +++ b/workbench/src/main/setupAddRemovePlugin.js @@ -11,7 +11,7 @@ import { settingsStore } from './settingsStore'; const logger = getLogger(__filename.split('/').slice(-1)[0]); -function customSpawn(cmd, args, options) { +function spawnWithLogging(cmd, args, options) { logger.info(cmd, args); const cmdProcess = spawn(cmd, args, options); if (cmdProcess.stdout) { @@ -19,14 +19,20 @@ function customSpawn(cmd, args, options) { cmdProcess.stdout.on('data', (data) => logger.info(data.toString())); } return new Promise((resolve, reject) => { - cmdProcess.on('close', (code) => { resolve(code); }); + cmdProcess.on('close', (code) => { + if (code === 0) { + resolve(code); + } else { + reject(code); + } + }); }); } export function setupAddPlugin() { ipcMain.handle( ipcMainChannels.ADD_PLUGIN, - (e, pluginURL) => { + async (e, pluginURL) => { logger.info('adding plugin at', pluginURL); const mamba = settingsStore.get('mamba'); @@ -38,12 +44,12 @@ export function setupAddPlugin() { try { // Create a temporary directory and check out the plugin's pyproject.toml const tmpPluginDir = fs.mkdtempSync(upath.join(tmpdir(), 'natcap-invest-')); - customSpawn( + await spawnWithLogging( 'git', ['clone', '--depth', '1', '--no-checkout', pluginURL, tmpPluginDir], { windowsHide: true } ).then(async () => { - await customSpawn( + await spawnWithLogging( 'git', ['checkout', 'HEAD', 'pyproject.toml'], { cwd: tmpPluginDir, windowsHide: true } @@ -62,14 +68,14 @@ export function setupAddPlugin() { // Create a conda env containing the plugin and its dependencies envName = `invest_plugin_${pluginID}`; - await customSpawn( + await spawnWithLogging( mamba, - ['create', '--yes', '--name', envName, '-c', 'conda-forge', '"python<3.12"', '"gdal<3.6"'], + ['createe', '--yes', '--name', envName, '-c', 'conda-forge', '"python<3.12"', '"gdal<3.6"'], { windowsHide: true } ); logger.info('created mamba env for plugin'); }).then(async () => { - await customSpawn( + await spawnWithLogging( mamba, ['run', '--verbose', '--no-capture-output', '--name', envName, 'pip', 'install', `git+${pluginURL}`], { windowsHide: true } diff --git a/workbench/src/renderer/components/PluginModal/index.jsx b/workbench/src/renderer/components/PluginModal/index.jsx index bfb3258972..d9b26a79d6 100644 --- a/workbench/src/renderer/components/PluginModal/index.jsx +++ b/workbench/src/renderer/components/PluginModal/index.jsx @@ -32,7 +32,7 @@ export default function PluginModal(props) { setLoading(false); updateInvestList(); if (addPluginErr) { - setErr(addPluginErr); + setErr(true); } else { setShowPluginModal(false); } @@ -119,10 +119,7 @@ export default function PluginModal(props) { if (err) { modalBody = ( - {t('Plugin installation failed')} -
-
- {err.toString()} + {t('Plugin installation failed. Check the workbench log for details.')} ); } From f234c01b4c5afbb8240fbde516b834832895c46e Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Mon, 11 Nov 2024 11:05:14 -0800 Subject: [PATCH 3/6] logging stdio for removing plugins --- workbench/src/main/setupAddRemovePlugin.js | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/workbench/src/main/setupAddRemovePlugin.js b/workbench/src/main/setupAddRemovePlugin.js index 8e40324021..dbc9645605 100644 --- a/workbench/src/main/setupAddRemovePlugin.js +++ b/workbench/src/main/setupAddRemovePlugin.js @@ -13,7 +13,7 @@ const logger = getLogger(__filename.split('/').slice(-1)[0]); function spawnWithLogging(cmd, args, options) { logger.info(cmd, args); - const cmdProcess = spawn(cmd, args, options); + const cmdProcess = spawn(cmd, args, { ...options, windowsHide: true }); if (cmdProcess.stdout) { cmdProcess.stderr.on('data', (data) => logger.info(data.toString())); cmdProcess.stdout.on('data', (data) => logger.info(data.toString())); @@ -46,13 +46,12 @@ export function setupAddPlugin() { const tmpPluginDir = fs.mkdtempSync(upath.join(tmpdir(), 'natcap-invest-')); await spawnWithLogging( 'git', - ['clone', '--depth', '1', '--no-checkout', pluginURL, tmpPluginDir], - { windowsHide: true } + ['clone', '--depth', '1', '--no-checkout', pluginURL, tmpPluginDir] ).then(async () => { await spawnWithLogging( 'git', ['checkout', 'HEAD', 'pyproject.toml'], - { cwd: tmpPluginDir, windowsHide: true } + { cwd: tmpPluginDir } ); }).then(async () => { // Read in the plugin's pyproject.toml, then delete it @@ -70,15 +69,13 @@ export function setupAddPlugin() { envName = `invest_plugin_${pluginID}`; await spawnWithLogging( mamba, - ['createe', '--yes', '--name', envName, '-c', 'conda-forge', '"python<3.12"', '"gdal<3.6"'], - { windowsHide: true } + ['create', '--yes', '--name', envName, '-c', 'conda-forge', '"python<3.12"', '"gdal<3.6"'] ); logger.info('created mamba env for plugin'); }).then(async () => { await spawnWithLogging( mamba, - ['run', '--verbose', '--no-capture-output', '--name', envName, 'pip', 'install', `git+${pluginURL}`], - { windowsHide: true } + ['run', '--verbose', '--no-capture-output', '--name', envName, 'pip', 'install', `git+${pluginURL}`] ); logger.info('installed plugin into its env'); }) @@ -86,10 +83,8 @@ export function setupAddPlugin() { // Write plugin metadata to the workbench's config.json const envInfo = execSync(`${mamba} info --envs`, { windowsHide: true }).toString(); logger.info(`env info:\n${envInfo}`); - const regex = new RegExp(String.raw`^${envName} +(.+)$`, 'm'); const envPath = envInfo.match(regex)[1]; - logger.info(`env path:\n${envPath}`); logger.info('writing plugin info to settings store'); settingsStore.set( @@ -114,16 +109,13 @@ export function setupAddPlugin() { export function setupRemovePlugin() { ipcMain.handle( ipcMainChannels.REMOVE_PLUGIN, - (e, pluginID) => { + async (e, pluginID) => { logger.info('removing plugin', pluginID); try { // Delete the plugin's conda env const env = settingsStore.get(`plugins.${pluginID}.env`); const mamba = settingsStore.get('mamba'); - execSync( - `${mamba} remove --yes --prefix ${env} --all`, - { stdio: 'inherit' } - ); + await spawnWithLogging(mamba, ['remove', '--yes', '--prefix', env, '--all']); // Delete the plugin's data from storage settingsStore.delete(`plugins.${pluginID}`); logger.info('successfully removed plugin'); From 6f82357c0bc85fc47dea6ee50a113bb69f8b48bc Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Mon, 11 Nov 2024 11:53:28 -0800 Subject: [PATCH 4/6] handle error in spawn --- workbench/src/main/setupAddRemovePlugin.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/workbench/src/main/setupAddRemovePlugin.js b/workbench/src/main/setupAddRemovePlugin.js index dbc9645605..cf7eec4752 100644 --- a/workbench/src/main/setupAddRemovePlugin.js +++ b/workbench/src/main/setupAddRemovePlugin.js @@ -19,6 +19,10 @@ function spawnWithLogging(cmd, args, options) { cmdProcess.stdout.on('data', (data) => logger.info(data.toString())); } return new Promise((resolve, reject) => { + cmdProcess.on('error', (err) => { + logger.error(err); + reject(err); + }); cmdProcess.on('close', (code) => { if (code === 0) { resolve(code); From ec6b7863d84947ce754c306af949f2458e5bc735 Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Tue, 12 Nov 2024 14:37:49 -0800 Subject: [PATCH 5/6] add docstring for spawnWithLogging function --- workbench/src/main/setupAddRemovePlugin.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/workbench/src/main/setupAddRemovePlugin.js b/workbench/src/main/setupAddRemovePlugin.js index cf7eec4752..1a38224c17 100644 --- a/workbench/src/main/setupAddRemovePlugin.js +++ b/workbench/src/main/setupAddRemovePlugin.js @@ -11,6 +11,20 @@ import { settingsStore } from './settingsStore'; const logger = getLogger(__filename.split('/').slice(-1)[0]); +/** + * Spawn a child process and log its stdout, stderr, and any error in spawning. + * + * child_process.spawn is called with the provided cmd, args, and options, + * and the windowsHide option set to true. + * + * Required properties missing from the store are initialized with defaults. + * Invalid properties are reset to defaults. + * @param {string} cmd - command to pass to spawn + * @param {Array} args - command arguments to pass to spawn + * @param {object} options - options to pass to spawn. + * @returns {Promise} resolves when the command finishes with exit code 0. + * Rejects with error otherwise. + */ function spawnWithLogging(cmd, args, options) { logger.info(cmd, args); const cmdProcess = spawn(cmd, args, { ...options, windowsHide: true }); From d60dea759444bf60eb91cf341ef71473daada37a Mon Sep 17 00:00:00 2001 From: Emily Soth Date: Wed, 13 Nov 2024 11:55:02 -0800 Subject: [PATCH 6/6] convert from .then syntax to await --- workbench/src/main/setupAddRemovePlugin.js | 105 +++++++++------------ 1 file changed, 47 insertions(+), 58 deletions(-) diff --git a/workbench/src/main/setupAddRemovePlugin.js b/workbench/src/main/setupAddRemovePlugin.js index 1a38224c17..1426fce86b 100644 --- a/workbench/src/main/setupAddRemovePlugin.js +++ b/workbench/src/main/setupAddRemovePlugin.js @@ -51,72 +51,61 @@ export function setupAddPlugin() { ipcMain.handle( ipcMainChannels.ADD_PLUGIN, async (e, pluginURL) => { - logger.info('adding plugin at', pluginURL); - - const mamba = settingsStore.get('mamba'); - let envName; - let pluginID; - let pluginName; - let pluginPyName; - try { + logger.info('adding plugin at', pluginURL); + const mamba = settingsStore.get('mamba'); // Create a temporary directory and check out the plugin's pyproject.toml const tmpPluginDir = fs.mkdtempSync(upath.join(tmpdir(), 'natcap-invest-')); await spawnWithLogging( 'git', ['clone', '--depth', '1', '--no-checkout', pluginURL, tmpPluginDir] - ).then(async () => { - await spawnWithLogging( - 'git', - ['checkout', 'HEAD', 'pyproject.toml'], - { cwd: tmpPluginDir } - ); - }).then(async () => { - // Read in the plugin's pyproject.toml, then delete it - const pyprojectTOML = toml.parse(fs.readFileSync( - upath.join(tmpPluginDir, 'pyproject.toml') - ).toString()); - fs.rmSync(tmpPluginDir, { recursive: true, force: true }); + ); + await spawnWithLogging( + 'git', + ['checkout', 'HEAD', 'pyproject.toml'], + { cwd: tmpPluginDir } + ); + // Read in the plugin's pyproject.toml, then delete it + const pyprojectTOML = toml.parse(fs.readFileSync( + upath.join(tmpPluginDir, 'pyproject.toml') + ).toString()); + fs.rmSync(tmpPluginDir, { recursive: true, force: true }); - // Access plugin metadata from the pyproject.toml - pluginID = pyprojectTOML.tool.natcap.invest.model_id; - pluginName = pyprojectTOML.tool.natcap.invest.model_name; - pluginPyName = pyprojectTOML.tool.natcap.invest.pyname; + // Access plugin metadata from the pyproject.toml + const pluginID = pyprojectTOML.tool.natcap.invest.model_id; + const pluginName = pyprojectTOML.tool.natcap.invest.model_name; + const pluginPyName = pyprojectTOML.tool.natcap.invest.pyname; - // Create a conda env containing the plugin and its dependencies - envName = `invest_plugin_${pluginID}`; - await spawnWithLogging( - mamba, - ['create', '--yes', '--name', envName, '-c', 'conda-forge', '"python<3.12"', '"gdal<3.6"'] - ); - logger.info('created mamba env for plugin'); - }).then(async () => { - await spawnWithLogging( - mamba, - ['run', '--verbose', '--no-capture-output', '--name', envName, 'pip', 'install', `git+${pluginURL}`] - ); - logger.info('installed plugin into its env'); - }) - .then(() => { - // Write plugin metadata to the workbench's config.json - const envInfo = execSync(`${mamba} info --envs`, { windowsHide: true }).toString(); - logger.info(`env info:\n${envInfo}`); - const regex = new RegExp(String.raw`^${envName} +(.+)$`, 'm'); - const envPath = envInfo.match(regex)[1]; - logger.info(`env path:\n${envPath}`); - logger.info('writing plugin info to settings store'); - settingsStore.set( - `plugins.${pluginID}`, - { - model_name: pluginName, - pyname: pluginPyName, - type: 'plugin', - source: pluginURL, - env: envPath, - } - ); - logger.info('successfully added plugin'); - }); + // Create a conda env containing the plugin and its dependencies + const envName = `invest_plugin_${pluginID}`; + await spawnWithLogging( + mamba, + ['create', '--yes', '--name', envName, '-c', 'conda-forge', '"python<3.12"', '"gdal<3.6"'] + ); + logger.info('created mamba env for plugin'); + await spawnWithLogging( + mamba, + ['run', '--verbose', '--no-capture-output', '--name', envName, 'pip', 'install', `git+${pluginURL}`] + ); + logger.info('installed plugin into its env'); + // Write plugin metadata to the workbench's config.json + const envInfo = execSync(`${mamba} info --envs`, { windowsHide: true }).toString(); + logger.info(`env info:\n${envInfo}`); + const regex = new RegExp(String.raw`^${envName} +(.+)$`, 'm'); + const envPath = envInfo.match(regex)[1]; + logger.info(`env path:\n${envPath}`); + logger.info('writing plugin info to settings store'); + settingsStore.set( + `plugins.${pluginID}`, + { + model_name: pluginName, + pyname: pluginPyName, + type: 'plugin', + source: pluginURL, + env: envPath, + } + ); + logger.info('successfully added plugin'); } catch (error) { return error; }