-
Notifications
You must be signed in to change notification settings - Fork 33
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #254 from NeuroML/experimental
To v1.1.0; Update to jnml v0.12.3 jar
- Loading branch information
Showing
17 changed files
with
512 additions
and
86 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,227 @@ | ||
| #!/usr/bin/env python3 | ||
| """ | ||
| Module for working with NSG. See https://github.com/OpenSourceBrain/pynsgr | ||
| File: pyneuroml/nsgr/__init__.py | ||
| Copyright 2023 NeuroML contributors | ||
| """ | ||
|
|
||
|
|
||
| import logging | ||
| import os | ||
| import pathlib | ||
| import shutil | ||
| import time | ||
| import typing | ||
| from zipfile import ZipFile | ||
|
|
||
| from pyneuroml.archive import get_model_file_list | ||
| from pyneuroml.pynml import run_lems_with | ||
| from pyneuroml.utils import get_files_generated_after, get_pyneuroml_tempdir | ||
| from pynsgr.commands.nsgr_submit import nsgr_submit | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
| logger.setLevel(logging.DEBUG) | ||
|
|
||
|
|
||
| def run_on_nsg( | ||
| engine: str, | ||
| lems_file_name: str, | ||
| nsg_sim_config: typing.Dict[typing.Any, typing.Any] = {}, | ||
| run_dir: typing.Optional[str] = None, | ||
| dry_run: bool = False, | ||
| *engine_args: typing.Any, | ||
| **engine_kwargs: typing.Any, | ||
| ): | ||
| """Run NeuroML/LEMS simulation on NSG. | ||
| Since the versions of tools on NSG may differ from what you have installed | ||
| locally, this method first generates the simulation engine specific files | ||
| (runner script for NEURON and mod files, for example) for the provided | ||
| engine in a new folder, zips them up, writes the config files, and then | ||
| submits the job to NSG using pynsgr. | ||
| The data generated by the job (if the command is not interrupted) is also | ||
| downloaded to the newly created directory. | ||
| Please ensure that you have set up an account and have your NSG | ||
| configuration file populated as noted in pynsgr. | ||
| - https://nsgr.sdsc.edu:8443/restusers/documentation | ||
| - https://nsgr.sdsc.edu:8443/restusers/docs/tools | ||
| Default for the nsg_sim_config is below, keys provided by the user in | ||
| nsg_sim_config overwrite these: | ||
| .. code:: python | ||
| nsg_sim_config_dict = { | ||
| "number_cores_": "1", | ||
| "number_nodes_": "1", | ||
| "tasks_per_node_": "1", | ||
| "runtime_": "0.5", | ||
| 'toolId': "PY_EXPANSE", | ||
| 'nrnivmodl_o_': "1" | ||
| } | ||
| .. versionadded:: 1.0.10 | ||
| :param engine: name of engine: suffixes of the run_lems_with.. functions | ||
| :type engine: str | ||
| :param lems_file_name: name of LEMS simulation file | ||
| :type lems_file_name: str | ||
| :param nsg_sim_config: dict containing params and values that will be | ||
| printed to testParam.properties | ||
| :type nsg_sim_config: dict | ||
| :param run_dir: directory in which model files are copied, backend specific | ||
| files are generated, and from which the archive is sent to NSG. Results | ||
| from the NSG run will also be downloaded to this directory. | ||
| By default, this is the directory that the command is run from (".") | ||
| It is good practice to separate directories where simulations are run | ||
| from the source of the model/simulations. | ||
| :type run_dir: str | ||
| :param *engine_args: positional args to be passed to the engine runner | ||
| function | ||
| :param **engine_kwargs: keyword args to be be passed to the engine runner | ||
| function | ||
| :param dry_run: do everything but do not submit | ||
| :type dry_run: bool | ||
| """ | ||
| supported_engines = ["jneuroml_neuron", "jneuroml_netpyne"] | ||
| if engine not in supported_engines: | ||
| print(f"Engine {engine} is not currently supported on NSG") | ||
| print(f"Supported engines are: {supported_engines}") | ||
| return | ||
|
|
||
| logger.debug(f"NSGR: engine is {engine}") | ||
|
|
||
| zipfile_name = lems_file_name.replace(".xml", "") + "_NSG.zip" | ||
| # default dictionary | ||
| nsg_sim_config_dict = { | ||
| "number_cores_": "1", | ||
| "number_nodes_": "1", | ||
| "tasks_per_node_": "1", | ||
| "runtime_": "0.5", | ||
| "toolId": "PY_EXPANSE", | ||
| "nrnivmodl_o_": "1", | ||
| } | ||
|
|
||
| # update dict based on user values | ||
| for key, val in nsg_sim_config.items(): | ||
| nsg_sim_config_dict[key] = val | ||
|
|
||
| if run_dir is None: | ||
| run_dir = "." | ||
|
|
||
| tdir = get_pyneuroml_tempdir(rootdir=run_dir, prefix="pyneuroml") | ||
| os.mkdir(tdir) | ||
|
|
||
| logger.debug("Getting list of model files") | ||
| model_file_list = [] # type: list | ||
| lems_def_dir = None | ||
| lems_def_dir = get_model_file_list( | ||
| lems_file_name, model_file_list, ".", lems_def_dir | ||
| ) | ||
|
|
||
| for model_file in model_file_list: | ||
| logger.debug(f"Copying: {model_file} -> {tdir + '/' + model_file}") | ||
| # if model file has directory structures in it, recreate the dirs in | ||
| # the temporary directory | ||
| if len(model_file.split("/")) > 1: | ||
| # throw error if files in parent directories are referred to | ||
| if "../" in model_file: | ||
| raise ValueError( | ||
| """ | ||
| Cannot handle parent directories because we | ||
| cannot create these directories correctly in | ||
| the temporary location. Please re-organize | ||
| your code such that all included files are in | ||
| sub-directories of this main directory. | ||
| """ | ||
| ) | ||
|
|
||
| model_file_path = pathlib.Path(tdir + "/" + model_file) | ||
| parent = model_file_path.parent | ||
| parent.mkdir(parents=True, exist_ok=True) | ||
| shutil.copy(model_file, tdir + "/" + model_file) | ||
|
|
||
| if lems_def_dir is not None: | ||
| logger.info(f"Removing LEMS definitions directory {lems_def_dir}") | ||
| shutil.rmtree(lems_def_dir) | ||
|
|
||
| os.chdir(tdir) | ||
| logger.info(f"Generating simulator specific files in {tdir}") | ||
| start_time = time.time() - 1.0 | ||
|
|
||
| if engine == "jneuroml_neuron": | ||
| run_lems_with( | ||
| engine, | ||
| lems_file_name=lems_file_name, | ||
| compile_mods=False, | ||
| only_generate_scripts=True, | ||
| *engine_args, | ||
| **engine_kwargs, | ||
| ) | ||
| elif engine == "jneuroml_netpyne": | ||
| run_lems_with( | ||
| engine, | ||
| lems_file_name=lems_file_name, | ||
| only_generate_scripts=True, | ||
| *engine_args, | ||
| **engine_kwargs, | ||
| ) | ||
|
|
||
| generated_files = get_files_generated_after( | ||
| start_time, ignore_suffixes=["xml", "nml"] | ||
| ) | ||
| logger.debug(f"Generated files are: {generated_files}") | ||
|
|
||
| logger.info("Generating zip file") | ||
| runner_file = "" | ||
| # NSG requires that the top level directory exist | ||
| nsg_dir = pathlib.Path(zipfile_name.replace(".zip", "")) | ||
| logger.debug(f"Creating directory and moving generated files to it: {nsg_dir}") | ||
|
|
||
| # remove it if it exists | ||
| if nsg_dir.is_dir(): | ||
| shutil.rmtree(str(nsg_dir)) | ||
| nsg_dir.mkdir() | ||
|
|
||
| with ZipFile(zipfile_name, "w") as archive: | ||
| for f in generated_files: | ||
| if engine == "jneuroml_neuron": | ||
| if f.endswith("_nrn.py"): | ||
| runner_file = f | ||
| elif engine == "jneuroml_netpyne": | ||
| if f.endswith("_netpyne.py"): | ||
| runner_file = f | ||
| fpath = pathlib.Path(f) | ||
| moved_path = fpath.rename(nsg_dir / fpath) | ||
| archive.write(str(moved_path)) | ||
|
|
||
| logger.debug("Printing testParam.properties") | ||
| nsg_sim_config_dict["filename_"] = runner_file | ||
| logger.debug(f"NSG sim config is: {nsg_sim_config_dict}") | ||
|
|
||
| with open("testParam.properties", "w") as file: | ||
| for key, val in nsg_sim_config_dict.items(): | ||
| print(f"{key}={val}", file=file) | ||
|
|
||
| logger.debug("Printing testInput.properties") | ||
| with open("testInput.properties", "w") as file: | ||
| print(f"infile_=@./{zipfile_name}", file=file) | ||
|
|
||
| print(f"{zipfile_name} generated") | ||
| # uses argv, where the first argument is the script itself, so we must pass | ||
| # something as the 0th index of the list | ||
| if not dry_run: | ||
| if nsgr_submit(["", ".", "validate"]) == 0: | ||
| print("Attempting to submit to NSGR") | ||
| return nsgr_submit(["", ".", "run"]) | ||
| else: | ||
| print("Dry run mode enabled. Not submitting to NSG.") | ||
|
|
||
| return True |
Oops, something went wrong.