From cc63e2c3f65d0faa6426169e322d4ab3ee3cef4c Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 27 May 2021 13:02:44 -0400 Subject: [PATCH 001/192] Added function to publish model presentations to the .presentations directory. --- stochss/handlers/util/stochss_file.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/stochss/handlers/util/stochss_file.py b/stochss/handlers/util/stochss_file.py index fbe99c91d3..425182d8e1 100644 --- a/stochss/handlers/util/stochss_file.py +++ b/stochss/handlers/util/stochss_file.py @@ -17,6 +17,7 @@ ''' import os +import stat import shutil import zipfile import traceback @@ -99,6 +100,30 @@ def duplicate(self): raise StochSSPermissionsError(message, traceback.format_exc()) from err + def publish_presentation(self): + ''' + Publish a model or spatial model presentation + + Attributes + ---------- + ''' + present_dir = os.path.join(self.user_dir, ".presentations") + if not os.path.exists(present_dir): + os.mkdir(present_dir) + dst = os.path.join(present_dir, self.get_file()) + if os.path.exists(dst): + message = "A publication with this name already exists" + raise StochSSFileExistsError(message) + src = self.get_path(full=True) + try: + shutil.copyfile(src, dst) + os.chmod(dst, stat.S_IREAD) + return {"message": f"Successfully published the {self.get_name()} presentation"} + except PermissionError as err: + message = f"You do not have permission to copy this file: {str(err)}" + raise StochSSPermissionsError(message, traceback.format_exc()) from err + + def move(self, location): ''' Moves a file to a new location. From 4852ef0787dfa6ff9807a192bf78352c1e6e8ef5 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 27 May 2021 13:04:03 -0400 Subject: [PATCH 002/192] Added api handler and route for publising model presentations. --- stochss/handlers/__init__.py | 1 + stochss/handlers/models.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/stochss/handlers/__init__.py b/stochss/handlers/__init__.py index f52ecc0747..635faa897c 100644 --- a/stochss/handlers/__init__.py +++ b/stochss/handlers/__init__.py @@ -73,6 +73,7 @@ def get_page_handlers(route_start): (r"/stochss/api/model/to-sbml\/?", ModelToSBMLAPIHandler), (r"/stochss/api/model/run\/?", RunModelAPIHandler), (r"/stochss/api/model/exists\/?", ModelExistsAPIHandler), + (r"/stochss/api/model/presentation\/?", ModelPresentationAPIHandler), (r"/stochss/api/spatial-model/domain-list\/?", LoadExternalDomains), (r"/stochss/api/spatial-model/types-list\/?", LoadParticleTypesDescriptions), (r"/stochss/api/spatial-model/domain-plot\/?", LoadDomainAPIHandler), diff --git a/stochss/handlers/models.py b/stochss/handlers/models.py index ee9c9284e1..988e486bce 100644 --- a/stochss/handlers/models.py +++ b/stochss/handlers/models.py @@ -28,7 +28,7 @@ # Use finish() for json, write() for text from .util import StochSSFolder, StochSSModel, StochSSSpatialModel, StochSSNotebook, \ - StochSSAPIError, report_error + StochSSFile, StochSSAPIError, report_error log = logging.getLogger('stochss') @@ -403,3 +403,32 @@ async def get(self): except StochSSAPIError as err: report_error(self, log, err) self.finish() + + +class ModelPresentationAPIHandler(APIHandler): + ''' + ################################################################################################ + Handler publishing model presentations. + ################################################################################################ + ''' + @web.authenticated + async def get(self): + ''' + Publish a model or spatial model presentation. + + Attributes + ---------- + ''' + self.set_header('Content-Type', 'application/json') + path = self.get_query_argument(name="path") + log.debug("Path to the file: %s", path) + try: + file = StochSSFile(path=path) + log.info("Publishing the %s presentation", file.get_name()) + resp = file.publish_presentation() + log.info(resp['message']) + log.debug("Response Message: %s", resp) + self.write(resp) + except StochSSAPIError as err: + report_error(self, log, err) + self.finish() From 2ffcbf2594432de8bd0b4a7a5494c083b35ab987 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 27 May 2021 14:41:37 -0400 Subject: [PATCH 003/192] removed the chmod block. --- stochss/handlers/util/stochss_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stochss/handlers/util/stochss_file.py b/stochss/handlers/util/stochss_file.py index 425182d8e1..83f49388ec 100644 --- a/stochss/handlers/util/stochss_file.py +++ b/stochss/handlers/util/stochss_file.py @@ -117,7 +117,7 @@ def publish_presentation(self): src = self.get_path(full=True) try: shutil.copyfile(src, dst) - os.chmod(dst, stat.S_IREAD) + # INSERT JUPYTER HUB CODE HERE return {"message": f"Successfully published the {self.get_name()} presentation"} except PermissionError as err: message = f"You do not have permission to copy this file: {str(err)}" From 39d578c6542941a20ec84389579ea22b5133bc20 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 27 May 2021 15:51:46 -0400 Subject: [PATCH 004/192] Setup the user interface to allow users to publish a model presentation. --- .../templates/includes/modelStateButtons.pug | 18 +++- client/views/model-state-buttons.js | 84 +++++++++++++++---- 2 files changed, 82 insertions(+), 20 deletions(-) diff --git a/client/templates/includes/modelStateButtons.pug b/client/templates/includes/modelStateButtons.pug index e5f340e678..32d0ce7f0d 100644 --- a/client/templates/includes/modelStateButtons.pug +++ b/client/templates/includes/modelStateButtons.pug @@ -1,14 +1,22 @@ div.mdl-edit-btn - div.mdl-edit-btn.saving-status(data-hook="saving-mdl") + div.mdl-edit-btn.saving-status(data-hook="mdl-action-start") div.spinner-grow - span Saving... + span(data-hook="saving" style="display: none") Saving ... - div.mdl-edit-btn.saved-status(data-hook="saved-mdl") + span(data-hook="publishing" style="display: none") Publishing ... - span Saved + div.mdl-edit-btn.saved-status(data-hook="mdl-action-end") + + span(data-hook="saved" style="display: none") Saved + + span(data-hook="published" style="display: none") Published + + div.mdl-edit-btn.save-error-status(data-hook="mdl-action-err") + + span Error button.btn.btn-primary.box-shadow(data-hook="save") Save @@ -29,4 +37,6 @@ div.mdl-edit-btn li.dropdown-item(id="stochss-es" data-hook="stochss-es" data-type="ensemble") Ensemble Simulation li.dropdown-item(id="stochss-ps" data-hook="stochss-ps" data-type="psweep") Parameter Sweep li.dropdown-item(data-hook="new-workflow" data-type="notebook") Jupyter Notebook + + button.btn.btn-primary.box-shadow(data-hook="presentation") Publish Presentation \ No newline at end of file diff --git a/client/views/model-state-buttons.js b/client/views/model-state-buttons.js index f7d5bebf8a..79fc02b4bb 100644 --- a/client/views/model-state-buttons.js +++ b/client/views/model-state-buttons.js @@ -19,6 +19,7 @@ along with this program. If not, see . var path = require('path'); var Plotly = require('../lib/plotly'); var $ = require('jquery'); +let _ = require('underscore'); //support file var app = require('../app'); var modals = require('../modals'); @@ -35,7 +36,8 @@ module.exports = View.extend({ "click [data-hook=stochss-es]" : "handleSimulateClick", "click [data-hook=stochss-ps]" : "handleSimulateClick", 'click [data-hook=new-workflow]' : 'handleSimulateClick', - 'click [data-hook=return-to-project-btn]' : 'clickReturnToProjectHandler' + 'click [data-hook=return-to-project-btn]' : 'clickReturnToProjectHandler', + 'click [data-hook=presentation]' : 'handlePresentationClick' }, initialize: function (attrs, options) { View.prototype.initialize.apply(this, arguments); @@ -51,7 +53,8 @@ module.exports = View.extend({ } }, clickSaveHandler: function (e) { - this.saveModel(this.saved.bind(this)); + let self = this; + this.saveModel(_.bind(self.endAction, self, "save")); }, clickRunHandler: function (e) { if(this.model.is_spatial && $(this.queryByHook("domain-plot-viewer-container")).css("display") !== "none") { @@ -71,7 +74,7 @@ module.exports = View.extend({ clickReturnToProjectHandler: function (e) { let self = this this.saveModel(function () { - self.saved() + self.endAction("save") var dirname = path.dirname(self.model.directory) if(dirname.endsWith(".wkgp")) { dirname = path.dirname(dirname) @@ -83,7 +86,7 @@ module.exports = View.extend({ clickNewWorkflowHandler: function (e) { let self = this this.saveModel(function () { - self.saved() + self.endAction("save") var queryString = "?path="+self.model.directory if(self.model.directory.includes('.proj')) { let wkgp = self.model.directory.includes('.wkgp') ? self.model.name + ".wkgp" : "WorkflowGroup1.wkgp" @@ -95,7 +98,7 @@ module.exports = View.extend({ }) }, getPreviewSpecies: function () { - this.saved(); + this.endAction("save"); let species = this.model.species.map(function (species) { return species.name }); @@ -125,7 +128,7 @@ module.exports = View.extend({ } }, saveModel: function (cb) { - this.saving(); + this.startAction("save"); // this.model is a ModelVersion, the parent of the collection is Model var model = this.model; if (cb) { @@ -134,24 +137,50 @@ module.exports = View.extend({ model.saveModel(); } }, - saving: function () { - var saving = this.queryByHook('saving-mdl'); - var saved = this.queryByHook('saved-mdl'); + startAction: function (action) { + if(action === "save") { + msg = $(this.queryByHook("saving")); + }else{ + msg = $(this.queryByHook("publishing")); + } + msg.css("display", "inline-block"); + var saving = this.queryByHook('mdl-action-start'); + var saved = this.queryByHook('mdl-action-end'); saved.style.display = "none"; saving.style.display = "inline-block"; }, - saved: function () { - var saving = this.queryByHook('saving-mdl'); - var saved = this.queryByHook('saved-mdl'); + errorAction: function () { + oldMsg = $(this.queryByHook("publishing")).css("display", "none"); + var saving = this.queryByHook('mdl-action-start'); + var error = this.queryByHook('mdl-action-err'); + saving.style.display = "none"; + error.style.display = "inline-block"; + setTimeout(function () { + error.style.display = "none"; + }, 5000); + }, + endAction: function (action) { + if(action === "save") { + oldMsg = $(this.queryByHook("saving")); + msg = $(this.queryByHook("saved")); + }else{ + oldMsg = $(this.queryByHook("publishing")); + msg = $(this.queryByHook("published")); + } + oldMsg.css("display", "none"); + msg.css("display", "inline-block"); + var saving = this.queryByHook('mdl-action-start'); + var saved = this.queryByHook('mdl-action-end'); saving.style.display = "none"; saved.style.display = "inline-block"; setTimeout(function () { saved.style.display = "none"; + msg.css("display", "none"); }, 5000); }, runModel: function (species=null) { if(typeof species !== "string") { - this.saved(); + this.endAction("save"); } this.running(); $(this.parent.queryByHook('model-run-container')).css("display", "block") @@ -238,12 +267,15 @@ module.exports = View.extend({ Plotly.newPlot(el, data); window.scrollTo(0, document.body.scrollHeight) }, + displayError: function (errorMsg, e) { + $(this.parent.queryByHook('toggle-preview-plot')).click() + errorMsg.css('display', 'block') + this.focusOnError(e) + }, handleSimulateClick: function (e) { var errorMsg = $(this.parent.queryByHook("error-detected-msg")) if(!this.model.valid) { - $(this.parent.queryByHook('toggle-preview-plot')).click() - errorMsg.css('display', 'block') - this.focusOnError(e) + this.displayError(errorMsg, e); }else{ errorMsg.css('display', 'none') let simType = e.target.dataset.type @@ -260,6 +292,26 @@ module.exports = View.extend({ } } }, + handlePresentationClick: function (e) { + var errorMsg = $(this.parent.queryByHook("error-detected-msg")); + if(!this.model.valid) { + this.displayError(errorMsg, e); + }else{ + let self = this; + this.startAction("publish") + let queryStr = "?path=" + this.model.directory; + let endpoint = path.join(app.getApiPath(), "model/presentation") + queryStr; + app.getXHR(endpoint, { + success: function (err, response, body) { + self.endAction("publish"); + }, + error: function (err, response, body) { + self.errorAction(); + $(modals.newProjectModelErrorHtml(body.Reason, body.Message)).modal(); + } + }); + } + }, focusOnError: function (e) { if(this.model.error) { let self = this From 89b677f8a31439e30b709fddf87ec7cfd6936e8c Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 27 May 2021 16:17:01 -0400 Subject: [PATCH 005/192] Added function to publish job, workflow, and project presentations. --- stochss/handlers/util/stochss_folder.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/stochss/handlers/util/stochss_folder.py b/stochss/handlers/util/stochss_folder.py index 8d9200e65b..2639a772a9 100644 --- a/stochss/handlers/util/stochss_folder.py +++ b/stochss/handlers/util/stochss_folder.py @@ -374,6 +374,30 @@ def move(self, location): raise StochSSPermissionsError(message, traceback.format_exc()) from err + def publish_presentation(self): + ''' + Publish a job, workflow, or project presentation. + + Attributes + ---------- + ''' + present_dir = os.path.join(self.user_dir, ".presentations") + if not os.path.exists(present_dir): + os.mkdir(present_dir) + dst = os.path.join(present_dir, self.get_file()) + if os.path.exists(dst): + message = "A presentation with this name already exists" + raise StochSSFileExistsError(message) + src = self.get_path(full=True) + try: + shutil.copytree(src, dst) + # INSERT JUPYTER HUB CODE HERE + return {"message": f"Successfully published the {self.get_name()} presentation"} + except PermissionError as err: + message = f"You do not have permission to publish this directory: {str(err)}" + raise StochSSPermissionsError(message, traceback.format_exc()) from err + + def upload(self, file_type, file, body, new_name=None): ''' Upload a file from a remote location to the users file system From 7f0c17d0224d90a7c26fbbc666efe657be363ecb Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 27 May 2021 16:18:21 -0400 Subject: [PATCH 006/192] Added api handler and routes for publishing job presentations. --- stochss/handlers/__init__.py | 3 ++- stochss/handlers/workflows.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/stochss/handlers/__init__.py b/stochss/handlers/__init__.py index f52ecc0747..c686c013b5 100644 --- a/stochss/handlers/__init__.py +++ b/stochss/handlers/__init__.py @@ -103,7 +103,8 @@ def get_page_handlers(route_start): (r"/stochss/api/workflow/edit-model\/?", GetWorkflowModelPathAPIHandler), (r"/stochss/api/workflow/save-plot\/?", SavePlotAPIHandler), (r"/stochss/api/workflow/save-annotation\/?", SaveAnnotationAPIHandler), - (r"/stochss/api/workflow/update-format\/?", UpadteWorkflowAPIHandler) + (r"/stochss/api/workflow/update-format\/?", UpadteWorkflowAPIHandler), + (r"/stochss/api/job/presentation\/?", JobPresentationAPIHandler) ] full_handlers = list(map(lambda h: (url_path_join(route_start, h[0]), h[1]), handlers)) return full_handlers diff --git a/stochss/handlers/workflows.py b/stochss/handlers/workflows.py index cb37ad715a..92eab198cb 100644 --- a/stochss/handlers/workflows.py +++ b/stochss/handlers/workflows.py @@ -28,7 +28,7 @@ # Use finish() for json, write() for text from .util import StochSSJob, StochSSModel, StochSSSpatialModel, StochSSNotebook, StochSSWorkflow, \ - StochSSAPIError, report_error + StochSSFolder, StochSSAPIError, report_error log = logging.getLogger('stochss') @@ -394,3 +394,32 @@ async def get(self): except StochSSAPIError as err: report_error(self, log, err) self.finish() + + +class JobPresentationAPIHandler(APIHandler): + ''' + ################################################################################################ + Handler for publishing job presentations. + ################################################################################################ + ''' + @web.authenticated + async def get(self): + ''' + Publish a job presentation. + + Attributes + ---------- + ''' + self.set_header('Content-Type', 'application/json') + path = self.get_query_argument(name="path") + log.debug("The path to the job: %s", path) + try: + folder = StochSSFolder(path=path) + log.info("Publishing the %s presentation", folder.get_name()) + resp = folder.publish_presentation() + log.info(resp['message']) + log.debug("Response Message: %s", resp) + self.write(resp) + except StochSSAPIError as err: + report_error(self, log, err) + self.finish() From d02eafc8c0b869c5378c942e168aa703c3678fa9 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 27 May 2021 16:22:30 -0400 Subject: [PATCH 007/192] Fixed the error messages and removed the stat import. --- stochss/handlers/util/stochss_file.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stochss/handlers/util/stochss_file.py b/stochss/handlers/util/stochss_file.py index 83f49388ec..695b442ef4 100644 --- a/stochss/handlers/util/stochss_file.py +++ b/stochss/handlers/util/stochss_file.py @@ -17,7 +17,6 @@ ''' import os -import stat import shutil import zipfile import traceback @@ -112,7 +111,7 @@ def publish_presentation(self): os.mkdir(present_dir) dst = os.path.join(present_dir, self.get_file()) if os.path.exists(dst): - message = "A publication with this name already exists" + message = "A presentation with this name already exists" raise StochSSFileExistsError(message) src = self.get_path(full=True) try: @@ -120,7 +119,7 @@ def publish_presentation(self): # INSERT JUPYTER HUB CODE HERE return {"message": f"Successfully published the {self.get_name()} presentation"} except PermissionError as err: - message = f"You do not have permission to copy this file: {str(err)}" + message = f"You do not have permission to publish this file: {str(err)}" raise StochSSPermissionsError(message, traceback.format_exc()) from err From 57146a4bef108d4d7bdffca4eb9b4a0466688bec Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 27 May 2021 17:14:15 -0400 Subject: [PATCH 008/192] Added arguments that will allow the job presentation to have the correct name. --- stochss/handlers/util/stochss_folder.py | 5 +++-- stochss/handlers/workflows.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/stochss/handlers/util/stochss_folder.py b/stochss/handlers/util/stochss_folder.py index 2639a772a9..3bac89337f 100644 --- a/stochss/handlers/util/stochss_folder.py +++ b/stochss/handlers/util/stochss_folder.py @@ -374,7 +374,7 @@ def move(self, location): raise StochSSPermissionsError(message, traceback.format_exc()) from err - def publish_presentation(self): + def publish_presentation(self, name=None): ''' Publish a job, workflow, or project presentation. @@ -384,7 +384,8 @@ def publish_presentation(self): present_dir = os.path.join(self.user_dir, ".presentations") if not os.path.exists(present_dir): os.mkdir(present_dir) - dst = os.path.join(present_dir, self.get_file()) + file = self.get_file() if name is None else name + dst = os.path.join(present_dir, file) if os.path.exists(dst): message = "A presentation with this name already exists" raise StochSSFileExistsError(message) diff --git a/stochss/handlers/workflows.py b/stochss/handlers/workflows.py index 92eab198cb..f5c3b3e06a 100644 --- a/stochss/handlers/workflows.py +++ b/stochss/handlers/workflows.py @@ -413,10 +413,12 @@ async def get(self): self.set_header('Content-Type', 'application/json') path = self.get_query_argument(name="path") log.debug("The path to the job: %s", path) + name = self.get_query_argument(name="name") + log.debug("Name of the job presentation: %s", name) try: folder = StochSSFolder(path=path) log.info("Publishing the %s presentation", folder.get_name()) - resp = folder.publish_presentation() + resp = folder.publish_presentation(name=name) log.info(resp['message']) log.debug("Response Message: %s", resp) self.write(resp) From 559d33eebef86c3b34e3bb42306a1405e2fc7aba Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 27 May 2021 17:15:31 -0400 Subject: [PATCH 009/192] Added the user interface that will allow user to publish job presentations. --- client/templates/includes/gillespyResults.pug | 16 ++++++++ .../includes/gillespyResultsEnsemble.pug | 16 ++++++++ .../includes/parameterScanResults.pug | 15 +++++++ .../includes/parameterSweepResults.pug | 16 ++++++++ client/views/workflow-results.js | 41 ++++++++++++++++++- 5 files changed, 103 insertions(+), 1 deletion(-) diff --git a/client/templates/includes/gillespyResults.pug b/client/templates/includes/gillespyResults.pug index 354ec74d2a..a37a6c6858 100644 --- a/client/templates/includes/gillespyResults.pug +++ b/client/templates/includes/gillespyResults.pug @@ -63,3 +63,19 @@ div#workflow-results.card.card-body button.btn.btn-primary.box-shadow(id="convert-to-notebook" data-hook="convert-to-notebook") Convert to Notebook button.btn.btn-primary.box-shadow(id="download-results-csv" data-hook="download-results-csv") Download Results as .csv + + button.btn.btn-primary.box-shadow(id="job-presentation" data-hook="job-presentation") Publish Presentation + + div.saving-status(data-hook="job-action-start") + + div.spinner-grow + + span Publishing ... + + div.saved-status(data-hook="job-action-end") + + span Published + + div.save-error-status(data-hook="job-action-err") + + span Error diff --git a/client/templates/includes/gillespyResultsEnsemble.pug b/client/templates/includes/gillespyResultsEnsemble.pug index ada2db9858..be9fa68f37 100644 --- a/client/templates/includes/gillespyResultsEnsemble.pug +++ b/client/templates/includes/gillespyResultsEnsemble.pug @@ -174,3 +174,19 @@ div#workflow-results.card.card-body button.btn.btn-primary.box-shadow(id="convert-to-notebook" data-hook="convert-to-notebook") Convert to Notebook button.btn.btn-primary.box-shadow(id="download-results-csv" data-hook="download-results-csv") Download Results as .csv + + button.btn.btn-primary.box-shadow(id="job-presentation" data-hook="job-presentation") Publish Presentation + + div.saving-status(data-hook="job-action-start") + + div.spinner-grow + + span Publishing ... + + div.saved-status(data-hook="job-action-end") + + span Published + + div.save-error-status(data-hook="job-action-err") + + span Error diff --git a/client/templates/includes/parameterScanResults.pug b/client/templates/includes/parameterScanResults.pug index 4b76a7a150..a6284002fe 100644 --- a/client/templates/includes/parameterScanResults.pug +++ b/client/templates/includes/parameterScanResults.pug @@ -78,3 +78,18 @@ div#workflow-results.card.card-body div.mt-2(data-hook="parameter-ranges") + button.btn.btn-primary.box-shadow(id="job-presentation" data-hook="job-presentation") Publish Presentation + + div.saving-status(data-hook="job-action-start") + + div.spinner-grow + + span Publishing ... + + div.saved-status(data-hook="job-action-end") + + span Published + + div.save-error-status(data-hook="job-action-err") + + span Error diff --git a/client/templates/includes/parameterSweepResults.pug b/client/templates/includes/parameterSweepResults.pug index 135f7f9121..ce1724f36c 100644 --- a/client/templates/includes/parameterSweepResults.pug +++ b/client/templates/includes/parameterSweepResults.pug @@ -158,3 +158,19 @@ div#workflow-results.card.card-body button.btn.btn-primary.box-shadow(id="convert-to-notebook" data-hook="convert-to-notebook") Convert to Notebook button.btn.btn-primary.box-shadow(id="download-results-csv" data-hook="download-results-csv") Download Results as .csv + + button.btn.btn-primary.box-shadow(id="job-presentation" data-hook="job-presentation") Publish Presentation + + div.saving-status(data-hook="job-action-start") + + div.spinner-grow + + span Publishing ... + + div.saved-status(data-hook="job-action-end") + + span Published + + div.save-error-status(data-hook="job-action-err") + + span Error diff --git a/client/views/workflow-results.js b/client/views/workflow-results.js index 7bcf2fe2f7..c28245c5d3 100644 --- a/client/views/workflow-results.js +++ b/client/views/workflow-results.js @@ -20,6 +20,7 @@ let $ = require('jquery'); let path = require('path'); //support files let app = require('../app'); +let modals = require('../modals'); let Tooltips = require('../tooltips'); let Plotly = require('../lib/plotly'); //views @@ -48,7 +49,8 @@ module.exports = View.extend({ 'click [data-target=download-png-custom]' : 'handleDownloadPNGClick', 'click [data-target=download-json]' : 'handleDownloadJSONClick', 'click [data-hook=convert-to-notebook]' : 'handleConvertToNotebookClick', - 'click [data-hook=download-results-csv]' : 'handleDownloadResultsCsvClick' + 'click [data-hook=download-results-csv]' : 'handleDownloadResultsCsvClick', + 'click [data-hook=job-presentation]' : 'handlePresentationClick' }, initialize: function (attrs, options) { View.prototype.initialize.apply(this, arguments); @@ -95,6 +97,22 @@ module.exports = View.extend({ changeCollapseButtonText: function (e) { app.changeCollapseButtonText(this, e); }, + endAction: function () { + $(this.queryByHook("job-action-start")).css("display", "none"); + let saved = $(this.queryByHook("job-action-end")); + saved.css("display", "inline-block"); + setTimeout(function () { + saved.css("display", "none"); + }, 5000); + }, + errorAction: function () { + $(this.queryByHook("job-action-start")).css("display", "none"); + let error = $(this.queryByHook("job-action-err")); + error.css("display", "inline-block"); + setTimeout(function () { + error.css("display", "none"); + }, 5000); + }, getPlot: function (type) { let self = this; let el = this.queryByHook(type + "-plot"); @@ -233,6 +251,22 @@ module.exports = View.extend({ } }); }, + handlePresentationClick: function (e) { + let self = this; + this.startAction(); + let name = this.parent.model.name + "_" + this.model.name; + let queryStr = "?path=" + this.model.directory + "&name=" + name; + let endpoint = path.join(app.getApiPath(), "job/presentation") + queryStr; + app.getXHR(endpoint, { + success: function (err, response, body) { + self.endAction(); + }, + error: function (err, response, body) { + self.errorAction(); + $(modals.newProjectModelErrorHtml(body.Reason, body.Message)).modal(); + } + }); + }, openPlotArgsSection: function (e) { $(this.queryByHook("edit-plot-args")).collapse("show"); $(document).ready(function () { @@ -342,6 +376,11 @@ module.exports = View.extend({ this.plotFigure(fig, type) } }, + startAction: function () { + $(this.queryByHook("job-action-start")).css("display", "inline-block"); + $(this.queryByHook("job-action-end")).css("display", "none"); + $(this.queryByHook("job-action-err")).css("display", "none"); + }, update: function () {}, updateValid: function () {}, subviews: { From bcaadbe156d03b56d32b03abbb06c255f8368842 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 11:40:59 -0400 Subject: [PATCH 010/192] Setup the base page for job presentations. --- client/pages/job-presentation.js | 93 ++++++++++++++++++++++ client/templates/pages/jobPresentation.pug | 22 +++++ 2 files changed, 115 insertions(+) create mode 100644 client/pages/job-presentation.js create mode 100644 client/templates/pages/jobPresentation.pug diff --git a/client/pages/job-presentation.js b/client/pages/job-presentation.js new file mode 100644 index 0000000000..1cd52015ad --- /dev/null +++ b/client/pages/job-presentation.js @@ -0,0 +1,93 @@ +/* +StochSS is a platform for simulating biochemical systems +Copyright (C) 2019-2020 StochSS developers. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +let $ = require('jquery'); +let domReady = require('domready'); +let bootstrap = require('bootstrap'); +//support files +let app = require("../app"); +//models +let Job = require("../models/job"); +//views +let PageView = require('./base'); +let ModelView = require('../views/model-viewer'); +let ResultsView = require('../views/workflow-results'); +let SettingsView = require('../views/settings-viewer'); +//templates +let template = require('../templates/pages/jobPresentation.pug'); + +import bootstrapStyles from '../styles/bootstrap.css'; +import styles from '../styles/styles.css'; +import fontawesomeStyles from '@fortawesome/fontawesome-free/css/svg-with-js.min.css' + +let JobPresentationPage = PageView.extend({ + template: template, + initialize: function () { + PageView.prototype.initialize.apply(this, arguments); + console.log("TODO: get the path to the job from the url") + // let urlParams = new URLSearchParams(window.location.search) + // this.model = new Job({ + // directory: urlParams.get("path") + // }); + console.log("TODO: get job from file system using the app.getXHR function") + // let self = this; + // let queryStr = "?path=" + this.model.directory; + // let endpoint = path.join(); + // app.getXHR(endpoint, { + // success: function (err, response, body) { + // self.model.set(body); + // self.renderSubviews(); + // } + // }); + console.log("TODO: generate the open link and store in this.open") + }, + renderSubviews: function () { + PageView.prototype.render.apply(this, arguments); + this.renderResultsContainer(); + this.renderSettingsContainer(); + this.renderModelContainer(); + }, + renderModelContainer: function () { + let modelView = new ModelView({ + model: this.model.model, + mode: "presentation" + }); + app.registerRenderSubview(this, modelView, "job-model"); + }, + renderResultsContainer: function () { + let resultsView = new ResultsView({ + model: this.model, + mode: "presentation" + }); + app.registerRenderSubview(this, resultsView, "job-results"); + }, + renderSettingsContainer: function () { + let settingsView = new SettingsView({ + model: this.model.settings, + mode: "presentation" + }); + app.registerRenderSubview(this, settingsView, "job-settings"); + } +}); + +domReady(() => { + let p = new JobPresentationPage({ + el: document.body + }); + p.render(); +}); \ No newline at end of file diff --git a/client/templates/pages/jobPresentation.pug b/client/templates/pages/jobPresentation.pug new file mode 100644 index 0000000000..bd2980d202 --- /dev/null +++ b/client/templates/pages/jobPresentation.pug @@ -0,0 +1,22 @@ +html + + head + meta(charset="utf-8") + title StochSS: Stochastic Simulation Service + meta(name="viewport", content="width=device-width, initial-scale=1.0") + meta(name="description", content="") + meta(name="author", content="") + + body + + div#job-presentation.container + + h2=this.model.name + + div(data-hook="job-results") + + div(data-hook="job-settings") + + div(data-hook="job-model") + + a.btn.btn-outline-secondary.box-shadow.text-break(href=this.open role="button") Open in StochSS \ No newline at end of file From c72b100c432fc1a3a276064802b9b31cc8b203a2 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 11:55:50 -0400 Subject: [PATCH 011/192] Added a check for a mode attribute that will hide elements that are not wanted for job presentations. --- client/views/workflow-results.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client/views/workflow-results.js b/client/views/workflow-results.js index 7bcf2fe2f7..6655cef574 100644 --- a/client/views/workflow-results.js +++ b/client/views/workflow-results.js @@ -55,6 +55,7 @@ module.exports = View.extend({ this.tooltips = Tooltips.parameterSweepResults; this.plots = {}; this.plotArgs = {}; + this.mode = Boolean(attrs.mode) ? attrs.mode : "edit"; }, render: function (attrs, options) { let isEnsemble = this.model.settings.simulationSettings.realizations > 1 && @@ -66,6 +67,12 @@ module.exports = View.extend({ this.template = isEnsemble ? gillespyResultsEnsembleTemplate : gillespyResultsTemplate; } View.prototype.render.apply(this, arguments); + if(this.mode === "presentation") { + $(this.queryByHook("job-presentation")).css("display", "none"); + if(!isParameterScan){ + $(this.queryByHook("convert-to-notebook")).css("display", "none"); + } + } if(this.parent.model.type === "Ensemble Simulation") { var type = isEnsemble ? "stddevran" : "trajectories"; }else{ From 8adefe739a4a2c932dc90fdbafddf4f32c830afc Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 12:01:21 -0400 Subject: [PATCH 012/192] Added a data hook to the section header to allow it to be changed. --- client/templates/includes/modelViewer.pug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/templates/includes/modelViewer.pug b/client/templates/includes/modelViewer.pug index 9559e2a1f8..cab8cb6844 100644 --- a/client/templates/includes/modelViewer.pug +++ b/client/templates/includes/modelViewer.pug @@ -2,7 +2,7 @@ div.card.card-body div(id="model-viewer-header") - h3.inline="Review Model: "+this.model.name + h3.inline(data-hook="job-model-name")="Review Model: "+this.model.name button.btn.btn-outline-collapse(data-toggle="collapse" data-target="#collapse-model" data-hook="collapse-model") + From f28f8156d8736d349fb57463a92cea5fb36624db Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 12:02:40 -0400 Subject: [PATCH 013/192] Added check for a mode attribute that will alter the section header for job presentations. --- client/views/model-viewer.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/views/model-viewer.js b/client/views/model-viewer.js index d35fe7885a..fec00e9ebc 100644 --- a/client/views/model-viewer.js +++ b/client/views/model-viewer.js @@ -16,6 +16,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ +let $ = require('jquery'); //support files let app = require('../app'); //views @@ -36,9 +37,13 @@ module.exports = View.extend({ }, initialize: function (attrs, options) { View.prototype.initialize.apply(this, arguments); + this.mode = Boolean(attrs.mode) ? attrs.mode : "edit"; }, render: function () { View.prototype.render.apply(this, arguments); + if(this.mode === "presentation") { + $(this.queryByHook("job-model-name")).html("Model: " + this.model.name) + } this.renderSpeciesView(); this.renderParametersView(); this.renderReactionsView(); From 703974b0a44e7bc5161cb2a89cc23b7486b10054 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 12:09:01 -0400 Subject: [PATCH 014/192] Added data hook to the section header to allow it to be changed. --- client/templates/includes/settingsViewer.pug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/templates/includes/settingsViewer.pug b/client/templates/includes/settingsViewer.pug index 90ba4f0528..b4875f85a5 100644 --- a/client/templates/includes/settingsViewer.pug +++ b/client/templates/includes/settingsViewer.pug @@ -2,7 +2,7 @@ div#workflow-settings-viewer.card.card-body div - h3.inline Review Settings + h3.inline(data-hook="job-settings-header") Review Settings button.btn.btn-outline-collapse(data-toggle="collapse" data-target="#collapse-wkfl-settings-viewer" data-hook="collapse-settings-viewer") + From 2a062290a78e230590840fdafa3620951f91f31d Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 12:33:46 -0400 Subject: [PATCH 015/192] Added check for a mode attribute that will alter the section header for job presentations. --- client/views/settings-viewer.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/views/settings-viewer.js b/client/views/settings-viewer.js index 918bfddf2b..f9312d7c1f 100644 --- a/client/views/settings-viewer.js +++ b/client/views/settings-viewer.js @@ -37,9 +37,13 @@ module.exports = View.extend({ this.algorithm = this.model.simulationSettings.isAutomatic ? "The algorithm was chosen based on your model." : this.model.simulationSettings.algorithm + this.mode = Boolean(attrs.mode) ? attrs.mode : "edit"; }, render: function (attrs, options) { View.prototype.render.apply(this, arguments); + if(this.mode === "presentation") { + $(this.queryByHook("job-settings-header")).html("Settings") + } if(!this.parent.model.newFormat) { $(this.queryByHook("timespan-settings-viewer-container")).css("display", "none"); } From 1657543c5a305bfa77822e24de78573e67227fa3 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 13:14:29 -0400 Subject: [PATCH 016/192] Added line to set the job models type to allow the current workflow results file to be compatable with the presentation page. --- client/pages/job-presentation.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client/pages/job-presentation.js b/client/pages/job-presentation.js index 1cd52015ad..1ee8e38f0a 100644 --- a/client/pages/job-presentation.js +++ b/client/pages/job-presentation.js @@ -51,6 +51,7 @@ let JobPresentationPage = PageView.extend({ // app.getXHR(endpoint, { // success: function (err, response, body) { // self.model.set(body); + // self.model.type = body.titleType; // self.renderSubviews(); // } // }); From 29fa9332ce616a1c61daa76453a84d3b266c1c36 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 13:15:54 -0400 Subject: [PATCH 017/192] Added the page api handler for the job presentation page. --- jupyterhub/handlers.py | 7 +++++++ jupyterhub/jupyterhub_config.py | 3 ++- webpack.hub.config.js | 8 ++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/jupyterhub/handlers.py b/jupyterhub/handlers.py index cd74cfb679..9b79473575 100644 --- a/jupyterhub/handlers.py +++ b/jupyterhub/handlers.py @@ -5,3 +5,10 @@ class HomeHandler(BaseHandler): async def get(self): html = self.render_template("stochss-home.html") self.finish(html) + + +class JobPresentationHandler(BaseHandler): + + async def get(self): + html = self.render_template("stochss-job-presentation.html") + self.finish(html) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 6b4103d270..0659af2c93 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -71,11 +71,12 @@ c.JupyterHub.default_url = '/stochss' # Page handlers -from handlers import HomeHandler +from handlers import HomeHandler, JobPresentationHandler # StochSS request handlers c.JupyterHub.extra_handlers = [ (r"/stochss\/?", HomeHandler), + (r"/stochss/job-presentation\/?", JobPresentationHandler) ] ## Paths to search for jinja templates, before using the default templates. diff --git a/webpack.hub.config.js b/webpack.hub.config.js index 99119e673d..77d0d2de4f 100644 --- a/webpack.hub.config.js +++ b/webpack.hub.config.js @@ -5,6 +5,7 @@ module.exports = { mode: 'development', entry: { home: './client/pages/home.js', + jobPresentation: './client/pages.job-presentation.js' }, output: { filename: 'stochss-[name].bundle.js', @@ -18,6 +19,13 @@ module.exports = { template: 'jupyterhub/home_template.pug', name: 'home', inject: false + }), + new HtmlWebpackPlugin({ + title: 'StochSS | Job Presentation', + filename: '../templates/stochss-job-presentation.html', + template: 'jupyterhub/home_template.pug', + name: 'jobPresentation', + inject: false }) ], module: { From 3250362e233e4b486dfbd2311b0c1a62007cd6c8 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 13:19:12 -0400 Subject: [PATCH 018/192] Added blank model presentation pug file. --- client/templates/pages/modelPresetation.pug | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 client/templates/pages/modelPresetation.pug diff --git a/client/templates/pages/modelPresetation.pug b/client/templates/pages/modelPresetation.pug new file mode 100644 index 0000000000..e69de29bb2 From 9e51e07430dc450965c6c9dd14ab036625152edd Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 13:32:30 -0400 Subject: [PATCH 019/192] pylint changes. --- jupyterhub/handlers.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/jupyterhub/handlers.py b/jupyterhub/handlers.py index 9b79473575..51903a707d 100644 --- a/jupyterhub/handlers.py +++ b/jupyterhub/handlers.py @@ -1,14 +1,22 @@ from jupyterhub.handlers.base import BaseHandler class HomeHandler(BaseHandler): - - async def get(self): - html = self.render_template("stochss-home.html") - self.finish(html) + ''' + ################################################################################################ + Handler for rendering jupyterhub home page. + ################################################################################################ + ''' + async def get(self): + html = self.render_template("stochss-home.html") + self.finish(html) class JobPresentationHandler(BaseHandler): - - async def get(self): - html = self.render_template("stochss-job-presentation.html") - self.finish(html) + ''' + ################################################################################################ + Handler for rendering jupyterhub job presentation page. + ################################################################################################ + ''' + async def get(self): + html = self.render_template("stochss-job-presentation.html") + self.finish(html) From 8816cd732d7b56bebda867da31756b8ae9745de3 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 13:46:24 -0400 Subject: [PATCH 020/192] More pylint changes. --- jupyterhub/handlers.py | 32 ++++ jupyterhub/jupyterhub_config.py | 306 ++++++++++++++++---------------- 2 files changed, 185 insertions(+), 153 deletions(-) diff --git a/jupyterhub/handlers.py b/jupyterhub/handlers.py index 51903a707d..e6023700f8 100644 --- a/jupyterhub/handlers.py +++ b/jupyterhub/handlers.py @@ -1,5 +1,25 @@ +''' +StochSS is a platform for simulating biochemical systems +Copyright (C) 2019-2020 StochSS developers. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +''' + from jupyterhub.handlers.base import BaseHandler +# pylint: disable=abstract-method +# pylint: disable=too-few-public-methods class HomeHandler(BaseHandler): ''' ################################################################################################ @@ -7,6 +27,12 @@ class HomeHandler(BaseHandler): ################################################################################################ ''' async def get(self): + ''' + Render the jupyterhub home page. + + Attributes + ---------- + ''' html = self.render_template("stochss-home.html") self.finish(html) @@ -18,5 +44,11 @@ class JobPresentationHandler(BaseHandler): ################################################################################################ ''' async def get(self): + ''' + Render the jupyterhub job presentation page. + + Attributes + ---------- + ''' html = self.render_template("stochss-job-presentation.html") self.finish(html) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 0659af2c93..9128aea405 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -22,22 +22,22 @@ import sys, os, os.path, shutil, logging ## Class for authenticating users. -# +# # This should be a subclass of :class:`jupyterhub.auth.Authenticator` -# +# # with an :meth:`authenticate` method that: -# +# # - is a coroutine (asyncio or tornado) # - returns username on success, None on failure # - takes two arguments: (handler, data), # where `handler` is the calling web.RequestHandler, # and `data` is the POST form data from the login page. -# +# # .. versionchanged:: 1.0 # authenticators may be registered via entry points, # e.g. `c.JupyterHub.authenticator_class = 'pam'` -# -# Currently installed: +# +# Currently installed: # - default: jupyterhub.auth.PAMAuthenticator # - dummy: jupyterhub.auth.DummyAuthenticator # - pam: jupyterhub.auth.PAMAuthenticator @@ -51,12 +51,12 @@ data_dir = os.environ.get('DATA_VOLUME_CONTAINER', '/data') c.JupyterHub.cookie_secret_file = os.path.join(data_dir, - 'jupyterhub_cookie_secret') + 'jupyterhub_cookie_secret') c.JupyterHub.db_url = 'postgresql://postgres:{password}@{host}/{db}'.format( - host=os.environ['POSTGRES_HOST'], - password=os.environ['POSTGRES_PASSWORD'], - db=os.environ['POSTGRES_DB'], + host=os.environ['POSTGRES_HOST'], + password=os.environ['POSTGRES_PASSWORD'], + db=os.environ['POSTGRES_DB'], ) ## An Application for starting a Multi-User Jupyter Notebook server. @@ -66,7 +66,7 @@ c.JupyterHub.log_level = 'DEBUG' ## The default URL for users when they arrive (e.g. when user directs to "/") -# +# # By default, redirects users to their own server. c.JupyterHub.default_url = '/stochss' @@ -75,57 +75,57 @@ # StochSS request handlers c.JupyterHub.extra_handlers = [ - (r"/stochss\/?", HomeHandler), - (r"/stochss/job-presentation\/?", JobPresentationHandler) + (r"/stochss\/?", HomeHandler), + (r"/stochss/job-presentation\/?", JobPresentationHandler) ] ## Paths to search for jinja templates, before using the default templates. c.JupyterHub.template_paths = [ - '/srv/jupyterhub/templates' + '/srv/jupyterhub/templates' ] ## The class to use for spawning single-user servers. -# +# # Should be a subclass of :class:`jupyterhub.spawner.Spawner`. -# +# # .. versionchanged:: 1.0 # spawners may be registered via entry points, # e.g. `c.JupyterHub.spawner_class = 'localprocess'` -# -# Currently installed: +# +# Currently installed: # - default: jupyterhub.spawner.LocalProcessSpawner # - localprocess: jupyterhub.spawner.LocalProcessSpawner # - simple: jupyterhub.spawner.SimpleLocalProcessSpawner c.JupyterHub.spawner_class = 'dockerspawner.DockerSpawner' ## The ip address for the Hub process to *bind* to. -# +# # By default, the hub listens on localhost only. This address must be accessible # from the proxy and user servers. You may need to set this to a public ip or '' # for all interfaces if the proxy or user servers are in containers or on a # different host. -# +# # See `hub_connect_ip` for cases where the bind and connect address should # differ, or `hub_bind_url` for setting the full bind URL. c.JupyterHub.hub_ip = os.environ.get('DOCKER_HUB_IMAGE') ## The internal port for the Hub process. -# +# # This is the internal port of the hub itself. It should never be accessed # directly. See JupyterHub.port for the public port to use when accessing # jupyterhub. It is rare that this port should be set except in cases of port # conflict. -# +# # See also `hub_ip` for the ip and `hub_bind_url` for setting the full bind URL. c.JupyterHub.hub_port = 8080 ## Services managed by JupyterHub c.JupyterHub.services = [ - { - 'name': 'cull-idle', - 'admin': True, - 'command': [sys.executable, '/srv/jupyterhub/cull_idle_servers.py', '--timeout=28800'], - } + { + 'name': 'cull-idle', + 'admin': True, + 'command': [sys.executable, '/srv/jupyterhub/cull_idle_servers.py', '--timeout=28800'], + } ] #------------------------------------------------------------------------------ @@ -147,7 +147,7 @@ # Pass the network name as argument to spawned containers # Pass cpu limit as extra config since dockerspawner does not natively support it c.DockerSpawner.extra_host_config = { - 'network_mode': network_name, + 'network_mode': network_name, } # Explicitly set notebook directory because we'll be mounting a host volume to # it. Most jupyter/docker-stacks *-notebook images run the Notebook server as @@ -160,7 +160,7 @@ c.DockerSpawner.volumes = { 'jupyterhub-user-{username}': notebook_dir } # Set extra environment variables c.DockerSpawner.environment = { - 'JUPYTER_CONFIG_DIR': os.environ['JUPYTER_CONFIG_DIR'] + 'JUPYTER_CONFIG_DIR': os.environ['JUPYTER_CONFIG_DIR'] } # Remove containers once they are stopped c.DockerSpawner.remove_containers = True @@ -172,123 +172,123 @@ #------------------------------------------------------------------------------ def get_user_cpu_count_or_fail(): - log = logging.getLogger() - reserve_count = int(os.environ['RESERVED_CPUS']) - log.info("RESERVED_CPUS environment variable is set to {}".format(reserve_count)) - # Round up to an even number of reserved cpus - if reserve_count % 2 > 0: - log.warn("Increasing reserved cpu count by one so it's an even number. This helps allocate logical cpus to users more easily.") - reserve_count += 1 - total_cpus = os.cpu_count() - log.info("Total cpu count as reported by os.count: {}".format(total_cpus)) - if reserve_count >= total_cpus: - e_message = "RESERVED_CPUS environment cannot be greater than or equal to the number of cpus returned by os.cpu_count()" - log.error(e_message) - raise ValueError(e_message) - user_cpu_count = total_cpus - reserve_count - # If (num logical cpus) - (num reserved cpus) is odd, - # use one less logical cpu for allocating user cpus - if user_cpu_count % 2 > 0 and user_cpu_count > 1: - user_cpu_count -= 1 - c.StochSS.reserved_cpu_count = reserve_count - log.info('Using {} logical cpus for user containers...'.format(user_cpu_count)) - log.info('Reserving {} logical cpus for hub container and underlying OS'.format(reserve_count)) - return user_cpu_count + log = logging.getLogger() + reserve_count = int(os.environ['RESERVED_CPUS']) + log.info("RESERVED_CPUS environment variable is set to {}".format(reserve_count)) + # Round up to an even number of reserved cpus + if reserve_count % 2 > 0: + log.warn("Increasing reserved cpu count by one so it's an even number. This helps allocate logical cpus to users more easily.") + reserve_count += 1 + total_cpus = os.cpu_count() + log.info("Total cpu count as reported by os.count: {}".format(total_cpus)) + if reserve_count >= total_cpus: + e_message = "RESERVED_CPUS environment cannot be greater than or equal to the number of cpus returned by os.cpu_count()" + log.error(e_message) + raise ValueError(e_message) + user_cpu_count = total_cpus - reserve_count + # If (num logical cpus) - (num reserved cpus) is odd, + # use one less logical cpu for allocating user cpus + if user_cpu_count % 2 > 0 and user_cpu_count > 1: + user_cpu_count -= 1 + c.StochSS.reserved_cpu_count = reserve_count + log.info('Using {} logical cpus for user containers...'.format(user_cpu_count)) + log.info('Reserving {} logical cpus for hub container and underlying OS'.format(reserve_count)) + return user_cpu_count c.StochSS.user_cpu_count = get_user_cpu_count_or_fail() c.StochSS.user_cpu_alloc = [0] * c.StochSS.user_cpu_count def get_power_users(): - power_users_file = os.environ.get('POWER_USERS_FILE') - log = logging.getLogger() - if not os.path.exists(power_users_file): - log.warn('No power users defined!') - return [] - with open(power_users_file) as f: - power_users = [ x.rstrip() for x in f.readlines() ] - return power_users + power_users_file = os.environ.get('POWER_USERS_FILE') + log = logging.getLogger() + if not os.path.exists(power_users_file): + log.warn('No power users defined!') + return [] + with open(power_users_file) as f: + power_users = [ x.rstrip() for x in f.readlines() ] + return power_users c.StochSS.power_users = get_power_users() def pre_spawn_hook(spawner): - '''Function that runs before DockerSpawner spawns a user container. - Limits the resources available to user containers, excluding a list of power users. - ''' - log = logging.getLogger() - # Remove the memory limit for power users - if spawner.user.name in c.StochSS.power_users: - spawner.mem_limit = None - return - palloc = c.StochSS.user_cpu_alloc - div = len(palloc) // 2 - reserved = c.StochSS.reserved_cpu_count - log.warn('Reserved CPUs: {}'.format(reserved)) - log.warn('Number of user containers using each logical core: {}'.format(palloc)) - # We want to allocate logical cores that are on the same physical core - # whenever possible. - # - # A hyper-threaded 4-core processor has 8 logical cpus, with - # two logical cpus per core. Logical cpus on the same cpu core are grouped - # such that if (#, #) represents two logical cpus on a single physical core, - # then the logical cores in this case are indexed on the system like this: - # - # (1, 5), (2, 6), (3, 7), (4, 8) - # - # To allocate two logical cpus per user container, we find the logical cpu that - # is being used by the least number of users in the first half of an - # array tracking which users are using what logical cores. - # - # The general formula for finding the index of the second logical core - # on the same physical core is then: - # - # index(matching logical core) = - # index(chosen logical core) + ( (number of logical cores) / 2 ) - # - avail_cpus = palloc[div:] - # If <= 4 cpus available then use 1 cpu for each user instead of 2 - if not len(avail_cpus): - log.warn("The host system only has 4 logical cpus, so we'll only reserve one logical cpu per user container, instead of the normal 2") - avail_cpus = palloc - least_used_cpu = min(avail_cpus) - cpu1_index = avail_cpus.index(least_used_cpu) - log.info("User {} to use logical cpu {}".format(spawner.user.name, str(cpu1_index))) - palloc[cpu1_index] += 1 - spawner.extra_host_config['cpuset_cpus'] = '{}'.format(cpu1_index) - else: - least_used_cpu = min(avail_cpus) - cpu1_index = avail_cpus.index(least_used_cpu) - palloc[cpu1_index] += 1 - cpu2_index = cpu1_index+div - palloc[cpu2_index] += 1 - log.info("User {} to use logical cpus {} and {}".format( - spawner.user.name, str(cpu1_index), str(cpu2_index))) - spawner.extra_host_config['cpuset_cpus'] = '{},{}'.format(cpu1_index, cpu2_index) + '''Function that runs before DockerSpawner spawns a user container. + Limits the resources available to user containers, excluding a list of power users. + ''' + log = logging.getLogger() + # Remove the memory limit for power users + if spawner.user.name in c.StochSS.power_users: + spawner.mem_limit = None + return + palloc = c.StochSS.user_cpu_alloc + div = len(palloc) // 2 + reserved = c.StochSS.reserved_cpu_count + log.warn('Reserved CPUs: {}'.format(reserved)) + log.warn('Number of user containers using each logical core: {}'.format(palloc)) + # We want to allocate logical cores that are on the same physical core + # whenever possible. + # + # A hyper-threaded 4-core processor has 8 logical cpus, with + # two logical cpus per core. Logical cpus on the same cpu core are grouped + # such that if (#, #) represents two logical cpus on a single physical core, + # then the logical cores in this case are indexed on the system like this: + # + # (1, 5), (2, 6), (3, 7), (4, 8) + # + # To allocate two logical cpus per user container, we find the logical cpu that + # is being used by the least number of users in the first half of an + # array tracking which users are using what logical cores. + # + # The general formula for finding the index of the second logical core + # on the same physical core is then: + # + # index(matching logical core) = + # index(chosen logical core) + ( (number of logical cores) / 2 ) + # + avail_cpus = palloc[div:] + # If <= 4 cpus available then use 1 cpu for each user instead of 2 + if not len(avail_cpus): + log.warn("The host system only has 4 logical cpus, so we'll only reserve one logical cpu per user container, instead of the normal 2") + avail_cpus = palloc + least_used_cpu = min(avail_cpus) + cpu1_index = avail_cpus.index(least_used_cpu) + log.info("User {} to use logical cpu {}".format(spawner.user.name, str(cpu1_index))) + palloc[cpu1_index] += 1 + spawner.extra_host_config['cpuset_cpus'] = '{}'.format(cpu1_index) + else: + least_used_cpu = min(avail_cpus) + cpu1_index = avail_cpus.index(least_used_cpu) + palloc[cpu1_index] += 1 + cpu2_index = cpu1_index+div + palloc[cpu2_index] += 1 + log.info("User {} to use logical cpus {} and {}".format( + spawner.user.name, str(cpu1_index), str(cpu2_index))) + spawner.extra_host_config['cpuset_cpus'] = '{},{}'.format(cpu1_index, cpu2_index) def post_stop_hook(spawner): - log = logging.getLogger() - reserved = c.StochSS.reserved_cpu_count - palloc = c.StochSS.user_cpu_alloc - try: - cpu1_index, cpu2_index = spawner.extra_host_config['cpuset_cpus'].split(',') - palloc[int(cpu1_index)] -= 1 - palloc[int(cpu2_index)] -= 1 - log.warn('Reserved CPUs: {}'.format(reserved)) - log.warn('Number of user containers using each logical core: {}'.format(palloc)) - except: - # Exception thrown due to cpuset_cpus not being set (power user) - pass + log = logging.getLogger() + reserved = c.StochSS.reserved_cpu_count + palloc = c.StochSS.user_cpu_alloc + try: + cpu1_index, cpu2_index = spawner.extra_host_config['cpuset_cpus'].split(',') + palloc[int(cpu1_index)] -= 1 + palloc[int(cpu2_index)] -= 1 + log.warn('Reserved CPUs: {}'.format(reserved)) + log.warn('Number of user containers using each logical core: {}'.format(palloc)) + except: + # Exception thrown due to cpuset_cpus not being set (power user) + pass c.Spawner.pre_spawn_hook = pre_spawn_hook c.Spawner.post_stop_hook = post_stop_hook ## The URL the single-user server should start in. -# +# # `{username}` will be expanded to the user's username -# +# # Example uses: -# +# # - You can set `notebook_dir` to `/` and `default_url` to `/tree/home/{username}` to allow people to # navigate the whole filesystem from their notebook server, but still start in their home directory. # - Start with `/notebooks` instead of `/tree` if `default_url` points to a notebook instead of a directory. @@ -296,17 +296,17 @@ def post_stop_hook(spawner): c.Spawner.default_url = '/stochss/models' ## Maximum number of bytes a single-user notebook server is allowed to use. -# +# # Allows the following suffixes: # - K -> Kilobytes # - M -> Megabytes # - G -> Gigabytes # - T -> Terabytes -# +# # If the single user server tries to allocate more memory than this, it will # fail. There is no guarantee that the single-user notebook server will be able # to allocate this much memory - only that it can not allocate more than this. -# +# # **This is a configuration setting. Your spawner must implement support for the # limit to work.** The default spawner, `LocalProcessSpawner`, does **not** # implement this support. A custom spawner **must** add support for this setting @@ -314,14 +314,14 @@ def post_stop_hook(spawner): c.Spawner.mem_limit = '4G' ## Maximum number of cpu-cores a single-user notebook server is allowed to use. -# +# # If this value is set to 0.5, allows use of 50% of one CPU. If this value is # set to 2, allows use of up to 2 CPUs. -# +# # The single-user notebook server will never be scheduled by the kernel to use # more cpu-cores than this. There is no guarantee that it can access this many # cpu-cores. -# +# # **This is a configuration setting. Your spawner must implement support for the # limit to work.** The default spawner, `LocalProcessSpawner`, does **not** # implement this support. A custom spawner **must** add support for this setting @@ -329,20 +329,20 @@ def post_stop_hook(spawner): #c.Spawner.cpu_limit = 2 ## Extra arguments to be passed to the single-user server. -# +# # Some spawners allow shell-style expansion here, allowing you to use # environment variables here. Most, including the default, do not. Consult the # documentation for your spawner to verify! #c.Spawner.args = [] ## The command used for starting the single-user server. -# +# # Provide either a string or a list containing the path to the startup script # command. Extra arguments, other than this path, should be provided via `args`. -# +# # This is usually set if you want to start the single-user server in a different # python environment (with virtualenv/conda) than JupyterHub itself. -# +# # Some spawners allow shell-style expansion here, allowing you to use # environment variables. Most, including the default, do not. Consult the # documentation for your spawner to verify! @@ -350,10 +350,10 @@ def post_stop_hook(spawner): ## Minimum number of cpu-cores a single-user notebook server is guaranteed to # have available. -# +# # If this value is set to 0.5, allows use of 50% of one CPU. If this value is # set to 2, allows use of up to 2 CPUs. -# +# # **This is a configuration setting. Your spawner must implement support for the # limit to work.** The default spawner, `LocalProcessSpawner`, does **not** # implement this support. A custom spawner **must** add support for this setting @@ -363,13 +363,13 @@ def post_stop_hook(spawner): ## Minimum number of bytes a single-user notebook server is guaranteed to have # available. -# +# # Allows the following suffixes: # - K -> Kilobytes # - M -> Megabytes # - G -> Gigabytes # - T -> Terabytes -# +# # **This is a configuration setting. Your spawner must implement support for the # limit to work.** The default spawner, `LocalProcessSpawner`, does **not** # implement this support. A custom spawner **must** add support for this setting @@ -383,28 +383,28 @@ def post_stop_hook(spawner): ## Base class for implementing an authentication provider for JupyterHub ## Set of users that will have admin rights on this JupyterHub. -# +# # Admin users have extra privileges: # - Use the admin panel to see list of users logged in # - Add / remove users in some authenticators # - Restart / halt the hub # - Start / stop users' single-user servers # - Can access each individual users' single-user server (if configured) -# +# # Admin access should be treated the same way root access is. -# +# # Defaults to an empty set, in which case no user has admin access. c.Authenticator.admin_users = admin = set([]) pwd = os.path.dirname(__file__) with open(os.path.join(pwd, 'userlist')) as f: - for line in f: - if not line: - continue - parts = line.split() - # in case of newline at the end of userlist file - if len(parts) >= 1: - name = parts[0] - #whitelist.add(name) - if len(parts) > 1 and parts[1] == 'admin': - admin.add(name) + for line in f: + if not line: + continue + parts = line.split() + # in case of newline at the end of userlist file + if len(parts) >= 1: + name = parts[0] + #whitelist.add(name) + if len(parts) > 1 and parts[1] == 'admin': + admin.add(name) From a84f80cfe00fc6444515c8566299dc89b0d9de86 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 15:11:28 -0400 Subject: [PATCH 021/192] Setup the base page for model presentations. --- client/pages/model-presentation.js | 117 +++++++++++++++++++ client/templates/pages/modelPresentation.pug | 30 +++++ client/templates/pages/modelPresetation.pug | 0 3 files changed, 147 insertions(+) create mode 100644 client/pages/model-presentation.js create mode 100644 client/templates/pages/modelPresentation.pug delete mode 100644 client/templates/pages/modelPresetation.pug diff --git a/client/pages/model-presentation.js b/client/pages/model-presentation.js new file mode 100644 index 0000000000..eb40d97a5e --- /dev/null +++ b/client/pages/model-presentation.js @@ -0,0 +1,117 @@ +/* +StochSS is a platform for simulating biochemical systems +Copyright (C) 2019-2020 StochSS developers. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +let $ = require('jquery'); +let path = require('path'); +let domReady = require('domready'); +let bootstrap = require('bootstrap'); +//support files +let app = require('../app'); +//models +let Model = require('../models/model'); +//views +let PageView = require('./base'); +let Rules = require('./rules-viewer'); +let Events = require('./events-viewer'); +let Species = require('./species-viewer'); +let Reactions = require('./reactions-viewer'); +let Parameters = require('./parameters-viewer'); +let SBMLComponents = require('./sbml-component-editor'); +//templates +let template = require('../templates/pages/modelPresentation.pug'); + +import bootstrapStyles from '../styles/bootstrap.css'; +import styles from '../styles/styles.css'; +import fontawesomeStyles from '@fortawesome/fontawesome-free/css/svg-with-js.min.css' + +let ModelPresentationPage = PageView.extend({ + template: template, + initialize: function (attrs, arguments) { + PageView.prototype.initialize.apply(this, arguments); + console.log("TODO: get the path to the model from the url") + // let urlParams = new URLSearchParams(window.location.search) + // this.model = new Model({ + // directory: urlParams.get("path"), + // for: "presentation" + // }); + console.log("TODO: get model from file system using the app.getXHR function") + // let self = this; + // let queryStr = "?path=" + this.model.directory; + // let endpoint = path.join(); + // app.getXHR(endpoint, { + // success: function (err, response, body) { + // self.model.set(body); + // self.renderSubviews(); + // } + // }); + console.log("TODO: generate the open link and store in this.open") + }, + renderSubviews: function () { + PageView.prototype.render.apply(this, arguments); + this.renderSpeciesContainer(); + this.renderParametersContainer(); + this.renderReactionsContainer(); + this.renderEventsContainer(); + this.renderRulesContainer(); + this.renderSBMLComponents(); + }, + renderEventsContainer: function () { + let events = new Events({ + collection: this.model.eventsCollection + }); + app.registerRenderSubview(this, events, "model-events"); + }, + renderParametersContainer: function () { + let parameters = new Parameters({ + collection: this.model.parameters + }); + app.registerRenderSubview(this, parameters, "model-parameters"); + }, + renderReactionsContainer: function () { + let reactions = new Reactions({ + collection: this.model.reactions + }); + app.registerRenderSubview(this, reactions, "model-reactions"); + }, + renderRulesContainer: function () { + let rules = new Rules({ + collection: this.model.rules + }); + app.registerRenderSubview(this, rules, "model-rules"); + }, + renderSBMLComponents: function () { + let sbmlComponents = new SBMLComponentsView({ + functionDefinitions: this.model.functionDefinitions, + viewMode: true + }); + app.registerRenderSubview(this, sbmlComponents, "model-sbml-components"); + }, + renderSpeciesContainer: function () { + let species = new Species({ + collection: this.model.species + }); + app.registerRenderSubview(this, species, "model-species"); + } +}); + +domReady(() => { + let p = new ModelPresentationPage({ + el: document.body + }); + p.render(); +}); diff --git a/client/templates/pages/modelPresentation.pug b/client/templates/pages/modelPresentation.pug new file mode 100644 index 0000000000..65b571444c --- /dev/null +++ b/client/templates/pages/modelPresentation.pug @@ -0,0 +1,30 @@ +html + + head + meta(charset="utf-8") + title StochSS: Stochastic Simulation Service + meta(name="viewport", content="width=device-width, initial-scale=1.0") + meta(name="description", content="") + meta(name="author", content="") + + body + + div#model-presentation.container + + h2=this.model.name + + div(data-hook="model-species") + + div(data-hook="model-parameters") + + div(data-hook="model-reactions") + + div(data-hook="model-events") + + div(data-hook="model-rules") + + div(data-hook="model-sbml-components") + + div.card.card-body="System Volume: "+this.model.volume + + a.btn.btn-outline-secondary.box-shadow.text-break(href=this.open role="button") Open in StochSS diff --git a/client/templates/pages/modelPresetation.pug b/client/templates/pages/modelPresetation.pug deleted file mode 100644 index e69de29bb2..0000000000 From eaf0b9a4cab70abd4fd1b024bf3b2cb5c88fe4d6 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 28 May 2021 15:15:06 -0400 Subject: [PATCH 022/192] Added the page handler for model presentations. --- client/pages/job-presentation.js | 1 + jupyterhub/handlers.py | 17 +++++++++++++++++ jupyterhub/jupyterhub_config.py | 5 +++-- webpack.hub.config.js | 10 +++++++++- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/client/pages/job-presentation.js b/client/pages/job-presentation.js index 1ee8e38f0a..b38c7c7fa6 100644 --- a/client/pages/job-presentation.js +++ b/client/pages/job-presentation.js @@ -17,6 +17,7 @@ along with this program. If not, see . */ let $ = require('jquery'); +let path = require('path'); let domReady = require('domready'); let bootstrap = require('bootstrap'); //support files diff --git a/jupyterhub/handlers.py b/jupyterhub/handlers.py index e6023700f8..b98c406d1d 100644 --- a/jupyterhub/handlers.py +++ b/jupyterhub/handlers.py @@ -52,3 +52,20 @@ async def get(self): ''' html = self.render_template("stochss-job-presentation.html") self.finish(html) + + +class ModelPresentationHandler(BaseHandler): + ''' + ################################################################################################ + Handler for rendering jupyterhub model presentation page. + ################################################################################################ + ''' + async def get(self): + ''' + Render the jupyterhub model presentation page. + + Attributes + ---------- + ''' + html = self.render_template("stochss-model-presentation.html") + self.finish(html) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 9128aea405..93998b6e88 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -71,12 +71,13 @@ c.JupyterHub.default_url = '/stochss' # Page handlers -from handlers import HomeHandler, JobPresentationHandler +from handlers import * # StochSS request handlers c.JupyterHub.extra_handlers = [ (r"/stochss\/?", HomeHandler), - (r"/stochss/job-presentation\/?", JobPresentationHandler) + (r"/stochss/job-presentation\/?", JobPresentationHandler), + (r"/stochss/model-presentation\/?", ModelPresentationHandler) ] ## Paths to search for jinja templates, before using the default templates. diff --git a/webpack.hub.config.js b/webpack.hub.config.js index 77d0d2de4f..658b62b9ff 100644 --- a/webpack.hub.config.js +++ b/webpack.hub.config.js @@ -5,7 +5,8 @@ module.exports = { mode: 'development', entry: { home: './client/pages/home.js', - jobPresentation: './client/pages.job-presentation.js' + jobPresentation: './client/pages/job-presentation.js', + modelPresentation: './client/pages/model-presentation.js' }, output: { filename: 'stochss-[name].bundle.js', @@ -26,6 +27,13 @@ module.exports = { template: 'jupyterhub/home_template.pug', name: 'jobPresentation', inject: false + }), + new HtmlWebpackPlugin({ + title: 'StochSS | Model Presentation', + filename: '../templates/stochss-model-presentation.html', + template: 'jupyterhub/home_template.pug', + name: 'modelPresentation', + inject: false }) ], module: { From dc902351a8417d9274e8e7a124fe7c877947f415 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 3 Jun 2021 15:08:40 -0400 Subject: [PATCH 023/192] updated the gillespy2 version to 1.6.0. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index eb3d418969..64cdace781 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ python-libsbml==5.18.0 python-libsedml==2.0.9 python-libcombine==0.2.7 -gillespy2==1.5.11 +gillespy2==1.6.0 sciope==0.4 pygmsh==5.0.2 meshio==2.3.10 From f3883a0be1c75128ab31110295b47e8495006e67 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Thu, 3 Jun 2021 15:41:09 -0400 Subject: [PATCH 024/192] Refactored the sbml convert-to-model function to use the new gillespy2 model-to-stochss export function. --- stochss/handlers/util/stochss_sbml.py | 218 +------------------------- 1 file changed, 4 insertions(+), 214 deletions(-) diff --git a/stochss/handlers/util/stochss_sbml.py b/stochss/handlers/util/stochss_sbml.py index 7f3cf29486..ff0fa7d67c 100644 --- a/stochss/handlers/util/stochss_sbml.py +++ b/stochss/handlers/util/stochss_sbml.py @@ -21,8 +21,7 @@ import traceback from gillespy2.sbml.SBMLimport import convert -from gillespy2.sbml.SBMLimport import __read_sbml_model as _read_sbml_model -from gillespy2.sbml.SBMLimport import __get_math as _get_math +from gillespy2.stochss.StochSSexport import export from .stochss_base import StochSSBase from .stochss_errors import StochSSFileNotFoundError @@ -57,205 +56,6 @@ def __init__(self, path, new=False, document=None): sbml_file.write(document) - @classmethod - def __build_element(cls, stoich_species): - if not stoich_species: - return "\\emptyset" - - elements = [] - for species in stoich_species: - name = species['specie']['name'] - ratio = species['ratio'] - element = f"{ratio}{name}" if ratio > 1 else name - elements.append(element) - return '+'.join(elements) - - - @classmethod - def __convert_assignments(cls, model, event, assignments): - for assignment in assignments: - name = assignment.variable.name - try: - variable = cls.__get_species(species=model['species'], name=name) - except IndexError: - variable = cls.__get_parameter(parameters=model['parameters'], name=name) - - s_assignment = {"variable": variable, - "expression": assignment.expression} - event['eventAssignments'].append(s_assignment) - - - @classmethod - def __convert_events(cls, model, events): - for name, event in events.items(): - s_event = {"compID":model['defaultID'], - "name": name, - "annotation": "", - "delay": event.delay, - "priority": event.priority, - "triggerExpression": event.trigger.expression, - "initialValue": event.trigger.value, - "persistent": event.trigger.persistent, - "useValuesFromTriggerTime": event.use_values_from_trigger_time, - "eventAssignments": []} - - cls.__convert_assignments(model=model, event=s_event, assignments=event.assignments) - - model['eventsCollection'].append(s_event) - model['defaultID'] += 1 - - - @classmethod - def __convert_function_definition(cls, model, function_definitions): - for function_definition in function_definitions: - name = function_definition["name"] - variables = ', '.join(function_definition["args"]) - expression = function_definition["function"] - function = "lambda({0}, {1})".format(variables, expression) - signature = "{0}({1})".format(name, variables) - - s_function_definition = {"compID":model['defaultID'], - "name":name, - "function":function, - "expression":expression, - "variables":variables, - "signature":signature, - "annotation": ""} - model['functionDefinitions'].append(s_function_definition) - model['defaultID'] += 1 - - - @classmethod - def __convert_parameters(cls, model, parameters): - for name, parameter in parameters.items(): - s_parameter = {"compID":model['defaultID'], - "name":name, - "expression":str(parameter.expression), - "annotation": ""} - model['parameters'].append(s_parameter) - model['defaultID'] += 1 - - - @classmethod - def __convert_stoich_species(cls, s_reaction, reaction, key, species): - source = reaction.reactants if key == "reactants" else reaction.products - for specie, ratio in source.items(): - stoich_species = {"ratio":ratio, - "specie":cls.__get_species(species=species, name=specie.name)} - s_reaction[key].append(stoich_species) - - - @classmethod - def __convert_reactions(cls, model, reactions): - for name, reaction in reactions.items(): - s_reaction = {"compID":model['defaultID'], - "name":name, - "reactionType": "custom-propensity", - "massaction": False, - "propensity": reaction.propensity_function, - "annotation": "", - "rate": {}, - "subdomains": [ - "subdomain 1: ", - "subdomain 2: " - ], - "reactants": [], - "products": []} - - for key in ['reactants', 'products']: - cls.__convert_stoich_species(s_reaction=s_reaction, reaction=reaction, - key=key, species=model['species']) - cls.__get_summary(reaction=s_reaction) - - model['reactions'].append(s_reaction) - model['defaultID'] += 1 - - - @classmethod - def __convert_rules(cls, model, r_type, rules): - for name, rule in rules.items(): - try: - variable = cls.__get_species(species=model['species'], name=rule.variable) - except IndexError: - variable = cls.__get_parameter(parameters=model['parameters'], name=rule.variable) - - s_rule = {"compID":model['defaultID'], - "name":name, - "expression":rule.formula, - "type":r_type, - "variable":variable, - "annotation": ""} - model['rules'].append(s_rule) - model['defaultID'] += 1 - - - @classmethod - def __convert_species(cls, model, species): - mode = "dynamic" - - # Get the model for all species - for _, specie in species.items(): - if specie.mode != mode: - mode = "continuous" - break - - for name, specie in species.items(): - s_species = {"compID":model['defaultID'], - "name":name, - "value":specie.initial_value, - "mode":mode, - "switchTol": 0.03, - "switchMin": 100, - "isSwitchTol": True, - "annotation": "", - "diffusionConst":0, - "subdomains": [ - "subdomain 1: ", - "subdomain 2: " - ]} - model['species'].append(s_species) - model['defaultID'] += 1 - - model['defaultMode'] = mode - - - def __get_function_definitions(self): - path = self.get_path(full=True) - sb_model = _read_sbml_model(path)[0] - function_definitions = [] - - for i in range(sb_model.getNumFunctionDefinitions()): - function = sb_model.getFunctionDefinition(i) - function_name = function.getId() - function_tree = function.getMath() - num_nodes = function_tree.getNumChildren() - function_args = [function_tree.getChild(i).getName() for i in range(num_nodes-1)] - function_string = _get_math(function_tree.getChild(num_nodes-1)) - s_function_definition = {"name":function_name, - "function":function_string, - "args":function_args} - function_definitions.append(s_function_definition) - - return function_definitions - - - @classmethod - def __get_parameter(cls, parameters, name): - return list(filter(lambda parameter: parameter['name'] == name, parameters))[0] - - - @classmethod - def __get_summary(cls, reaction): - r_summary = cls.__build_element(reaction['reactants']) - p_summary = cls.__build_element(reaction['products']) - reaction['summary'] = f"{r_summary} \\rightarrow {p_summary}" - - - @classmethod - def __get_species(cls, species, name): - return list(filter(lambda specie: specie['name'] == name, species))[0] - - def convert_to_gillespy(self): ''' Convert the sbml model to a gillespy model and return it @@ -282,14 +82,14 @@ def convert_to_model(self, name=None, wkgp=False): Attributes ---------- ''' - s_model = self.get_model_template() # StochSS Model in json format - self.log("debug", f"Model template: \n{json.dumps(s_model)}") - g_model, errors = self.convert_to_gillespy() # GillesPy2 Model object if g_model is None: message = "ERROR! We were unable to convert the SBML Model into a StochSS Model." return {"message":message, "errors":errors, "model":None} + s_model = export(model=g_model, return_stochss_model=True) # StochSS Model in json format + self.log("debug", f"Model: \n{json.dumps(s_model)}") + s_file = f"{g_model.name}.mdl" if name is None else f"{name}.mdl" if wkgp: wkgp_path, changed = self.get_unique_path(name=f"{self.get_name(path=s_file)}.wkgp", @@ -300,15 +100,5 @@ def convert_to_model(self, name=None, wkgp=False): else: s_path = os.path.join(self.get_dir_name(), s_file) - self.__convert_species(model=s_model, species=g_model.get_all_species()) - self.__convert_parameters(model=s_model, parameters=g_model.get_all_parameters()) - self.__convert_reactions(model=s_model, reactions=g_model.get_all_reactions()) - self.__convert_events(model=s_model, events=g_model.get_all_events()) - self.__convert_rules(model=s_model, r_type='Rate Rule', rules=g_model.get_all_rate_rules()) - self.__convert_rules(model=s_model, r_type='Assignment Rule', - rules=g_model.get_all_assignment_rules()) - self.__convert_function_definition(model=s_model, - function_definitions=self.__get_function_definitions()) - message = "The SBML Model was successfully converted to a StochSS Model." return {"message":message, "errors":errors, "model":s_model, "path":s_path} From ad0d1d157f84a830986e5c968d55495e23ef2154 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Wed, 9 Jun 2021 14:30:58 -0400 Subject: [PATCH 025/192] Added the presentation base class to handle common backend functions for presentations. Added the presentation error module to hold all of the presentation system backend errors. --- jupyterhub/presentation_base.py | 57 +++++++++++++++ jupyterhub/presentation_error.py | 118 +++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 jupyterhub/presentation_base.py create mode 100644 jupyterhub/presentation_error.py diff --git a/jupyterhub/presentation_base.py b/jupyterhub/presentation_base.py new file mode 100644 index 0000000000..bf201c867a --- /dev/null +++ b/jupyterhub/presentation_base.py @@ -0,0 +1,57 @@ +''' +StochSS is a platform for simulating biochemical systems +Copyright (C) 2019-2020 StochSS developers. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +''' + +import os + +class StochSSBase(): + ''' + ################################################################################################ + StochSS base object + ################################################################################################ + ''' + user_dir = os.path.expanduser("~") # returns the path to the users home directory + + def __init__(self, path): + ''' + Intitialize a file object + + Attributes + ---------- + path : str + Path to the folder + ''' + self.path = path + self.logs = [] + + + def get_name(self, path=None): + ''' + Get the name from the file object's path or provided path + + Attributes + ---------- + path : str + Path to a file object + ''' + name = self.path if path is None else path + if name.endswith("/"): + name = name[:-1] + name = name.split('/').pop() + if "." not in name: + return name + return '.'.join(name.split('.')[:-1]) diff --git a/jupyterhub/presentation_error.py b/jupyterhub/presentation_error.py new file mode 100644 index 0000000000..ead4cbb8ef --- /dev/null +++ b/jupyterhub/presentation_error.py @@ -0,0 +1,118 @@ +''' +StochSS is a platform for simulating biochemical systems +Copyright (C) 2019-2020 StochSS developers. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +''' + +import traceback + +def report_error(handler, log, err): + ''' + Report a stochss error to the front end + + Attributes + ---------- + handler : obj + Jupyter Notebook API Handler + log : obj + StochSS log + ''' + handler.set_status(err.status_code) + error = {"Reason":err.reason, "Message":err.message} + if err.traceback is None: + trace = traceback.format_exc() + else: + trace = err.traceback + log.error("Exception information: %s\n%s", error, trace) + error['Traceback'] = trace + handler.write(error) + + +class StochSSAPIError(Exception): + ''' + ################################################################################################ + StochSS Base Api Handler Error + ################################################################################################ + ''' + + def __init__(self, status_code, reason, msg, trace): + ''' + Base error for all stochss api errors + + Attributes + ---------- + status_code : int + XML request status code + reason : str + Reason for the error + msg : str + Details on what caused the error + trace : str + Error traceback for the error + ''' + super().__init__() + self.status_code = status_code + self.reason = reason + self.message = msg + self.traceback = trace + + +#################################################################################################### +# File System Errors +#################################################################################################### + +class StochSSFileNotFoundError(StochSSAPIError): + ''' + ################################################################################################ + StochSS File/Folder Not Found API Handler Error + ################################################################################################ + ''' + + def __init__(self, msg, trace=None): + ''' + Indicates that the file/folder with the given path does not exist + + Attributes + ---------- + msg : str + Details on what caused the error + trace : str + Error traceback for the error + ''' + super().__init__(404, "StochSS File or Directory Not Found", msg, trace) + +#################################################################################################### +# Model Errors +#################################################################################################### + +class FileNotJSONFormatError(StochSSAPIError): + ''' + ################################################################################################ + StochSS Model/Template Not In JSON Format + ################################################################################################ + ''' + + def __init__(self, msg, trace=None): + ''' + Indicates that the model or template file is not in proper JSON format + + Attributes + ---------- + msg : str + Details on what caused the error + trace : str + Error traceback for the error + ''' + super().__init__(406, "File Data Not JSON Format", msg, trace) From 15d414544f3c6d4d4e89451da94857bb92be5739 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Wed, 9 Jun 2021 14:33:22 -0400 Subject: [PATCH 026/192] Added the load model api handler to load models and spatial models for presentation. Added the model and spatial model presentation classes to load the models form disc. --- jupyterhub/model_presentation.py | 235 +++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 jupyterhub/model_presentation.py diff --git a/jupyterhub/model_presentation.py b/jupyterhub/model_presentation.py new file mode 100644 index 0000000000..3c5108f57a --- /dev/null +++ b/jupyterhub/model_presentation.py @@ -0,0 +1,235 @@ +''' +StochSS is a platform for simulating biochemical systems +Copyright (C) 2019-2020 StochSS developers. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +''' + +import ast +import json +import logging +import traceback +from tornado import web +from notebook.base.handlers import APIHandler +# APIHandler documentation: +# https://github.com/jupyter/notebook/blob/master/notebook/base/handlers.py#L583 +# Note APIHandler.finish() sets Content-Type handler to 'application/json' +# Use finish() for json, write() for text + +from presentation_base import StochSSBase +from presentation_error import StochSSAPIError, report_error, \ + StochSSFileNotFoundError, FileNotJSONFormatError + +log = logging.getLogger('stochss') + +# pylint: disable=abstract-method +# pylint: disable=too-few-public-methods +class JsonFileAPIHandler(APIHandler): + ''' + ################################################################################################ + Base Handler for interacting with Model file Get/Post Requests and + downloading json formatted files. + ################################################################################################ + ''' + @web.authenticated + async def get(self): + ''' + Retrieve model data from User's file system if it exists and + create new models using a model template if they don't. Also + retrieves JSON files for download. + + Attributes + ---------- + ''' + purpose = self.get_query_argument(name="for") + log.debug("Purpose of the handler: %s", purpose) + path = self.get_query_argument(name="path") + log.debug("Path to the file: %s", path) + self.set_header('Content-Type', 'application/json') + file_objs = {"mdl":StochSSModel, "smdl":StochSSSpatialModel} + ext = path.split(".").pop() + try: + file = file_objs[ext](path=path) + data = file.load() + log.debug("Contents of the json file: %s", data) + file.print_logs(log) + self.write(data) + except StochSSAPIError as load_err: + report_error(self, log, load_err) + self.finish() + + +def __read_model_file(model): + try: + with open(model.get_path(full=True), "r") as mdl_file: + return json.load(mdl_file) + except FileNotFoundError as err: + message = f"Could not find the model file: {str(err)}" + raise StochSSFileNotFoundError(message, traceback.format_exc()) from err + except json.decoder.JSONDecodeError as err: + message = f"The model is not JSON decobable: {str(err)}" + raise FileNotJSONFormatError(message, traceback.format_exc()) from err + + +class StochSSModel(StochSSBase): + ''' + ################################################################################################ + StochSS model object + ################################################################################################ + ''' + + def __init__(self, path): + ''' + Intitialize a model object + + Attributes + ---------- + path : str + Path to the model + ''' + super().__init__(path=path) + self.model = __read_model_file(self) + + + @classmethod + def __update_event_assignments(cls, event, param_ids): + if "eventAssignments" not in event.keys(): + return + for assignment in event['eventAssignments']: + try: + if assignment['variable']['compID'] in param_ids: + expression = ast.literal_eval(assignment['variable']['expression']) + assignment['variable']['expression'] = expression + except KeyError: + pass + except ValueError: + pass + + + def __update_events(self, param_ids): + if "eventsCollection" not in self.model.keys() or not param_ids: + return + for event in self.model['eventsCollection']: + self.__update_event_assignments(event=event, param_ids=param_ids) + + + def __update_parameters(self): + if "parameters" not in self.model.keys(): + return [] + param_ids = [] + for param in self.model['parameters']: + try: + param_ids.append(param['compID']) + if isinstance(param['expression'], str): + param['expression'] = ast.literal_eval(param['expression']) + except KeyError: + pass + except ValueError: + pass + return param_ids + + + def __update_reactions(self): + if "reactions" not in self.model.keys(): + return + for reaction in self.model['reactions']: + try: + if reaction['rate'].keys() and isinstance(reaction['rate']['expression'], str): + expression = ast.literal_eval(reaction['rate']['expression']) + reaction['rate']['expression'] = expression + except KeyError: + pass + except ValueError: + pass + + + def __update_rules(self, param_ids): + if "rules" not in self.model.keys() or not param_ids: + return + for rule in self.model['rules']: + try: + if rule['variable']['compID'] in param_ids: + expression = ast.literal_eval(rule['variable']['expression']) + rule['variable']['expression'] = expression + except KeyError: + pass + except ValueError: + pass + + + def load(self): + ''' + Reads the model file, updates the model to the current format, and stores it in self.model + + Attributes + ---------- + ''' + if "annotation" not in self.model.keys(): + self.model['annotation'] = "" + if "volume" not in self.model.keys(): + if "volume" in self.model['modelSettings'].keys(): + self.model['volume'] = self.model['modelSettings']['volume'] + else: + self.model['volume'] = 1 + param_ids = self.__update_parameters() + self.__update_reactions() + self.__update_events(param_ids=param_ids) + self.__update_rules(param_ids=param_ids) + self.model['name'] = self.get_name() + self.model['directory'] = self.path + return self.model + + +class StochSSSpatialModel(StochSSBase): + ''' + ################################################################################################ + StochSS spatial model object + ################################################################################################ + ''' + + def __init__(self, path): + ''' + Intitialize a spatial model object + + Attributes + ---------- + path : str + Path to the spatial model + ''' + super().__init__(path=path) + self.model = __read_model_file(self) + + + def load(self): + ''' + Reads the spatial model file, updates it to the current format, and stores it in self.model + + Attributes + ---------- + ''' + self.model['name'] = self.get_name() + if not self.model['defaultMode']: + self.model['defaultMode'] = "discrete" + if "static" not in self.model['domain'].keys(): + self.model['domain']['static'] = True + for species in self.model['species']: + if "types" not in species.keys(): + species['types'] = list(range(1, len(self.model['domain']['types']))) + if "diffusionConst" not in species.keys(): + diff = 0.0 if "diffusionCoeff" not in species.keys() else species['diffusionCoeff'] + species['diffusionConst'] = diff + for reaction in self.model['reactions']: + if "types" not in reaction.keys(): + reaction['types'] = list(range(1, len(self.model['domain']['types']))) + return self.model From 0b7a87e599eb203a8ede4bbd710dd084c8117cf4 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Wed, 9 Jun 2021 14:36:18 -0400 Subject: [PATCH 027/192] Added the api handler route for loading models. Pylint fixes. --- jupyterhub/jupyterhub_config.py | 81 +++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 93998b6e88..df6b88a19c 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -18,8 +18,15 @@ #------------------------------------------------------------------------------ # JupyterHub(Application) configuration #------------------------------------------------------------------------------ +import os +import os.path +import sys +import logging -import sys, os, os.path, shutil, logging +# Page handlers +from handlers import * +# API Handlers +from model_presentation import JsonFileAPIHandler ## Class for authenticating users. # @@ -70,14 +77,12 @@ # By default, redirects users to their own server. c.JupyterHub.default_url = '/stochss' -# Page handlers -from handlers import * - # StochSS request handlers c.JupyterHub.extra_handlers = [ (r"/stochss\/?", HomeHandler), (r"/stochss/job-presentation\/?", JobPresentationHandler), - (r"/stochss/model-presentation\/?", ModelPresentationHandler) + (r"/stochss/model-presentation\/?", ModelPresentationHandler), + (r"/stochss/api/file/json-data\/?", JsonFileAPIHandler) ] ## Paths to search for jinja templates, before using the default templates. @@ -173,15 +178,18 @@ #------------------------------------------------------------------------------ def get_user_cpu_count_or_fail(): + ''' + Get the user cpu count or raise error + ''' log = logging.getLogger() reserve_count = int(os.environ['RESERVED_CPUS']) - log.info("RESERVED_CPUS environment variable is set to {}".format(reserve_count)) + log.info("RESERVED_CPUS environment variable is set to %s", reserve_count) # Round up to an even number of reserved cpus if reserve_count % 2 > 0: - log.warn("Increasing reserved cpu count by one so it's an even number. This helps allocate logical cpus to users more easily.") + log.warning("Increasing reserved cpu count by one so it's an even number. This helps allocate logical cpus to users more easily.") reserve_count += 1 total_cpus = os.cpu_count() - log.info("Total cpu count as reported by os.count: {}".format(total_cpus)) + log.info("Total cpu count as reported by os.count: %s", total_cpus) if reserve_count >= total_cpus: e_message = "RESERVED_CPUS environment cannot be greater than or equal to the number of cpus returned by os.cpu_count()" log.error(e_message) @@ -192,28 +200,32 @@ def get_user_cpu_count_or_fail(): if user_cpu_count % 2 > 0 and user_cpu_count > 1: user_cpu_count -= 1 c.StochSS.reserved_cpu_count = reserve_count - log.info('Using {} logical cpus for user containers...'.format(user_cpu_count)) - log.info('Reserving {} logical cpus for hub container and underlying OS'.format(reserve_count)) + log.info('Using %s logical cpus for user containers...', user_cpu_count) + log.info('Reserving %s logical cpus for hub container and underlying OS', reserve_count) return user_cpu_count c.StochSS.user_cpu_count = get_user_cpu_count_or_fail() c.StochSS.user_cpu_alloc = [0] * c.StochSS.user_cpu_count def get_power_users(): + ''' + Get the list of power users + ''' power_users_file = os.environ.get('POWER_USERS_FILE') log = logging.getLogger() if not os.path.exists(power_users_file): - log.warn('No power users defined!') + log.warning('No power users defined!') return [] - with open(power_users_file) as f: - power_users = [ x.rstrip() for x in f.readlines() ] + with open(power_users_file) as file: + power_users = [ x.rstrip() for x in file.readlines() ] return power_users c.StochSS.power_users = get_power_users() def pre_spawn_hook(spawner): - '''Function that runs before DockerSpawner spawns a user container. + ''' + Function that runs before DockerSpawner spawns a user container. Limits the resources available to user containers, excluding a list of power users. ''' log = logging.getLogger() @@ -224,8 +236,8 @@ def pre_spawn_hook(spawner): palloc = c.StochSS.user_cpu_alloc div = len(palloc) // 2 reserved = c.StochSS.reserved_cpu_count - log.warn('Reserved CPUs: {}'.format(reserved)) - log.warn('Number of user containers using each logical core: {}'.format(palloc)) + log.warning('Reserved CPUs: %s', reserved) + log.warning('Number of user containers using each logical core: %s', palloc) # We want to allocate logical cores that are on the same physical core # whenever possible. # @@ -248,12 +260,12 @@ def pre_spawn_hook(spawner): # avail_cpus = palloc[div:] # If <= 4 cpus available then use 1 cpu for each user instead of 2 - if not len(avail_cpus): - log.warn("The host system only has 4 logical cpus, so we'll only reserve one logical cpu per user container, instead of the normal 2") + if not avail_cpus: + log.warning("The host system only has 4 logical cpus, so we'll only reserve one logical cpu per user container, instead of the normal 2") avail_cpus = palloc least_used_cpu = min(avail_cpus) cpu1_index = avail_cpus.index(least_used_cpu) - log.info("User {} to use logical cpu {}".format(spawner.user.name, str(cpu1_index))) + log.info("User %s to use logical cpu %s", spawner.user.name, str(cpu1_index)) palloc[cpu1_index] += 1 spawner.extra_host_config['cpuset_cpus'] = '{}'.format(cpu1_index) else: @@ -262,12 +274,15 @@ def pre_spawn_hook(spawner): palloc[cpu1_index] += 1 cpu2_index = cpu1_index+div palloc[cpu2_index] += 1 - log.info("User {} to use logical cpus {} and {}".format( - spawner.user.name, str(cpu1_index), str(cpu2_index))) + log.info("User %s to use logical cpus %s and %s", + spawner.user.name, str(cpu1_index), str(cpu2_index)) spawner.extra_host_config['cpuset_cpus'] = '{},{}'.format(cpu1_index, cpu2_index) def post_stop_hook(spawner): + ''' + Post stop hook + ''' log = logging.getLogger() reserved = c.StochSS.reserved_cpu_count palloc = c.StochSS.user_cpu_alloc @@ -275,8 +290,8 @@ def post_stop_hook(spawner): cpu1_index, cpu2_index = spawner.extra_host_config['cpuset_cpus'].split(',') palloc[int(cpu1_index)] -= 1 palloc[int(cpu2_index)] -= 1 - log.warn('Reserved CPUs: {}'.format(reserved)) - log.warn('Number of user containers using each logical core: {}'.format(palloc)) + log.warning('Reserved CPUs: %s', reserved) + log.warning('Number of user containers using each logical core: %s', palloc) except: # Exception thrown due to cpuset_cpus not being set (power user) pass @@ -399,13 +414,13 @@ def post_stop_hook(spawner): pwd = os.path.dirname(__file__) with open(os.path.join(pwd, 'userlist')) as f: - for line in f: - if not line: - continue - parts = line.split() - # in case of newline at the end of userlist file - if len(parts) >= 1: - name = parts[0] - #whitelist.add(name) - if len(parts) > 1 and parts[1] == 'admin': - admin.add(name) + for line in f: + if not line: + continue + parts = line.split() + # in case of newline at the end of userlist file + if len(parts) >= 1: + name = parts[0] + #whitelist.add(name) + if len(parts) > 1 and parts[1] == 'admin': + admin.add(name) From 0baf9cd5d79bb5252bafb9f880d105c5d25b1eb6 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Wed, 9 Jun 2021 14:39:08 -0400 Subject: [PATCH 028/192] Finalized the initialize function for the model presentation page. --- client/pages/model-presentation.js | 32 ++++++++++++++---------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/client/pages/model-presentation.js b/client/pages/model-presentation.js index eb40d97a5e..3a681b64ec 100644 --- a/client/pages/model-presentation.js +++ b/client/pages/model-presentation.js @@ -43,23 +43,21 @@ let ModelPresentationPage = PageView.extend({ template: template, initialize: function (attrs, arguments) { PageView.prototype.initialize.apply(this, arguments); - console.log("TODO: get the path to the model from the url") - // let urlParams = new URLSearchParams(window.location.search) - // this.model = new Model({ - // directory: urlParams.get("path"), - // for: "presentation" - // }); - console.log("TODO: get model from file system using the app.getXHR function") - // let self = this; - // let queryStr = "?path=" + this.model.directory; - // let endpoint = path.join(); - // app.getXHR(endpoint, { - // success: function (err, response, body) { - // self.model.set(body); - // self.renderSubviews(); - // } - // }); - console.log("TODO: generate the open link and store in this.open") + let urlParams = new URLSearchParams(window.location.search); + this.model = new Model({ + directory: urlParams.get("path"), + for: "presentation" + }); + let self = this; + let queryStr = "?path=" + this.model.directory; + let endpoint = path.join(); + app.getXHR(endpoint, { + success: function (err, response, body) { + self.model.set(body); + self.renderSubviews(); + } + }); + this.open = "open.stochss.org?open=" + this.model.directory; }, renderSubviews: function () { PageView.prototype.render.apply(this, arguments); From fddd31646aa5f99cb17db51493802eeeafcbee97 Mon Sep 17 00:00:00 2001 From: Matthew Geiger Date: Thu, 10 Jun 2021 11:27:35 -0400 Subject: [PATCH 029/192] Skeleton for copying files via docker from user container to jupyterhub --- client/pages/model-presentation.js | 16 ++++++------- jupyterhub/Dockerfile.jupyterhub | 3 ++- jupyterhub/handlers.py | 24 +++++++++++++++++++ jupyterhub/jupyterhub_config.py | 5 +++- .../templates/stochss-job-presentation.html | 1 + .../templates/stochss-model-presentation.html | 1 + 6 files changed, 40 insertions(+), 10 deletions(-) create mode 100644 jupyterhub/templates/stochss-job-presentation.html create mode 100644 jupyterhub/templates/stochss-model-presentation.html diff --git a/client/pages/model-presentation.js b/client/pages/model-presentation.js index 3a681b64ec..66f18cd24b 100644 --- a/client/pages/model-presentation.js +++ b/client/pages/model-presentation.js @@ -26,12 +26,12 @@ let app = require('../app'); let Model = require('../models/model'); //views let PageView = require('./base'); -let Rules = require('./rules-viewer'); -let Events = require('./events-viewer'); -let Species = require('./species-viewer'); -let Reactions = require('./reactions-viewer'); -let Parameters = require('./parameters-viewer'); -let SBMLComponents = require('./sbml-component-editor'); +let Rules = require('../views/rules-viewer'); +let Events = require('../views/events-viewer'); +let Species = require('../views/species-viewer'); +let Reactions = require('../views/reactions-viewer'); +let Parameters = require('../views/parameters-viewer'); +let SBMLComponents = require('../views/sbml-component-editor'); //templates let template = require('../templates/pages/modelPresentation.pug'); @@ -41,7 +41,7 @@ import fontawesomeStyles from '@fortawesome/fontawesome-free/css/svg-with-js.min let ModelPresentationPage = PageView.extend({ template: template, - initialize: function (attrs, arguments) { + initialize: function (attrs, options) { PageView.prototype.initialize.apply(this, arguments); let urlParams = new URLSearchParams(window.location.search); this.model = new Model({ @@ -60,7 +60,7 @@ let ModelPresentationPage = PageView.extend({ this.open = "open.stochss.org?open=" + this.model.directory; }, renderSubviews: function () { - PageView.prototype.render.apply(this, arguments); + PageView.prototype.render.apply(this, options); this.renderSpeciesContainer(); this.renderParametersContainer(); this.renderReactionsContainer(); diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub index ab0119b137..38df4d10b2 100644 --- a/jupyterhub/Dockerfile.jupyterhub +++ b/jupyterhub/Dockerfile.jupyterhub @@ -39,7 +39,8 @@ RUN python3 -m pip install --no-cache-dir \ jupyterhub==1.1.0 \ oauthenticator==0.11.* \ dockerspawner==0.11.* \ - psycopg2==2.7.* + psycopg2==2.7.* \ + notebook COPY static/* /usr/local/share/jupyterhub/static/ diff --git a/jupyterhub/handlers.py b/jupyterhub/handlers.py index b98c406d1d..3b7da6f570 100644 --- a/jupyterhub/handlers.py +++ b/jupyterhub/handlers.py @@ -37,6 +37,30 @@ async def get(self): self.finish(html) +class PublishUserModel(BaseHandler): + ''' + ################################################################################################ + Handler for copying a user model from a user container into a presentations folder + managed by jupyterhub. + ################################################################################################ + ''' + async def get(self): + pass + # Pass the username as a query parameter + # username = + # Pass the model filename as a query parameter + # filename = + client = docker.from_env() + containers = client.containers.list() + user_container = [ c in containers if c.name == f'jupyter-{username}' ][0] + user_model_path = f'/home/jovyan/.presentations/{filename}' + jupyterhub_tar_path = f'/srv/jupyterhub/.presentations/{filename}.tar' + bits, stat = user_container.get_archive(user_model_path) + with open(jupyterhub_tar_path, 'w') as tar_file: + for chunk in bits: + tar_file.write(chunk) + + class JobPresentationHandler(BaseHandler): ''' ################################################################################################ diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index df6b88a19c..aa4ef947e8 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -23,6 +23,8 @@ import sys import logging +sys.path.append('/srv/jupyterhub/') + # Page handlers from handlers import * # API Handlers @@ -82,7 +84,8 @@ (r"/stochss\/?", HomeHandler), (r"/stochss/job-presentation\/?", JobPresentationHandler), (r"/stochss/model-presentation\/?", ModelPresentationHandler), - (r"/stochss/api/file/json-data\/?", JsonFileAPIHandler) + (r"/stochss/api/file/json-data\/?", JsonFileAPIHandler), + (r"/stochss/api/publish/model\/?", PublishUserModel) ] ## Paths to search for jinja templates, before using the default templates. diff --git a/jupyterhub/templates/stochss-job-presentation.html b/jupyterhub/templates/stochss-job-presentation.html new file mode 100644 index 0000000000..47c4ea2ab8 --- /dev/null +++ b/jupyterhub/templates/stochss-job-presentation.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/jupyterhub/templates/stochss-model-presentation.html b/jupyterhub/templates/stochss-model-presentation.html new file mode 100644 index 0000000000..3092353528 --- /dev/null +++ b/jupyterhub/templates/stochss-model-presentation.html @@ -0,0 +1 @@ + \ No newline at end of file From 4c8adda0a3a22eb686d2af7992e29ce526054b4e Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 11:55:11 -0400 Subject: [PATCH 030/192] Moved the publish presentation function to the stochss_model file. --- stochss/handlers/util/stochss_file.py | 24 --------------------- stochss/handlers/util/stochss_model.py | 30 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/stochss/handlers/util/stochss_file.py b/stochss/handlers/util/stochss_file.py index 695b442ef4..fbe99c91d3 100644 --- a/stochss/handlers/util/stochss_file.py +++ b/stochss/handlers/util/stochss_file.py @@ -99,30 +99,6 @@ def duplicate(self): raise StochSSPermissionsError(message, traceback.format_exc()) from err - def publish_presentation(self): - ''' - Publish a model or spatial model presentation - - Attributes - ---------- - ''' - present_dir = os.path.join(self.user_dir, ".presentations") - if not os.path.exists(present_dir): - os.mkdir(present_dir) - dst = os.path.join(present_dir, self.get_file()) - if os.path.exists(dst): - message = "A presentation with this name already exists" - raise StochSSFileExistsError(message) - src = self.get_path(full=True) - try: - shutil.copyfile(src, dst) - # INSERT JUPYTER HUB CODE HERE - return {"message": f"Successfully published the {self.get_name()} presentation"} - except PermissionError as err: - message = f"You do not have permission to publish this file: {str(err)}" - raise StochSSPermissionsError(message, traceback.format_exc()) from err - - def move(self, location): ''' Moves a file to a new location. diff --git a/stochss/handlers/util/stochss_model.py b/stochss/handlers/util/stochss_model.py index bb2b9ba9d9..b884dacc95 100644 --- a/stochss/handlers/util/stochss_model.py +++ b/stochss/handlers/util/stochss_model.py @@ -19,6 +19,7 @@ import os import ast import json +import haslib import tempfile import traceback @@ -459,6 +460,35 @@ def load(self): return self.model + def publish_presentation(self): + ''' + Publish a model or spatial model presentation + + Attributes + ---------- + ''' + present_dir = os.path.join(self.user_dir, ".presentations") + if not os.path.exists(present_dir): + os.mkdir(present_dir) + dst = os.path.join(present_dir, self.get_file()) + if os.path.exists(dst): + message = "A presentation with this name already exists" + raise StochSSFileExistsError(message) + src = self.get_path(full=True) + try: + shutil.copyfile(src, dst) + hostname = os.uname()[1] + model = json.jumps(self.load(), sort_keys=True) + file = f"{hashlib.md5(model.encode('utf-8')).hexdigest()}.mdl" # replace with gillespy2.Model.to_json + present_link = f"live.stochss.org/present-model?owner={hostname}&file={file}" + download_link = f"live.stochss.org/download_presentation?owner={hostname}&file={file}" + return {"message": f"Successfully published the {self.get_name()} presentation", + "links": {"presentation": present_link, "download": download_link}} + except PermissionError as err: + message = f"You do not have permission to publish this file: {str(err)}" + raise StochSSPermissionsError(message, traceback.format_exc()) from err + + def save(self, model): ''' Saves the model to an existing file From d12cd8ff3b3bf4d2e8c4478407a444efa2b25d01 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 12:04:16 -0400 Subject: [PATCH 031/192] Generated links for presentation and download. --- stochss/handlers/util/stochss_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stochss/handlers/util/stochss_model.py b/stochss/handlers/util/stochss_model.py index b884dacc95..361d138652 100644 --- a/stochss/handlers/util/stochss_model.py +++ b/stochss/handlers/util/stochss_model.py @@ -480,8 +480,8 @@ def publish_presentation(self): hostname = os.uname()[1] model = json.jumps(self.load(), sort_keys=True) file = f"{hashlib.md5(model.encode('utf-8')).hexdigest()}.mdl" # replace with gillespy2.Model.to_json - present_link = f"live.stochss.org/present-model?owner={hostname}&file={file}" - download_link = f"live.stochss.org/download_presentation?owner={hostname}&file={file}" + present_link = f"live.stochss.org/stochss/present-model?owner={hostname}&file={file}" + download_link = f"live.stochss.org/stochss/download_presentation?owner={hostname}&file={file}" return {"message": f"Successfully published the {self.get_name()} presentation", "links": {"presentation": present_link, "download": download_link}} except PermissionError as err: From 37d5bdc5a54ed3022e4d46633dd80658c14c499c Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 12:49:34 -0400 Subject: [PATCH 032/192] Refactor the model presentation handler to load a model and presentation links, create a new model in the presentation directory and return the links. --- stochss/handlers/models.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/stochss/handlers/models.py b/stochss/handlers/models.py index b2d5ec321d..603632aa77 100644 --- a/stochss/handlers/models.py +++ b/stochss/handlers/models.py @@ -425,13 +425,19 @@ async def get(self): self.set_header('Content-Type', 'application/json') path = self.get_query_argument(name="path") log.debug("Path to the file: %s", path) - try: - file = StochSSFile(path=path) - log.info("Publishing the %s presentation", file.get_name()) - resp = file.publish_presentation() - log.info(resp['message']) - log.debug("Response Message: %s", resp) - self.write(resp) - except StochSSAPIError as err: - report_error(self, log, err) + file_objs = {"mdl":StochSSModel, "smdl":StochSSSpatialModel} + ext = path.split(".").pop() + if ext == "mdl": + try: + model = file_objs[ext](path=path) + log.info("Publishing the %s presentation", file.get_name()) + links, data = model.publish_presentation() + presentation_model = file_objs[ext](**data) + resp = {"message": f"Successfully published the {self.get_name()} presentation", + "links": links} + log.info(resp['message']) + log.debug("Response Message: %s", resp) + self.write(resp) + except StochSSAPIError as err: + report_error(self, log, err) self.finish() From b66fd0d5ac5f9b0d02ea649a23c57937d4b5cc44 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 12:52:34 -0400 Subject: [PATCH 033/192] Refactored the publish presentation function to generate presentation links, load the model, and generate a md5 hash file. --- stochss/handlers/util/stochss_model.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/stochss/handlers/util/stochss_model.py b/stochss/handlers/util/stochss_model.py index 361d138652..c9c1ceff39 100644 --- a/stochss/handlers/util/stochss_model.py +++ b/stochss/handlers/util/stochss_model.py @@ -470,20 +470,21 @@ def publish_presentation(self): present_dir = os.path.join(self.user_dir, ".presentations") if not os.path.exists(present_dir): os.mkdir(present_dir) - dst = os.path.join(present_dir, self.get_file()) - if os.path.exists(dst): - message = "A presentation with this name already exists" - raise StochSSFileExistsError(message) src = self.get_path(full=True) try: - shutil.copyfile(src, dst) + self.load() hostname = os.uname()[1] - model = json.jumps(self.load(), sort_keys=True) + model = json.jumps(self.model, sort_keys=True) file = f"{hashlib.md5(model.encode('utf-8')).hexdigest()}.mdl" # replace with gillespy2.Model.to_json + dst = os.path.join(present_dir, file) + if os.path.exists(dst): + message = "A presentation with this name already exists" + raise StochSSFileExistsError(message) present_link = f"live.stochss.org/stochss/present-model?owner={hostname}&file={file}" download_link = f"live.stochss.org/stochss/download_presentation?owner={hostname}&file={file}" - return {"message": f"Successfully published the {self.get_name()} presentation", - "links": {"presentation": present_link, "download": download_link}} + links = {"presentation": present_link, "download": download_link} + data = {"path": dst, "new":True, "model":self.model} + return links, data except PermissionError as err: message = f"You do not have permission to publish this file: {str(err)}" raise StochSSPermissionsError(message, traceback.format_exc()) from err From 3fc4d10e2ad2d97443c5120c289a2b61a8c82ffe Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 12:54:24 -0400 Subject: [PATCH 034/192] Refactored the presentation page initialize function to pass on the ower and file query arguments to the load handler. --- client/pages/model-presentation.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client/pages/model-presentation.js b/client/pages/model-presentation.js index 66f18cd24b..1d998e55c1 100644 --- a/client/pages/model-presentation.js +++ b/client/pages/model-presentation.js @@ -44,13 +44,15 @@ let ModelPresentationPage = PageView.extend({ initialize: function (attrs, options) { PageView.prototype.initialize.apply(this, arguments); let urlParams = new URLSearchParams(window.location.search); + let owner = urlParams.get("owner") + let file = urlParams.get("file"); this.model = new Model({ - directory: urlParams.get("path"), + directory: file, for: "presentation" }); let self = this; - let queryStr = "?path=" + this.model.directory; - let endpoint = path.join(); + let queryStr = "?file=" + this.model.directory + "&owner=" + owner; + let endpoint = "stochss/api/file/json-data" + queryStr; app.getXHR(endpoint, { success: function (err, response, body) { self.model.set(body); From e3db1687d1b95bd881e58812ad8f4df51c12a516 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 12:55:57 -0400 Subject: [PATCH 035/192] Updated the page routes for the presentation pages. --- jupyterhub/jupyterhub_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index aa4ef947e8..f9598a2053 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -82,8 +82,8 @@ # StochSS request handlers c.JupyterHub.extra_handlers = [ (r"/stochss\/?", HomeHandler), - (r"/stochss/job-presentation\/?", JobPresentationHandler), - (r"/stochss/model-presentation\/?", ModelPresentationHandler), + (r"/stochss/present-job\/?", JobPresentationHandler), + (r"/stochss/present-model\/?", ModelPresentationHandler), (r"/stochss/api/file/json-data\/?", JsonFileAPIHandler), (r"/stochss/api/publish/model\/?", PublishUserModel) ] From ff4e90c67dce2912f8496ea01c0bc832fa356f35 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 13:00:48 -0400 Subject: [PATCH 036/192] Refactor the model presentation handler to load a model and presentation links, create a new model in the presentation directory and return the links. --- stochss/handlers/models.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/stochss/handlers/models.py b/stochss/handlers/models.py index 603632aa77..ff167604e5 100644 --- a/stochss/handlers/models.py +++ b/stochss/handlers/models.py @@ -427,17 +427,16 @@ async def get(self): log.debug("Path to the file: %s", path) file_objs = {"mdl":StochSSModel, "smdl":StochSSSpatialModel} ext = path.split(".").pop() - if ext == "mdl": - try: - model = file_objs[ext](path=path) - log.info("Publishing the %s presentation", file.get_name()) - links, data = model.publish_presentation() - presentation_model = file_objs[ext](**data) - resp = {"message": f"Successfully published the {self.get_name()} presentation", - "links": links} - log.info(resp['message']) - log.debug("Response Message: %s", resp) - self.write(resp) - except StochSSAPIError as err: - report_error(self, log, err) + try: + model = file_objs[ext](path=path) + log.info("Publishing the %s presentation", file.get_name()) + links, data = model.publish_presentation() + presentation_model = file_objs[ext](**data) + resp = {"message": f"Successfully published the {self.get_name()} presentation", + "links": links} + log.info(resp['message']) + log.debug("Response Message: %s", resp) + self.write(resp) + except StochSSAPIError as err: + report_error(self, log, err) self.finish() From a87dea472e46806fd09f03511e17ba0336783aa7 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 15:09:42 -0400 Subject: [PATCH 037/192] Fixed bugs and did some pylint cleanup. --- stochss/handlers/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stochss/handlers/models.py b/stochss/handlers/models.py index ff167604e5..de9c8aacf0 100644 --- a/stochss/handlers/models.py +++ b/stochss/handlers/models.py @@ -28,7 +28,7 @@ # Use finish() for json, write() for text from .util import StochSSFolder, StochSSModel, StochSSSpatialModel, StochSSNotebook, \ - StochSSFile, StochSSAPIError, report_error + StochSSAPIError, report_error log = logging.getLogger('stochss') @@ -429,9 +429,9 @@ async def get(self): ext = path.split(".").pop() try: model = file_objs[ext](path=path) - log.info("Publishing the %s presentation", file.get_name()) + log.info("Publishing the %s presentation", model.get_name()) links, data = model.publish_presentation() - presentation_model = file_objs[ext](**data) + file_objs[ext](**data) resp = {"message": f"Successfully published the {self.get_name()} presentation", "links": links} log.info(resp['message']) From be28bcdab233a744d7485213e48ee9354aea914d Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 15:11:34 -0400 Subject: [PATCH 038/192] Finalized the return for the publish presentation function. Added missing imports. pylint cleanup. --- stochss/handlers/util/stochss_model.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/stochss/handlers/util/stochss_model.py b/stochss/handlers/util/stochss_model.py index c9c1ceff39..6fce11ca1d 100644 --- a/stochss/handlers/util/stochss_model.py +++ b/stochss/handlers/util/stochss_model.py @@ -19,7 +19,7 @@ import os import ast import json -import haslib +import hashlib import tempfile import traceback @@ -30,7 +30,7 @@ from .stochss_base import StochSSBase from .stochss_errors import StochSSAPIError, StochSSFileNotFoundError, FileNotJSONFormatError, \ - StochSSModelFormatError + StochSSModelFormatError, StochSSFileExistsError, StochSSPermissionsError class StochSSModel(StochSSBase): ''' @@ -470,18 +470,19 @@ def publish_presentation(self): present_dir = os.path.join(self.user_dir, ".presentations") if not os.path.exists(present_dir): os.mkdir(present_dir) - src = self.get_path(full=True) try: self.load() hostname = os.uname()[1] - model = json.jumps(self.model, sort_keys=True) - file = f"{hashlib.md5(model.encode('utf-8')).hexdigest()}.mdl" # replace with gillespy2.Model.to_json + model = json.dumps(self.model, sort_keys=True) + # replace with gillespy2.Model.to_json + file = f"{hashlib.md5(model.encode('utf-8')).hexdigest()}.mdl" dst = os.path.join(present_dir, file) if os.path.exists(dst): message = "A presentation with this name already exists" raise StochSSFileExistsError(message) - present_link = f"live.stochss.org/stochss/present-model?owner={hostname}&file={file}" - download_link = f"live.stochss.org/stochss/download_presentation?owner={hostname}&file={file}" + query_str = f"?owner={hostname}&file={file}" + present_link = f"live.stochss.org/stochss/present-model{query_str}" + download_link = f"live.stochss.org/stochss/download_presentation{query_str}" links = {"presentation": present_link, "download": download_link} data = {"path": dst, "new":True, "model":self.model} return links, data From 941e400b109acb10ec6765fcca867ddb189a958c Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 15:14:21 -0400 Subject: [PATCH 039/192] Integrated the PublishUserModel handler code into the model presentation python file. --- jupyterhub/handlers.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/jupyterhub/handlers.py b/jupyterhub/handlers.py index 3b7da6f570..b98c406d1d 100644 --- a/jupyterhub/handlers.py +++ b/jupyterhub/handlers.py @@ -37,30 +37,6 @@ async def get(self): self.finish(html) -class PublishUserModel(BaseHandler): - ''' - ################################################################################################ - Handler for copying a user model from a user container into a presentations folder - managed by jupyterhub. - ################################################################################################ - ''' - async def get(self): - pass - # Pass the username as a query parameter - # username = - # Pass the model filename as a query parameter - # filename = - client = docker.from_env() - containers = client.containers.list() - user_container = [ c in containers if c.name == f'jupyter-{username}' ][0] - user_model_path = f'/home/jovyan/.presentations/{filename}' - jupyterhub_tar_path = f'/srv/jupyterhub/.presentations/{filename}.tar' - bits, stat = user_container.get_archive(user_model_path) - with open(jupyterhub_tar_path, 'w') as tar_file: - for chunk in bits: - tar_file.write(chunk) - - class JobPresentationHandler(BaseHandler): ''' ################################################################################################ From 4ea24a085e5240fed7f11f1adf050a022a2f0d8a Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 15:15:50 -0400 Subject: [PATCH 040/192] Removed the route for the PublishUserModel handler. --- jupyterhub/jupyterhub_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index f9598a2053..10f2594d4f 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -85,7 +85,6 @@ (r"/stochss/present-job\/?", JobPresentationHandler), (r"/stochss/present-model\/?", ModelPresentationHandler), (r"/stochss/api/file/json-data\/?", JsonFileAPIHandler), - (r"/stochss/api/publish/model\/?", PublishUserModel) ] ## Paths to search for jinja templates, before using the default templates. From f45ecfbc6b047d3486dabb9c809f97e54254ea84 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 15:21:55 -0400 Subject: [PATCH 041/192] Created a function to execute the docker commands to the get the presentation model from the user container. Added functions to extract the model presentation from the tar bits returned from the get_archive function. Refactored the StochSSModel and StochSSSpatialModel classes fit the new presentation functionality. --- jupyterhub/model_presentation.py | 98 +++++++++++++++++++------------- 1 file changed, 57 insertions(+), 41 deletions(-) diff --git a/jupyterhub/model_presentation.py b/jupyterhub/model_presentation.py index 3c5108f57a..7dfaaeb423 100644 --- a/jupyterhub/model_presentation.py +++ b/jupyterhub/model_presentation.py @@ -16,10 +16,14 @@ along with this program. If not, see . ''' +import os import ast import json import logging -import traceback +import tarfile +import tempfile + +import docker from tornado import web from notebook.base.handlers import APIHandler # APIHandler documentation: @@ -27,9 +31,7 @@ # Note APIHandler.finish() sets Content-Type handler to 'application/json' # Use finish() for json, write() for text -from presentation_base import StochSSBase -from presentation_error import StochSSAPIError, report_error, \ - StochSSFileNotFoundError, FileNotJSONFormatError +from presentation_error import StochSSAPIError, report_error log = logging.getLogger('stochss') @@ -52,54 +54,72 @@ async def get(self): Attributes ---------- ''' - purpose = self.get_query_argument(name="for") - log.debug("Purpose of the handler: %s", purpose) - path = self.get_query_argument(name="path") - log.debug("Path to the file: %s", path) + owner = self.get_query_argument(name="owner") + log.debug("Container id of the owner: %s", owner) + file = self.get_query_argument(name="file") + log.debug("Name to the file: %s", file) self.set_header('Content-Type', 'application/json') file_objs = {"mdl":StochSSModel, "smdl":StochSSSpatialModel} - ext = path.split(".").pop() + ext = file.split(".").pop() try: - file = file_objs[ext](path=path) - data = file.load() - log.debug("Contents of the json file: %s", data) - file.print_logs(log) - self.write(data) + model = get_presentation_from_user(owner=owner, file=file) + file_obj = file_objs[ext](model=model) + model = file_obj.load() + log.debug("Contents of the json file: %s", model) + file_obj.print_logs(log) + self.write(model) except StochSSAPIError as load_err: report_error(self, log, load_err) self.finish() -def __read_model_file(model): - try: - with open(model.get_path(full=True), "r") as mdl_file: - return json.load(mdl_file) - except FileNotFoundError as err: - message = f"Could not find the model file: {str(err)}" - raise StochSSFileNotFoundError(message, traceback.format_exc()) from err - except json.decoder.JSONDecodeError as err: - message = f"The model is not JSON decobable: {str(err)}" - raise FileNotJSONFormatError(message, traceback.format_exc()) from err - - -class StochSSModel(StochSSBase): +def get_presentation_from_user(owner, file): + ''' + Get the model presentation from the users container + + Attributes + ---------- + owner : str + Hostname of the user container + file : str + Name of the model presentation file + ''' + client = docker.from_env() + containers = client.containers.list() + user_container = list(filter(lambda container: container.id.startswith(owner), + containers))[0] + user_model_path = f'/home/jovyan/.presentations/{file}' + tar_mdl = tempfile.TemporaryFile() + bits, _ = user_container.get_archive(user_model_path) + for chunk in bits: + tar_mdl.write(chunk) + tar_mdl.seek(0) + tar_file = tarfile.TarFile(fileobj=tar_mdl) + tmp_dir = tempfile.TemporaryDirectory() + tar_file.extractall(tmp_dir.name) + tar_mdl.close() + mdl_path = os.path.join(tmp_dir.name, file) + with open(mdl_path, "r") as mdl_file: + return json.load(mdl_file) + + +class StochSSModel(): ''' ################################################################################################ StochSS model object ################################################################################################ ''' - def __init__(self, path): + def __init__(self, model): ''' Intitialize a model object Attributes ---------- - path : str - Path to the model + model : dict + Existing model data ''' - super().__init__(path=path) - self.model = __read_model_file(self) + self.model = model @classmethod @@ -186,29 +206,26 @@ def load(self): self.__update_reactions() self.__update_events(param_ids=param_ids) self.__update_rules(param_ids=param_ids) - self.model['name'] = self.get_name() - self.model['directory'] = self.path return self.model -class StochSSSpatialModel(StochSSBase): +class StochSSSpatialModel(): ''' ################################################################################################ StochSS spatial model object ################################################################################################ ''' - def __init__(self, path): + def __init__(self, model): ''' Intitialize a spatial model object Attributes ---------- - path : str - Path to the spatial model + model : dict + Existing model data ''' - super().__init__(path=path) - self.model = __read_model_file(self) + self.model = model def load(self): @@ -218,7 +235,6 @@ def load(self): Attributes ---------- ''' - self.model['name'] = self.get_name() if not self.model['defaultMode']: self.model['defaultMode'] = "discrete" if "static" not in self.model['domain'].keys(): From b4d26dcb63ed0eb0565f01e939299232089c7871 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 15:35:42 -0400 Subject: [PATCH 042/192] removed the presentation buttons for local deployments of stochss. removed the model presentation button for spatial models. disabled the job presentation button. --- client/views/model-state-buttons.js | 3 +++ client/views/workflow-results.js | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/client/views/model-state-buttons.js b/client/views/model-state-buttons.js index 79fc02b4bb..90474e6534 100644 --- a/client/views/model-state-buttons.js +++ b/client/views/model-state-buttons.js @@ -50,6 +50,9 @@ module.exports = View.extend({ if(this.model.is_spatial) { $(this.queryByHook("stochss-es")).addClass("disabled"); $(this.queryByHook("stochss-ps")).addClass("disabled"); + $(this.queryByHook("presentation")).css("display", "none"); + }else if(app.getBasePath() === "/") { + $(this.queryByHook("presentation")).css("display", "none"); } }, clickSaveHandler: function (e) { diff --git a/client/views/workflow-results.js b/client/views/workflow-results.js index 2dc10c1d3a..6fe988d7cb 100644 --- a/client/views/workflow-results.js +++ b/client/views/workflow-results.js @@ -75,6 +75,10 @@ module.exports = View.extend({ if(!isParameterScan){ $(this.queryByHook("convert-to-notebook")).css("display", "none"); } + }else if(app.getBasePath() === "/") { + $(this.queryByHook("job-presentation")).css("display", "none"); + }else{ + $(this.queryByHook("job-presentation")).prop("disabled", true); } if(this.parent.model.type === "Ensemble Simulation") { var type = isEnsemble ? "stddevran" : "trajectories"; From 2bcb890a54c514da71ebcd76295173d46f828c4d Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 15:53:24 -0400 Subject: [PATCH 043/192] Created a model to present the presentation links to the user. --- client/modals.js | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/client/modals.js b/client/modals.js index c55db1f6b8..3c9ef0f7de 100644 --- a/client/modals.js +++ b/client/modals.js @@ -223,6 +223,30 @@ let templates = { ` + }, + presentationLinks : (modalID, title, name, headers, links) => { + return ` + ` } } @@ -530,5 +554,10 @@ module.exports = { ` + }, + presentationLinks : (title, name, headers, links) => { + let modalID = "presentationLinksModal" + + return templates.presentationLinks(modalID, title, name, headers, links); } } \ No newline at end of file From 6e31d9d570e3233c53f351984ec29e94ad64e2e9 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Fri, 11 Jun 2021 15:54:35 -0400 Subject: [PATCH 044/192] Added function to launch the presentation links modal when publishing is successful. --- client/views/model-state-buttons.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/views/model-state-buttons.js b/client/views/model-state-buttons.js index 90474e6534..6ab8276c85 100644 --- a/client/views/model-state-buttons.js +++ b/client/views/model-state-buttons.js @@ -307,6 +307,11 @@ module.exports = View.extend({ app.getXHR(endpoint, { success: function (err, response, body) { self.endAction("publish"); + let title = body.message; + let linkHeaders = ["Presentation Link", "Download Link"]; + let links = body.links; + let name = self.model.name + $(modals.presentationLinks(title, name, linkHeaders, links)).modal(); }, error: function (err, response, body) { self.errorAction(); From 6bb685e7f40f01a2b33fc643271092ccd645210c Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Sun, 13 Jun 2021 12:35:04 -0400 Subject: [PATCH 045/192] Added the download api handler for model presentations. --- jupyterhub/model_presentation.py | 38 +++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/jupyterhub/model_presentation.py b/jupyterhub/model_presentation.py index 7dfaaeb423..1b31cd41db 100644 --- a/jupyterhub/model_presentation.py +++ b/jupyterhub/model_presentation.py @@ -40,16 +40,13 @@ class JsonFileAPIHandler(APIHandler): ''' ################################################################################################ - Base Handler for interacting with Model file Get/Post Requests and - downloading json formatted files. + Base Handler for getting model presentations from user containers. ################################################################################################ ''' @web.authenticated async def get(self): ''' - Retrieve model data from User's file system if it exists and - create new models using a model template if they don't. Also - retrieves JSON files for download. + Load the model presentation from User's presentations directory. Attributes ---------- @@ -73,6 +70,37 @@ async def get(self): self.finish() +class DownModelPresentationAPIHandler(APIHandler): + ''' + ################################################################################################ + Base Handler for downloading model presentations from user containers. + ################################################################################################ + ''' + @web.authenticated + async def get(self): + ''' + Download the model presentation from User's presentations directory. + + Attributes + ---------- + ''' + owner = self.get_query_argument(name="owner") + log.debug("Container id of the owner: %s", owner) + file = self.get_query_argument(name="file") + log.debug("Name to the file: %s", file) + self.set_header('Content-Type', 'application/json') + file_objs = {"mdl":StochSSModel, "smdl":StochSSSpatialModel} + ext = file.split(".").pop() + try: + model = get_presentation_from_user(owner=owner, file=file) + self.set_header('Content-Disposition', f'attachment; filename="{model["name"]}.{ext}"') + log.debug("Contents of the json file: %s", model) + self.write(model) + except StochSSAPIError as load_err: + report_error(self, log, load_err) + self.finish() + + def get_presentation_from_user(owner, file): ''' Get the model presentation from the users container From c7a3716317af79cf375933c263effddae74b818f Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Sun, 13 Jun 2021 12:45:18 -0400 Subject: [PATCH 046/192] Refactored the download model presentation link to use link args instead of query paramters. Added route for downloding presentations. --- jupyterhub/jupyterhub_config.py | 1 + jupyterhub/model_presentation.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 10f2594d4f..1b2626b315 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -85,6 +85,7 @@ (r"/stochss/present-job\/?", JobPresentationHandler), (r"/stochss/present-model\/?", ModelPresentationHandler), (r"/stochss/api/file/json-data\/?", JsonFileAPIHandler), + (r"/stochss/presentation-download/(\w+)/(.+)\/?", DownModelPresentationAPIHandler) ] ## Paths to search for jinja templates, before using the default templates. diff --git a/jupyterhub/model_presentation.py b/jupyterhub/model_presentation.py index 1b31cd41db..4a3b490e8f 100644 --- a/jupyterhub/model_presentation.py +++ b/jupyterhub/model_presentation.py @@ -77,16 +77,14 @@ class DownModelPresentationAPIHandler(APIHandler): ################################################################################################ ''' @web.authenticated - async def get(self): + async def get(self, owner, file): ''' Download the model presentation from User's presentations directory. Attributes ---------- ''' - owner = self.get_query_argument(name="owner") log.debug("Container id of the owner: %s", owner) - file = self.get_query_argument(name="file") log.debug("Name to the file: %s", file) self.set_header('Content-Type', 'application/json') file_objs = {"mdl":StochSSModel, "smdl":StochSSSpatialModel} From e62dd35bf0cc6e731cecd9069054fba32cf2f2cf Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Sun, 13 Jun 2021 14:19:26 -0400 Subject: [PATCH 047/192] Added missing import. Pylint fixes. --- jupyterhub/jupyterhub_config.py | 2 +- jupyterhub/model_presentation.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py index 1b2626b315..64c02e7605 100644 --- a/jupyterhub/jupyterhub_config.py +++ b/jupyterhub/jupyterhub_config.py @@ -28,7 +28,7 @@ # Page handlers from handlers import * # API Handlers -from model_presentation import JsonFileAPIHandler +from model_presentation import JsonFileAPIHandler, DownModelPresentationAPIHandler ## Class for authenticating users. # diff --git a/jupyterhub/model_presentation.py b/jupyterhub/model_presentation.py index 4a3b490e8f..a8e52ea197 100644 --- a/jupyterhub/model_presentation.py +++ b/jupyterhub/model_presentation.py @@ -87,10 +87,9 @@ async def get(self, owner, file): log.debug("Container id of the owner: %s", owner) log.debug("Name to the file: %s", file) self.set_header('Content-Type', 'application/json') - file_objs = {"mdl":StochSSModel, "smdl":StochSSSpatialModel} - ext = file.split(".").pop() try: model = get_presentation_from_user(owner=owner, file=file) + ext = file.split(".").pop() self.set_header('Content-Disposition', f'attachment; filename="{model["name"]}.{ext}"') log.debug("Contents of the json file: %s", model) self.write(model) From bf56951a8b4c0ee5a1a288b5ad875906dfc1cf54 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Sun, 13 Jun 2021 14:23:34 -0400 Subject: [PATCH 048/192] Open links are now generated when presentations are published. --- stochss/handlers/util/stochss_model.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stochss/handlers/util/stochss_model.py b/stochss/handlers/util/stochss_model.py index 6fce11ca1d..63a620074c 100644 --- a/stochss/handlers/util/stochss_model.py +++ b/stochss/handlers/util/stochss_model.py @@ -482,8 +482,10 @@ def publish_presentation(self): raise StochSSFileExistsError(message) query_str = f"?owner={hostname}&file={file}" present_link = f"live.stochss.org/stochss/present-model{query_str}" - download_link = f"live.stochss.org/stochss/download_presentation{query_str}" - links = {"presentation": present_link, "download": download_link} + download_link = os.path.join("live.stochss.org/stochss/download_presentation", + hostname, file) + open_link = f"live.stochss.org?open={download_link}" + links = {"presentation": present_link, "download": download_link, "open": open_link} data = {"path": dst, "new":True, "model":self.model} return links, data except PermissionError as err: From cb3e33c122f36aa99d9ead0623df31c9a3a74c82 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Sun, 13 Jun 2021 14:42:05 -0400 Subject: [PATCH 049/192] Fixed issue with generating the response message. --- stochss/handlers/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stochss/handlers/models.py b/stochss/handlers/models.py index de9c8aacf0..bca85af6ac 100644 --- a/stochss/handlers/models.py +++ b/stochss/handlers/models.py @@ -432,7 +432,7 @@ async def get(self): log.info("Publishing the %s presentation", model.get_name()) links, data = model.publish_presentation() file_objs[ext](**data) - resp = {"message": f"Successfully published the {self.get_name()} presentation", + resp = {"message": f"Successfully published the {model.get_name()} presentation", "links": links} log.info(resp['message']) log.debug("Response Message: %s", resp) From d171b52fca7dba1688713e9150499a2cebd1f843 Mon Sep 17 00:00:00 2001 From: Bryan Rumsey Date: Sun, 13 Jun 2021 14:43:38 -0400 Subject: [PATCH 050/192] Added the open link block to the success modal and updated the styling. --- client/modals.js | 8 ++++++-- client/views/model-state-buttons.js | 10 +++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/client/modals.js b/client/modals.js index 3c9ef0f7de..aad8b627af 100644 --- a/client/modals.js +++ b/client/modals.js @@ -236,10 +236,14 @@ let templates = {