diff --git a/DAFNI-wrappers/Readme.md b/DAFNI-wrappers/Readme.md new file mode 100644 index 00000000..2a98f6a5 --- /dev/null +++ b/DAFNI-wrappers/Readme.md @@ -0,0 +1,14 @@ +# DAFNI Wrappers + +These are the Docker images that wrap the NISMOD2 models to allow them to be run on +DAFNI. + +Each of the images has their own readme file. Here are just some things to keep in mind +when changing the images: + +- The extract_data.py scripts currently do reference the data packs by file name + directly so if you update the data packs you will also need to update the + extract_data.py scripts to make sure they are correct too. +- The decide_step and transforms wrappers define their own run_nismod.py script that + overwrites the one in the base image, this means any changes to the script in the base + image won't apply to those wrappers. diff --git a/DAFNI-wrappers/base-image/README.md b/DAFNI-wrappers/base-image/README.md index 586a2c27..7b1d93a6 100644 --- a/DAFNI-wrappers/base-image/README.md +++ b/DAFNI-wrappers/base-image/README.md @@ -1,3 +1,19 @@ -# transport +# Base Image -Base image for all nismod models \ No newline at end of file +This is the base Docker image for all the other DAFNI-wrappers for NISMOD2 models. This +image contains all of the packages and scripts that are common between the models. + +The settings.py file sets some commonly used paths and also grabs the parameters from +the environment variables that DAFNI sets on the container when the Model is run. This +is how users can specify the parameter values via Workflows. + +utils.py just contains some useful functions that are used by the wrapper scripts. + +run_nismod.py contains the code that actually runs the NISMOD2 model based on the +parameters the user provided. + +job_processing_wrapper.py is the script DAFNI calls to run the models, it simply calls +run_nismod.py and then copies the results of the model into the proper output location. + +You shouldn't need to build this image locally in order to build the other models as +they all use the image in the github registry as their base. diff --git a/DAFNI-wrappers/decide_step/README.md b/DAFNI-wrappers/decide_step/README.md index fae9dedf..85ab0361 100644 --- a/DAFNI-wrappers/decide_step/README.md +++ b/DAFNI-wrappers/decide_step/README.md @@ -1,3 +1,10 @@ # Decide Step -NIMS wrapper for NISMOD decision steps \ No newline at end of file +This is the DAFNI wrapper for the NISMOD decision step. The reason this step has been +extracted out is so that each of the other NISMOD models can only require the datasets +they directly use. If this step wasn't extracted out then every model would need a copy +of every dataset in order to be properly used in a SOS Workflow. + +This is a very simple image, it just extracts all the datasets into their proper +locations then runs the decision step. In order to create a SOS Workflow this should be +the first step of the Workflow. diff --git a/DAFNI-wrappers/decide_step/dockerFiles/extract_data.py b/DAFNI-wrappers/decide_step/dockerFiles/extract_data.py index 65b8571f..9e6cd815 100644 --- a/DAFNI-wrappers/decide_step/dockerFiles/extract_data.py +++ b/DAFNI-wrappers/decide_step/dockerFiles/extract_data.py @@ -36,7 +36,7 @@ def extract(): "dest": str(NISMOD_SCENARIOS_PATH.joinpath("ev_transport_trips/")), }, { - "src": "/data/energy_supply/energy_supply_data_v0.9.10.zip", + "src": "/data/energy_supply/energy_supply_data_v0.9.12.zip", "dest": str(NISMOD_DATA_PATH.joinpath("energy_supply/")), }, { diff --git a/DAFNI-wrappers/decide_step/model_definition.yaml b/DAFNI-wrappers/decide_step/model_definition.yaml index 1e34f5b9..bd7cf8fc 100644 --- a/DAFNI-wrappers/decide_step/model_definition.yaml +++ b/DAFNI-wrappers/decide_step/model_definition.yaml @@ -1,7 +1,7 @@ kind: Model apiVersion: v1alpha4 metadata: - displayName: NISMOD - Decision Module (model v2.4.0 build 2) + displayName: NISMOD - Decision Module (model v2.4.0 build 5) name: nismod-decide summary: National Infrastructure Systems Model type: systems of systems @@ -15,29 +15,15 @@ metadata: spec: inputs: env: - - name: sector_model - title: NISMOD2 Sector Model - desc: Pick one of the NISMOD2 Sector Models to run the decide step for - type: string - name: model_to_run title: Model Run desc: The name of the model run that is being executed type: string - default: energy_supply_constrained - - name: part_of_sos_model - title: Part of SOS Model - desc: "Is this model being run as part of a NISMOD2 Systems of Systems Model?" - type: boolean - default: false - name: use_generated_scenario title: Use generated scenario desc: "Is this run using a new generated scenario file?" type: boolean default: false - - name: timestep - title: Timestep - desc: "Timestep to run the Model at if this is part of an SoS Model" - type: integer dataslots: - name: Scenario Data description: > @@ -93,8 +79,8 @@ spec: vehicle trip starts and consumption scenarios (test outputs from the Transport model used to test running the et_module standalone). default: - - uid: dbfe6814-3fd3-4e94-8c30-ccfec60c9989 - versionUid: df234857-a622-4b83-bec1-c72b1c7ed8b6 + - uid: ed671698-069c-40f4-8645-bf08e45634fb + versionUid: 66382de9-bef7-45ab-b729-dbfc1d01007f path: et_module/ required: true - name: LAD boundaries diff --git a/DAFNI-wrappers/energy_demand/README.md b/DAFNI-wrappers/energy_demand/README.md index b462d731..701d9002 100644 --- a/DAFNI-wrappers/energy_demand/README.md +++ b/DAFNI-wrappers/energy_demand/README.md @@ -1,3 +1,4 @@ # energy_demand -NIMS wrapper for NISMOD energy_demand model \ No newline at end of file +The DAFNI wrapper for NISMOD2's energy_demand model. Again this is a fairly simple +wrapper, that extracts the energy demand datasets and then runs the Model. diff --git a/DAFNI-wrappers/energy_demand/dockerFiles/extract_data.py b/DAFNI-wrappers/energy_demand/dockerFiles/extract_data.py index 458ad9cf..b8b0a86d 100644 --- a/DAFNI-wrappers/energy_demand/dockerFiles/extract_data.py +++ b/DAFNI-wrappers/energy_demand/dockerFiles/extract_data.py @@ -35,7 +35,7 @@ def extract(): "dest": str(NISMOD_SCENARIOS_PATH.joinpath("ev_transport_trips/")), }, # { - # "src": "/data/energy_supply/energy_supply_data_v0.9.10.zip", + # "src": "/data/energy_supply/energy_supply_data_v0.9.12.zip", # "dest": str(NISMOD_DATA_PATH.joinpath("energy_supply/")), # }, { diff --git a/DAFNI-wrappers/energy_demand/model_definition.yaml b/DAFNI-wrappers/energy_demand/model_definition.yaml index 066298c2..455875e8 100644 --- a/DAFNI-wrappers/energy_demand/model_definition.yaml +++ b/DAFNI-wrappers/energy_demand/model_definition.yaml @@ -3,7 +3,7 @@ apiVersion: v1alpha4 metadata: displayName: NISMOD - Energy Demand (model v1.0.1 build 1) name: nismod-energy-demand - summary: National Infrastructure Systems Model + summary: National Infrastructure Systems Model (preliminary upload No.9) type: systems of systems description: > HIRE (HIgh-Resolution Energy demand model) simulates energy demand for the @@ -48,14 +48,14 @@ spec: title: Timestep desc: "Simulation year (used if run as part of a NISMOD Systems of Systems Model)" type: integer - datasets: + dataslots: - name: Scenario Data description: > NISMOD scenario data pack. Should contain socio-economic scenarios (population, GVA) and energy (fuel) price scenarios. default: - uid: 1237868f-52ea-416c-9087-eef9dbde429d - versionUid: cef750da-b26d-4da1-b67b-871281c3d551 + versionUid: 5afbfaa2-a351-4a04-9b79-efa6a194aa60 path: scenarios/ required: true - name: Energy Demand Model Data diff --git a/DAFNI-wrappers/energy_supply/README.md b/DAFNI-wrappers/energy_supply/README.md index a704d65e..39430da3 100644 --- a/DAFNI-wrappers/energy_supply/README.md +++ b/DAFNI-wrappers/energy_supply/README.md @@ -1,3 +1,23 @@ # energy_supply -NIMS wrapper for NISMOD energy_supply model \ No newline at end of file +The DAFNI wrapper for NISMOD2's energy_supply model. This model is quite complex, the +current setup on DAFNI has the postgres database running in a sidecar (a container that +runs along side the main model container but that is linked by a local network). The +image for this sidecar is in the db folder, unfortunately DAFNI doesn't currently have a +good process to allow users to customise and specify sidecars themselves so this image +is provided purely for completeness and to allow you to run the model locally. + +I have separated out the model image into a new model_base image (which adds packages to +the generic base image) and the model image which actually containers the wrapper +scripts. I have done this because the install for FICO and the other packages in the +model_base image takes a long time and also because these are unlikely to ever change. +The model image itself is quite simple, it contains the wrapper scripts and a built +version of the energy supply model. The model and the datasets will be extracted to the +right places and the model will then be run. + +I have provided a docker-compose file that should allow you to build and run the energy +supply locally with minimal hassle. + +One thing that hasn't been included in the repository is the license file for FICO that +this wrapper needs in order to run the energy supply model. You will unfortunately have +to provide this yourself as FICO is commercial software. diff --git a/DAFNI-wrappers/energy_supply/model/dockerFiles/extract_data.py b/DAFNI-wrappers/energy_supply/model/dockerFiles/extract_data.py index 855615ff..94ab7ba2 100644 --- a/DAFNI-wrappers/energy_supply/model/dockerFiles/extract_data.py +++ b/DAFNI-wrappers/energy_supply/model/dockerFiles/extract_data.py @@ -108,7 +108,7 @@ def extract(): "dest": str(NISMOD_SCENARIOS_PATH.joinpath("ev_transport_trips/")), }, { - "src": "/data/energy_supply/energy_supply_data_v0.9.10.zip", + "src": "/data/energy_supply/energy_supply_data_v0.9.12.zip", "dest": str(NISMOD_DATA_PATH.joinpath("energy_supply/")), }, # { diff --git a/DAFNI-wrappers/et_module/README.md b/DAFNI-wrappers/et_module/README.md index bdbea5de..d03ffaa4 100644 --- a/DAFNI-wrappers/et_module/README.md +++ b/DAFNI-wrappers/et_module/README.md @@ -1,3 +1,4 @@ # et_module -Docker image for the et_module in nismod2 \ No newline at end of file +The DAFNI wrapper for NISMOD2's et_module model. Again this is a fairly simple +wrapper, that extracts the et module datasets and then runs the model. diff --git a/DAFNI-wrappers/et_module/model_definition.yaml b/DAFNI-wrappers/et_module/model_definition.yaml index 6f9946ee..28176135 100644 --- a/DAFNI-wrappers/et_module/model_definition.yaml +++ b/DAFNI-wrappers/et_module/model_definition.yaml @@ -65,8 +65,8 @@ spec: vehicle trip starts and consumption scenarios (test outputs from the Transport model used to test running the et_module standalone). default: - - uid: dbfe6814-3fd3-4e94-8c30-ccfec60c9989 - versionUid: df234857-a622-4b83-bec1-c72b1c7ed8b6 + - uid: ed671698-069c-40f4-8645-bf08e45634fb + versionUid: 66382de9-bef7-45ab-b729-dbfc1d01007f path: et_module/ required: true - name: LAD boundaries diff --git a/DAFNI-wrappers/transforms/README.md b/DAFNI-wrappers/transforms/README.md index d7703332..d93446b3 100644 --- a/DAFNI-wrappers/transforms/README.md +++ b/DAFNI-wrappers/transforms/README.md @@ -1,3 +1,5 @@ # nismod-transforms -Model that runs the transformation / conversion steps for nismod2. \ No newline at end of file +This DAFNI wrapper allows a user to run one (or more) of the various adaptors / +transform operations that are required to convert data to the appropriate forms when +going from model to model in a SOS Workflow. diff --git a/DAFNI-wrappers/transforms/dockerFiles/extract_data.py b/DAFNI-wrappers/transforms/dockerFiles/extract_data.py index a4828fc8..09febe4f 100644 --- a/DAFNI-wrappers/transforms/dockerFiles/extract_data.py +++ b/DAFNI-wrappers/transforms/dockerFiles/extract_data.py @@ -48,7 +48,7 @@ def extract(): "dest": str(NISMOD_SCENARIOS_PATH.joinpath("ev_transport_trips/")), }, { - "src": "/data/energy_supply/energy_supply_data_v0.9.10.zip", + "src": "/data/energy_supply/energy_supply_data_v0.9.12.zip", "dest": str(NISMOD_DATA_PATH.joinpath("energy_supply/")), }, { diff --git a/DAFNI-wrappers/transforms/model_definition.yaml b/DAFNI-wrappers/transforms/model_definition.yaml index 8ffbe115..dd86b386 100644 --- a/DAFNI-wrappers/transforms/model_definition.yaml +++ b/DAFNI-wrappers/transforms/model_definition.yaml @@ -1,7 +1,7 @@ kind: Model apiVersion: v1alpha4 metadata: - displayName: NISMOD - Adaptors (model v2.4.0 build 1) + displayName: NISMOD - Adaptors (model v2.4.0 build 3) name: nismod-transforms summary: National Infrastructure Systems Model type: systems of systems @@ -93,8 +93,8 @@ spec: vehicle trip starts and consumption scenarios (test outputs from the Transport model used to test running the et_module standalone). default: - - uid: dbfe6814-3fd3-4e94-8c30-ccfec60c9989 - versionUid: df234857-a622-4b83-bec1-c72b1c7ed8b6 + - uid: ed671698-069c-40f4-8645-bf08e45634fb + versionUid: 66382de9-bef7-45ab-b729-dbfc1d01007f path: et_module/ required: true - name: LAD boundaries diff --git a/DAFNI-wrappers/transport/README.md b/DAFNI-wrappers/transport/README.md index aa7f00fd..ca994df6 100644 --- a/DAFNI-wrappers/transport/README.md +++ b/DAFNI-wrappers/transport/README.md @@ -1,3 +1,4 @@ # transport -Docker image for the NISMOD2 transport model \ No newline at end of file +The DAFNI wrapper for NISMOD2's transport model. Again this is a fairly simple wrapper, +that extracts the transport datasets and then runs the model.