From d227905fa6d81bcccc7a0f6306c66c170799dc00 Mon Sep 17 00:00:00 2001 From: Joan Date: Tue, 22 Oct 2024 09:07:03 +0200 Subject: [PATCH 01/50] Added support for new definition of the connection --- configuration/modelling-example.yaml | 61 +++++++++++++++++++++------- configuration/parse_yaml.py | 21 +++++++--- 2 files changed, 61 insertions(+), 21 deletions(-) diff --git a/configuration/modelling-example.yaml b/configuration/modelling-example.yaml index 41e1f5b..8909e83 100644 --- a/configuration/modelling-example.yaml +++ b/configuration/modelling-example.yaml @@ -22,11 +22,9 @@ models: # list of models for the energy network - capacity - soc_min scenario_data: 'path/to/file' #path to the scenario data file for the model. This should be optional - method: - type: connect # can be python for local or connect for remote - location: null - connect_ip: 192.168.0.1 - connect_port: 5123 + connect: + ip: 192.168.0.1 + port: 5000 - name: Battery2 type: Battery # models can reuse the same type inputs: @@ -38,11 +36,47 @@ models: # list of models for the energy network charge_power_max: 100 states: soc: 0.5 # initial value for the state, optional - method: - type: connect # can be python for local or connect for remote - location: null - connect_ip: 192.168.0.2 - connect_port: 5123 + connect: + ip: 192.168.0.2 + port: 5000 +- name: Battery3 + type: Battery # models can reuse the same type + inputs: + input1: 0 # input-name: initial value, default value will be used if not defined + outputs: + output1: 0 + output2: null + parameters: + charge_power_max: 100 + states: + soc: 0.5 # initial value for the state, optional + connect: + port: 5000 +- name: Battery4 + type: Battery # models can reuse the same type + inputs: + input1: 0 # input-name: initial value, default value will be used if not defined + outputs: + output1: 0 + output2: null + parameters: + charge_power_max: 100 + states: + soc: 0.5 # initial value for the state, optional + connect: + ip: + port: 5000 +- name: Battery5 + type: Battery # models can reuse the same type + inputs: + input1: 0 # input-name: initial value, default value will be used if not defined + outputs: + output1: 0 + output2: null + parameters: + charge_power_max: 100 + states: + soc: 0.5 # initial value for the state, optional - name: PV1 type: PV inputs: @@ -54,11 +88,8 @@ models: # list of models for the energy network power_max: 100 states: initial_soc: 0.5 - method: - type: connect # can be python for local or connect for remote - location: null - connect_ip: 192.168.0.3 - connect_port: 5123 + connect: + ip: 192.168.0.3 connections: - from: Battery1.output2 # start model, pattern: model_name.output_name/input_name to: PV1.input1 # end model diff --git a/configuration/parse_yaml.py b/configuration/parse_yaml.py index 60c75d0..2a524b0 100644 --- a/configuration/parse_yaml.py +++ b/configuration/parse_yaml.py @@ -1,5 +1,12 @@ import yaml +""" +This script reads a yaml file of an illuminator model. It extracts the connection information. +If an IP port is not specified it will default to 5123. +If both IP address and port are known than a line will be prined that can be used in the run.sh file. +The run.sh file is used to start the processes on remote pi's. +""" + # Open and load the YAML file with open('modelling-example.yaml', 'r') as _file: data = yaml.safe_load(_file) @@ -10,13 +17,15 @@ for model in data['models']: model_type = model.get('type') - method = model.get('method', {}) - method_type = method.get('type') - location = method.get('location') - connect_ip = method.get('connect_ip') - connect_port = method.get('connect_port') + connect = model.get('connect', {}) + + if connect: + connect_ip = connect.get('ip') + connect_port = connect.get('port') + + if not connect_port: + connect_port = 5123 - if method_type == 'connect': if connect_ip and connect_port: print(f"lxterminal -e ssh illuminator@{connect_ip} '{run_path}run{model_type}.sh {connect_ip} {connect_port} {run_model}'&") From ca1ed92489a027dec99fa70a347115c3d375a9ef Mon Sep 17 00:00:00 2001 From: Joan Date: Tue, 22 Oct 2024 09:39:43 +0200 Subject: [PATCH 02/50] Added ChatGPT version of the script. --- configuration/parse_yaml_CGPT.py | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 configuration/parse_yaml_CGPT.py diff --git a/configuration/parse_yaml_CGPT.py b/configuration/parse_yaml_CGPT.py new file mode 100644 index 0000000..c45e67d --- /dev/null +++ b/configuration/parse_yaml_CGPT.py @@ -0,0 +1,53 @@ +import yaml +import os + +# Constants +DEFAULT_PORT = 5123 +RUN_PATH = './Desktop/illuminatorclient/configuration/runshfile/' +RUN_MODEL = '/home/illuminator/Desktop/Final_illuminator' + +def load_yaml(file_path): + """Loads a YAML file and returns its content.""" + try: + with open(file_path, 'r') as file: + return yaml.safe_load(file) + except FileNotFoundError: + print(f"Error: The file {file_path} was not found.") + return None + except yaml.YAMLError as exc: + print(f"Error while parsing YAML: {exc}") + return None + +def build_run_command(model_type, connect_ip, connect_port): + """Builds the SSH run command for a given model.""" + return f"lxterminal -e ssh illuminator@{connect_ip} '{RUN_PATH}run{model_type}.sh {connect_ip} {connect_port} {RUN_MODEL}'&" + +def process_models(data): + """Processes each model in the YAML data.""" + if 'models' not in data: + print("Error: No 'models' key found in the YAML data.") + return + + for model in data['models']: + model_type = model.get('type', 'unknown') + + # Extract connection details + connect = model.get('connect', {}) + connect_ip = connect.get('ip') + connect_port = connect.get('port', DEFAULT_PORT) + + # Ensure both IP and port are available + if connect_ip: + run_command = build_run_command(model_type, connect_ip, connect_port) + print(run_command) + +def main(): + yaml_file = 'modelling-example.yaml' + data = load_yaml(yaml_file) + + if data: + process_models(data) + +if __name__ == "__main__": + main() + From 566200dfb482127c634c50076d6a30064dac6e15 Mon Sep 17 00:00:00 2001 From: Joan Date: Tue, 5 Nov 2024 09:48:16 +0100 Subject: [PATCH 03/50] Improved the script and renamed to build_runshfile --- configuration/build_runshfile.py | 67 ++++++++++++++++++++++++++++++++ configuration/parse_yaml.py | 31 --------------- configuration/parse_yaml_CGPT.py | 53 ------------------------- configuration/run.sh | 7 ++-- 4 files changed, 70 insertions(+), 88 deletions(-) create mode 100644 configuration/build_runshfile.py delete mode 100644 configuration/parse_yaml.py delete mode 100644 configuration/parse_yaml_CGPT.py diff --git a/configuration/build_runshfile.py b/configuration/build_runshfile.py new file mode 100644 index 0000000..448befa --- /dev/null +++ b/configuration/build_runshfile.py @@ -0,0 +1,67 @@ +import yaml +import os +import sys + +# Constants +DEFAULT_PORT = 5123 +RUN_PATH = './Desktop/illuminatorclient/configuration/runshfile/' +RUN_MODEL = '/home/illuminator/Desktop/Final_illuminator' +RUN_FILE = 'run.sh' + +def load_yaml(file_path): + """Loads a YAML file and returns its content.""" + try: + with open(file_path, 'r') as file: + return yaml.safe_load(file) + except FileNotFoundError: + print(f"Error: The file {file_path} was not found.") + return None + except yaml.YAMLError as exc: + print(f"Error while parsing YAML: {exc}") + return None + +def build_run_command(model_type, connect_ip, connect_port): + """Builds the SSH run command for a given model.""" + return f"lxterminal -e ssh illuminator@{connect_ip} '{RUN_PATH}run{model_type}.sh {connect_ip} {connect_port} {RUN_MODEL}'&" + +def process_models(data, output_file): + """Processes each model in the YAML data and writes the commands to a file.""" + if 'models' not in data: + print("Error: No 'models' key found in the YAML data.") + return + + with open(output_file, 'w') as file: + + file.write('#! /bin/bash\n') + + for model in data['models']: + model_type = model.get('type', 'unknown') + + # Extract connection details + connect = model.get('connect', {}) + connect_ip = connect.get('ip') + connect_port = connect.get('port', DEFAULT_PORT) + + # Ensure both IP and port are available + if connect_ip: + run_command = build_run_command(model_type, connect_ip, connect_port) + file.write(run_command + '\n') + else: + print(f"Warning: Model '{model_type}' has no IP address specified.") + +def main(): + if len(sys.argv) != 2: + print("Usage: python script.py ") + sys.exit(1) + + yaml_file = sys.argv[1] + output_file = RUN_FILE + data = load_yaml(yaml_file) + + if data: + process_models(data, output_file) + print(f"Commands have been written to {output_file}") + +if __name__ == "__main__": + main() + diff --git a/configuration/parse_yaml.py b/configuration/parse_yaml.py deleted file mode 100644 index 2a524b0..0000000 --- a/configuration/parse_yaml.py +++ /dev/null @@ -1,31 +0,0 @@ -import yaml - -""" -This script reads a yaml file of an illuminator model. It extracts the connection information. -If an IP port is not specified it will default to 5123. -If both IP address and port are known than a line will be prined that can be used in the run.sh file. -The run.sh file is used to start the processes on remote pi's. -""" - -# Open and load the YAML file -with open('modelling-example.yaml', 'r') as _file: - data = yaml.safe_load(_file) - -run_path='./Desktop/illuminatorclient/configuration/runshfile/' -run_model='/home/illuminator/Desktop/Final_illuminator' - -for model in data['models']: - model_type = model.get('type') - - connect = model.get('connect', {}) - - if connect: - connect_ip = connect.get('ip') - connect_port = connect.get('port') - - if not connect_port: - connect_port = 5123 - - if connect_ip and connect_port: - print(f"lxterminal -e ssh illuminator@{connect_ip} '{run_path}run{model_type}.sh {connect_ip} {connect_port} {run_model}'&") - diff --git a/configuration/parse_yaml_CGPT.py b/configuration/parse_yaml_CGPT.py deleted file mode 100644 index c45e67d..0000000 --- a/configuration/parse_yaml_CGPT.py +++ /dev/null @@ -1,53 +0,0 @@ -import yaml -import os - -# Constants -DEFAULT_PORT = 5123 -RUN_PATH = './Desktop/illuminatorclient/configuration/runshfile/' -RUN_MODEL = '/home/illuminator/Desktop/Final_illuminator' - -def load_yaml(file_path): - """Loads a YAML file and returns its content.""" - try: - with open(file_path, 'r') as file: - return yaml.safe_load(file) - except FileNotFoundError: - print(f"Error: The file {file_path} was not found.") - return None - except yaml.YAMLError as exc: - print(f"Error while parsing YAML: {exc}") - return None - -def build_run_command(model_type, connect_ip, connect_port): - """Builds the SSH run command for a given model.""" - return f"lxterminal -e ssh illuminator@{connect_ip} '{RUN_PATH}run{model_type}.sh {connect_ip} {connect_port} {RUN_MODEL}'&" - -def process_models(data): - """Processes each model in the YAML data.""" - if 'models' not in data: - print("Error: No 'models' key found in the YAML data.") - return - - for model in data['models']: - model_type = model.get('type', 'unknown') - - # Extract connection details - connect = model.get('connect', {}) - connect_ip = connect.get('ip') - connect_port = connect.get('port', DEFAULT_PORT) - - # Ensure both IP and port are available - if connect_ip: - run_command = build_run_command(model_type, connect_ip, connect_port) - print(run_command) - -def main(): - yaml_file = 'modelling-example.yaml' - data = load_yaml(yaml_file) - - if data: - process_models(data) - -if __name__ == "__main__": - main() - diff --git a/configuration/run.sh b/configuration/run.sh index 8a3a0a3..b186b61 100644 --- a/configuration/run.sh +++ b/configuration/run.sh @@ -1,5 +1,4 @@ #! /bin/bash -lxterminal -e ssh illuminator@192.168.0.1 './Desktop/illuminatorclient/configuration/runshfile/runWind.sh 192.168.0.1 5123 /home/illuminator/Desktop/Final_illuminator'& -lxterminal -e ssh illuminator@192.168.0.2 './Desktop/illuminatorclient/configuration/runshfile/runPV.sh 192.168.0.2 5123 /home/illuminator/Desktop/Final_illuminator'& -lxterminal -e ssh illuminator@192.168.0.3 './Desktop/illuminatorclient/configuration/runshfile/runLoad.sh 192.168.0.3 5123 /home/illuminator/Desktop/Final_illuminator'& -lxterminal -e ssh illuminator@192.168.0.4 './Desktop/illuminatorclient/configuration/runshfile/runBattery.sh 192.168.0.4 5123 /home/illuminator/Desktop/Final_illuminator'& \ No newline at end of file +lxterminal -e ssh illuminator@192.168.0.1 './Desktop/illuminatorclient/configuration/runshfile/runBattery.sh 192.168.0.1 5000 /home/illuminator/Desktop/Final_illuminator'& +lxterminal -e ssh illuminator@192.168.0.2 './Desktop/illuminatorclient/configuration/runshfile/runBattery.sh 192.168.0.2 5000 /home/illuminator/Desktop/Final_illuminator'& +lxterminal -e ssh illuminator@192.168.0.3 './Desktop/illuminatorclient/configuration/runshfile/runPV.sh 192.168.0.3 5123 /home/illuminator/Desktop/Final_illuminator'& From 17b6ddcc42215551a21424b0cd9ecd992ca97dd9 Mon Sep 17 00:00:00 2001 From: Joan Date: Tue, 5 Nov 2024 10:05:30 +0100 Subject: [PATCH 04/50] Corrected error message --- configuration/build_runshfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/build_runshfile.py b/configuration/build_runshfile.py index 448befa..e286ded 100644 --- a/configuration/build_runshfile.py +++ b/configuration/build_runshfile.py @@ -51,7 +51,7 @@ def process_models(data, output_file): def main(): if len(sys.argv) != 2: - print("Usage: python script.py ") + print("Usage: python3 "+sys.argv[0]+" ") sys.exit(1) yaml_file = sys.argv[1] From 573bf4a0447720286e784b05f16b48f31ba637d7 Mon Sep 17 00:00:00 2001 From: Joan Date: Tue, 5 Nov 2024 10:44:49 +0100 Subject: [PATCH 05/50] Adjusted the readme.md file --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 934370e..3e42104 100644 --- a/readme.md +++ b/readme.md @@ -80,7 +80,7 @@ The **master** provides a Dashboard to viazulize the results, and saves them to ``` [TODO: This suggest that all Pi's need a user with the name 'illuminator'] -6. Run the `buildcilentremoterun.py` file on each *client* and give all users execute permission to all the documents in `runshfile/` in order +6. Run the `build_runshfile.py` file in the configuration directory on *master* to generate a run.sh file and give all users execute permission to all the documents in `runshfile/` in order to make sure the leader can access the *client* model. ```shell From c8364a88e2e76a7c4e85ba71965312a85542a709 Mon Sep 17 00:00:00 2001 From: Joan Date: Tue, 5 Nov 2024 10:51:37 +0100 Subject: [PATCH 06/50] Corrected the readme.md file --- readme.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index 3e42104..f8045bd 100644 --- a/readme.md +++ b/readme.md @@ -54,6 +54,13 @@ The **master** provides a Dashboard to viazulize the results, and saves them to interface etho static ip_address=192.168.0.1/24 # change the IP address as you want ``` + + Give all users execute permission to all the documents in `runshfile/` in order to make sure the leader can access the *client* model. + + ```shell + chmod -R a+X *dir* + ``` + Finally, reboot the Raspberry Pi suing `sudo reboot` on the terminal. 3. [Configure SSH connections so that the *master* can connect to the *clients* without a password.](https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys-2) 4. Install the following Python packages. @@ -80,11 +87,10 @@ The **master** provides a Dashboard to viazulize the results, and saves them to ``` [TODO: This suggest that all Pi's need a user with the name 'illuminator'] -6. Run the `build_runshfile.py` file in the configuration directory on *master* to generate a run.sh file and give all users execute permission to all the documents in `runshfile/` in order -to make sure the leader can access the *client* model. +6. Run the `build_runshfile.py` file in the configuration directory on *master* to generate a run.sh. Give the appropiate yaml file for the model as input: ```shell - chmod -R a+X *dir* + python3 build_runshfile.py ``` More detialed instructions are given in the [user guide document](docs/user/user-guide.md) and the [model build up document](Models.md). From 33a0debbabdeb3e11acd90808907e9972bd50a62 Mon Sep 17 00:00:00 2001 From: Joan Date: Tue, 5 Nov 2024 10:53:19 +0100 Subject: [PATCH 07/50] Corrected the readme.md file --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index f8045bd..e54a178 100644 --- a/readme.md +++ b/readme.md @@ -87,7 +87,7 @@ The **master** provides a Dashboard to viazulize the results, and saves them to ``` [TODO: This suggest that all Pi's need a user with the name 'illuminator'] -6. Run the `build_runshfile.py` file in the configuration directory on *master* to generate a run.sh. Give the appropiate yaml file for the model as input: +6. Run the `build_runshfile.py` file in the configuration directory on *master* to generate a run.sh script. Give the appropiate yaml file for the model as input: ```shell python3 build_runshfile.py From 50e0e283552451f6bce17572cbd8091b37a8a5cc Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 17:31:35 +0100 Subject: [PATCH 08/50] Update cluster setup documentation to clarify server/client roles and improve instructions --- docs/cluster-setup.md | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md index 3fc123b..2c015f6 100644 --- a/docs/cluster-setup.md +++ b/docs/cluster-setup.md @@ -1,35 +1,41 @@ -# Cluster Pi Set Up +# Cluster Pi Setup - -## Raspberry Pi Setup - -The setup for the Illuminator requires one **master** Raspberry Pi and several **clients** Raspberry Pi's. +The setup for the Illuminator requires one Raspberry Pi acting as a **server** and several **clients** Raspberry Pi's. Raspberry Pi's must be connected and configured as a local network, and the -*master* must be configured to have permissions to access and control the *clients* through Secure Shell Protocol (SSH). +*server* must be configured to have permissions to access and control the *clients* through the Secure Shell Protocol (SSH). -During simulation, the *master* engage with the *clients* to run the simulations defined in the *simulation configuration*, and +During simulation, the *server* engage with the *clients* to run the simulations defined in the *simulation configuration*, and information is exchanged between Rasberry Pi's using network sockets. -The **master** provides a Dashboard to viazulize the results, and saves them to a `.csv` files for later analysis. +The **server** provides a Dashboard to viazulize the results, and saves them to a `.csv` files for later analysis.
- +
-### Set up a Raspberry Pi cluster +### Set up + +On every raspberrry Pi: -1. [Install Raspberry pi OS using Raspberry Pi imager.](https://www.raspberrypi.com/software/) -2. Set an static IP addresse for each Raspberry Pi. Use the following command on the terminal to open the `dhcpcd.conf` file: +1. [Install Raspberry pi OS using Raspberry Pi imager.](https://www.raspberrypi.com/software/). +2. Set an static IP address for each Raspberry Pi. Use the following command on the terminal to open the `dhcpcd.conf` file: ``` - sudo nano /etc/dhcpcd.conf + sudo nano /etc/dhcpcd.confß ``` - In the `dhcpcd.conf` file, find the information to change the IP address as static as following: + In the `dhcpcd.conf` file, find the information to change the IP address to static as following: ``` interface etho static ip_address=192.168.0.1/24 # change the IP address as you want ``` + + Give all users execute permission to all the documents in `runshfile/` in order to make sure the *server* can access the *client* model. + + ```shell + chmod -R a+X *dir* + ``` + Finally, reboot the Raspberry Pi suing `sudo reboot` on the terminal. 3. [Configure SSH connections so that the *master* can connect to the *clients* without a password.](https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys-2) 4. Install the following Python packages. @@ -56,16 +62,12 @@ The **master** provides a Dashboard to viazulize the results, and saves them to ``` [TODO: This suggest that all Pi's need a user with the name 'illuminator'] -6. Run the `buildcilentremoterun.py` file on each *client* and give all users execute permission to all the documents in `runshfile/` in order -to make sure the leader can access the *client* model. +6. Run the `build_runshfile.py` file in the configuration directory on *master* to generate a run.sh script. Give the appropiate yaml file for the model as input: ```shell - chmod -R a+X *dir* + python3 build_runshfile.py ``` -More detialed instructions are given in the [user guide document](docs/user/user-guide.md) and the [model build up document](Models.md). - - ## Contact and Support From d2a7db09055749f0a7e57748f18d34ad26e1e708 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 17:35:17 +0100 Subject: [PATCH 09/50] add comments --- src/illuminator/cli/state_space_simulator.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/illuminator/cli/state_space_simulator.py b/src/illuminator/cli/state_space_simulator.py index 1279b19..80664e0 100644 --- a/src/illuminator/cli/state_space_simulator.py +++ b/src/illuminator/cli/state_space_simulator.py @@ -2,6 +2,8 @@ import importlib.util class StateSpaceSimulator(mosaik_api_v3.Simulator): + + #[Manuel] # this can be standardized and must be provided by the model class meta = { 'models': { 'StateSpaceModel': { @@ -35,6 +37,7 @@ def init(self, sid, time_resolution, **sim_params): self.model = self.load_model_class(self.model_data['model_path'], self.model_data['model_type']) return self.model_data['meta'] + def load_model_class(self, model_path, model_type): # Get the module name from the model path, e.g., 'Models/adder_model' -> 'adder_model' module_name = model_path.replace('/', '.').rstrip('.py').split('.')[-1] @@ -49,6 +52,7 @@ def load_model_class(self, model_path, model_type): return model_class + # [Manuel] # this can be standardized? def create(self, num, model, **model_params): if num != 1: raise ValueError("Can only create one instance of the model.") @@ -71,6 +75,7 @@ def create(self, num, model, **model_params): return entities + # [Manuel] most be implemented by the model class def step(self, time, inputs, max_advance): # Update inputs for eid, entity_inputs in inputs.items(): @@ -84,6 +89,8 @@ def step(self, time, inputs, max_advance): return time + self.model_data['step_size'] + + # [Manuel] this can be standardized def get_data(self, outputs): data = {} From 469c8811c4ad251c578a8ec10e21544295fdd6d2 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 17:35:44 +0100 Subject: [PATCH 10/50] remove user guide from documentation --- docs/user/{user-guide.md => depricated_user-guide.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/user/{user-guide.md => depricated_user-guide.md} (100%) diff --git a/docs/user/user-guide.md b/docs/user/depricated_user-guide.md similarity index 100% rename from docs/user/user-guide.md rename to docs/user/depricated_user-guide.md From 6eb08ff404dd2fe054cb81f2f76bc0dd8161ccc8 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 17:35:57 +0100 Subject: [PATCH 11/50] Refactor AdderModel step method to accept time parameter and print result --- examples/adder_model.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/examples/adder_model.py b/examples/adder_model.py index 8dd2349..be955ba 100644 --- a/examples/adder_model.py +++ b/examples/adder_model.py @@ -19,15 +19,29 @@ # construct the model class AdderModel(ModelConstructor): - def step(self) -> None: + def step(self, time) -> None: self._model.outputs["out1"] = self._model.inputs["in1"] + self._model.inputs["in2"] - return self._model.outputs["out1"] + print("result", self._model.outputs["out1"]) + + return time + self._model.time_step_size # todo: adopt get_data() as defined by Mosaik if __name__ == '__main__': + + import mosaik + # Sim config + SIM_CONFIG: mosaik.SimConfig = { + 'ExampleSim': { + 'python': 'simulator_mosaik:ExampleSim', + }, + 'Collector': { + 'cmd': '%(python)s collector.py %(addr)s', + }, + } + END = 10 # 10 seconds # [Manuel] this can be standardized # Create a model by inheriting from ModelConstructor # and implementing the step method adder_model = AdderModel(adder) - print(adder_model.step()) \ No newline at end of file + adder_model.step(time=1) \ No newline at end of file From e0391846794af2d60fd5296c527548eccc4410fe Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 17:36:10 +0100 Subject: [PATCH 12/50] Refactor builder_model_example.py to improve code readability and comment out unused PV model code --- examples/builder_model_example.py | 127 +++++++++++++++--------------- 1 file changed, 64 insertions(+), 63 deletions(-) diff --git a/examples/builder_model_example.py b/examples/builder_model_example.py index 75324ed..9459e9f 100644 --- a/examples/builder_model_example.py +++ b/examples/builder_model_example.py @@ -22,6 +22,7 @@ def step(self) -> None: # step 3: create an instance of the model battery_model = BatteryModel(battery) +print(battery_model) # step 4: properties can be overriden battery.inputs["voltage"] = 300 @@ -31,66 +32,66 @@ def step(self) -> None: pass -# New PV model -import pandas as pd -import itertools - -pv_adapter = IlluminatorModel( - parameters={"panel_data": None, - 'm_tilt': None, - 'm_az': None, - 'cap': None, - 'sim_start': None, - 'output_type': None - }, - inputs={ - 'G_Gh': None, - 'G_Dh': None, - 'G_Bn': None, - 'Ta': None, - 'hs': None, - 'FF': None, - 'Az': None - }, - - outputs={"pv_gen": None, "total_irr": None}, - states={"voltage": 10} -) - - -class NewPvAdapter(ModelConstructor): - - def step(self, time, inputs) -> None: - # in this method, we call the python file at every data interval and perform the calculations. - current_time = (self.start + pd.Timedelta(time * self.time_resolution, - unit='seconds')) # timedelta represents a duration of time - print('from pv %%%%%%%%%', current_time) - # print('#inouts: ', inputs) - for eid, attrs in inputs.items(): - # print('#eid: ', eid) - # print('#attrs: ', attrs) - # and relate it with the information in mosaik document. - v = [] # we create this empty list to hold all the input values we want to give since we have more than 2 - for attr, vals in attrs.items(): - - # print('#attr: ', attr) - # print('#vals: ', vals) - # inputs is a dictionary, which contains another dictionary. - # value of U is a list. we need to combine all the values into a single list. But is we just simply - # append them in v, we have a nested list, hence just 1 list. that creates a problem as it just - # gives all 7 values to only sun_az in the python model and we get an error that other 6 values are missing. - u = list(vals.values()) - # print('#u: ', u) - v.append(u) # we append every value of u to v from this command. - # print('#v: ', v) - - # the following code helps us to convert the nested list into a simple plain list and we can use that simply - v_merged = list(itertools.chain(*v)) - # print('#v_merged: ', v_merged) - self._cache[eid] = self.model_entities[eid].connect(v_merged[0], v_merged[1], v_merged[2], v_merged[3], - v_merged[4], v_merged[5], v_merged[6]) # PV1 - # print(self._cache) - # print('# cache[eid]: ', self._cache[eid]) - # the following code desnt work because it just put one value 7 times :/! Dumb move - # self._cache[eid] = self.entities[eid].connect(u, u, u, u, u, u, u) - return None \ No newline at end of file +# # New PV model +# import pandas as pd +# import itertools + +# pv_adapter = IlluminatorModel( +# parameters={"panel_data": None, +# 'm_tilt': None, +# 'm_az': None, +# 'cap': None, +# 'sim_start': None, +# 'output_type': None +# }, +# inputs={ +# 'G_Gh': None, +# 'G_Dh': None, +# 'G_Bn': None, +# 'Ta': None, +# 'hs': None, +# 'FF': None, +# 'Az': None +# }, + +# outputs={"pv_gen": None, "total_irr": None}, +# states={"voltage": 10} +# ) + + +# class NewPvAdapter(ModelConstructor): + +# def step(self, time, inputs) -> None: +# # in this method, we call the python file at every data interval and perform the calculations. +# current_time = (self.start + pd.Timedelta(time * self.time_resolution, +# unit='seconds')) # timedelta represents a duration of time +# print('from pv %%%%%%%%%', current_time) +# # print('#inouts: ', inputs) +# for eid, attrs in inputs.items(): +# # print('#eid: ', eid) +# # print('#attrs: ', attrs) +# # and relate it with the information in mosaik document. +# v = [] # we create this empty list to hold all the input values we want to give since we have more than 2 +# for attr, vals in attrs.items(): + +# # print('#attr: ', attr) +# # print('#vals: ', vals) +# # inputs is a dictionary, which contains another dictionary. +# # value of U is a list. we need to combine all the values into a single list. But is we just simply +# # append them in v, we have a nested list, hence just 1 list. that creates a problem as it just +# # gives all 7 values to only sun_az in the python model and we get an error that other 6 values are missing. +# u = list(vals.values()) +# # print('#u: ', u) +# v.append(u) # we append every value of u to v from this command. +# # print('#v: ', v) + +# # the following code helps us to convert the nested list into a simple plain list and we can use that simply +# v_merged = list(itertools.chain(*v)) +# # print('#v_merged: ', v_merged) +# self._cache[eid] = self.model_entities[eid].connect(v_merged[0], v_merged[1], v_merged[2], v_merged[3], +# v_merged[4], v_merged[5], v_merged[6]) # PV1 +# # print(self._cache) +# # print('# cache[eid]: ', self._cache[eid]) +# # the following code desnt work because it just put one value 7 times :/! Dumb move +# # self._cache[eid] = self.entities[eid].connect(u, u, u, u, u, u, u) +# return None \ No newline at end of file From 2d68ac1755b70ea7c83ef6dd4753b1bbbb12a886 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 17:36:18 +0100 Subject: [PATCH 13/50] Update adder.yaml to change model type from AdderModel to Adder for consistency --- examples/adder.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/adder.yaml b/examples/adder.yaml index be83701..818c3ce 100644 --- a/examples/adder.yaml +++ b/examples/adder.yaml @@ -5,7 +5,7 @@ scenario: time_resolution: 900 # time step in seconds. Defaults to 15 minutes (900 s) models: # list of models for the energy network - name: Adder1 # name for the model (must be unique) - type: AdderModel # most match the class inheriting from IlluminatorConstructor + type: Adder # most match the class inheriting from IlluminatorConstructor inputs: in1: 10 # input-name: initial value, default value will be used if not defined in2: 20 @@ -14,7 +14,7 @@ models: # list of models for the energy network parameters: param1: "adding tens" - name: Adder2 - type: AdderModel # models can reuse the same type + type: Adder # models can reuse the same type inputs: in1: 100 # input-name: initial value, default value will be used if not defined in2: 200 From 9a2a58996331404697e217ec3add8e71a3721630 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 17:50:43 +0100 Subject: [PATCH 14/50] Update documentation structure: remove deprecated user guide, rename start.md to set-up.md, and enhance setup instructions --- docs/conf.py | 2 +- docs/developer/{start.md => set-up.md} | 4 ++-- .../{depricated_user-guide.md => depricated-user-guide.md} | 0 docs/user/start.md | 3 --- 4 files changed, 3 insertions(+), 6 deletions(-) rename docs/developer/{start.md => set-up.md} (89%) rename docs/user/{depricated_user-guide.md => depricated-user-guide.md} (100%) delete mode 100644 docs/user/start.md diff --git a/docs/conf.py b/docs/conf.py index 8fc8700..14cb71c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,7 +38,7 @@ ] templates_path = ['_templates'] -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'user/depricated-user-guide.md'] diff --git a/docs/developer/start.md b/docs/developer/set-up.md similarity index 89% rename from docs/developer/start.md rename to docs/developer/set-up.md index 8bc40d3..8f21749 100644 --- a/docs/developer/start.md +++ b/docs/developer/set-up.md @@ -1,10 +1,10 @@ # Set Up Environment +Follow these steps to set up a development environment. **Requirements** * Python >= 3.11 * Recent version of PIP - To set up a development environment: 1. Clone the repository: @@ -29,7 +29,7 @@ pip install -e .e We use `Pytest` to write and test the source code. To run all unit-tests, run the following command at the root of the repository: -``` +```shell pytest tests/ ``` diff --git a/docs/user/depricated_user-guide.md b/docs/user/depricated-user-guide.md similarity index 100% rename from docs/user/depricated_user-guide.md rename to docs/user/depricated-user-guide.md diff --git a/docs/user/start.md b/docs/user/start.md deleted file mode 100644 index eea37d6..0000000 --- a/docs/user/start.md +++ /dev/null @@ -1,3 +0,0 @@ -# User Documentation - -> Comming soon... \ No newline at end of file From 25652b8b5b45f0f240aab6cc3346b799b880f1f1 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 17:52:07 +0100 Subject: [PATCH 15/50] Remove deprecated user guide references from documentation index --- docs/index.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 5b90f5a..e7810ac 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,8 +25,6 @@ and the simulation engine is based on `Mosaik. `_ :maxdepth: 2 :caption: User's Documentation - user/start - user/user-guide user/models .. toctree:: From a2f603162db5ea5af10f557f8a04339440a3d4ac Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 17:54:13 +0100 Subject: [PATCH 16/50] Add developer/set-up.md to documentation index --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index e7810ac..5e8cf0f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,6 +37,7 @@ and the simulation engine is based on `Mosaik. `_ :maxdepth: 2 :caption: Developer's Documentation + developer/set-up.md developer/developer-docstrings.md developer/testing-explained.md developer/writing-tests.md From 163806a1e5c0e56bc0ecf3cc30f5b84406ab1cf4 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 18:46:49 +0100 Subject: [PATCH 17/50] Update cluster-setup.md and set-up.md for clarity and accuracy in setup instructions --- docs/cluster-setup.md | 47 ++++++++++++++++++++-------------------- docs/developer/set-up.md | 4 +--- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md index 2c015f6..bb75ff9 100644 --- a/docs/cluster-setup.md +++ b/docs/cluster-setup.md @@ -13,19 +13,20 @@ The **server** provides a Dashboard to viazulize the results, and saves them to -### Set up +## Set up On every raspberrry Pi: 1. [Install Raspberry pi OS using Raspberry Pi imager.](https://www.raspberrypi.com/software/). 2. Set an static IP address for each Raspberry Pi. Use the following command on the terminal to open the `dhcpcd.conf` file: - ``` - sudo nano /etc/dhcpcd.confß + + ```shell + sudo nano /etc/dhcpcd.conf ``` In the `dhcpcd.conf` file, find the information to change the IP address to static as following: - ``` + ```shell interface etho static ip_address=192.168.0.1/24 # change the IP address as you want ``` @@ -38,34 +39,32 @@ On every raspberrry Pi: Finally, reboot the Raspberry Pi suing `sudo reboot` on the terminal. 3. [Configure SSH connections so that the *master* can connect to the *clients* without a password.](https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys-2) -4. Install the following Python packages. + +4. Install the Illuminator Python package, and the addional dependencies: + + ```shell + # if connected to the internet + pip install illuminator + + # or, if from source code + pip install Illuminator/ ``` - pandas - tk - python-csv - datetime - python-math - numpy - scipy - arrow - mosaik - mosaik_api - mosaik.util - wandb - matplotlib - itertools + + ```shell + # aditional dependencies + pip install tk python-csv python-math scipy wandb itertools ``` -5. Send the Illuminator package [TODO: What is the illuminator package?] to all *clients*. Use the following command on the *master's* terminal to check the connection between *master* and the *clients* +5. Use the following command on the *master's* terminal to check the connection between *master* and the *clients* ```shell + # notice that the followng assumes that each client has a + # user named 'illuminator' ssh illuminator@ip #ip represent your follower IP address ``` - [TODO: This suggest that all Pi's need a user with the name 'illuminator'] - -6. Run the `build_runshfile.py` file in the configuration directory on *master* to generate a run.sh script. Give the appropiate yaml file for the model as input: +6. Run the `build_runshfile.py` file in the configuration directory on *server*, this will generate a to generate a `run.sh` script. Give the appropiate `config.yaml` file containing the simulation scenario definition: ```shell - python3 build_runshfile.py + python3 build_runshfile.py ``` diff --git a/docs/developer/set-up.md b/docs/developer/set-up.md index 8f21749..80ab7a2 100644 --- a/docs/developer/set-up.md +++ b/docs/developer/set-up.md @@ -5,8 +5,6 @@ Follow these steps to set up a development environment. * Python >= 3.11 * Recent version of PIP -To set up a development environment: - 1. Clone the repository: ```shell @@ -22,7 +20,7 @@ cd Illuminator/ 3. install the development dependencies in editable mode: ```shell -pip install -e .e +pip install -e . ``` ## Running Unit Tests From 12b07a5e07ac3b578da3a2800d749e67448ab8ad Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 18:55:24 +0100 Subject: [PATCH 18/50] create cluster package --- src/illuminator/cluster/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/illuminator/cluster/__init__.py diff --git a/src/illuminator/cluster/__init__.py b/src/illuminator/cluster/__init__.py new file mode 100644 index 0000000..e69de29 From 52227415d76610e20f30a291ad61ebd1c021a6e0 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 20:24:23 +0100 Subject: [PATCH 19/50] Refactor CLI to use Typer for improved command handling and add scenario and cluster subcommands --- src/illuminator/cli/main.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/illuminator/cli/main.py b/src/illuminator/cli/main.py index eb17281..a2c4728 100644 --- a/src/illuminator/cli/main.py +++ b/src/illuminator/cli/main.py @@ -3,16 +3,22 @@ By: M. Rom & M. Garcia Alvarez """ -import argparse +import typer +from typing_extensions import Annotated import illuminator.engine as engine -def main(): - parser = argparse.ArgumentParser(description='Run the simulation with the specified scenario file.') - parser.add_argument('file_path', nargs='?', default='simple_test2.yaml', - help='Path to the scenario file. [Default: config.yaml]') - args = parser.parse_args() - file_path = args.file_path +app = typer.Typer() +scenario_app = typer.Typer() +app.add_typer(scenario_app, name="scenario", help="Run a simulation scenario.") +cluster_app = typer.Typer() +app.add_typer(cluster_app, name="cluster", help="Utilities for a RaspberryPi cluster.") + + +@scenario_app.command("run") +def scenario_run(config: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): + + file_path = config # load and validate configuration file config = engine.validate_config_data(file_path) @@ -64,6 +70,10 @@ def main(): print(f"Running simulation from") world.run(until=mosaik_end_time) -if __name__ == '__main__': +@cluster_app.command("build") +def cluster_build(config: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): + print('building sh') + - main() +if __name__ == "__main__": + app() \ No newline at end of file From 9fa3a60c56793b56967f894ab81fe67a305ff89d Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 20:38:26 +0100 Subject: [PATCH 20/50] Enhance CLI help messages for scenario and cluster commands --- src/illuminator/cli/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/illuminator/cli/main.py b/src/illuminator/cli/main.py index a2c4728..efff767 100644 --- a/src/illuminator/cli/main.py +++ b/src/illuminator/cli/main.py @@ -10,13 +10,14 @@ app = typer.Typer() scenario_app = typer.Typer() -app.add_typer(scenario_app, name="scenario", help="Run a simulation scenario.") +app.add_typer(scenario_app, name="scenario", help="Run simulation scenarios.") cluster_app = typer.Typer() app.add_typer(cluster_app, name="cluster", help="Utilities for a RaspberryPi cluster.") @scenario_app.command("run") def scenario_run(config: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): + "Runs a simulation scenario using a YAML file." file_path = config @@ -70,8 +71,10 @@ def scenario_run(config: Annotated[str, typer.Argument(help="Path to scenario co print(f"Running simulation from") world.run(until=mosaik_end_time) + @cluster_app.command("build") def cluster_build(config: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): + """Builds the run.sh files for a cluster of Raspberry Pi's.""" print('building sh') From e04334968b7262a8c24c5379dd987d0f643e707a Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Wed, 6 Nov 2024 20:39:21 +0100 Subject: [PATCH 21/50] Update pyproject.toml to add Typer dependency and modify entry point for CLI --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4cb195..7deca27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "mosaik>=3.3.3", "ruamel.yaml>= 0.18.5", "schema>=0.7.7", + "typer>=0.12.5" ] keywords = [ "energy", @@ -55,7 +56,7 @@ dev = [ ] [project.scripts] -illuminator = "illuminator.cli.main:main" +illuminator = "illuminator.cli.main:app" [project.urls] Documentation = "https://illuminator-team.github.io/Illuminator/" From 0504fc474eb0df410875b9ede1a3be931f116def Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 06:55:34 +0100 Subject: [PATCH 22/50] clean up test_engine.py --- tests/test_engine.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 4d8048f..4594e83 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -31,9 +31,6 @@ def csv_model(): } } -# CONTINUE FROM HERE -# ADD example data to the data.csv file - @pytest.fixture def yaml_models(): return [{'name': 'CSVB', # this name must match the name in the mosaik configuration @@ -83,8 +80,6 @@ def test_approximation_to_floor(self, start_timestamp): assert computed_steps == 17 - - class TestStartSimulators: """ Tests for the start_simulators function. @@ -123,5 +118,3 @@ def test_datafile_value_error(self, mosaik_world, yaml_models): with pytest.raises(ValueError): start_simulators(mosaik_world, yaml_models) - - From 07f30845e96a9f6642eb0c37e8386e1d7adff66f Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 06:56:14 +0100 Subject: [PATCH 23/50] Remove unused validate_config_data function and its YAML import --- src/illuminator/engine.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/illuminator/engine.py b/src/illuminator/engine.py index 60ae5b4..8d9230d 100644 --- a/src/illuminator/engine.py +++ b/src/illuminator/engine.py @@ -8,7 +8,6 @@ import importlib.util from mosaik.scenario import Entity as MosaikEntity from mosaik.scenario import World as MosaikWorld -from ruamel.yaml import YAML from illuminator.schemas.simulation import schema from datetime import datetime @@ -34,17 +33,6 @@ def create_world(sim_config: dict, time_resolution: int) -> MosaikWorld: return world -def validate_config_data(config_file: str) -> dict: - """Returns the content of an scenerio file writtent in YAML after - validating its content against the Illuminator's Schema. - """ - - _file = open(config_file, 'r') - yaml = YAML(typ='safe') - data = yaml.load(_file) - return schema.validate(data) - - def get_collector_path() -> str: """Returns the path to the default collector script.""" From a3202baa3cc37dadb3e376518659f00268bd732a Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 06:56:33 +0100 Subject: [PATCH 24/50] Add validate_config_data function for YAML configuration validation --- src/illuminator/schemas/simulation.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/illuminator/schemas/simulation.py b/src/illuminator/schemas/simulation.py index 7231cc6..a56e7d4 100644 --- a/src/illuminator/schemas/simulation.py +++ b/src/illuminator/schemas/simulation.py @@ -6,6 +6,7 @@ import datetime import re import os +from ruamel.yaml import YAML from schema import Schema, And, Use, Regex, Optional, SchemaError # valid format for start and end times: YYYY-MM-DD HH:MM:SS" @@ -50,6 +51,29 @@ def validate_directory_path(file_path: str) -> str: return file_path +def validate_config_data(config_file: str) -> dict: + """Returns the content of an scenerio file written as YAML after + validating its content against the Illuminator's Schema. + + Parameters + ---------- + config_file : str + The path to the YAML configuration file. + + Returns + ------- + dict + The content of the configuration file as a dictionary. + """ + + _file = open(config_file, 'r') + yaml = YAML(typ='safe') + data = yaml.load(_file) + + return schema.validate(data) + + + class ScenarioSchema(Schema): """ A schema for validating the scenario section of the simulation From 54c81fef6fcb0a9c303c3d53d900eb2a548cb739 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 06:58:05 +0100 Subject: [PATCH 25/50] Remove unused import of schema from illuminator.schemas.simulation in engine.py --- src/illuminator/engine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/illuminator/engine.py b/src/illuminator/engine.py index 8d9230d..d2ccc31 100644 --- a/src/illuminator/engine.py +++ b/src/illuminator/engine.py @@ -8,7 +8,6 @@ import importlib.util from mosaik.scenario import Entity as MosaikEntity from mosaik.scenario import World as MosaikWorld -from illuminator.schemas.simulation import schema from datetime import datetime From 7ac78e61fc1c1b2304dc8ce531ce832ced0db67f Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 07:06:27 +0100 Subject: [PATCH 26/50] Rename schemas directory to 'schema' and update validate_config_data to support JSON output --- src/illuminator/{schemas => schema}/__init__.py | 0 src/illuminator/{schemas => schema}/config.yaml | 0 src/illuminator/{schemas => schema}/scenario_data.csv | 0 .../{schemas => schema}/simulation.example.yaml | 0 src/illuminator/{schemas => schema}/simulation.py | 8 +++++++- src/illuminator/{schemas => schema}/validate_yaml.py | 7 ++++--- 6 files changed, 11 insertions(+), 4 deletions(-) rename src/illuminator/{schemas => schema}/__init__.py (100%) rename src/illuminator/{schemas => schema}/config.yaml (100%) rename src/illuminator/{schemas => schema}/scenario_data.csv (100%) rename src/illuminator/{schemas => schema}/simulation.example.yaml (100%) rename src/illuminator/{schemas => schema}/simulation.py (96%) rename src/illuminator/{schemas => schema}/validate_yaml.py (68%) diff --git a/src/illuminator/schemas/__init__.py b/src/illuminator/schema/__init__.py similarity index 100% rename from src/illuminator/schemas/__init__.py rename to src/illuminator/schema/__init__.py diff --git a/src/illuminator/schemas/config.yaml b/src/illuminator/schema/config.yaml similarity index 100% rename from src/illuminator/schemas/config.yaml rename to src/illuminator/schema/config.yaml diff --git a/src/illuminator/schemas/scenario_data.csv b/src/illuminator/schema/scenario_data.csv similarity index 100% rename from src/illuminator/schemas/scenario_data.csv rename to src/illuminator/schema/scenario_data.csv diff --git a/src/illuminator/schemas/simulation.example.yaml b/src/illuminator/schema/simulation.example.yaml similarity index 100% rename from src/illuminator/schemas/simulation.example.yaml rename to src/illuminator/schema/simulation.example.yaml diff --git a/src/illuminator/schemas/simulation.py b/src/illuminator/schema/simulation.py similarity index 96% rename from src/illuminator/schemas/simulation.py rename to src/illuminator/schema/simulation.py index a56e7d4..e2e8781 100644 --- a/src/illuminator/schemas/simulation.py +++ b/src/illuminator/schema/simulation.py @@ -6,6 +6,7 @@ import datetime import re import os +import json from ruamel.yaml import YAML from schema import Schema, And, Use, Regex, Optional, SchemaError @@ -51,7 +52,7 @@ def validate_directory_path(file_path: str) -> str: return file_path -def validate_config_data(config_file: str) -> dict: +def validate_config_data(config_file: str, json:bool=False) -> dict | str: """Returns the content of an scenerio file written as YAML after validating its content against the Illuminator's Schema. @@ -59,6 +60,8 @@ def validate_config_data(config_file: str) -> dict: ---------- config_file : str The path to the YAML configuration file. + json : bool + If True, the content of the configuration file is returned as a JSON string. Returns ------- @@ -69,6 +72,9 @@ def validate_config_data(config_file: str) -> dict: _file = open(config_file, 'r') yaml = YAML(typ='safe') data = yaml.load(_file) + + if json: + return json.dumps(data, indent=4) return schema.validate(data) diff --git a/src/illuminator/schemas/validate_yaml.py b/src/illuminator/schema/validate_yaml.py similarity index 68% rename from src/illuminator/schemas/validate_yaml.py rename to src/illuminator/schema/validate_yaml.py index 5ab669b..b9360f8 100644 --- a/src/illuminator/schemas/validate_yaml.py +++ b/src/illuminator/schema/validate_yaml.py @@ -1,10 +1,10 @@ from ruamel.yaml import YAML import sys import json -from illuminator.schemas.simulation import schema +from illuminator.schema.simulation import schema # this if a yaml file complies with the modelling schema -_file = open('simple_test.yaml', 'r') +_file = open('simple_test2.yaml', 'r') yaml = YAML(typ='safe') data = yaml.load(_file) @@ -13,5 +13,6 @@ print(val) json_data = json.dumps(data, indent=4) -print(json_data) + +print(type(json_data)) From 1a316f2a562539d4b34c9cf22aec5e061e46804b Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 07:07:16 +0100 Subject: [PATCH 27/50] Remove validate_yaml.py as it is no longer needed --- src/illuminator/schema/validate_yaml.py | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 src/illuminator/schema/validate_yaml.py diff --git a/src/illuminator/schema/validate_yaml.py b/src/illuminator/schema/validate_yaml.py deleted file mode 100644 index b9360f8..0000000 --- a/src/illuminator/schema/validate_yaml.py +++ /dev/null @@ -1,18 +0,0 @@ -from ruamel.yaml import YAML -import sys -import json -from illuminator.schema.simulation import schema - -# this if a yaml file complies with the modelling schema -_file = open('simple_test2.yaml', 'r') - -yaml = YAML(typ='safe') -data = yaml.load(_file) - -val = schema.validate(data) -print(val) - -json_data = json.dumps(data, indent=4) - -print(type(json_data)) - From 94f7426ba191ef2f166e450a582a53f14108b4f1 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 07:25:07 +0100 Subject: [PATCH 28/50] rename alidate_config_data() to load_config_file() --- src/illuminator/schema/simulation.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/illuminator/schema/simulation.py b/src/illuminator/schema/simulation.py index e2e8781..55a2063 100644 --- a/src/illuminator/schema/simulation.py +++ b/src/illuminator/schema/simulation.py @@ -52,7 +52,7 @@ def validate_directory_path(file_path: str) -> str: return file_path -def validate_config_data(config_file: str, json:bool=False) -> dict | str: +def load_config_file(config_file: str, json:bool=False) -> dict | str: """Returns the content of an scenerio file written as YAML after validating its content against the Illuminator's Schema. @@ -69,17 +69,23 @@ def validate_config_data(config_file: str, json:bool=False) -> dict | str: The content of the configuration file as a dictionary. """ - _file = open(config_file, 'r') - yaml = YAML(typ='safe') - data = yaml.load(_file) - + try: + with open(config_file, 'r') as _file: + yaml = YAML(typ='safe') + data = yaml.load(_file) + except FileNotFoundError: + print(f"Error: The file {config_file} was not found.") + return None + # except yaml.YAMLError as exc: + # print(f"Error while parsing YAML: {exc}") + # return None + if json: return json.dumps(data, indent=4) return schema.validate(data) - class ScenarioSchema(Schema): """ A schema for validating the scenario section of the simulation From f0b102f6c81c8c417918c71791027664b8b60f7a Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 07:59:42 +0100 Subject: [PATCH 29/50] Update function to load config from file --- src/illuminator/schema/simulation.py | 74 +++++++++++++++------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/src/illuminator/schema/simulation.py b/src/illuminator/schema/simulation.py index 55a2063..b9c9cf7 100644 --- a/src/illuminator/schema/simulation.py +++ b/src/illuminator/schema/simulation.py @@ -8,7 +8,7 @@ import os import json from ruamel.yaml import YAML -from schema import Schema, And, Use, Regex, Optional, SchemaError +from schema import Schema, And, Use, Regex, Optional, SchemaError, SchemaUnexpectedTypeError # valid format for start and end times: YYYY-MM-DD HH:MM:SS" valid_start_time = r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$' @@ -19,6 +19,44 @@ valid_model_item_format = r'^\w+\.\w+$' + +def load_config_file(config_file: str, json:bool=False) -> dict | str: + """Returns the content of an scenerio file written as YAML after + validating its content against the Illuminator's Schema. + + Parameters + ---------- + config_file : str + The path to the YAML configuration file. + json : bool + If True, the content of the configuration file is returned as a JSON string. + + Returns + ------- + dict + The content of the configuration file as a dictionary. + """ + + try: + with open(config_file, 'r') as _file: + yaml = YAML(typ='safe') + data = yaml.load(_file) + except FileNotFoundError: + print(f"Error: The file {config_file} was not found.") + return None + + try: + valid_data = schema.validate(data) + except SchemaUnexpectedTypeError as exc: + print(f"Error while parsing YAML file. \n Are you passing a file written in YAML?: {exc}") + return None + + if json: + return json.dumps(valid_data, indent=4) + + return valid_data + + def validate_model_item_format(items: list) -> list: """ Validates the monitor section of the simulation configuration file, as @@ -52,40 +90,6 @@ def validate_directory_path(file_path: str) -> str: return file_path -def load_config_file(config_file: str, json:bool=False) -> dict | str: - """Returns the content of an scenerio file written as YAML after - validating its content against the Illuminator's Schema. - - Parameters - ---------- - config_file : str - The path to the YAML configuration file. - json : bool - If True, the content of the configuration file is returned as a JSON string. - - Returns - ------- - dict - The content of the configuration file as a dictionary. - """ - - try: - with open(config_file, 'r') as _file: - yaml = YAML(typ='safe') - data = yaml.load(_file) - except FileNotFoundError: - print(f"Error: The file {config_file} was not found.") - return None - # except yaml.YAMLError as exc: - # print(f"Error while parsing YAML: {exc}") - # return None - - if json: - return json.dumps(data, indent=4) - - return schema.validate(data) - - class ScenarioSchema(Schema): """ A schema for validating the scenario section of the simulation From 3efa3011ff160fef62d7fe7bd689363e0ac4c5be Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 17:26:36 +0100 Subject: [PATCH 30/50] Fix typos and update terminology in cluster setup documentation --- docs/cluster-setup.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md index bb75ff9..510e547 100644 --- a/docs/cluster-setup.md +++ b/docs/cluster-setup.md @@ -37,8 +37,8 @@ On every raspberrry Pi: chmod -R a+X *dir* ``` - Finally, reboot the Raspberry Pi suing `sudo reboot` on the terminal. -3. [Configure SSH connections so that the *master* can connect to the *clients* without a password.](https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys-2) + Finally, reboot the Raspberry Pi using `sudo reboot` on the terminal. +3. [Configure SSH connections so that the *server* can connect to the *clients* without a password.](https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys-2) 4. Install the Illuminator Python package, and the addional dependencies: @@ -54,14 +54,14 @@ On every raspberrry Pi: # aditional dependencies pip install tk python-csv python-math scipy wandb itertools ``` -5. Use the following command on the *master's* terminal to check the connection between *master* and the *clients* +5. Use the following command on the *server's* terminal to check the connection between *server* and the *clients* ```shell # notice that the followng assumes that each client has a # user named 'illuminator' ssh illuminator@ip #ip represent your follower IP address ``` -6. Run the `build_runshfile.py` file in the configuration directory on *server*, this will generate a to generate a `run.sh` script. Give the appropiate `config.yaml` file containing the simulation scenario definition: +6. Run the `build_runshfile.py` file in the configuration directory on the *server*, this will generate a `run.sh` script. Give the appropiate `config.yaml` file containing the simulation scenario definition: ```shell python3 build_runshfile.py From dd4cc0d0fdd084e9453ec56b0933eb1fe7be7c63 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 7 Nov 2024 17:27:02 +0100 Subject: [PATCH 31/50] Refactor scenario_run and cluster_build commands to use load_config_file and improve run.sh generation --- src/illuminator/cli/main.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/illuminator/cli/main.py b/src/illuminator/cli/main.py index efff767..7242895 100644 --- a/src/illuminator/cli/main.py +++ b/src/illuminator/cli/main.py @@ -6,7 +6,15 @@ import typer from typing_extensions import Annotated import illuminator.engine as engine +from pathlib import Path +from illuminator.cluster import build_runshfile +from illuminator.schema.simulation import load_config_file +APP_NAME = "illuminator" +DEFAULT_PORT = 5123 +RUN_PATH = './Desktop/illuminatorclient/configuration/runshfile/' +RUN_MODEL = '/home/illuminator/Desktop/Final_illuminator' +RUN_FILE = 'run.sh' app = typer.Typer() scenario_app = typer.Typer() @@ -14,12 +22,11 @@ cluster_app = typer.Typer() app.add_typer(cluster_app, name="cluster", help="Utilities for a RaspberryPi cluster.") - @scenario_app.command("run") -def scenario_run(config: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): +def scenario_run(config_file: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): "Runs a simulation scenario using a YAML file." - file_path = config + file_path = config_file # load and validate configuration file config = engine.validate_config_data(file_path) @@ -73,9 +80,19 @@ def scenario_run(config: Annotated[str, typer.Argument(help="Path to scenario co @cluster_app.command("build") -def cluster_build(config: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): +def cluster_build(config_file: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): """Builds the run.sh files for a cluster of Raspberry Pi's.""" - print('building sh') + app_dir = typer.get_app_dir(APP_NAME) + runsh_path: Path = Path(app_dir) / "runshfile" + + runsh_path.mkdir(parents=True, exist_ok=True) + + output_file = RUN_FILE + data = load_config_file(config_file) + + if data: + build_runshfile.process_models(data, output_file) + print(f"Commands have been written to {output_file}") if __name__ == "__main__": From 252b3daa79e4d3bc1b4dd849319f04e00e050638 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Mon, 11 Nov 2024 13:48:38 +0100 Subject: [PATCH 32/50] add TODO --- src/illuminator/builder/model.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/illuminator/builder/model.py b/src/illuminator/builder/model.py index 50023de..4e399a0 100644 --- a/src/illuminator/builder/model.py +++ b/src/illuminator/builder/model.py @@ -9,6 +9,13 @@ class SimulatorType(Enum): EVENT_BASED = 'event-based' HYBRID = 'hybrid' +# TOOD: IMPORTANT +# each model file must provide a way to run it remotely. In the current implementation, +# that is achieved by including a the folloowing code on the simulator file: +# def main(): +# mosaik_api.start_simulation(eboilerSim(), 'eboiler Simulator') +# if __name__ == "__main__": +# main() @dataclass class IlluminatorModel(): From 6af3162e07decc89c28419546aa9aa4aab2ea59a Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Mon, 11 Nov 2024 13:59:11 +0100 Subject: [PATCH 33/50] fix error with depricated validate_config_file() --- src/illuminator/cli/main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/illuminator/cli/main.py b/src/illuminator/cli/main.py index 7242895..4854241 100644 --- a/src/illuminator/cli/main.py +++ b/src/illuminator/cli/main.py @@ -29,7 +29,7 @@ def scenario_run(config_file: Annotated[str, typer.Argument(help="Path to scenar file_path = config_file # load and validate configuration file - config = engine.validate_config_data(file_path) + config = load_config_file(file_path) config = engine.apply_default_values(config) print("config: ", config['models']) @@ -96,4 +96,11 @@ def cluster_build(config_file: Annotated[str, typer.Argument(help="Path to scena if __name__ == "__main__": + + # import importlib.util + + # package_spec = importlib.util.find_spec("illuminator.models.Battery") + # print(package_spec.origin) + + app() \ No newline at end of file From 33c33291a9d25d83f0cc09713b2ae2f235fb7716 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Mon, 11 Nov 2024 17:30:03 +0100 Subject: [PATCH 34/50] Update cluster setup documentation for clarity and completeness --- docs/cluster-setup.md | 55 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/docs/cluster-setup.md b/docs/cluster-setup.md index 510e547..4c9c49e 100644 --- a/docs/cluster-setup.md +++ b/docs/cluster-setup.md @@ -1,24 +1,29 @@ # Cluster Pi Setup -The setup for the Illuminator requires one Raspberry Pi acting as a **server** and several **clients** Raspberry Pi's. +The Illuminator can be deployed to a cluster of Raspberry Pi's. It requires one Raspberry Pi acting as a **server** or 'master' and several **clients** Raspberry Pi's. Raspberry Pi's must be connected and configured as a local network, and the *server* must be configured to have permissions to access and control the *clients* through the Secure Shell Protocol (SSH). -During simulation, the *server* engage with the *clients* to run the simulations defined in the *simulation configuration*, and +During simulation, the *server* engage with the *clients* to run the simulations defined in the [simulation configuration file](user/config-file.md), and information is exchanged between Rasberry Pi's using network sockets. -The **server** provides a Dashboard to viazulize the results, and saves them to a `.csv` files for later analysis. +The **server** provides a Dashboard to visualize the results, and saves them to a `.csv` files for later analysis.
+## Hardware Requirements +- A Raspberry Pi to use as a *server*. +- One or more Raspebrry Pi's to use as *clients*. +- A networkw switch to connect all Rasberry Pi's as a local network. + ## Set up -On every raspberrry Pi: +Conduct the following steps on each Raspberry Pi to deploy the illuminator in the cluster. -1. [Install Raspberry pi OS using Raspberry Pi imager.](https://www.raspberrypi.com/software/). -2. Set an static IP address for each Raspberry Pi. Use the following command on the terminal to open the `dhcpcd.conf` file: +1. [Install Raspberry pi OS using Raspberry Pi imager.](https://www.raspberrypi.com/software/) +2. Set an static IP address for the Raspberry Pi. Use the following command on the terminal to open the `dhcpcd.conf` file: ```shell sudo nano /etc/dhcpcd.conf @@ -54,12 +59,12 @@ On every raspberrry Pi: # aditional dependencies pip install tk python-csv python-math scipy wandb itertools ``` -5. Use the following command on the *server's* terminal to check the connection between *server* and the *clients* +5. Use the following command on the *server's* terminal to check the connections for each of the *clients:* ```shell # notice that the followng assumes that each client has a # user named 'illuminator' - ssh illuminator@ip #ip represent your follower IP address + ssh illuminator@ip #ip represent your follower IP address set in step 2 ``` 6. Run the `build_runshfile.py` file in the configuration directory on the *server*, this will generate a `run.sh` script. Give the appropiate `config.yaml` file containing the simulation scenario definition: @@ -67,6 +72,40 @@ On every raspberrry Pi: python3 build_runshfile.py ``` +The `runs.sh` file contains a list of commands that will start the models required by a simulation defined in the `config.yaml`, such as: + +```shell +# add example +``` + +TODO: integrate this: + + The lx terminal command does the following: + +Lxterminal itself starts a terminal on a remote machine. + +So ‘lxterminal -e ssh illuminator@192.168.0.1’ would use SSH to login to machine 192.168.0.1 with the user illuminator which has no password (room for improvement 😉 ). + +The second part: + +'./Desktop/illuminatorclient/configuration/runshfile/runWind.sh 192.168.0.1 5123 /home/illuminator/Desktop/Final_illuminator'& + +Is starting the script ./Desktop/illuminatorclient/configuration/runshfile/runWind.sh on the remote machine with the following parameters: + +IP address 192.168.0.1 +Port 5123 +Path of mosaik /home/illuminator/Desktop/Final_illuminator + +The & at the end starts the process in the background, so that the run.sh script does not wait for the command to finish but executes the next command immediately. + +For example runWind.sh looks like this: + +#! /bin/bash +cd $3/Wind +python wind_mosaik.py $1:$2 –remote + +There you see the three parameters in action. + ## Contact and Support From 58017713ce7ab0009f21534342683e1860c61840 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Thu, 14 Nov 2024 20:57:04 +0100 Subject: [PATCH 35/50] Add simulation configuration file documentation with YAML structure and examples --- docs/user/config-file.md | 101 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 docs/user/config-file.md diff --git a/docs/user/config-file.md b/docs/user/config-file.md new file mode 100644 index 0000000..27e5ccc --- /dev/null +++ b/docs/user/config-file.md @@ -0,0 +1,101 @@ +# Simulation Configuration File + +Simulation scenarios for the *Illuminator* are define using configuration files written in YAML. The structure of a configuration must be as in the example below. + +A *simulation file* has four main sections: + +- `scenario`: defines metadata and global variable for the simulation. +- `models`: defines which models are included in the simulation. +- `connections`: defines how the models in the `models` section must be connected for a particular simulation. +- `monitor`: defines which inputs, outputs, and states of a particular model must be monitored and logged during simulation. These are consider the simulation results and they will be saved to the file in `results`. + +## Example + +See the table below a description of each keyword and their default values. Optinal keywords can be ommitted, in those case the defaults will be used. + + +```yaml +scenario: + name: "ExampleScenario" # a name for the simulation + start_time: '2012-01-02 00:00:00' # start and end times of the simulation + end_time: '2012-01-02 00:00:10' + time_resolution: 900 # duration of each step in second + results: './out.csv' # a file to store the results (optional) +models: # list of models for the energy network +- name: Battery1 # name for the model (must be unique) + type: Battery # name of the model registered in the Illuminator + inputs: + input1: 0 # input-name: initial value, default value will be used if not declaredd + outputs: + output1: 0 + output2: null + parameters: + charge_power_max: 100 + discharge_power_max: 200 + soc_min: 0.1 + soc_max: 0.9 + capacity: 1000 + states: + initial_soc: 0.5 + triggers: # list of triggers for the of another model??. It must be an input, output or state of the model + - capacity + - soc_min + scenario_data: './src/illuminator/schemas/scenario_data.csv' #path to the scenario data file for the model. This should be optional + connect: # this is optional. If not defined, the model will be assumed to be local + ip: 127.0.0.1 # Ip version 4 + port: 5000 # optional, if not defined, the default port will be used +- name: Battery2 + type: Battery # models can reuse the same type + inputs: + input1: 0 # input-name: initial value, default value will be used if not defined + outputs: + output1: 0 + output2: null + parameters: + charge_power_max: 100 + states: + soc: 0.5 # initial value for the state, optional +- name: PV1 + type: PVModel + inputs: + input1: 0 # input-name: initial value, default value will be used if not defined + input2: 0 + outputs: + output1: 0 + parameters: + power_max: 100 + states: + initial_soc: 0.5 +connections: +- from: Battery1.output2 # start model, pattern: model_name.output_name/input_name + to: PV1.input1 # end model +- from: Battery2.output + to: PV1.input2 +monitor: # a list of models, its inputs, output and states to be monitored and logged +- Battery1.input1 # pattern model_name.state_name +- Battery2.output1 # no duplicates allowed +- PV1.soc # pattern model_name.state_name +- PV1.output1 +``` + + +| Keyword | Description | Optional | Default | +|---------|-------------|----------|---------| +| **scenario:** | a set of global values for a simulation. | | | +|`name` | A name for the simulation, internally this name will be asssigned to what the Mosaik World created during runtime. | | | +| `start_time` | start time for the simulation. Must be a timestamp in ISO 8601 format | | | +| `end_time` | end time for the simulation. Must be a timestamp in ISO 8601 format. | | | +| `time_resolution` | number of seconds between simulation steps | ☑ | 900 (15 min) +| `results` | path to a CSV file to store the results of the simulation. The values in the `monitor` section determine which values are saved as results.| ☑ | a `out.csv` file saved to the current directory | +| **models:** | a list of models for the simulation | | | +| `name` | a name for the model. Must be unique in each simulation | | | +| `type` | type of model. This must correspond with the name of the model registered in the Illuminator. | | | +| `inputs` | a set of input-names and initial values for the model. The model types determine which names and values are applicable to each model, and they must be declared accordingly. At least one input per model must be declared | | If the value is set to `null`, the default value will be used. See the respective model type for details.| +| `outputs` | a set of output-names and initial values for the model. Similar to *inputs* valid names and values for each model are determined by the model *type*. See the respective model type for details. | | If the value is set to `null`, the default value will be used. | +| `parameters` | a set of name-value pairs for the model. Parameters declared constants for a model during runtime. | ☑ | If ommited, the default values will be used. See the respective model type for details. | +| `states` | a set of name-value pairs considered as states for the model. Values allow to modify the internal initial values of a state. | ☑ | If ommited, the default values will be used. See the respective model type for details. | +| `triggers` | names of inputs, output or states that are use as triggers for a particular model. Triggers can only be declaredd by models that implement the *event-based paradigm*. See the respective model type to know if it accepts triggers. | ☑ | | +| **connections:** | how models connect to each other. | | | +| `from` | origin of the connection declared as `.`. Input names use here must also appear as *inputs* in the models section. | | | +| `to` | destination of the connection declared as `.`. Output names use here must also appear as *outputs* in the models section. | | +| **monitor:** | a list of which inputs, outputs or states of the modles must be monitored and logged during runtime. These are consired *results* of the simulation and they will be saved to the file in `results`. Items must be declared as `.`, where *name* is an input, output or stated clared int the *models* section. No duplicated values are allowed | | | From d813a1162dc941426c0c0006360562857a7ca62a Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 10:37:17 +0100 Subject: [PATCH 36/50] remove results from simulation schema --- src/illuminator/schema/simulation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/illuminator/schema/simulation.py b/src/illuminator/schema/simulation.py index b9c9cf7..b87dad0 100644 --- a/src/illuminator/schema/simulation.py +++ b/src/illuminator/schema/simulation.py @@ -6,7 +6,7 @@ import datetime import re import os -import json +import json as json_module from ruamel.yaml import YAML from schema import Schema, And, Use, Regex, Optional, SchemaError, SchemaUnexpectedTypeError @@ -52,7 +52,7 @@ def load_config_file(config_file: str, json:bool=False) -> dict | str: return None if json: - return json.dumps(valid_data, indent=4) + return json_module.dumps(valid_data, indent=4) return valid_data @@ -128,7 +128,6 @@ def validate(self, data, _is_scenario_schema=True): # _is_scenario_schema Optional("time_resolution"): And(int, lambda n: n > 0, error="time resolution must be a " "positive integer"), - Optional("results"): And(str, len, error="results must be a non-empty string"), } ), "models": Schema( # a sequence of mappings From 652c69fa314bc2f0c1da43fdd2d63d82bbbfb856 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 10:37:51 +0100 Subject: [PATCH 37/50] update valid example for testing --- tests/schema/scenario.example.yaml | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/schema/scenario.example.yaml diff --git a/tests/schema/scenario.example.yaml b/tests/schema/scenario.example.yaml new file mode 100644 index 0000000..2d70b86 --- /dev/null +++ b/tests/schema/scenario.example.yaml @@ -0,0 +1,42 @@ +scenario: + name: "SimpleTest" # in mosaik so called world + start_time: '2012-01-01 00:00:00' # ISO 8601 start time of the simulation + end_time: '2012-01-01 01:00:00' # duration in seconds + time_resolution: 900 # time step in seconds (optional). Defaults to 15 minutes (900 s) +models: # list of models for the energy network +- name: CSVB # name for the model (must be unique) + type: CSV # name of the model registered in the Illuminator + parameters: # a CSV model must have a start time and a file as parameters + start: '2012-01-01 00:00:00' # ISO 8601 start time of the simulation + datafile: './tests/data/solar-sample.csv' # path to the file with the data +- name: PV + type: PvAdapter # models can reuse the same type + inputs: + G_Gh: null + G_Dh: null + G_Bn: null + Ta: null + hs: null + FF: null + Az: null + outputs: + G_Gh: null +connections: +- from: CSVB.G_Gh # start model, pattern: model_name.output_name/input_name + to: PV.G_Gh # end model +- from: CSVB.G_Dh + to: PV.G_Dh +- from: CSVB.G_Bn + to: PV.G_Bn +- from: CSVB.Ta + to: PV.Ta +- from: CSVB.hs + to: PV.hs +- from: CSVB.FF + to: PV.FF +- from: CSVB.Az + to: PV.Az +monitor: + file: './out.csv' # optional with default, path to the results file for the scenario. This should be optional # a list of models, its inputs, output and states to be monitored and logged + items: + - PV.pv_gen \ No newline at end of file From 32e0682f4d477a7c64a87f58d102a93cfce0d334 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 10:40:44 +0100 Subject: [PATCH 38/50] update unit tests for engine and simulation --- tests/schema/test_simulation.py | 17 +++++++++ tests/schemas/schema.example.yaml | 59 ------------------------------- tests/schemas/test_simulation.py | 18 ---------- tests/test_engine.py | 3 +- 4 files changed, 18 insertions(+), 79 deletions(-) create mode 100644 tests/schema/test_simulation.py delete mode 100644 tests/schemas/schema.example.yaml delete mode 100644 tests/schemas/test_simulation.py diff --git a/tests/schema/test_simulation.py b/tests/schema/test_simulation.py new file mode 100644 index 0000000..00d832f --- /dev/null +++ b/tests/schema/test_simulation.py @@ -0,0 +1,17 @@ +""" +Unit tests for the simulation.py of the schemas module. +""" + +from illuminator.schema.simulation import load_config_file +from ruamel.yaml import YAML + +SCENARIO_FILE = './tests/schema/scenario.example.yaml' + +def test_load_config_file(): + """Test if a YAML file contains a valid definition for a simulation scenario""" + + load_config_file(SCENARIO_FILE) + + +# TOOD: add tests to check if exceptions are raised + \ No newline at end of file diff --git a/tests/schemas/schema.example.yaml b/tests/schemas/schema.example.yaml deleted file mode 100644 index adc06e6..0000000 --- a/tests/schemas/schema.example.yaml +++ /dev/null @@ -1,59 +0,0 @@ -scenario: - name: "ExampleScenario" # in mosaik so called world - start_time: '2012-01-02 00:00:00' # ISO 8601 start time of the simulation - end_time: '2012-01-02 00:00:10' # duration in seconds -models: # list of models for the energy network -- name: Battery1 # name for the model (must be unique) - type: BatteryModel # name of the model registered in the Illuminator - inputs: - input1: 0 # input-name: initial value, default value will be used if not defined - outputs: - output1: 0 - output2: null - parameters: - charge_power_max: 100 - discharge_power_max: 200 - soc_min: 0.1 - soc_max: 0.9 - capacity: 1000 - states: - initial_soc: 0.5 - triggers: # list of triggers for the of another model??. It must be an input, output or state of the model - - capacity - - soc_min - scenario_data: '/Users/mgarciaalvarez/devel/Illuminator/src/illuminator/schemas/scenario_data.csv' #path to the scenario data file for the model. This should be optional - connect: # this is optional. If not defined, the model will be assumed to be local - ip: 127.0.0.1 # Ip version 4 or 6 - port: 5000 # optional, if not defined, the default port will be used -- name: Battery2 - type: BatteryModel # models can reuse the same type - inputs: - input1: 0 # input-name: initial value, default value will be used if not defined - outputs: - output1: 0 - output2: null - parameters: - charge_power_max: 100 - states: - soc: 0.5 # initial value for the state, optional -- name: PV1 - type: PVModel - inputs: - input1: 0 # input-name: initial value, default value will be used if not defined - input2: 0 - outputs: - output1: 0 - parameters: - power_max: 100 - states: - initial_soc: 0.5 -connections: -- from: Battery1.output2 # start model, pattern: model_name.output_name/input_name - to: PV1.input1 # end model -- from: Battery2.output - to: PV1.input2 -monitor: # a list of models, its inputs, output and states to be monitored and logged -- Battery1.input1 # pattern model_name.state_name -- Battery2.output1 # no duplicates allowed -- PV1.soc # pattern model_name.state_name -- PV1.output1 diff --git a/tests/schemas/test_simulation.py b/tests/schemas/test_simulation.py deleted file mode 100644 index 6f14d82..0000000 --- a/tests/schemas/test_simulation.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Unit tests for the simulation.py of the schemas module. -""" - -from illuminator.schemas.simulation import schema -from ruamel.yaml import YAML - -SCENARIO_FILE = open('./src/illuminator/schemas/simulation.example.yaml', 'r') - - -def test_valid_scenario_schema(): - """Test if a YAML file contains a valid definition for a simulation scenario""" - - yaml = YAML(typ='safe') - data = yaml.load(SCENARIO_FILE) - - schema.validate(data) - \ No newline at end of file diff --git a/tests/test_engine.py b/tests/test_engine.py index 4594e83..2667797 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -3,7 +3,6 @@ """ import pytest -import copy import mosaik from illuminator.engine import start_simulators, compute_mosaik_end_time @@ -36,7 +35,7 @@ def yaml_models(): return [{'name': 'CSVB', # this name must match the name in the mosaik configuration 'type': 'CSV', 'parameters': - {'start': '2012-01-01 00:00:00', 'datafile': 'tests/data/data.csv'}}, + {'start': '2012-01-01 00:00:00', 'datafile': 'tests/data/solar-sample.csv'}}, {'name': 'PV', 'type': 'PvAdapter', 'inputs': From 205a32ab41f5e1b1a7e91edd6f133f4e20a590a8 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 12:59:02 +0100 Subject: [PATCH 39/50] Update configuration file example with new scenario and model definitions --- docs/user/config-file.md | 124 ++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 66 deletions(-) diff --git a/docs/user/config-file.md b/docs/user/config-file.md index 27e5ccc..25fcb05 100644 --- a/docs/user/config-file.md +++ b/docs/user/config-file.md @@ -11,71 +11,58 @@ A *simulation file* has four main sections: ## Example -See the table below a description of each keyword and their default values. Optinal keywords can be ommitted, in those case the defaults will be used. +The following is an example to explain the basic format of a configuration file. +See the table below a description of each keyword and their default values. Optinal keywords can be ommitted, in those case the defaults will be used. ```yaml +# An example of a configuration file for a simuation. Won't run successfully. scenario: - name: "ExampleScenario" # a name for the simulation - start_time: '2012-01-02 00:00:00' # start and end times of the simulation - end_time: '2012-01-02 00:00:10' - time_resolution: 900 # duration of each step in second - results: './out.csv' # a file to store the results (optional) -models: # list of models for the energy network -- name: Battery1 # name for the model (must be unique) - type: Battery # name of the model registered in the Illuminator - inputs: - input1: 0 # input-name: initial value, default value will be used if not declaredd - outputs: - output1: 0 - output2: null - parameters: - charge_power_max: 100 - discharge_power_max: 200 - soc_min: 0.1 - soc_max: 0.9 - capacity: 1000 - states: - initial_soc: 0.5 - triggers: # list of triggers for the of another model??. It must be an input, output or state of the model - - capacity - - soc_min - scenario_data: './src/illuminator/schemas/scenario_data.csv' #path to the scenario data file for the model. This should be optional - connect: # this is optional. If not defined, the model will be assumed to be local - ip: 127.0.0.1 # Ip version 4 - port: 5000 # optional, if not defined, the default port will be used -- name: Battery2 - type: Battery # models can reuse the same type - inputs: - input1: 0 # input-name: initial value, default value will be used if not defined - outputs: - output1: 0 - output2: null - parameters: - charge_power_max: 100 - states: - soc: 0.5 # initial value for the state, optional -- name: PV1 - type: PVModel - inputs: - input1: 0 # input-name: initial value, default value will be used if not defined - input2: 0 - outputs: - output1: 0 - parameters: - power_max: 100 - states: - initial_soc: 0.5 + name: "ScenarioTest" # name for the similation + start_time: '2012-01-01 00:00:00' # ISO 8601 start time + end_time: '2012-01-01 01:00:00' + time_resolution: 900 # time step in seconds (optional). +models: # list of models for the energy system +- name: CSVB # name for the model (must be unique) + type: CSV # name the model type in the Illuminator + parameters: # vary per model type + start: '2012-01-01 00:00:00' + datafile: './tests/data/solar-sample.csv' +- name: PV + type: PvAdapter + inputs: # vary per model type (optional) + G_Gh: null + G_Dh: null + outputs: + G_Gh: null + states: + state1: null + state2: null + triggers: + - D_Dh + - state2 + connect: # necessary for running a simulation in a Raspberry Pi cluster + ip: 168.192.0.3 # IP of client machine + port: 5000 connections: -- from: Battery1.output2 # start model, pattern: model_name.output_name/input_name - to: PV1.input1 # end model -- from: Battery2.output - to: PV1.input2 -monitor: # a list of models, its inputs, output and states to be monitored and logged -- Battery1.input1 # pattern model_name.state_name -- Battery2.output1 # no duplicates allowed -- PV1.soc # pattern model_name.state_name -- PV1.output1 +- from: CSVB.G_Gh # origin model, format: model_name.output_name + to: PV.G_Gh # destinatioin model, format: model_name.input_name +- from: CSVB.G_Dh + to: PV.G_Dh +- from: CSVB.G_Bn + to: PV.G_Bn +- from: CSVB.Ta + to: PV.Ta +- from: CSVB.hs + to: PV.hs +- from: CSVB.FF + to: PV.FF +- from: CSVB.Az + to: PV.Az +monitor: + file: './out.csv' # file where items are saved during simualation (optional) + items: + - PV.pv_gen # List of inputs, outputs or states to monitor ``` @@ -86,16 +73,21 @@ monitor: # a list of models, its inputs, output and states to be monitored and | `start_time` | start time for the simulation. Must be a timestamp in ISO 8601 format | | | | `end_time` | end time for the simulation. Must be a timestamp in ISO 8601 format. | | | | `time_resolution` | number of seconds between simulation steps | ☑ | 900 (15 min) -| `results` | path to a CSV file to store the results of the simulation. The values in the `monitor` section determine which values are saved as results.| ☑ | a `out.csv` file saved to the current directory | | **models:** | a list of models for the simulation | | | -| `name` | a name for the model. Must be unique in each simulation | | | +| `name` | a name for the model. Must be unique for each simulation | | | | `type` | type of model. This must correspond with the name of the model registered in the Illuminator. | | | -| `inputs` | a set of input-names and initial values for the model. The model types determine which names and values are applicable to each model, and they must be declared accordingly. At least one input per model must be declared | | If the value is set to `null`, the default value will be used. See the respective model type for details.| +| `inputs` | a set of input-names and initial values for the model. The model type determines which names and values are applicable to each model, and they must be declared accordingly. Inputs are optional | | If the value is set to `null`, the default value will be used. See the respective model type for details.| | `outputs` | a set of output-names and initial values for the model. Similar to *inputs* valid names and values for each model are determined by the model *type*. See the respective model type for details. | | If the value is set to `null`, the default value will be used. | | `parameters` | a set of name-value pairs for the model. Parameters declared constants for a model during runtime. | ☑ | If ommited, the default values will be used. See the respective model type for details. | -| `states` | a set of name-value pairs considered as states for the model. Values allow to modify the internal initial values of a state. | ☑ | If ommited, the default values will be used. See the respective model type for details. | -| `triggers` | names of inputs, output or states that are use as triggers for a particular model. Triggers can only be declaredd by models that implement the *event-based paradigm*. See the respective model type to know if it accepts triggers. | ☑ | | +| `states` | a set of name-value pairs considered as states for the model. The values modify the internal initial values of a state. | ☑ | If ommited, the default values will be used. See the respective model type for details. | +| `triggers` | names of inputs, output or states that are use as triggers for a particular model. Triggers can only be declared by models that implement the *event-based paradigm*. See the respective model type to know if it accepts triggers. | ☑ | | +| `connect` | to declare in which client a model runs when using a Raspberry Pi cluster. | ☑ | | +| `ip` | Ip of the client manchine that will run the model. Only IP version 4 format. | | | +| `port` | TCP port to use to connect to the client machine| ☑ | | | **connections:** | how models connect to each other. | | | | `from` | origin of the connection declared as `.`. Input names use here must also appear as *inputs* in the models section. | | | | `to` | destination of the connection declared as `.`. Output names use here must also appear as *outputs* in the models section. | | -| **monitor:** | a list of which inputs, outputs or states of the modles must be monitored and logged during runtime. These are consired *results* of the simulation and they will be saved to the file in `results`. Items must be declared as `.`, where *name* is an input, output or stated clared int the *models* section. No duplicated values are allowed | | | +| **monitor:** | +| `file` | path to a CSV file to store results of the simulation. File will be created if necessary. | ☑ | a `out.csv` file saved to the current directory | +|`items` | a list of which inputs, outputs or states of models that most be monitored during runtime. Items must be declared as `.`, where *name* is an input, output or stated clared in the *models* section. No duplicated values are allowed | | | + From 65971334325d077e578036ebe464847500581a18 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 13:07:45 +0100 Subject: [PATCH 40/50] add build_runshfile.py to cluster package --- src/illuminator/cluster/build_runshfile.py | 56 ++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 src/illuminator/cluster/build_runshfile.py diff --git a/src/illuminator/cluster/build_runshfile.py b/src/illuminator/cluster/build_runshfile.py new file mode 100644 index 0000000..5f7ae5a --- /dev/null +++ b/src/illuminator/cluster/build_runshfile.py @@ -0,0 +1,56 @@ +import yaml +import os +import sys + +# Constants +DEFAULT_PORT = 5123 +RUN_PATH = './Desktop/illuminatorclient/configuration/runshfile/' +RUN_MODEL = '/home/illuminator/Desktop/Final_illuminator' +RUN_FILE = 'run.sh' + + +def build_run_command(model_type, connect_ip, connect_port): + """Builds the SSH run command for a given model.""" + return f"lxterminal -e ssh illuminator@{connect_ip} '{RUN_PATH}run{model_type}.sh {connect_ip} {connect_port} {RUN_MODEL}'&" + +def process_models(data, output_file): + """Processes each model in the YAML data and writes the commands to a file.""" + if 'models' not in data: + print("Error: No 'models' key found in the YAML data.") + return + + with open(output_file, 'w') as file: + file.write('#! /bin/bash\n') + + for model in data['models']: + model_type = model.get('type', 'unknown') + + # Extract connection details + connect = model.get('connect', {}) + connect_ip = connect.get('ip') + connect_port = connect.get('port', DEFAULT_PORT) + + # Ensure both IP and port are available + if connect_ip: + run_command = build_run_command(model_type, connect_ip, connect_port) + file.write(run_command + '\n') + else: + print(f"Warning: Model '{model_type}' has no IP address specified.") + +def main(): + if len(sys.argv) != 2: + print("Usage: python3 "+sys.argv[0]+" ") + sys.exit(1) + + yaml_file = sys.argv[1] + output_file = RUN_FILE + data = load_yaml(yaml_file) + + if data: + process_models(data, output_file) + print(f"Commands have been written to {output_file}") + + +if __name__ == "__main__": + main() + From f959d0049410ed72476ea7291a36f8f70779ebbc Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 14:36:06 +0100 Subject: [PATCH 41/50] Refactor configuration keys in engine and simulation schema for clarity --- src/illuminator/engine.py | 7 ++++--- src/illuminator/schema/simulation.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/illuminator/engine.py b/src/illuminator/engine.py index d2ccc31..467219e 100644 --- a/src/illuminator/engine.py +++ b/src/illuminator/engine.py @@ -44,6 +44,7 @@ def get_collector_path() -> str: return collector_path # TODO: write a unit test for this + def apply_default_values(config_simulation: dict) -> dict: """Applies Illuminator default values to the configuration if they are not specified. @@ -61,14 +62,14 @@ def apply_default_values(config_simulation: dict) -> dict: # defaults time_resolution = {'time_resolution': 900} # 15 minutes - results = {'results': './out.csv'} + out_file = {'file': './out.csv'} # TODO: set other default values if 'time_resolution' not in config_simulation['scenario']: config_simulation.update(time_resolution) # file to store the results - if 'results' not in config_simulation['scenario']: - config_simulation.update(results) + if 'file' not in config_simulation['monitor']: + config_simulation.update(out_file) #TODO: Write a unit test for this return config_simulation diff --git a/src/illuminator/schema/simulation.py b/src/illuminator/schema/simulation.py index b87dad0..1a36d4c 100644 --- a/src/illuminator/schema/simulation.py +++ b/src/illuminator/schema/simulation.py @@ -161,7 +161,7 @@ def validate(self, data, _is_scenario_schema=True): # _is_scenario_schema ), "monitor": Schema( { - Optional("file"): And(str, len, Use(validate_directory_path, error="Path for 'results' does not exists..."), error="you must provide a non-empty string for 'results'"), + Optional("file"): And(str, len, Use(validate_directory_path, error="Path for 'file' does not exists..."), error="you must provide a non-empty string for 'file'"), "items": And(list, len, Use(validate_model_item_format, error="Items in 'monitor' must have the format: ."), error="you must provide at least one item to monitor") } From e5aa4f38eb3f0b6ceed535721779910578f9dcd0 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 14:36:23 +0100 Subject: [PATCH 42/50] Update results file path in scenario_run function to use monitor configuration --- src/illuminator/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/illuminator/cli/main.py b/src/illuminator/cli/main.py index 4854241..c705a3c 100644 --- a/src/illuminator/cli/main.py +++ b/src/illuminator/cli/main.py @@ -44,7 +44,7 @@ def scenario_run(config_file: Annotated[str, typer.Argument(help="Path to scenar _end_time = config['scenario']['end_time'] _time_resolution = config['scenario']['time_resolution'] # output file with forecast results - _results_file = config['scenario']['results'] + _results_file = config['monitor']['file'] # Initialize the Mosaik worlds world = engine.create_world(sim_config, time_resolution=_time_resolution) From bb220a1f8c0be025e5c343f004a671d572b1d7f5 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 14:36:31 +0100 Subject: [PATCH 43/50] Replace load_yaml with load_config_file in build_runshfile.py for improved configuration handling --- src/illuminator/cluster/build_runshfile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/illuminator/cluster/build_runshfile.py b/src/illuminator/cluster/build_runshfile.py index 5f7ae5a..165ab4c 100644 --- a/src/illuminator/cluster/build_runshfile.py +++ b/src/illuminator/cluster/build_runshfile.py @@ -1,6 +1,7 @@ import yaml import os import sys +from illuminator.schema.simulation import load_config_file # Constants DEFAULT_PORT = 5123 @@ -37,6 +38,7 @@ def process_models(data, output_file): else: print(f"Warning: Model '{model_type}' has no IP address specified.") + def main(): if len(sys.argv) != 2: print("Usage: python3 "+sys.argv[0]+" ") @@ -44,7 +46,7 @@ def main(): yaml_file = sys.argv[1] output_file = RUN_FILE - data = load_yaml(yaml_file) + data = load_config_file(yaml_file) if data: process_models(data, output_file) From 5254596c4a888c54b80e84de1acac5bf82cdef4a Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 14:37:25 +0100 Subject: [PATCH 44/50] Update README to replace 'master' with 'server' for clarity in Raspberry Pi setup instructions --- readme.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/readme.md b/readme.md index c387d5c..fb3867e 100644 --- a/readme.md +++ b/readme.md @@ -27,13 +27,13 @@ conda activate Ecosystem ## Raspberry Pi Setup -The setup for the Illuminator requires one **master** Raspberry Pi and several **clients** Raspberry Pi's. +The setup for the Illuminator requires one **server** Raspberry Pi and several **clients** Raspberry Pi's. Raspberry Pi's must be connected and configured as a local network, and the -*master* must be configured to have permissions to access and control the *clients* through Secure Shell Protocol (SSH). +*server* must be configured to have permissions to access and control the *clients* through Secure Shell Protocol (SSH). -During simulation, the *master* engage with the *clients* to run the simulations defined in the *simulation configuration*, and +During simulation, the *server* engage with the *clients* to run the simulations defined in the *simulation configuration*, and information is exchanged between Rasberry Pi's using network sockets. -The **master** provides a Dashboard to viazulize the results, and saves them to a `.csv` files for later analysis. +The **server** provides a Dashboard to viazulize the results, and saves them to a `.csv` files for later analysis.
@@ -44,25 +44,27 @@ The **master** provides a Dashboard to viazulize the results, and saves them to 1. [Install Raspberry pi OS using Raspberry Pi imager.](https://www.raspberrypi.com/software/) 2. Set an static IP addresse for each Raspberry Pi. Use the following command on the terminal to open the `dhcpcd.conf` file: - ``` + + ```shell sudo nano /etc/dhcpcd.conf ``` In the `dhcpcd.conf` file, find the information to change the IP address as static as following: - ``` + ```shell interface etho static ip_address=192.168.0.1/24 # change the IP address as you want ``` - Give all users execute permission to all the documents in `runshfile/` in order to make sure the leader can access the *client* model. + Give all users execute permission to all the documents in `runshfile/` in order to make sure the *server* can access the *client* model. ```shell chmod -R a+X *dir* ``` Finally, reboot the Raspberry Pi suing `sudo reboot` on the terminal. -3. [Configure SSH connections so that the *master* can connect to the *clients* without a password.](https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys-2) + +3. [Configure SSH connections so that the *server* can connect to the *clients* without a password.](https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys-2) 4. Install the following Python packages. ``` pandas @@ -80,14 +82,13 @@ The **master** provides a Dashboard to viazulize the results, and saves them to matplotlib itertools ``` -5. Send the Illuminator package [TODO: What is the illuminator package?] to all *clients*. Use the following command on the *master's* terminal to check the connection between *master* and the *clients* +5. Send the Illuminator package to all *clients*. Then, use the following command on the *server's* terminal to check the connection between *server* and the *clients* ```shell ssh illuminator@ip #ip represent your follower IP address ``` - [TODO: This suggest that all Pi's need a user with the name 'illuminator'] -6. Run the `build_runshfile.py` file in the configuration directory on *master* to generate a run.sh script. Give the appropiate yaml file for the model as input: +6. Run the `build_runshfile.py` file in the configuration directory on the *server* to generate a `run.sh` script. Give the appropiate yaml file for the model as input: ```shell python3 build_runshfile.py From 206728700ffc58e9eb28c917307a425231424ce5 Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 14:41:25 +0100 Subject: [PATCH 45/50] Clarify monitor section in config-file.md by removing redundant information about simulation results --- docs/user/config-file.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/config-file.md b/docs/user/config-file.md index 25fcb05..9ec4afe 100644 --- a/docs/user/config-file.md +++ b/docs/user/config-file.md @@ -7,7 +7,7 @@ A *simulation file* has four main sections: - `scenario`: defines metadata and global variable for the simulation. - `models`: defines which models are included in the simulation. - `connections`: defines how the models in the `models` section must be connected for a particular simulation. -- `monitor`: defines which inputs, outputs, and states of a particular model must be monitored and logged during simulation. These are consider the simulation results and they will be saved to the file in `results`. +- `monitor`: defines which inputs, outputs, and states of a particular model must be monitored and logged during simulation. ## Example From 2654899d55c31f2e90bf96161b3bce56e826fade Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 16:03:30 +0100 Subject: [PATCH 46/50] Implement Simulation class to manage and run simulations with configuration file loading --- src/illuminator/engine.py | 62 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/illuminator/engine.py b/src/illuminator/engine.py index 467219e..eadcae1 100644 --- a/src/illuminator/engine.py +++ b/src/illuminator/engine.py @@ -9,6 +9,7 @@ from mosaik.scenario import Entity as MosaikEntity from mosaik.scenario import World as MosaikWorld from datetime import datetime +from illuminator.schema.simulation import load_config_file def create_world(sim_config: dict, time_resolution: int) -> MosaikWorld: @@ -316,3 +317,64 @@ def connect_monitor(world: MosaikWorld, model_entities: dict[MosaikEntity], # TODO: write a unit test for this. Cases: 1) all connections were established, 2) exception raised return world + + +class Simulation: + """A simplified interface to run simulations with Illuminator.""" + + def __init__(self, config_file:str) -> None: + """Loads and validates the configuration file for the simulation.""" + self.config_file = load_config_file(config_file) + + + def run(self): + """Runs a simulation scenario""" + + config = apply_default_values(self.config_file) + + # Define the Mosaik simulation configuration + sim_config = generate_mosaik_configuration(config) + + # simulation time + _start_time = config['scenario']['start_time'] + _end_time = config['scenario']['end_time'] + _time_resolution = config['scenario']['time_resolution'] + # output file with forecast results + _results_file = config['monitor']['file'] + + # Initialize the Mosaik worlds + world = create_world(sim_config, time_resolution=_time_resolution) + # TODO: collectors are also customisable simulators, define in the same way as models. + # A way to define custom collectors should be provided by the Illuminator. + collector = world.start('Collector', + time_resolution=_time_resolution, + start_date=_start_time, + results_show={'write2csv':True, 'dashboard_show':False, + 'Finalresults_show':False,'database':False, 'mqtt':False}, + output_file=_results_file) + + # initialize monitor + monitor = collector.Monitor() + + # Dictionary to keep track of created model entities + model_entities = start_simulators(world, config['models']) + + # Connect the models based on the connections specified in the configuration + world = build_connections(world, model_entities, config['connections']) + + # Connect monitor + world = connect_monitor(world, model_entities, monitor, config['monitor']) + + # Run the simulation until the specified end time + mosaik_end_time = compute_mosaik_end_time(_start_time, + _end_time, + _time_resolution + ) + + world.run(until=mosaik_end_time) + + @property + def config(self)-> dict: + """Returns the configuration file for the simulation.""" + return self.config_file + \ No newline at end of file From 5d2d41de7035a21a78b2787111662671cd90c07c Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 16:03:47 +0100 Subject: [PATCH 47/50] Refactor scenario_run function to utilize Simulation class --- src/illuminator/cli/main.py | 54 +++---------------------------------- 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/src/illuminator/cli/main.py b/src/illuminator/cli/main.py index c705a3c..92e3a89 100644 --- a/src/illuminator/cli/main.py +++ b/src/illuminator/cli/main.py @@ -9,6 +9,7 @@ from pathlib import Path from illuminator.cluster import build_runshfile from illuminator.schema.simulation import load_config_file +from illuminator.engine import Simulation APP_NAME = "illuminator" DEFAULT_PORT = 5123 @@ -26,58 +27,9 @@ def scenario_run(config_file: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): "Runs a simulation scenario using a YAML file." - file_path = config_file - - # load and validate configuration file - config = load_config_file(file_path) - config = engine.apply_default_values(config) - - print("config: ", config['models']) - - # Define the Mosaik simulation configuration - sim_config = engine.generate_mosaik_configuration(config) - - print("mosaik conf: ", sim_config) - - # simulation time - _start_time = config['scenario']['start_time'] - _end_time = config['scenario']['end_time'] - _time_resolution = config['scenario']['time_resolution'] - # output file with forecast results - _results_file = config['monitor']['file'] - - # Initialize the Mosaik worlds - world = engine.create_world(sim_config, time_resolution=_time_resolution) - # TODO: collectors are also customisable simulators, define in the same way as models. - # A way to define custom collectors should be provided by the Illuminator. - collector = world.start('Collector', - time_resolution=_time_resolution, - start_date=_start_time, - results_show={'write2csv':True, 'dashboard_show':False, - 'Finalresults_show':False,'database':False, 'mqtt':False}, - output_file=_results_file) + simulation = Simulation(config_file) + simulation.run() - # initialize monitor - monitor = collector.Monitor() - - # Dictionary to keep track of created model entities - model_entities = engine.start_simulators(world, config['models']) - - # Connect the models based on the connections specified in the configuration - world = engine.build_connections(world, model_entities, config['connections']) - - # Connect monitor - world = engine.connect_monitor(world, model_entities, monitor, config['monitor']) - - # Run the simulation until the specified end time - mosaik_end_time = engine.compute_mosaik_end_time(_start_time, - _end_time, - _time_resolution - ) - - print(f"Running simulation from") - world.run(until=mosaik_end_time) - @cluster_app.command("build") def cluster_build(config_file: Annotated[str, typer.Argument(help="Path to scenario configuration file.")] = "config.yaml"): From 28fa80b0e404aeae498cff2bd5bc39301ca0f02b Mon Sep 17 00:00:00 2001 From: Manuel Garcia Date: Fri, 15 Nov 2024 16:17:56 +0100 Subject: [PATCH 48/50] Add simulations documentation and update index.rst for navigation --- docs/index.rst | 2 ++ docs/user/simulations.md | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 docs/user/simulations.md diff --git a/docs/index.rst b/docs/index.rst index 5e8cf0f..e087a23 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,6 +26,8 @@ and the simulation engine is based on `Mosaik. `_ :caption: User's Documentation user/models + user/config-file.md + user/simulations.md .. toctree:: :maxdepth: 2 diff --git a/docs/user/simulations.md b/docs/user/simulations.md new file mode 100644 index 0000000..412fb0a --- /dev/null +++ b/docs/user/simulations.md @@ -0,0 +1,24 @@ +# Simulations + +Simulations can be directly from Python or suing the *command line interface* (CLI). + +## Python Interface + +To run a simulation from Python, you need to provide a [configuration file](./config-file.md). Then you can start the simulation as follows: + +```python +from illuminator.engine import Simulation + +simulation = Simulation('') +simulation.run() + +``` + +## CLI + +You can use the commands `scenario run` to start a simulation from the terminal: + + +```shell +illuminator scenario run +``` From 2070e7391bfc0b2fad7c1dde84b385de4f8047ca Mon Sep 17 00:00:00 2001 From: Josip Grguric Date: Fri, 15 Nov 2024 16:58:34 +0100 Subject: [PATCH 49/50] Current status: No longer has error on missing parameter --- examples/adder_copy.yaml | 39 ++++++++++++++++++++++++++++++++ src/illuminator/builder/model.py | 19 ++++++++++++---- src/illuminator/cli/main.py | 2 +- src/illuminator/engine.py | 14 ++++++++++++ 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 examples/adder_copy.yaml diff --git a/examples/adder_copy.yaml b/examples/adder_copy.yaml new file mode 100644 index 0000000..127c603 --- /dev/null +++ b/examples/adder_copy.yaml @@ -0,0 +1,39 @@ +scenario: + name: "AddingNumbers" # in mosaik so called world + start_time: '2012-01-02 00:00:00' # ISO 8601 start time of the simulation + end_time: '2012-01-02 00:00:10' # duration in seconds + time_resolution: 900 # time step in seconds. Defaults to 15 minutes (900 s) + results: './out.csv' +models: # list of models for the energy network +- name: Adder1 # name for the model (must be unique) + type: Adder # most match the class inheriting from IlluminatorConstructor + inputs: + in1: 10 # input-name: initial value, default value will be used if not defined + in2: 20 + outputs: + out1: 0 + parameters: + param1: "adding tens" +- name: Adder2 + type: Adder # models can reuse the same type + inputs: + in1: 100 # input-name: initial value, default value will be used if not defined + in2: 200 + outputs: + out1: 0 + parameters: + param1: "adding hundreds" +connections: +- from: Adder1.out1 # start model, pattern: model_name.output_name/input_name + to: Adder2.in1 # end model + +#monitor: # a list of models, its inputs, output and states to be monitored and logged +#- Adder2.out2 # pattern model_name.state_name +monitor: + file: './out.csv' + items: + - Adder2.out2 + + + + diff --git a/src/illuminator/builder/model.py b/src/illuminator/builder/model.py index 4e399a0..3265cad 100644 --- a/src/illuminator/builder/model.py +++ b/src/illuminator/builder/model.py @@ -3,6 +3,7 @@ from dataclasses import dataclass, field from enum import Enum from datetime import datetime +import illuminator.engine as engine class SimulatorType(Enum): TIME_BASED = 'time-based' @@ -56,7 +57,7 @@ class IlluminatorModel(): simulator_type: SimulatorType = SimulatorType.HYBRID time_step_size: int = 15 # This is closely related to logic in the step method. Currently, all models have the same time step size (15 minutes). This is a global setting for the simulation, not a model-specific setting. time: Optional[datetime] = None # Shouldn't be modified by the user. - + model_type: Optional[str] = "Model" def __post_init__(self): self.validate_states() @@ -70,7 +71,7 @@ def simulator_meta(self) -> dict: meta = { 'type': self.simulator_type.value, 'models': { - 'Model': { + self.model_type : { 'public': True, 'params': list(self.parameters.keys()), 'attrs': list(self.outputs.keys()) @@ -111,7 +112,16 @@ def validate_simulator_type(self): class ModelConstructor(ABC, Simulator): """A common interface for constructing models in the Illuminator""" - def __init__(self, model: IlluminatorModel) -> None: + def __init__(self, **kwargs) -> None: + #model: IlluminatorModel + model_vals = engine.current_model + model = IlluminatorModel( + parameters=model_vals['parameters'], + inputs=model_vals["inputs"], + outputs=model_vals["outputs"], + states={}, + model_type=model_vals["type"] + ) super().__init__(meta=model.simulator_meta) self._model = model self.model_entities = {} @@ -140,12 +150,13 @@ def step(self, time:int, inputs:dict=None, max_advance:int=None) -> int: """ pass - def init(self, sid, time_resolution, **sim_params): # can be use to update model parameters set in __init__ + def init(self, sid, time_resolution=1, **sim_params): # can be use to update model parameters set in __init__ print(f"running extra init") # This is the standard Mosaik init method signature self.sid = sid self.time_resolution = time_resolution + # This bit of code is unused. # Assuming sim_params is structured as {'sim_params': {model_name: model_data}} sim_params = sim_params.get('sim_params', {}) if len(sim_params) != 1: diff --git a/src/illuminator/cli/main.py b/src/illuminator/cli/main.py index 4854241..b077fd8 100644 --- a/src/illuminator/cli/main.py +++ b/src/illuminator/cli/main.py @@ -7,7 +7,7 @@ from typing_extensions import Annotated import illuminator.engine as engine from pathlib import Path -from illuminator.cluster import build_runshfile +# from illuminator.cluster import build_runshfile from illuminator.schema.simulation import load_config_file APP_NAME = "illuminator" diff --git a/src/illuminator/engine.py b/src/illuminator/engine.py index d2ccc31..2e1cecb 100644 --- a/src/illuminator/engine.py +++ b/src/illuminator/engine.py @@ -10,6 +10,7 @@ from mosaik.scenario import World as MosaikWorld from datetime import datetime +current_model = {} def create_world(sim_config: dict, time_resolution: int) -> MosaikWorld: """ @@ -156,6 +157,7 @@ def start_simulators(world: MosaikWorld, models: list) -> dict: for model in models: model_name = model['name'] model_type = model['type'] + set_current_model(model) if 'parameters' in model: @@ -177,6 +179,8 @@ def start_simulators(world: MosaikWorld, models: list) -> dict: else: simulator = world.start(sim_name=model_name, # **model_parameters + model_name = model_name, + sim_params= {model_name: model} # This value gets picked up in the init() function # Some items must be passed here, and some other at create() ) @@ -270,6 +274,16 @@ def compute_mosaik_end_time(start_time:str, end_time:str, time_resolution:int = return steps +# def get_current_model(): +# return current_model + +def set_current_model(model): + global current_model + current_model["type"] = model['type'] + current_model['parameters']=model['parameters'] + current_model['inputs']=model["inputs"] + current_model['outputs']=model["outputs"] + def connect_monitor(world: MosaikWorld, model_entities: dict[MosaikEntity], monitor:MosaikEntity, monitor_config: dict) -> MosaikWorld: From 67d6dfed46a2f2ace8c3e6765728afe8166ab48d Mon Sep 17 00:00:00 2001 From: Josip Grguric Date: Mon, 18 Nov 2024 15:27:11 +0100 Subject: [PATCH 50/50] Creation stage has issues fixed. Currently on build_connections. --- examples/adder_copy.yaml | 5 ++++- src/illuminator/builder/model.py | 13 ++++++++----- src/illuminator/engine.py | 21 +++++++++++++-------- src/illuminator/schema/simulation.py | 1 + 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/examples/adder_copy.yaml b/examples/adder_copy.yaml index 127c603..f1a3a54 100644 --- a/examples/adder_copy.yaml +++ b/examples/adder_copy.yaml @@ -25,7 +25,10 @@ models: # list of models for the energy network param1: "adding hundreds" connections: - from: Adder1.out1 # start model, pattern: model_name.output_name/input_name - to: Adder2.in1 # end model + to: Adder2.in1n # end model +# TODO: Below is what Illuminator looks for, but is not valid. Needs to be fixed +# - from: Adder1-0.hybrid_0.out1 +# to: Adder2-0.hybrid_0.in1 #monitor: # a list of models, its inputs, output and states to be monitored and logged #- Adder2.out2 # pattern model_name.state_name diff --git a/src/illuminator/builder/model.py b/src/illuminator/builder/model.py index 3265cad..9d8adaa 100644 --- a/src/illuminator/builder/model.py +++ b/src/illuminator/builder/model.py @@ -167,12 +167,15 @@ def init(self, sid, time_resolution=1, **sim_params): # can be use to update mo # self.model = self.load_model_class(self.model_data['model_path'], self.model_data['model_type']) return self._model.simulator_meta - def create(self, num: int) -> List: + def create(self, num:int, model:str, **model_params) -> List[dict]: # This change is mandatory. It MUST contain num and model as parameters otherwise it receives an incorrect number of parameters """Creates an instance of the model""" - for i in range(num): - eid = f"{self._model.simulator_type.value}_{i}" - self.model_entities[eid] = self._model - return list(self.model_entities.keys()) + new_entities = [] # See below why this was created + for i in range(num): # this seems ok + eid = f"{self._model.simulator_type.value}_{i}" # this seems ok + self.model_entities[eid] = self._model # this seems fine + # return list(self.model_entities.keys()) # I removed this bit for now. Create is expected to return a list of dictionaries + new_entities.append({'eid': eid, 'type': model}) # So basically, like this. Later on we can look into other alternatives if needed. + return new_entities def current_time(self): """Returns the current time of the simulation""" diff --git a/src/illuminator/engine.py b/src/illuminator/engine.py index 2e1cecb..5ec3c49 100644 --- a/src/illuminator/engine.py +++ b/src/illuminator/engine.py @@ -196,14 +196,19 @@ def start_simulators(world: MosaikWorld, models: list) -> dict: # TODO: # this is a temporary solution to continue developing the CLI - entity = model_factory.create(num=1, sim_start='2012-01-01 00:00:00', - panel_data={'Module_area': 1.26, 'NOCT': 44, 'Module_Efficiency': 0.198, 'Irradiance_at_NOCT': 800, - 'Power_output_at_STC': 250,'peak_power':600}, - m_tilt=14, - m_az=180, - cap=500, - output_type='power' - ) + + # TODO: If we wish to use different values here, we must define the parameters used here within the appropriate .yaml file. + # Right now adder.yaml defines the custom parameters as "param1" + + # entity = model_factory.create(num=1, sim_start='2012-01-01 00:00:00', + # panel_data={'Module_area': 1.26, 'NOCT': 44, 'Module_Efficiency': 0.198, 'Irradiance_at_NOCT': 800, + # 'Power_output_at_STC': 250,'peak_power':600}, + # m_tilt=14, + # m_az=180, + # cap=500, + # output_type='power' + # ) + entity = model_factory.create(num=1, param1="Not in use") model_entities[model_name] = entity print(model_entities) diff --git a/src/illuminator/schema/simulation.py b/src/illuminator/schema/simulation.py index b9c9cf7..649ca10 100644 --- a/src/illuminator/schema/simulation.py +++ b/src/illuminator/schema/simulation.py @@ -17,6 +17,7 @@ # monitor and connections sections enforce a format such as # . valid_model_item_format = r'^\w+\.\w+$' +# valid_model_item_format = r'^([\w-]+\.?)+$' # This is kept here as an alternative. I believe this might be useful later on