From bfd639de09979f32b489201636432ccc4210974e Mon Sep 17 00:00:00 2001 From: Drew Johnston <31270488+drewjj@users.noreply.github.com> Date: Tue, 1 Oct 2024 15:44:22 -0600 Subject: [PATCH 1/7] local commit --- docker-compose.yml | 58 +++++++++---------- ...mware_manager.py => firmware_scheduler.py} | 0 .../images/firmware_manager/requirements.txt | 2 +- .../images/firmware_manager/upgrade_init.py | 7 +++ 4 files changed, 37 insertions(+), 30 deletions(-) rename services/addons/images/firmware_manager/{firmware_manager.py => firmware_scheduler.py} (100%) create mode 100644 services/addons/images/firmware_manager/upgrade_init.py diff --git a/docker-compose.yml b/docker-compose.yml index d9362828..eebc78d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,35 +55,35 @@ services: max-size: '10m' max-file: '5' - cvmanager_webapp: - build: - context: webapp - dockerfile: Dockerfile - args: - API_URI: http://${WEBAPP_DOMAIN}:8081 - MAPBOX_TOKEN: ${MAPBOX_TOKEN} - KEYCLOAK_HOST_URL: http://${KEYCLOAK_DOMAIN}:8084/ - COUNT_MESSAGE_TYPES: ${COUNTS_MSG_TYPES} - VIEWER_MESSAGE_TYPES: ${VIEWER_MSG_TYPES} - DOT_NAME: ${DOT_NAME} - MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE} - MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE} - MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM} - CVIZ_API_SERVER_URL: ${CVIZ_API_SERVER_URL} - CVIZ_API_WS_URL: ${CVIZ_API_WS_URL} - image: jpo_cvmanager_webapp:latest - restart: always - depends_on: - cvmanager_keycloak: - condition: service_healthy - extra_hosts: - ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} - ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} - ports: - - '80:80' - logging: - options: - max-size: '10m' + # cvmanager_webapp: + # build: + # context: webapp + # dockerfile: Dockerfile + # args: + # API_URI: http://${WEBAPP_DOMAIN}:8081 + # MAPBOX_TOKEN: ${MAPBOX_TOKEN} + # KEYCLOAK_HOST_URL: http://${KEYCLOAK_DOMAIN}:8084/ + # COUNT_MESSAGE_TYPES: ${COUNTS_MSG_TYPES} + # VIEWER_MESSAGE_TYPES: ${VIEWER_MSG_TYPES} + # DOT_NAME: ${DOT_NAME} + # MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE} + # MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE} + # MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM} + # CVIZ_API_SERVER_URL: ${CVIZ_API_SERVER_URL} + # CVIZ_API_WS_URL: ${CVIZ_API_WS_URL} + # image: jpo_cvmanager_webapp:latest + # restart: always + # depends_on: + # cvmanager_keycloak: + # condition: service_healthy + # extra_hosts: + # ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} + # ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} + # ports: + # - '80:80' + # logging: + # options: + # max-size: '10m' cvmanager_postgres: image: postgis/postgis:15-master diff --git a/services/addons/images/firmware_manager/firmware_manager.py b/services/addons/images/firmware_manager/firmware_scheduler.py similarity index 100% rename from services/addons/images/firmware_manager/firmware_manager.py rename to services/addons/images/firmware_manager/firmware_scheduler.py diff --git a/services/addons/images/firmware_manager/requirements.txt b/services/addons/images/firmware_manager/requirements.txt index 8411b997..5a26ee63 100644 --- a/services/addons/images/firmware_manager/requirements.txt +++ b/services/addons/images/firmware_manager/requirements.txt @@ -1,7 +1,7 @@ APScheduler==3.10.4 google-cloud-storage==2.14.0 flask==3.0.0 -paramiko==3.3.1 +paramiko==3.5.0 pg8000==1.30.2 requests==2.31.0 scp==0.14.5 diff --git a/services/addons/images/firmware_manager/upgrade_init.py b/services/addons/images/firmware_manager/upgrade_init.py new file mode 100644 index 00000000..cd9cbe51 --- /dev/null +++ b/services/addons/images/firmware_manager/upgrade_init.py @@ -0,0 +1,7 @@ +from flask import Flask, jsonify, request +from subprocess import Popen, DEVNULL +from threading import Lock +from waitress import serve +import json +import logging +import os From 3ff70e7860e93ddae6566d3b4e5cb093899cb85d Mon Sep 17 00:00:00 2001 From: Drew Johnston <31270488+drewjj@users.noreply.github.com> Date: Wed, 2 Oct 2024 02:54:01 -0600 Subject: [PATCH 2/7] Add firmware manager updates --- docker-compose-addons.yml | 27 ++++- sample.env | 1 + ...rfile.firmware_manager => Dockerfile.fmur} | 4 +- services/Dockerfile.fmus | 13 +++ .../images/firmware_manager/upgrade_init.py | 7 -- .../commsignia_upgrader.py | 39 +++++--- .../{ => upgrade_runner}/download_blob.py | 0 .../{ => upgrade_runner}/sample.env | 7 -- .../upgrade_runner/upgrade_runner.py | 99 +++++++++++++++++++ .../{ => upgrade_runner}/upgrader.py | 0 .../{ => upgrade_runner}/yunex_upgrader.py | 20 +++- .../upgrade_scheduler/sample.env | 12 +++ .../upgrade_scheduler.py} | 37 ++++--- 13 files changed, 209 insertions(+), 57 deletions(-) rename services/{Dockerfile.firmware_manager => Dockerfile.fmur} (76%) create mode 100644 services/Dockerfile.fmus delete mode 100644 services/addons/images/firmware_manager/upgrade_init.py rename services/addons/images/firmware_manager/{ => upgrade_runner}/commsignia_upgrader.py (80%) rename services/addons/images/firmware_manager/{ => upgrade_runner}/download_blob.py (100%) rename services/addons/images/firmware_manager/{ => upgrade_runner}/sample.env (76%) create mode 100644 services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py rename services/addons/images/firmware_manager/{ => upgrade_runner}/upgrader.py (100%) rename services/addons/images/firmware_manager/{ => upgrade_runner}/yunex_upgrader.py (88%) create mode 100644 services/addons/images/firmware_manager/upgrade_scheduler/sample.env rename services/addons/images/firmware_manager/{firmware_scheduler.py => upgrade_scheduler/upgrade_scheduler.py} (90%) diff --git a/docker-compose-addons.yml b/docker-compose-addons.yml index b0092d77..2b925fa8 100644 --- a/docker-compose-addons.yml +++ b/docker-compose-addons.yml @@ -123,11 +123,11 @@ services: max-size: '10m' max-file: '5' - firmware_manager: + firmware_manager_upgrade_scheduler: build: context: services - dockerfile: Dockerfile.firmware_manager - image: jpo_firmware_manager:latest + dockerfile: Dockerfile.fmus + image: jpo_firmware_manager_upgrade_scheduler:latest restart: on-failure:3 ports: @@ -138,6 +138,27 @@ services: PG_DB_USER: ${PG_DB_USER} PG_DB_PASS: ${PG_DB_PASS} + UPGRADE_RUNNER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_RUNNER_ENDPOINT} + + LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL} + volumes: + - ${GOOGLE_APPLICATION_CREDENTIALS}:/google/gcp_credentials.json + - ${HOST_BLOB_STORAGE_DIRECTORY}:/mnt/blob_storage + logging: + options: + max-size: '10m' + max-file: '5' + + firmware_manager_upgrade_runner: + build: + context: services + dockerfile: Dockerfile.fmur + image: jpo_firmware_manager_upgrade_runner:latest + restart: on-failure:3 + + ports: + - '8090:8080' + environment: BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER} BLOB_STORAGE_BUCKET: ${BLOB_STORAGE_BUCKET} diff --git a/sample.env b/sample.env index 1ab17337..9d1102d1 100644 --- a/sample.env +++ b/sample.env @@ -8,6 +8,7 @@ KC_HOST_IP=${DOCKER_HOST_IP} # Firmware Manager connectivity in the format 'http://endpoint:port' FIRMWARE_MANAGER_ENDPOINT=http://${DOCKER_HOST_IP}:8089 +FIRMWARE_MANAGER_UPGRADE_RUNNER_ENDPOINT=http://${DOCKER_HOST_IP}:8090 # Allowed CORS domain for accessing the CV Manager API from (set to the web application hostname) # Make sure to include http:// or https:// diff --git a/services/Dockerfile.firmware_manager b/services/Dockerfile.fmur similarity index 76% rename from services/Dockerfile.firmware_manager rename to services/Dockerfile.fmur index 28172fb9..7c0bf9a1 100644 --- a/services/Dockerfile.firmware_manager +++ b/services/Dockerfile.fmur @@ -4,7 +4,7 @@ WORKDIR /home ADD addons/images/firmware_manager/requirements.txt . ADD addons/images/firmware_manager/resources/xfer_yunex.jar ./tools/ -ADD addons/images/firmware_manager/*.py . +ADD addons/images/firmware_manager/upgrade_runner/*.py . ADD common/*.py ./common/ RUN pip3 install -r requirements.txt @@ -12,5 +12,5 @@ RUN apt-get update RUN apt-get install -y default-jdk RUN apt-get install -y iputils-ping -CMD ["/home/firmware_manager.py"] +CMD ["/home/upgrade_runner.py"] ENTRYPOINT ["python3"] \ No newline at end of file diff --git a/services/Dockerfile.fmus b/services/Dockerfile.fmus new file mode 100644 index 00000000..2017df2d --- /dev/null +++ b/services/Dockerfile.fmus @@ -0,0 +1,13 @@ +FROM python:3.12.2-slim + +WORKDIR /home + +ADD addons/images/firmware_manager/requirements.txt . +ADD addons/images/firmware_manager/upgrade_scheduler/*.py . +ADD common/*.py ./common/ + +RUN pip3 install -r requirements.txt +RUN apt-get update + +CMD ["/home/upgrade_scheduler.py"] +ENTRYPOINT ["python3"] \ No newline at end of file diff --git a/services/addons/images/firmware_manager/upgrade_init.py b/services/addons/images/firmware_manager/upgrade_init.py deleted file mode 100644 index cd9cbe51..00000000 --- a/services/addons/images/firmware_manager/upgrade_init.py +++ /dev/null @@ -1,7 +0,0 @@ -from flask import Flask, jsonify, request -from subprocess import Popen, DEVNULL -from threading import Lock -from waitress import serve -import json -import logging -import os diff --git a/services/addons/images/firmware_manager/commsignia_upgrader.py b/services/addons/images/firmware_manager/upgrade_runner/commsignia_upgrader.py similarity index 80% rename from services/addons/images/firmware_manager/commsignia_upgrader.py rename to services/addons/images/firmware_manager/upgrade_runner/commsignia_upgrader.py index cc25c78f..ca8070e1 100644 --- a/services/addons/images/firmware_manager/commsignia_upgrader.py +++ b/services/addons/images/firmware_manager/upgrade_runner/commsignia_upgrader.py @@ -11,7 +11,9 @@ class CommsigniaUpgrader(upgrader.UpgraderAbstractClass): def __init__(self, upgrade_info): # set file/blob location for post_upgrade script - self.post_upgrade_file_name = f"/home/{upgrade_info['ipv4_address']}/post_upgrade.sh" + self.post_upgrade_file_name = ( + f"/home/{upgrade_info['ipv4_address']}/post_upgrade.sh" + ) self.post_upgrade_blob_name = f"{upgrade_info['manufacturer']}/{upgrade_info['model']}/{upgrade_info['target_firmware_version']}/post_upgrade.sh" super().__init__(upgrade_info, firmware_extension=".tar.sig") @@ -54,7 +56,9 @@ def upgrade(self): ssh.close() # If post_upgrade script exists execute it - if (self.download_blob(self.post_upgrade_blob_name, self.post_upgrade_file_name, ".sh")): + if self.download_blob( + self.post_upgrade_blob_name, self.post_upgrade_file_name, ".sh" + ): self.post_upgrade() # Delete local installation package and its parent directory so it doesn't take up storage space @@ -64,7 +68,9 @@ def upgrade(self): self.notify_firmware_manager(success=True) except Exception as err: # If something goes wrong, cleanup anything left and report failure if possible - logging.error(f"Failed to perform firmware upgrade for {self.rsu_ip}: {err}") + logging.error( + f"Failed to perform firmware upgrade for {self.rsu_ip}: {err}" + ) self.cleanup() self.notify_firmware_manager(success=False) # send email to support team with the rsu and error @@ -72,7 +78,9 @@ def upgrade(self): def post_upgrade(self): if self.wait_until_online() == -1: - raise Exception("RSU " + self.rsu_ip + " offline for too long after firmware upgrade") + raise Exception( + "RSU " + self.rsu_ip + " offline for too long after firmware upgrade" + ) try: time.sleep(60) # Make connection with the target device @@ -95,25 +103,28 @@ def post_upgrade(self): # Change permissions and execute post upgrade script logging.info("Running post upgrade script for " + self.rsu_ip + "...") - ssh.exec_command( - f"chmod +x /tmp/post_upgrade.sh" - ) - _stdin, _stdout, _stderr = ssh.exec_command( - f"/tmp/post_upgrade.sh" - ) + ssh.exec_command(f"chmod +x /tmp/post_upgrade.sh") + _stdin, _stdout, _stderr = ssh.exec_command(f"/tmp/post_upgrade.sh") decoded_stdout = _stdout.read().decode() logging.info(decoded_stdout) if "ALL OK" not in decoded_stdout: ssh.close() - logging.error(f"Failed to execute post upgrade script for rsu {self.rsu_ip}: {decoded_stdout}") + logging.error( + f"Failed to execute post upgrade script for rsu {self.rsu_ip}: {decoded_stdout}" + ) return ssh.close() - logging.info(f"Post upgrade script executed successfully for rsu: {self.rsu_ip}.") + logging.info( + f"Post upgrade script executed successfully for rsu: {self.rsu_ip}." + ) except Exception as err: - logging.error(f"Failed to execute post upgrade script for rsu {self.rsu_ip}: {err}") + logging.error( + f"Failed to execute post upgrade script for rsu {self.rsu_ip}: {err}" + ) # send email to support team with the rsu and error self.send_error_email("Post-Upgrade Script", err) + # sys.argv[1] - JSON string with the following key-values: # - ipv4_address # - manufacturer @@ -129,7 +140,7 @@ def post_upgrade(self): # Trimming outer single quotes from the json.loads upgrade_info = json.loads(sys.argv[1][1:-1]) commsignia_upgrader = CommsigniaUpgrader(upgrade_info) - if (commsignia_upgrader.check_online()): + if commsignia_upgrader.check_online(): commsignia_upgrader.upgrade() else: logging.error(f"RSU {upgrade_info['ipv4_address']} is offline") diff --git a/services/addons/images/firmware_manager/download_blob.py b/services/addons/images/firmware_manager/upgrade_runner/download_blob.py similarity index 100% rename from services/addons/images/firmware_manager/download_blob.py rename to services/addons/images/firmware_manager/upgrade_runner/download_blob.py diff --git a/services/addons/images/firmware_manager/sample.env b/services/addons/images/firmware_manager/upgrade_runner/sample.env similarity index 76% rename from services/addons/images/firmware_manager/sample.env rename to services/addons/images/firmware_manager/upgrade_runner/sample.env index 2d8d8069..168fd089 100644 --- a/services/addons/images/firmware_manager/sample.env +++ b/services/addons/images/firmware_manager/upgrade_runner/sample.env @@ -1,11 +1,4 @@ LOGGING_LEVEL="INFO" -ACTIVE_UPGRADE_LIMIT=20 - -# PostgreSQL database variables -PG_DB_HOST="" -PG_DB_NAME="" -PG_DB_USER="" -PG_DB_PASS="" # Blob storage variables (only 'GCP' and 'DOCKER' are supported at this time) BLOB_STORAGE_PROVIDER=DOCKER diff --git a/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py b/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py new file mode 100644 index 00000000..920b546f --- /dev/null +++ b/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py @@ -0,0 +1,99 @@ +from flask import Flask, jsonify, request, abort +from subprocess import Popen, DEVNULL +from waitress import serve +from marshmallow import Schema, fields +import json +import logging +import os + +app = Flask(__name__) + +log_level = os.environ.get("LOGGING_LEVEL", "INFO") +logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) + +manufacturer_upgrade_scripts = { + "Commsignia": "commsignia_upgrader.py", + "Yunex": "yunex_upgrader.py", +} + + +def start_upgrade_task(rsu_upgrade_data): + try: + Popen( + [ + "python3", + f'/home/{manufacturer_upgrade_scripts[rsu_upgrade_data["manufacturer"]]}', + f"'{json.dumps(rsu_upgrade_data)}'", + ], + stdout=DEVNULL, + ) + + return ( + jsonify( + { + "message": f"Firmware upgrade started successfully for '{rsu_upgrade_data['ipv4_address']}'" + } + ), + 201, + ) + except Exception as err: + # If this case occurs, only log it since there may not be a listener. + # Since the upgrade_queue and upgrade_queue_info will no longer have the RSU present, + # the hourly check_for_upgrades() will pick up the firmware upgrade again to retry the upgrade. + logging.error( + f"Encountered error of type {type(err)} while starting automatic upgrade process for {rsu_upgrade_data['ipv4_address']}: {err}" + ) + + return ( + jsonify( + { + "message": f"Firmware upgrade failed to start for '{rsu_upgrade_data['ipv4_address']}'" + } + ), + 500, + ) + + +class RunFirmwareUpgradeSchema(Schema): + ipv4_address = fields.IPv4(required=True) + manufacturer = fields.Str(required=True) + model = fields.Str(required=True) + ssh_username = fields.Str(required=True) + ssh_password = fields.Str(required=True) + target_firmware_id = fields.Int(required=True) + target_firmware_version = fields.Str(required=True) + install_package = fields.Str(required=True) + + +# REST endpoint to manually start firmware upgrades for a single targeted RSU +# Required request body values: +# - ipv4_address +# - manufacturer +# - model +# - ssh_username +# - ssh_password +# - target_firmware_id +# - target_firmware_version +# - install_package +@app.route("/run_firmware_upgrade", methods=["POST"]) +def run_firmware_upgrade(): + # Verify HTTP body JSON object + request_args = request.get_json() + schema = RunFirmwareUpgradeSchema() + errors = schema.validate(request.json) + if errors: + logging.error(str(errors)) + abort(400, str(errors)) + + # Start the RSU upgrade task + return start_upgrade_task(request_args) + + +def serve_rest_api(): + # Run Flask app + logging.info("Initiating the Firmware Manager Upgrade Runner REST API...") + serve(app, host="0.0.0.0", port=8080) + + +if __name__ == "__main__": + serve_rest_api() diff --git a/services/addons/images/firmware_manager/upgrader.py b/services/addons/images/firmware_manager/upgrade_runner/upgrader.py similarity index 100% rename from services/addons/images/firmware_manager/upgrader.py rename to services/addons/images/firmware_manager/upgrade_runner/upgrader.py diff --git a/services/addons/images/firmware_manager/yunex_upgrader.py b/services/addons/images/firmware_manager/upgrade_runner/yunex_upgrader.py similarity index 88% rename from services/addons/images/firmware_manager/yunex_upgrader.py rename to services/addons/images/firmware_manager/upgrade_runner/yunex_upgrader.py index 352fcf92..75836f09 100644 --- a/services/addons/images/firmware_manager/yunex_upgrader.py +++ b/services/addons/images/firmware_manager/upgrade_runner/yunex_upgrader.py @@ -26,7 +26,12 @@ def run_xfer_upgrade(self, file_name): # If the command ends with a non-successful status code, return -1 if code != 0: - logging.error("Firmware not successful for " + self.rsu_ip + ": " + stderr.decode("utf-8")) + logging.error( + "Firmware not successful for " + + self.rsu_ip + + ": " + + stderr.decode("utf-8") + ) return -1 output_lines = stdout.decode("utf-8").split("\n")[:-1] @@ -35,7 +40,12 @@ def run_xfer_upgrade(self, file_name): 'TEXT: {"success":{"upload":"Processing OK. Rebooting now ..."}}' not in output_lines ): - logging.error("Firmware not successful for " + self.rsu_ip + ": " + stderr.decode("utf-8")) + logging.error( + "Firmware not successful for " + + self.rsu_ip + + ": " + + stderr.decode("utf-8") + ) return -1 # If everything goes as expected, the XFER upgrade was complete @@ -95,7 +105,9 @@ def upgrade(self): # If something goes wrong, cleanup anything left and report failure if possible. # Yunex RSUs can handle having the same firmware upgraded over again. # There is no issue with starting from the beginning even with a partially complete upgrade. - logging.error(f"Failed to perform firmware upgrade for {self.rsu_ip}: {err}") + logging.error( + f"Failed to perform firmware upgrade for {self.rsu_ip}: {err}" + ) self.cleanup() self.notify_firmware_manager(success=False) # send email to support team with the rsu and error @@ -117,7 +129,7 @@ def upgrade(self): # Trimming outer single quotes from the json.loads upgrade_info = json.loads(sys.argv[1][1:-1]) yunex_upgrader = YunexUpgrader(upgrade_info) - if (yunex_upgrader.check_online()): + if yunex_upgrader.check_online(): yunex_upgrader.upgrade() else: logging.error(f"RSU {upgrade_info['ipv4_address']} is offline") diff --git a/services/addons/images/firmware_manager/upgrade_scheduler/sample.env b/services/addons/images/firmware_manager/upgrade_scheduler/sample.env new file mode 100644 index 00000000..836e6702 --- /dev/null +++ b/services/addons/images/firmware_manager/upgrade_scheduler/sample.env @@ -0,0 +1,12 @@ +LOGGING_LEVEL="INFO" +ACTIVE_UPGRADE_LIMIT=20 + +# PostgreSQL database variables +PG_DB_HOST="" +PG_DB_NAME="" +PG_DB_USER="" +PG_DB_PASS="" + +# Must specify this endpoint to wherever the Upgrade Runner is hosted +# Must include 'http://' or 'https://' +UPGRADE_RUNNER_ENDPOINT="http://" \ No newline at end of file diff --git a/services/addons/images/firmware_manager/firmware_scheduler.py b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py similarity index 90% rename from services/addons/images/firmware_manager/firmware_scheduler.py rename to services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py index 733f9265..37a52e5c 100644 --- a/services/addons/images/firmware_manager/firmware_scheduler.py +++ b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py @@ -2,10 +2,9 @@ from common import pgquery from collections import deque from flask import Flask, jsonify, request -from subprocess import Popen, DEVNULL from threading import Lock from waitress import serve -import json +import requests import logging import os @@ -14,16 +13,10 @@ log_level = os.environ.get("LOGGING_LEVEL", "INFO") logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) -manufacturer_upgrade_scripts = { - "Commsignia": "commsignia_upgrader.py", - "Yunex": "yunex_upgrader.py", -} - # Tracker for active firmware upgrades # Key: IPv4 string of target device # Value: Dictionary with the following key-values: -# - process # - manufacturer # - model # - ssh_username @@ -44,6 +37,7 @@ def get_upgrade_limit() -> int: except ValueError: raise ValueError("The environment variable 'ACTIVE_UPGRADE_LIMIT' must be an integer.") + # Function to query the CV Manager PostgreSQL database for RSUs that have: # - A different target version than their current version # - A target firmware that complies with an existing upgrade rule relative to the RSU's current version @@ -81,18 +75,21 @@ def start_tasks_from_queue(): try: rsu_upgrade_info = upgrade_queue_info[rsu_to_upgrade] del upgrade_queue_info[rsu_to_upgrade] - p = Popen( - [ - "python3", - f'/home/{manufacturer_upgrade_scripts[rsu_upgrade_info["manufacturer"]]}', - f"'{json.dumps(rsu_upgrade_info)}'", - ], - stdout=DEVNULL, - ) - rsu_upgrade_info["process"] = p + + # Begin the firmware upgrade using the Upgrade Runner API + upgrade_runner_endpoint = os.environ.get("UPGRADE_RUNNER_ENDPOINT", "UNDEFINED") + response = requests.post(f"{upgrade_runner_endpoint}/run_firmware_upgrade", json=rsu_upgrade_info) + # Remove redundant ipv4_address from rsu since it is the key for active_upgrades del rsu_upgrade_info["ipv4_address"] - active_upgrades[rsu_to_upgrade] = rsu_upgrade_info + + # If the POST response includes a 201 code, add it to the active upgrades + if response.status_code == 201: + active_upgrades[rsu_to_upgrade] = rsu_upgrade_info + else: + logging.error( + f"Firmware upgrade runner request failed for {rsu_to_upgrade}. Check Upgrade Runner logs for details." + ) except Exception as err: # If this case occurs, only log it since there may not be a listener. # Since the upgrade_queue and upgrade_queue_info will no longer have the RSU present, @@ -273,12 +270,12 @@ def check_for_upgrades(): def serve_rest_api(): # Run Flask app for manually initiated firmware upgrades - logging.info("Initiating Firmware Manager REST API...") + logging.info("Initiating the Firmware Manager Upgrade Scheduler REST API...") serve(app, host="0.0.0.0", port=8080) def init_background_task(): - logging.info("Initiating Firmware Manager background checker...") + logging.info("Initiating the Firmware Manager Upgrade Scheduler background checker...") # Run scheduler for async RSU firmware upgrade checks scheduler = BackgroundScheduler({"apscheduler.timezone": "UTC"}) scheduler.add_job(check_for_upgrades, "cron", minute="0") From e796d2743c122e6b01585c0cd3ce2129f177e84d Mon Sep 17 00:00:00 2001 From: Drew Johnston <31270488+drewjj@users.noreply.github.com> Date: Thu, 3 Oct 2024 09:06:58 -0600 Subject: [PATCH 3/7] Unit tests for the upgrade scheduler and runner scripts --- .../addons/images/firmware_manager/README.md | 10 +- .../images/firmware_manager/requirements.txt | 1 + .../upgrade_runner/upgrade_runner.py | 2 +- .../upgrade_scheduler/upgrade_scheduler.py | 3 +- .../firmware_manager/test_firmware_manager.py | 645 -------------- .../test_commsignia_upgrader.py | 75 +- .../test_download_blob.py | 5 +- .../upgrade_runner/test_upgrade_runner.py | 106 +++ .../test_upgrade_runner_values.py | 20 + .../{ => upgrade_runner}/test_upgrader.py | 50 +- .../test_yunex_upgrader.py | 105 ++- .../test_upgrade_scheduler.py | 790 ++++++++++++++++++ .../test_upgrade_scheduler_values.py} | 0 services/pytest.ini | 3 +- 14 files changed, 1063 insertions(+), 752 deletions(-) delete mode 100644 services/addons/tests/firmware_manager/test_firmware_manager.py rename services/addons/tests/firmware_manager/{ => upgrade_runner}/test_commsignia_upgrader.py (80%) rename services/addons/tests/firmware_manager/{ => upgrade_runner}/test_download_blob.py (85%) create mode 100644 services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner.py create mode 100644 services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner_values.py rename services/addons/tests/firmware_manager/{ => upgrade_runner}/test_upgrader.py (75%) rename services/addons/tests/firmware_manager/{ => upgrade_runner}/test_yunex_upgrader.py (77%) create mode 100644 services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py rename services/addons/tests/firmware_manager/{test_firmware_manager_values.py => upgrade_scheduler/test_upgrade_scheduler_values.py} (100%) diff --git a/services/addons/images/firmware_manager/README.md b/services/addons/images/firmware_manager/README.md index ee6721dd..2bfcea63 100644 --- a/services/addons/images/firmware_manager/README.md +++ b/services/addons/images/firmware_manager/README.md @@ -12,20 +12,20 @@ ## About -This directory contains a microservice that runs within the CV Manager GKE Cluster. The firmware manager monitors the CV Manager PostgreSQL database to determine if there are any RSUs that are targeted for a firmware upgrade. This monitoring is a once-per-hour, scheduled occurrence. Alternatively, this micro-service hosts a REST API for directly initiating firmware upgrades - this is used by the CV Manager API. Firmware upgrades are then run in parallel and tracked until completion. +This directory contains two microservices that run within the CV Manager GKE Cluster. The firmware manager upgrade scheduler monitors the CV Manager PostgreSQL database to determine if there are any RSUs that are targeted for a firmware upgrade. This monitoring is a once-per-hour, scheduled occurrence. Alternatively, this micro-service hosts a REST API for directly initiating firmware upgrades - this is used by the CV Manager API. Firmware upgrades then schedule off tasks to the firmware manager upgrade runner that is initiated through an HTTP request. This allows for better scaling for more parallel upgrades. An RSU is determined to be ready for upgrade if its entry in the "rsus" table in PostgreSQL has its "target_firmware_version" set to be different than its "firmware_version". The Firmware Manager will ignore all devices with incompatible firmware upgrades set as their target firmware based on the "firmware_upgrade_rules" table. The CV Manager API will only offer CV Manager webapp users compatible options so this generally is a precaution. Hosting firmware files is recommended to be done via the cloud. GCP cloud storage is the currently supported method, but a directory mounted as a docker volume can also be used. Alternative cloud support can be added via the [download_blob.py](download_blob.py) script. Firmware storage must be organized by: `vendor/rsu-model/firmware-version/install_package`. -Firmware upgrades have unique procedures based on RSU vendor/manufacturer. To avoid requiring a unique bash script for every single firmware upgrade, the Firmware Manager has been written to use vendor based upgrade scripts that have been thoroughly tested. An interface-like abstract class, [base_upgrader.py](base_upgrader.py), has been made for helping create upgrade scripts for vendors not yet supported. The Firmware Manager selects the script to use based off the RSU's "model" column in the "rsus" table. These scripts report back to the Firmware Manager on completion with a status of whether the upgrade was a success or failure. Regardless, the Firmware Manager will remove the process from its tracking and update the PostgreSQL database accordingly. +Firmware upgrades have unique procedures based on RSU vendor/manufacturer. To avoid requiring a unique bash script for every single firmware upgrade, the firmware manager upgrade runner has been written to use vendor based upgrade scripts that have been thoroughly tested. An interface-like abstract class, [base_upgrader.py](base_upgrader.py), has been made for helping create upgrade scripts for vendors not yet supported. The firmware manager upgrade runner selects the script to use based off the RSU's "model" column in the "rsus" table. These scripts report back to the firmware manager upgrade scheduler on completion with a status of whether the upgrade was a success or failure. Regardless, the Firmware Manager will remove the process from its tracking and update the PostgreSQL database accordingly. List of currently supported vendors: - Commsignia - Yunex -Available REST endpoints: +Available Firmware Manager Upgrade Scheduler REST endpoints: - /init_firmware_upgrade [ **POST** ] `{ "rsu_ip": "" }` - `rsu_ip` is the target RSU being upgraded (The target firmware is separately updated in PostgreSQL, this is just to get the Firmware Manager to immediately go look) @@ -36,6 +36,10 @@ Available REST endpoints: - Used to list all active upgrades in the form: `{"active_upgrades": {"1.1.1.1": {"manufacturer": "Commsignia", "model": "ITS-RS4-M", "target_firmware_id": 2, "target_firmware_version": "y20.39.0", "install_package": "blob.blob"}}}` +Available Firmware Manager Upgrade Runner REST endpoints: + +- /run_firmware_upgrade [ **POST** ] `{ "ipv4_address": "", "manufacturer": "", "model": "", "ssh_username": "", "ssh_password": "","target_firmware_id": "", "target_firmware_version": "", "install_package": ""}` + ## Requirements To properly run the firmware_manager microservice the following services are also required: diff --git a/services/addons/images/firmware_manager/requirements.txt b/services/addons/images/firmware_manager/requirements.txt index 5a26ee63..943a42d3 100644 --- a/services/addons/images/firmware_manager/requirements.txt +++ b/services/addons/images/firmware_manager/requirements.txt @@ -1,6 +1,7 @@ APScheduler==3.10.4 google-cloud-storage==2.14.0 flask==3.0.0 +marshmallow==3.20.1 paramiko==3.5.0 pg8000==1.30.2 requests==2.31.0 diff --git a/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py b/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py index 920b546f..fc5b5762 100644 --- a/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py +++ b/services/addons/images/firmware_manager/upgrade_runner/upgrade_runner.py @@ -80,7 +80,7 @@ def run_firmware_upgrade(): # Verify HTTP body JSON object request_args = request.get_json() schema = RunFirmwareUpgradeSchema() - errors = schema.validate(request.json) + errors = schema.validate(request_args) if errors: logging.error(str(errors)) abort(400, str(errors)) diff --git a/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py index 37a52e5c..1993b571 100644 --- a/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py +++ b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py @@ -85,10 +85,11 @@ def start_tasks_from_queue(): # If the POST response includes a 201 code, add it to the active upgrades if response.status_code == 201: + logging.info(f"Firmware upgrade runner successfully requested to begin the upgrade for {rsu_to_upgrade}") active_upgrades[rsu_to_upgrade] = rsu_upgrade_info else: logging.error( - f"Firmware upgrade runner request failed for {rsu_to_upgrade}. Check Upgrade Runner logs for details." + f"Firmware upgrade runner request failed for {rsu_to_upgrade}, check Upgrade Runner logs for details" ) except Exception as err: # If this case occurs, only log it since there may not be a listener. diff --git a/services/addons/tests/firmware_manager/test_firmware_manager.py b/services/addons/tests/firmware_manager/test_firmware_manager.py deleted file mode 100644 index bd760a32..00000000 --- a/services/addons/tests/firmware_manager/test_firmware_manager.py +++ /dev/null @@ -1,645 +0,0 @@ -from unittest.mock import call, patch, MagicMock -from subprocess import DEVNULL -from collections import deque -import test_firmware_manager_values as fmv -import pytest -from addons.images.firmware_manager import firmware_manager - - -@patch("addons.images.firmware_manager.firmware_manager.active_upgrades", {}) -@patch("addons.images.firmware_manager.firmware_manager.pgquery.query_db") -def test_get_rsu_upgrade_data_all(mock_querydb): - mock_querydb.return_value = [ - ({"ipv4_address": "8.8.8.8"}, ""), - ({"ipv4_address": "9.9.9.9"}, ""), - ] - - result = firmware_manager.get_rsu_upgrade_data() - - mock_querydb.assert_called_with(fmv.all_rsus_query) - assert result == [{"ipv4_address": "8.8.8.8"}, {"ipv4_address": "9.9.9.9"}] - - -@patch("addons.images.firmware_manager.firmware_manager.active_upgrades", {}) -@patch("addons.images.firmware_manager.firmware_manager.pgquery.query_db") -def test_get_rsu_upgrade_data_one(mock_querydb): - mock_querydb.return_value = [(fmv.rsu_info, "")] - - result = firmware_manager.get_rsu_upgrade_data(rsu_ip="8.8.8.8") - - expected_result = [fmv.rsu_info] - mock_querydb.assert_called_with(fmv.one_rsu_query) - assert result == expected_result - - -# start_tasks_from_queue tests - - -@patch("addons.images.firmware_manager.firmware_manager.active_upgrades", {}) -@patch( - "addons.images.firmware_manager.firmware_manager.upgrade_queue", deque(["8.8.8.8"]) -) -@patch( - "addons.images.firmware_manager.firmware_manager.upgrade_queue_info", - { - "8.8.8.8": { - "ipv4_address": "8.8.8.8", - "manufacturer": "Commsignia", - "model": "ITS-RS4-M", - "ssh_username": "user", - "ssh_password": "psw", - "target_firmware_id": 2, - "target_firmware_version": "y20.39.0", - "install_package": "install_package.tar", - } - }, -) -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch( - "addons.images.firmware_manager.firmware_manager.Popen", - side_effect=Exception("Process failed to start"), -) -def test_start_tasks_from_queue_popen_fail(mock_popen, mock_logging): - firmware_manager.start_tasks_from_queue() - - # Assert firmware upgrade process was started with expected arguments - expected_json_str = ( - '\'{"ipv4_address": "8.8.8.8", "manufacturer": "Commsignia", "model": "ITS-RS4-M", ' - '"ssh_username": "user", "ssh_password": "psw", "target_firmware_id": 2, "target_firmware_version": "y20.39.0", ' - '"install_package": "install_package.tar"}\'' - ) - mock_popen.assert_called_with( - ["python3", f"/home/commsignia_upgrader.py", expected_json_str], - stdout=DEVNULL, - ) - - # Assert logging - mock_logging.info.assert_not_called() - mock_logging.error.assert_called_with( - f"Encountered error of type {Exception} while starting automatic upgrade process for 8.8.8.8: Process failed to start" - ) - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch("addons.images.firmware_manager.firmware_manager.active_upgrades", {}) -@patch( - "addons.images.firmware_manager.firmware_manager.upgrade_queue", deque(["8.8.8.8"]) -) -@patch( - "addons.images.firmware_manager.firmware_manager.upgrade_queue_info", - { - "8.8.8.8": { - "ipv4_address": "8.8.8.8", - "manufacturer": "Commsignia", - "model": "ITS-RS4-M", - "ssh_username": "user", - "ssh_password": "psw", - "target_firmware_id": 2, - "target_firmware_version": "y20.39.0", - "install_package": "install_package.tar", - } - }, -) -@patch("addons.images.firmware_manager.firmware_manager.Popen") -def test_start_tasks_from_queue_popen_success(mock_popen, mock_logging): - mock_popen_obj = mock_popen.return_value - - firmware_manager.start_tasks_from_queue() - - # Assert firmware upgrade process was started with expected arguments - expected_json_str = ( - '\'{"ipv4_address": "8.8.8.8", "manufacturer": "Commsignia", "model": "ITS-RS4-M", ' - '"ssh_username": "user", "ssh_password": "psw", "target_firmware_id": 2, "target_firmware_version": "y20.39.0", ' - '"install_package": "install_package.tar"}\'' - ) - mock_popen.assert_called_with( - ["python3", f"/home/commsignia_upgrader.py", expected_json_str], - stdout=DEVNULL, - ) - # Assert the process reference is successfully tracked in the active_upgrades dictionary - assert firmware_manager.active_upgrades["8.8.8.8"]["process"] == mock_popen_obj - - mock_logging.info.assert_not_called() - mock_logging.error.assert_not_called() - - -# init_firmware_upgrade tests - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch("addons.images.firmware_manager.firmware_manager.active_upgrades", {}) -def test_init_firmware_upgrade_missing_rsu_ip(mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = {} - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.init_firmware_upgrade() - - mock_flask_jsonify.assert_called_with( - {"error": "Missing 'rsu_ip' parameter"} - ) - assert code == 400 - - mock_logging.info.assert_not_called() - mock_logging.error.assert_not_called() - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch( - "addons.images.firmware_manager.firmware_manager.active_upgrades", {"8.8.8.8": {}} -) -def test_init_firmware_upgrade_already_running(mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.init_firmware_upgrade() - - mock_flask_jsonify.assert_called_with( - { - "error": f"Firmware upgrade failed to start for '8.8.8.8': an upgrade is already underway or queued for the target device" - } - ) - assert code == 500 - - # Assert logging - mock_logging.info.assert_called_with("Checking if existing upgrade is running or queued for '8.8.8.8'") - mock_logging.error.assert_not_called() - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch("addons.images.firmware_manager.firmware_manager.active_upgrades", {}) -@patch( - "addons.images.firmware_manager.firmware_manager.get_rsu_upgrade_data", - MagicMock(return_value=[]), -) -def test_init_firmware_upgrade_no_eligible_upgrade(mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.init_firmware_upgrade() - - mock_flask_jsonify.assert_called_with( - { - "error": f"Firmware upgrade failed to start for '8.8.8.8': the target firmware is already installed or is an invalid upgrade from the current firmware" - } - ) - assert code == 500 - - # Assert logging - mock_logging.info.assert_has_calls( - [ - call("Checking if existing upgrade is running or queued for '8.8.8.8'"), - call("Querying RSU data for '8.8.8.8'") - ] - ) - mock_logging.error.assert_not_called() - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch("addons.images.firmware_manager.firmware_manager.active_upgrades", {}) -@patch( - "addons.images.firmware_manager.firmware_manager.get_rsu_upgrade_data", - MagicMock(return_value=[fmv.rsu_info]), -) -@patch("addons.images.firmware_manager.firmware_manager.start_tasks_from_queue") -def test_init_firmware_upgrade_success(mock_stfq, mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.init_firmware_upgrade() - - # Assert start_tasks_from_queue is called - mock_stfq.assert_called_with() - - # Assert the process reference is successfully tracked in the upgrade_queue - assert firmware_manager.upgrade_queue[0] == "8.8.8.8" - - # Assert REST response is as expected from a successful run - mock_flask_jsonify.assert_called_with( - {"message": f"Firmware upgrade started successfully for '8.8.8.8'"} - ) - assert code == 201 - - # Assert logging - mock_logging.info.assert_has_calls( - [ - call("Checking if existing upgrade is running or queued for '8.8.8.8'"), - call("Querying RSU data for '8.8.8.8'"), - call("Adding '8.8.8.8' to the firmware manager upgrade queue") - ] - ) - mock_logging.error.assert_not_called() - - firmware_manager.upgrade_queue = deque([]) - - -# firmware_upgrade_completed tests - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch("addons.images.firmware_manager.firmware_manager.active_upgrades", {}) -def test_firmware_upgrade_completed_missing_rsu_ip(mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = {} - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.firmware_upgrade_completed() - - mock_flask_jsonify.assert_called_with( - {"error": "Missing 'rsu_ip' parameter"} - ) - assert code == 400 - - # Assert logging - mock_logging.info.assert_not_called() - mock_logging.error.assert_not_called() - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch("addons.images.firmware_manager.firmware_manager.active_upgrades", {}) -def test_firmware_upgrade_completed_unknown_process(mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = { - "rsu_ip": "8.8.8.8", - "status": "success", - } - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.firmware_upgrade_completed() - - mock_flask_jsonify.assert_called_with( - { - "error": "Specified device is not actively being upgraded or was already completed" - } - ) - assert code == 400 - - # Assert logging - mock_logging.info.assert_not_called() - mock_logging.error.assert_not_called() - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch( - "addons.images.firmware_manager.firmware_manager.active_upgrades", - {"8.8.8.8": fmv.upgrade_info}, -) -def test_firmware_upgrade_completed_missing_status(mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.firmware_upgrade_completed() - - mock_flask_jsonify.assert_called_with( - {"error": "Missing 'status' parameter"} - ) - assert code == 400 - - # Assert logging - mock_logging.info.assert_not_called() - mock_logging.error.assert_not_called() - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch( - "addons.images.firmware_manager.firmware_manager.active_upgrades", - {"8.8.8.8": fmv.upgrade_info}, -) -def test_firmware_upgrade_completed_illegal_status(mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8", "status": "frog"} - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.firmware_upgrade_completed() - - mock_flask_jsonify.assert_called_with( - { - "error": "Wrong value for 'status' parameter - must be either 'success' or 'fail'" - } - ) - assert code == 400 - - # Assert logging - mock_logging.info.assert_not_called() - mock_logging.error.assert_not_called() - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch( - "addons.images.firmware_manager.firmware_manager.active_upgrades", - {"8.8.8.8": fmv.upgrade_info}, -) -def test_firmware_upgrade_completed_fail_status(mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8", "status": "fail"} - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.firmware_upgrade_completed() - - assert "8.8.8.8" not in firmware_manager.active_upgrades - mock_flask_jsonify.assert_called_with( - {"message": "Firmware upgrade successfully marked as complete"} - ) - assert code == 204 - - # Assert logging - mock_logging.info.assert_called_with("Marking firmware upgrade as complete for '8.8.8.8'") - mock_logging.error.assert_not_called() - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch( - "addons.images.firmware_manager.firmware_manager.active_upgrades", - {"8.8.8.8": fmv.upgrade_info}, -) -@patch("addons.images.firmware_manager.firmware_manager.pgquery.write_db") -def test_firmware_upgrade_completed_success_status(mock_writedb, mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = { - "rsu_ip": "8.8.8.8", - "status": "success", - } - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.firmware_upgrade_completed() - - mock_writedb.assert_called_with( - "UPDATE public.rsus SET firmware_version=2 WHERE ipv4_address='8.8.8.8'" - ) - assert "8.8.8.8" not in firmware_manager.active_upgrades - mock_flask_jsonify.assert_called_with( - {"message": "Firmware upgrade successfully marked as complete"} - ) - assert code == 204 - - # Assert logging - mock_logging.info.assert_called_with("Marking firmware upgrade as complete for '8.8.8.8'") - mock_logging.error.assert_not_called() - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch( - "addons.images.firmware_manager.firmware_manager.active_upgrades", - {"8.8.8.8": fmv.upgrade_info}, -) -@patch( - "addons.images.firmware_manager.firmware_manager.pgquery.write_db", - side_effect=Exception("Failure to query PostgreSQL"), -) -def test_firmware_upgrade_completed_success_status_exception(mock_writedb, mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = { - "rsu_ip": "8.8.8.8", - "status": "success", - } - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.firmware_upgrade_completed() - - mock_writedb.assert_called_with( - "UPDATE public.rsus SET firmware_version=2 WHERE ipv4_address='8.8.8.8'" - ) - mock_flask_jsonify.assert_called_with( - { - "error": "Unexpected error occurred while querying the PostgreSQL database - firmware upgrade not marked as complete" - } - ) - assert code == 500 - - - # Assert logging - mock_logging.info.assert_not_called() - mock_logging.error.assert_called_with("Encountered error of type while querying the PostgreSQL database: Failure to query PostgreSQL") - - -# list_active_upgrades tests - - -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch( - "addons.images.firmware_manager.firmware_manager.active_upgrades", - {"8.8.8.8": fmv.upgrade_info}, -) -def test_list_active_upgrades(mock_logging): - mock_flask_request = MagicMock() - mock_flask_request.get_json.return_value = { - "rsu_ip": "8.8.8.8", - "status": "success", - } - mock_flask_jsonify = MagicMock() - with patch( - "addons.images.firmware_manager.firmware_manager.request", mock_flask_request - ): - with patch( - "addons.images.firmware_manager.firmware_manager.jsonify", - mock_flask_jsonify, - ): - message, code = firmware_manager.list_active_upgrades() - - expected_active_upgrades = { - "8.8.8.8": { - "manufacturer": "Commsignia", - "model": "ITS-RS4-M", - "target_firmware_id": 2, - "target_firmware_version": "y20.39.0", - "install_package": "install_package.tar", - } - } - mock_flask_jsonify.assert_called_with( - {"active_upgrades": expected_active_upgrades, "upgrade_queue": []} - ) - assert code == 200 - - # Assert logging - mock_logging.info.assert_not_called() - mock_logging.error.assert_not_called() - -# check_for_upgrades tests - - -@patch( - "addons.images.firmware_manager.firmware_manager.active_upgrades", - {}, -) -@patch( - "addons.images.firmware_manager.firmware_manager.get_rsu_upgrade_data", - MagicMock(return_value=fmv.single_rsu_info), -) -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch( - "addons.images.firmware_manager.firmware_manager.Popen", - side_effect=Exception("Process failed to start"), -) -@patch("addons.images.firmware_manager.firmware_manager.get_upgrade_limit") -def test_check_for_upgrades_exception(mock_upgrade_limit, mock_popen, mock_logging): - mock_upgrade_limit.return_value = 5 - firmware_manager.check_for_upgrades() - - # Assert firmware upgrade process was started with expected arguments - expected_json_str = ( - '\'{"ipv4_address": "9.9.9.9", "manufacturer": "Commsignia", "model": "ITS-RS4-M", ' - '"ssh_username": "user", "ssh_password": "psw", "target_firmware_id": 2, "target_firmware_version": "y20.39.0", ' - '"install_package": "install_package.tar"}\'' - ) - mock_popen.assert_called_once_with( - ["python3", f"/home/commsignia_upgrader.py", expected_json_str], stdout=DEVNULL - ) - - # Assert the process reference is successfully tracked in the active_upgrades dictionary - assert "9.9.9.9" not in firmware_manager.active_upgrades - mock_logging.info.assert_has_calls( - [ - call("Checking PostgreSQL DB for RSUs with new target firmware"), - call("Adding '9.9.9.9' to the firmware manager upgrade queue"), - call("Firmware upgrade successfully started for '9.9.9.9'") - ] - ) - mock_logging.error.assert_called_with( - f"Encountered error of type {Exception} while starting automatic upgrade process for 9.9.9.9: Process failed to start" - ) - - -@patch( - "addons.images.firmware_manager.firmware_manager.active_upgrades", - {}, -) -@patch( - "addons.images.firmware_manager.firmware_manager.get_rsu_upgrade_data", - MagicMock(return_value=fmv.multi_rsu_info), -) -@patch("addons.images.firmware_manager.firmware_manager.logging") -@patch("addons.images.firmware_manager.firmware_manager.start_tasks_from_queue") -@patch("addons.images.firmware_manager.firmware_manager.get_upgrade_limit") -def test_check_for_upgrades(mock_upgrade_limit, mock_stfq, mock_logging): - mock_upgrade_limit.return_value = 5 - firmware_manager.check_for_upgrades() - - # Assert firmware upgrade process was started with expected arguments - mock_stfq.assert_called_once_with() - - # Assert the process reference is successfully tracked in the active_upgrades dictionary - assert firmware_manager.upgrade_queue[1] == "9.9.9.9" - mock_logging.info.assert_has_calls( - [ - call("Checking PostgreSQL DB for RSUs with new target firmware"), - call("Adding '8.8.8.8' to the firmware manager upgrade queue"), - call("Firmware upgrade successfully started for '8.8.8.8'"), - call("Adding '9.9.9.9' to the firmware manager upgrade queue"), - call("Firmware upgrade successfully started for '9.9.9.9'") - ] - ) - mock_logging.info.assert_called_with( - "Firmware upgrade successfully started for '9.9.9.9'" - ) - - -# Other tests - - -@patch("addons.images.firmware_manager.firmware_manager.serve") -def test_serve_rest_api(mock_serve): - firmware_manager.serve_rest_api() - mock_serve.assert_called_with(firmware_manager.app, host="0.0.0.0", port=8080) - - -@patch("addons.images.firmware_manager.firmware_manager.BackgroundScheduler") -def test_init_background_task(mock_bgscheduler): - mock_bgscheduler_obj = mock_bgscheduler.return_value - - firmware_manager.init_background_task() - - mock_bgscheduler.assert_called_with({"apscheduler.timezone": "UTC"}) - mock_bgscheduler_obj.add_job.assert_called_with( - firmware_manager.check_for_upgrades, "cron", minute="0" - ) - mock_bgscheduler_obj.start.assert_called_with() - - -def test_get_upgrade_limit_no_env(): - limit = firmware_manager.get_upgrade_limit() - assert limit == 1 - - -@patch.dict("os.environ", {"ACTIVE_UPGRADE_LIMIT": "5"}) -def test_get_upgrade_limit_with_env(): - limit = firmware_manager.get_upgrade_limit() - assert limit == 5 - - -@patch.dict("os.environ", {"ACTIVE_UPGRADE_LIMIT": "bad_value"}) -def test_get_upgrade_limit_with_bad_env(): - with pytest.raises( - ValueError, - match="The environment variable 'ACTIVE_UPGRADE_LIMIT' must be an integer.", - ): - firmware_manager.get_upgrade_limit() diff --git a/services/addons/tests/firmware_manager/test_commsignia_upgrader.py b/services/addons/tests/firmware_manager/upgrade_runner/test_commsignia_upgrader.py similarity index 80% rename from services/addons/tests/firmware_manager/test_commsignia_upgrader.py rename to services/addons/tests/firmware_manager/upgrade_runner/test_commsignia_upgrader.py index cf107fc8..58340292 100644 --- a/services/addons/tests/firmware_manager/test_commsignia_upgrader.py +++ b/services/addons/tests/firmware_manager/upgrade_runner/test_commsignia_upgrader.py @@ -1,7 +1,9 @@ from unittest.mock import call, patch, MagicMock from paramiko import WarningPolicy -from addons.images.firmware_manager.commsignia_upgrader import CommsigniaUpgrader +from addons.images.firmware_manager.upgrade_runner.commsignia_upgrader import ( + CommsigniaUpgrader, +) test_upgrade_info = { "ipv4_address": "8.8.8.8", @@ -31,10 +33,12 @@ def test_commsignia_upgrader_init(): assert test_commsignia_upgrader.ssh_password == "test-psw" -@patch("addons.images.firmware_manager.commsignia_upgrader.logging") -@patch("addons.images.firmware_manager.commsignia_upgrader.SCPClient") -@patch("addons.images.firmware_manager.commsignia_upgrader.SSHClient") -def test_commsignia_upgrader_upgrade_success_no_post_update(mock_sshclient, mock_scpclient, mock_logging): +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SCPClient") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SSHClient") +def test_commsignia_upgrader_upgrade_success_no_post_update( + mock_sshclient, mock_scpclient, mock_logging +): # Mock SSH Client and successful firmware upgrade return value sshclient_obj = mock_sshclient.return_value _stdout = MagicMock() @@ -85,16 +89,19 @@ def test_commsignia_upgrader_upgrade_success_no_post_update(mock_sshclient, mock call("Making SSH connection with 8.8.8.8..."), call("Copying installation package to 8.8.8.8..."), call("Running firmware upgrade for 8.8.8.8..."), - call("ALL OK") + call("ALL OK"), ] ) mock_logging.error.assert_not_called() -@patch("addons.images.firmware_manager.commsignia_upgrader.logging") -@patch("addons.images.firmware_manager.commsignia_upgrader.SCPClient") -@patch("addons.images.firmware_manager.commsignia_upgrader.SSHClient") -@patch("addons.images.firmware_manager.commsignia_upgrader.time") -def test_commsignia_upgrader_upgrade_success_post_update(mock_time, mock_sshclient, mock_scpclient, mock_logging): + +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SCPClient") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SSHClient") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.time") +def test_commsignia_upgrader_upgrade_success_post_update( + mock_time, mock_sshclient, mock_scpclient, mock_logging +): # Mock SSH Client and successful firmware upgrade return value sshclient_obj = mock_sshclient.return_value _stdout = MagicMock() @@ -137,7 +144,7 @@ def test_commsignia_upgrader_upgrade_success_post_update(mock_time, mock_sshclie # Assert SSH firmware upgrade run sshclient_obj.exec_command.assert_has_calls( [ - call("signedUpgrade.sh /tmp/firmware_package.tar"), + call("signedUpgrade.sh /tmp/firmware_package.tar"), call("reboot"), call("chmod +x /tmp/post_upgrade.sh"), call("/tmp/post_upgrade.sh"), @@ -159,16 +166,19 @@ def test_commsignia_upgrader_upgrade_success_post_update(mock_time, mock_sshclie call("Copying post upgrade script to 8.8.8.8..."), call("Running post upgrade script for 8.8.8.8..."), call("ALL OK"), - call("Post upgrade script executed successfully for rsu: 8.8.8.8.") + call("Post upgrade script executed successfully for rsu: 8.8.8.8."), ] ) mock_logging.error.assert_not_called() -@patch("addons.images.firmware_manager.commsignia_upgrader.SCPClient") -@patch("addons.images.firmware_manager.commsignia_upgrader.SSHClient") -@patch("addons.images.firmware_manager.commsignia_upgrader.time") -@patch("addons.images.firmware_manager.commsignia_upgrader.logging") -def test_commsignia_upgrader_upgrade_post_update_fail(mock_logging, mock_time, mock_sshclient, mock_scpclient): + +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SCPClient") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SSHClient") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.time") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.logging") +def test_commsignia_upgrader_upgrade_post_update_fail( + mock_logging, mock_time, mock_sshclient, mock_scpclient +): # Mock SSH Client and successful firmware upgrade return value sshclient_obj = mock_sshclient.return_value _stdout = MagicMock() @@ -207,21 +217,21 @@ def test_commsignia_upgrader_upgrade_post_update_fail(mock_logging, mock_time, m # Assert SCP file transfer mock_scpclient.assert_called_with(sshclient_obj.get_transport()) scpclient_obj.put.assert_called_with( - '/home/8.8.8.8/post_upgrade.sh', remote_path='/tmp/' + "/home/8.8.8.8/post_upgrade.sh", remote_path="/tmp/" ) scpclient_obj.close.assert_called_with() # Assert SSH firmware upgrade run sshclient_obj.exec_command.assert_has_calls( [ - call("signedUpgrade.sh /tmp/firmware_package.tar"), + call("signedUpgrade.sh /tmp/firmware_package.tar"), call("reboot"), call("chmod +x /tmp/post_upgrade.sh"), call("/tmp/post_upgrade.sh"), ] ) sshclient_obj.close.assert_called_with() - + # Assert logging mock_logging.info.assert_has_calls( [ @@ -235,14 +245,17 @@ def test_commsignia_upgrader_upgrade_post_update_fail(mock_logging, mock_time, m call("NOT OK TEST"), ] ) - mock_logging.error.assert_called_with("Failed to execute post upgrade script for rsu 8.8.8.8: NOT OK TEST") + mock_logging.error.assert_called_with( + "Failed to execute post upgrade script for rsu 8.8.8.8: NOT OK TEST" + ) # Assert notified success value notify.assert_called_with(success=True) -@patch("addons.images.firmware_manager.commsignia_upgrader.logging") -@patch("addons.images.firmware_manager.commsignia_upgrader.SCPClient") -@patch("addons.images.firmware_manager.commsignia_upgrader.SSHClient") + +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SCPClient") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SSHClient") def test_commsignia_upgrader_upgrade_fail(mock_sshclient, mock_scpclient, mock_logging): # Mock SSH Client and failed firmware upgrade return value sshclient_obj = mock_sshclient.return_value @@ -291,7 +304,7 @@ def test_commsignia_upgrader_upgrade_fail(mock_sshclient, mock_scpclient, mock_l call("Making SSH connection with 8.8.8.8..."), call("Copying installation package to 8.8.8.8..."), call("Running firmware upgrade for 8.8.8.8..."), - call("NOT OK TEST") + call("NOT OK TEST"), ] ) mock_logging.error.assert_not_called() @@ -300,9 +313,9 @@ def test_commsignia_upgrader_upgrade_fail(mock_sshclient, mock_scpclient, mock_l notify.assert_called_with(success=False) -@patch("addons.images.firmware_manager.commsignia_upgrader.logging") -@patch("addons.images.firmware_manager.commsignia_upgrader.SCPClient") -@patch("addons.images.firmware_manager.commsignia_upgrader.SSHClient") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SCPClient") +@patch("addons.images.firmware_manager.upgrade_runner.commsignia_upgrader.SSHClient") def test_commsignia_upgrader_upgrade_exception( mock_sshclient, mock_scpclient, mock_logging ): @@ -338,7 +351,9 @@ def test_commsignia_upgrader_upgrade_exception( # Assert logging mock_logging.info.assert_called_with("Making SSH connection with 8.8.8.8...") - mock_logging.error.assert_called_with("Failed to perform firmware upgrade for 8.8.8.8: Exception occurred during upgrade") + mock_logging.error.assert_called_with( + "Failed to perform firmware upgrade for 8.8.8.8: Exception occurred during upgrade" + ) # Assert exception was cleaned up and firmware manager was notified of upgrade failure cleanup.assert_called_with() diff --git a/services/addons/tests/firmware_manager/test_download_blob.py b/services/addons/tests/firmware_manager/upgrade_runner/test_download_blob.py similarity index 85% rename from services/addons/tests/firmware_manager/test_download_blob.py rename to services/addons/tests/firmware_manager/upgrade_runner/test_download_blob.py index a266c44d..8cf711ff 100644 --- a/services/addons/tests/firmware_manager/test_download_blob.py +++ b/services/addons/tests/firmware_manager/upgrade_runner/test_download_blob.py @@ -1,11 +1,10 @@ from unittest.mock import MagicMock, patch import os -import pytest -from addons.images.firmware_manager import download_blob +from addons.images.firmware_manager.upgrade_runner import download_blob -@patch("addons.images.firmware_manager.download_blob.logging") +@patch("addons.images.firmware_manager.upgrade_runner.download_blob.logging") def test_download_docker_blob(mock_logging): # prepare os.system = MagicMock() diff --git a/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner.py b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner.py new file mode 100644 index 00000000..fff9b3e9 --- /dev/null +++ b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner.py @@ -0,0 +1,106 @@ +from unittest.mock import call, patch, MagicMock +from subprocess import DEVNULL +from addons.images.firmware_manager.upgrade_runner import upgrade_runner +from werkzeug.exceptions import BadRequest +import services.addons.tests.firmware_manager.upgrade_runner.test_upgrade_runner_values as fmv + +# start_upgrade_task tests + + +@patch("addons.images.firmware_manager.upgrade_runner.upgrade_runner.Popen") +def test_start_upgrade_task_success(mock_popen): + with upgrade_runner.app.app_context(): + try: + response = upgrade_runner.start_upgrade_task(fmv.request_body_good) + + # Assert + expected_json_str = ( + '\'{"ipv4_address": "8.8.8.8", "manufacturer": "Commsignia", "model": "ITS-RS4-M", ' + '"ssh_username": "user", "ssh_password": "psw", "target_firmware_id": 2, "target_firmware_version": "y20.39.0", ' + '"install_package": "install_package.tar"}\'' + ) + mock_popen.assert_called_with( + ["python3", f"/home/commsignia_upgrader.py", expected_json_str], + stdout=DEVNULL, + ) + assert response[1] == 201 + except Exception as e: + assert False + + +@patch( + "addons.images.firmware_manager.upgrade_runner.upgrade_runner.Popen", + side_effect=Exception("Process failed to start"), +) +def test_start_upgrade_task_fail(mock_popen): + with upgrade_runner.app.app_context(): + try: + upgrade_runner.start_upgrade_task(fmv.request_body_good) + assert False + except Exception as e: + # Assert + expected_json_str = ( + '\'{"ipv4_address": "8.8.8.8", "manufacturer": "Commsignia", "model": "ITS-RS4-M", ' + '"ssh_username": "user", "ssh_password": "psw", "target_firmware_id": 2, "target_firmware_version": "y20.39.0", ' + '"install_package": "install_package.tar"}\'' + ) + mock_popen.assert_called_with( + ["python3", f"/home/commsignia_upgrader.py", expected_json_str], + stdout=DEVNULL, + ) + + +# run_firmware_upgrade tests + + +@patch( + "addons.images.firmware_manager.upgrade_runner.upgrade_runner.start_upgrade_task", + MagicMock(), +) +@patch("addons.images.firmware_manager.upgrade_runner.upgrade_runner.logging") +def test_run_firmware_upgrade_missing_rsu_ip(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = fmv.request_body_bad + + with upgrade_runner.app.app_context(): + with patch( + "addons.images.firmware_manager.upgrade_runner.upgrade_runner.request", + mock_flask_request, + ): + try: + upgrade_runner.run_firmware_upgrade() + assert False + except BadRequest as e: + mock_logging.error.assert_called_with( + "{'ipv4_address': ['Missing data for required field.']}" + ) + + +@patch( + "addons.images.firmware_manager.upgrade_runner.upgrade_runner.start_upgrade_task" +) +@patch("addons.images.firmware_manager.upgrade_runner.upgrade_runner.logging") +def test_run_firmware_upgrade_success(mock_logging, mock_start_upgrade_task): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = fmv.request_body_good + + with upgrade_runner.app.app_context(): + with patch( + "addons.images.firmware_manager.upgrade_runner.upgrade_runner.request", + mock_flask_request, + ): + try: + upgrade_runner.run_firmware_upgrade() + mock_logging.error.assert_not_called() + mock_start_upgrade_task.assert_called_with(fmv.request_body_good) + except BadRequest as e: + assert False + + +# Other tests + + +@patch("addons.images.firmware_manager.upgrade_runner.upgrade_runner.serve") +def test_serve_rest_api(mock_serve): + upgrade_runner.serve_rest_api() + mock_serve.assert_called_with(upgrade_runner.app, host="0.0.0.0", port=8080) diff --git a/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner_values.py b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner_values.py new file mode 100644 index 00000000..221660e9 --- /dev/null +++ b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner_values.py @@ -0,0 +1,20 @@ +request_body_good = { + "ipv4_address": "8.8.8.8", + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", +} + +request_body_bad = { + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", +} diff --git a/services/addons/tests/firmware_manager/test_upgrader.py b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrader.py similarity index 75% rename from services/addons/tests/firmware_manager/test_upgrader.py rename to services/addons/tests/firmware_manager/upgrade_runner/test_upgrader.py index 40ce9be1..65f2970c 100644 --- a/services/addons/tests/firmware_manager/test_upgrader.py +++ b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrader.py @@ -2,8 +2,10 @@ import os import pytest -from addons.images.firmware_manager import upgrader -from addons.images.firmware_manager.upgrader import StorageProviderNotSupportedException +from addons.images.firmware_manager.upgrade_runner import upgrader +from addons.images.firmware_manager.upgrade_runner.upgrader import ( + StorageProviderNotSupportedException, +) # Test class for testing the abstract class @@ -44,8 +46,8 @@ def test_upgrader_init(): assert test_upgrader.ssh_password == "test-psw" -@patch("addons.images.firmware_manager.upgrader.shutil") -@patch("addons.images.firmware_manager.upgrader.Path") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.shutil") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.Path") def test_cleanup_exists(mock_Path, mock_shutil): mock_path_obj = mock_Path.return_value mock_path_obj.exists.return_value = True @@ -58,8 +60,8 @@ def test_cleanup_exists(mock_Path, mock_shutil): mock_shutil.rmtree.assert_called_with(mock_path_obj) -@patch("addons.images.firmware_manager.upgrader.shutil") -@patch("addons.images.firmware_manager.upgrader.Path") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.shutil") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.Path") def test_cleanup_not_exist(mock_Path, mock_shutil): mock_path_obj = mock_Path.return_value mock_path_obj.exists.return_value = False @@ -74,7 +76,7 @@ def test_cleanup_not_exist(mock_Path, mock_shutil): @patch.dict(os.environ, {"BLOB_STORAGE_PROVIDER": "GCP"}) @patch("common.gcs_utils.download_gcp_blob") -@patch("addons.images.firmware_manager.upgrader.Path") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.Path") def test_download_blob_gcp(mock_Path, mock_download_gcp_blob): mock_path_obj = mock_Path.return_value test_upgrader = TestUpgrader(test_upgrade_info) @@ -84,12 +86,16 @@ def test_download_blob_gcp(mock_Path, mock_download_gcp_blob): mock_path_obj.mkdir.assert_called_with(exist_ok=True) mock_download_gcp_blob.assert_called_with( "test-manufacturer/test-model/1.0.0/firmware_package.tar", - "/home/8.8.8.8/firmware_package.tar", '' + "/home/8.8.8.8/firmware_package.tar", + "", ) + @patch.dict(os.environ, {"BLOB_STORAGE_PROVIDER": "DOCKER"}) -@patch("addons.images.firmware_manager.upgrader.download_blob.download_docker_blob") -@patch("addons.images.firmware_manager.upgrader.Path") +@patch( + "addons.images.firmware_manager.upgrade_runner.upgrader.download_blob.download_docker_blob" +) +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.Path") def test_download_blob_docker(mock_Path, mock_download_docker_blob): mock_path_obj = mock_Path.return_value test_upgrader = TestUpgrader(test_upgrade_info) @@ -104,9 +110,9 @@ def test_download_blob_docker(mock_Path, mock_download_docker_blob): @patch.dict(os.environ, {"BLOB_STORAGE_PROVIDER": "Test"}) -@patch("addons.images.firmware_manager.upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.logging") @patch("common.gcs_utils.download_gcp_blob") -@patch("addons.images.firmware_manager.upgrader.Path") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.Path") def test_download_blob_not_supported(mock_Path, mock_download_gcp_blob, mock_logging): mock_path_obj = mock_Path.return_value test_upgrader = TestUpgrader(test_upgrade_info) @@ -119,8 +125,8 @@ def test_download_blob_not_supported(mock_Path, mock_download_gcp_blob, mock_log mock_logging.error.assert_called_with("Unsupported blob storage provider") -@patch("addons.images.firmware_manager.upgrader.logging") -@patch("addons.images.firmware_manager.upgrader.requests") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.requests") def test_notify_firmware_manager_success(mock_requests, mock_logging): test_upgrader = TestUpgrader(test_upgrade_info) @@ -135,8 +141,8 @@ def test_notify_firmware_manager_success(mock_requests, mock_logging): mock_requests.post.assert_called_with(expected_url, json=expected_body) -@patch("addons.images.firmware_manager.upgrader.logging") -@patch("addons.images.firmware_manager.upgrader.requests") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.requests") def test_notify_firmware_manager_fail(mock_requests, mock_logging): test_upgrader = TestUpgrader(test_upgrade_info) @@ -151,8 +157,8 @@ def test_notify_firmware_manager_fail(mock_requests, mock_logging): mock_requests.post.assert_called_with(expected_url, json=expected_body) -@patch("addons.images.firmware_manager.upgrader.logging") -@patch("addons.images.firmware_manager.upgrader.requests") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.requests") def test_notify_firmware_manager_exception(mock_requests, mock_logging): mock_requests.post.side_effect = Exception("Exception occurred during upgrade") test_upgrader = TestUpgrader(test_upgrade_info) @@ -164,8 +170,8 @@ def test_notify_firmware_manager_exception(mock_requests, mock_logging): ) -@patch("addons.images.firmware_manager.upgrader.time") -@patch("addons.images.firmware_manager.upgrader.subprocess") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.time") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.subprocess") def test_upgrader_wait_until_online_success(mock_subprocess, mock_time): run_response_obj = MagicMock() run_response_obj.returncode = 0 @@ -178,8 +184,8 @@ def test_upgrader_wait_until_online_success(mock_subprocess, mock_time): assert mock_time.sleep.call_count == 1 -@patch("addons.images.firmware_manager.upgrader.time") -@patch("addons.images.firmware_manager.upgrader.subprocess") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.time") +@patch("addons.images.firmware_manager.upgrade_runner.upgrader.subprocess") def test_upgrader_wait_until_online_timeout(mock_subprocess, mock_time): run_response_obj = MagicMock() run_response_obj.returncode = 1 diff --git a/services/addons/tests/firmware_manager/test_yunex_upgrader.py b/services/addons/tests/firmware_manager/upgrade_runner/test_yunex_upgrader.py similarity index 77% rename from services/addons/tests/firmware_manager/test_yunex_upgrader.py rename to services/addons/tests/firmware_manager/upgrade_runner/test_yunex_upgrader.py index e1dc8624..87a9b1b6 100644 --- a/services/addons/tests/firmware_manager/test_yunex_upgrader.py +++ b/services/addons/tests/firmware_manager/upgrade_runner/test_yunex_upgrader.py @@ -1,6 +1,6 @@ from unittest.mock import call, patch, MagicMock, mock_open -from addons.images.firmware_manager.yunex_upgrader import YunexUpgrader +from addons.images.firmware_manager.upgrade_runner.yunex_upgrader import YunexUpgrader test_upgrade_info = { "ipv4_address": "8.8.8.8", @@ -34,8 +34,8 @@ def test_yunex_upgrader_init(): assert test_yunex_upgrader.ssh_password == "test-psw" -@patch("addons.images.firmware_manager.yunex_upgrader.logging") -@patch("addons.images.firmware_manager.yunex_upgrader.subprocess") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.subprocess") def test_yunex_upgrader_run_xfer_upgrade_success(mock_subprocess, mock_logging): run_response_obj = MagicMock() run_response_obj.returncode = 0 @@ -55,8 +55,8 @@ def test_yunex_upgrader_run_xfer_upgrade_success(mock_subprocess, mock_logging): mock_logging.error.assert_not_called() -@patch("addons.images.firmware_manager.yunex_upgrader.logging") -@patch("addons.images.firmware_manager.yunex_upgrader.subprocess") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.subprocess") def test_yunex_upgrader_run_xfer_upgrade_fail_code(mock_subprocess, mock_logging): run_response_obj = MagicMock() run_response_obj.returncode = 2 @@ -67,16 +67,18 @@ def test_yunex_upgrader_run_xfer_upgrade_fail_code(mock_subprocess, mock_logging test_yunex_upgrader = YunexUpgrader(test_upgrade_info) code = test_yunex_upgrader.run_xfer_upgrade("core-file-name") - + assert code == -1 # Assert logging mock_logging.info.assert_not_called() - mock_logging.error.assert_called_with("Firmware not successful for 8.8.8.8: test-error") + mock_logging.error.assert_called_with( + "Firmware not successful for 8.8.8.8: test-error" + ) -@patch("addons.images.firmware_manager.yunex_upgrader.logging") -@patch("addons.images.firmware_manager.yunex_upgrader.subprocess") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.subprocess") def test_yunex_upgrader_run_xfer_upgrade_fail_output(mock_subprocess, mock_logging): run_response_obj = MagicMock() run_response_obj.returncode = 0 @@ -94,16 +96,17 @@ def test_yunex_upgrader_run_xfer_upgrade_fail_output(mock_subprocess, mock_loggi # Assert logging mock_logging.info.assert_not_called() - mock_logging.error.assert_called_with("Firmware not successful for 8.8.8.8: test-error") - + mock_logging.error.assert_called_with( + "Firmware not successful for 8.8.8.8: test-error" + ) -@patch("addons.images.firmware_manager.yunex_upgrader.logging") -@patch("addons.images.firmware_manager.yunex_upgrader.time") -@patch("addons.images.firmware_manager.yunex_upgrader.json") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.time") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.json") @patch("builtins.open", new_callable=mock_open, read_data="data") @patch( - "addons.images.firmware_manager.yunex_upgrader.tarfile.open", + "addons.images.firmware_manager.upgrade_runner.yunex_upgrader.tarfile.open", return_value=MagicMock(), ) def test_yunex_upgrader_upgrade_success( @@ -149,18 +152,18 @@ def test_yunex_upgrader_upgrade_success( call("Unpacking TAR file prior to upgrading 8.8.8.8..."), call("Running Core firmware upgrade for 8.8.8.8..."), call("Running SDK firmware upgrade for 8.8.8.8..."), - call("Running application provisioning for 8.8.8.8...") + call("Running application provisioning for 8.8.8.8..."), ] ) mock_logging.error.assert_not_called() -@patch("addons.images.firmware_manager.yunex_upgrader.logging") -@patch("addons.images.firmware_manager.yunex_upgrader.time") -@patch("addons.images.firmware_manager.yunex_upgrader.json") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.time") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.json") @patch("builtins.open", new_callable=mock_open, read_data="data") @patch( - "addons.images.firmware_manager.yunex_upgrader.tarfile.open", + "addons.images.firmware_manager.upgrade_runner.yunex_upgrader.tarfile.open", return_value=MagicMock(), ) def test_yunex_upgrader_core_upgrade_fail( @@ -200,18 +203,20 @@ def test_yunex_upgrader_core_upgrade_fail( mock_logging.info.assert_has_calls( [ call("Unpacking TAR file prior to upgrading 8.8.8.8..."), - call("Running Core firmware upgrade for 8.8.8.8...") + call("Running Core firmware upgrade for 8.8.8.8..."), ] ) - mock_logging.error.assert_called_with("Failed to perform firmware upgrade for 8.8.8.8: Yunex RSU Core upgrade failed") + mock_logging.error.assert_called_with( + "Failed to perform firmware upgrade for 8.8.8.8: Yunex RSU Core upgrade failed" + ) -@patch("addons.images.firmware_manager.yunex_upgrader.logging") -@patch("addons.images.firmware_manager.yunex_upgrader.time") -@patch("addons.images.firmware_manager.yunex_upgrader.json") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.time") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.json") @patch("builtins.open", new_callable=mock_open, read_data="data") @patch( - "addons.images.firmware_manager.yunex_upgrader.tarfile.open", + "addons.images.firmware_manager.upgrade_runner.yunex_upgrader.tarfile.open", return_value=MagicMock(), ) def test_yunex_upgrader_core_ping_fail( @@ -251,18 +256,20 @@ def test_yunex_upgrader_core_ping_fail( mock_logging.info.assert_has_calls( [ call("Unpacking TAR file prior to upgrading 8.8.8.8..."), - call("Running Core firmware upgrade for 8.8.8.8...") + call("Running Core firmware upgrade for 8.8.8.8..."), ] ) - mock_logging.error.assert_called_with("Failed to perform firmware upgrade for 8.8.8.8: RSU offline for too long after Core upgrade") + mock_logging.error.assert_called_with( + "Failed to perform firmware upgrade for 8.8.8.8: RSU offline for too long after Core upgrade" + ) -@patch("addons.images.firmware_manager.yunex_upgrader.logging") -@patch("addons.images.firmware_manager.yunex_upgrader.time") -@patch("addons.images.firmware_manager.yunex_upgrader.json") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.time") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.json") @patch("builtins.open", new_callable=mock_open, read_data="data") @patch( - "addons.images.firmware_manager.yunex_upgrader.tarfile.open", + "addons.images.firmware_manager.upgrade_runner.yunex_upgrader.tarfile.open", return_value=MagicMock(), ) def test_yunex_upgrader_sdk_upgrade_fail( @@ -303,18 +310,20 @@ def test_yunex_upgrader_sdk_upgrade_fail( [ call("Unpacking TAR file prior to upgrading 8.8.8.8..."), call("Running Core firmware upgrade for 8.8.8.8..."), - call("Running SDK firmware upgrade for 8.8.8.8...") + call("Running SDK firmware upgrade for 8.8.8.8..."), ] ) - mock_logging.error.assert_called_with("Failed to perform firmware upgrade for 8.8.8.8: Yunex RSU SDK upgrade failed") + mock_logging.error.assert_called_with( + "Failed to perform firmware upgrade for 8.8.8.8: Yunex RSU SDK upgrade failed" + ) -@patch("addons.images.firmware_manager.yunex_upgrader.logging") -@patch("addons.images.firmware_manager.yunex_upgrader.time") -@patch("addons.images.firmware_manager.yunex_upgrader.json") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.time") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.json") @patch("builtins.open", new_callable=mock_open, read_data="data") @patch( - "addons.images.firmware_manager.yunex_upgrader.tarfile.open", + "addons.images.firmware_manager.upgrade_runner.yunex_upgrader.tarfile.open", return_value=MagicMock(), ) def test_yunex_upgrader_sdk_ping_fail( @@ -355,18 +364,20 @@ def test_yunex_upgrader_sdk_ping_fail( [ call("Unpacking TAR file prior to upgrading 8.8.8.8..."), call("Running Core firmware upgrade for 8.8.8.8..."), - call("Running SDK firmware upgrade for 8.8.8.8...") + call("Running SDK firmware upgrade for 8.8.8.8..."), ] ) - mock_logging.error.assert_called_with("Failed to perform firmware upgrade for 8.8.8.8: RSU offline for too long after SDK upgrade") + mock_logging.error.assert_called_with( + "Failed to perform firmware upgrade for 8.8.8.8: RSU offline for too long after SDK upgrade" + ) -@patch("addons.images.firmware_manager.yunex_upgrader.logging") -@patch("addons.images.firmware_manager.yunex_upgrader.time") -@patch("addons.images.firmware_manager.yunex_upgrader.json") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.logging") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.time") +@patch("addons.images.firmware_manager.upgrade_runner.yunex_upgrader.json") @patch("builtins.open", new_callable=mock_open, read_data="data") @patch( - "addons.images.firmware_manager.yunex_upgrader.tarfile.open", + "addons.images.firmware_manager.upgrade_runner.yunex_upgrader.tarfile.open", return_value=MagicMock(), ) def test_yunex_upgrader_provision_upgrade_fail( @@ -412,7 +423,9 @@ def test_yunex_upgrader_provision_upgrade_fail( call("Unpacking TAR file prior to upgrading 8.8.8.8..."), call("Running Core firmware upgrade for 8.8.8.8..."), call("Running SDK firmware upgrade for 8.8.8.8..."), - call("Running application provisioning for 8.8.8.8...") + call("Running application provisioning for 8.8.8.8..."), ] ) - mock_logging.error.assert_called_with("Failed to perform firmware upgrade for 8.8.8.8: Yunex RSU application provisioning upgrade failed") + mock_logging.error.assert_called_with( + "Failed to perform firmware upgrade for 8.8.8.8: Yunex RSU application provisioning upgrade failed" + ) diff --git a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py new file mode 100644 index 00000000..eeb35135 --- /dev/null +++ b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py @@ -0,0 +1,790 @@ +from unittest.mock import call, patch, MagicMock +from collections import deque +import services.addons.tests.firmware_manager.upgrade_scheduler.test_upgrade_scheduler_values as fmv +import pytest +from addons.images.firmware_manager.upgrade_scheduler import upgrade_scheduler + + +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" +) +def test_get_rsu_upgrade_data_all(mock_querydb): + mock_querydb.return_value = [ + ({"ipv4_address": "8.8.8.8"}, ""), + ({"ipv4_address": "9.9.9.9"}, ""), + ] + + result = upgrade_scheduler.get_rsu_upgrade_data() + + mock_querydb.assert_called_with(fmv.all_rsus_query) + assert result == [{"ipv4_address": "8.8.8.8"}, {"ipv4_address": "9.9.9.9"}] + + +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" +) +def test_get_rsu_upgrade_data_one(mock_querydb): + mock_querydb.return_value = [(fmv.rsu_info, "")] + + result = upgrade_scheduler.get_rsu_upgrade_data(rsu_ip="8.8.8.8") + + expected_result = [fmv.rsu_info] + mock_querydb.assert_called_with(fmv.one_rsu_query) + assert result == expected_result + + +# start_tasks_from_queue tests + + +@patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.upgrade_queue", + deque(["8.8.8.8"]), +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.upgrade_queue_info", + { + "8.8.8.8": { + "ipv4_address": "8.8.8.8", + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", + } + }, +) +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.requests.post", + side_effect=Exception("Exception during request"), +) +def test_start_tasks_from_queue_post_exception(mock_post, mock_logging): + upgrade_scheduler.start_tasks_from_queue() + + # Assert firmware upgrade process was started with expected arguments + mock_post.assert_called_with( + "http://test-endpoint/run_firmware_upgrade", + json={ + "ipv4_address": "8.8.8.8", + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", + }, + ) + + # Assert logging + mock_logging.info.assert_not_called() + mock_logging.error.assert_called_with( + f"Encountered error of type {Exception} while starting automatic upgrade process for 8.8.8.8: Exception during request" + ) + + +@patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.upgrade_queue", + deque(["8.8.8.8"]), +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.upgrade_queue_info", + { + "8.8.8.8": { + "ipv4_address": "8.8.8.8", + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", + } + }, +) +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.requests.post" +) +def test_start_tasks_from_queue_post_success(mock_post, mock_logging): + mock_post_response = MagicMock() + mock_post_response.status_code = 201 + mock_post.return_value = mock_post_response + + upgrade_scheduler.start_tasks_from_queue() + + # Assert firmware upgrade process was started with expected arguments + mock_post.assert_called_with( + "http://test-endpoint/run_firmware_upgrade", + json={ + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", + }, + ) + + mock_logging.info.assert_called_with( + f"Firmware upgrade runner successfully requested to begin the upgrade for 8.8.8.8" + ) + mock_logging.error.assert_not_called() + + +@patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.upgrade_queue", + deque(["8.8.8.8"]), +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.upgrade_queue_info", + { + "8.8.8.8": { + "ipv4_address": "8.8.8.8", + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", + } + }, +) +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.requests.post" +) +def test_start_tasks_from_queue_post_fail(mock_post, mock_logging): + mock_post_response = MagicMock() + mock_post_response.status_code = 500 + mock_post.return_value = mock_post_response + + upgrade_scheduler.start_tasks_from_queue() + + # Assert firmware upgrade process was started with expected arguments + mock_post.assert_called_with( + "http://test-endpoint/run_firmware_upgrade", + json={ + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", + }, + ) + + mock_logging.info.assert_not_called() + mock_logging.error.assert_called_with( + f"Firmware upgrade runner request failed for 8.8.8.8, check Upgrade Runner logs for details" + ) + + +# init_firmware_upgrade tests + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +def test_init_firmware_upgrade_missing_rsu_ip(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {} + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.init_firmware_upgrade() + + mock_flask_jsonify.assert_called_with( + {"error": "Missing 'rsu_ip' parameter"} + ) + assert code == 400 + + mock_logging.info.assert_not_called() + mock_logging.error.assert_not_called() + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {"8.8.8.8": {}}, +) +def test_init_firmware_upgrade_already_running(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.init_firmware_upgrade() + + mock_flask_jsonify.assert_called_with( + { + "error": f"Firmware upgrade failed to start for '8.8.8.8': an upgrade is already underway or queued for the target device" + } + ) + assert code == 500 + + # Assert logging + mock_logging.info.assert_called_with( + "Checking if existing upgrade is running or queued for '8.8.8.8'" + ) + mock_logging.error.assert_not_called() + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_rsu_upgrade_data", + MagicMock(return_value=[]), +) +def test_init_firmware_upgrade_no_eligible_upgrade(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.init_firmware_upgrade() + + mock_flask_jsonify.assert_called_with( + { + "error": f"Firmware upgrade failed to start for '8.8.8.8': the target firmware is already installed or is an invalid upgrade from the current firmware" + } + ) + assert code == 500 + + # Assert logging + mock_logging.info.assert_has_calls( + [ + call( + "Checking if existing upgrade is running or queued for '8.8.8.8'" + ), + call("Querying RSU data for '8.8.8.8'"), + ] + ) + mock_logging.error.assert_not_called() + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_rsu_upgrade_data", + MagicMock(return_value=[fmv.rsu_info]), +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.start_tasks_from_queue" +) +def test_init_firmware_upgrade_success(mock_stfq, mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.init_firmware_upgrade() + + # Assert start_tasks_from_queue is called + mock_stfq.assert_called_with() + + # Assert the process reference is successfully tracked in the upgrade_queue + assert upgrade_scheduler.upgrade_queue[0] == "8.8.8.8" + + # Assert REST response is as expected from a successful run + mock_flask_jsonify.assert_called_with( + {"message": f"Firmware upgrade started successfully for '8.8.8.8'"} + ) + assert code == 201 + + # Assert logging + mock_logging.info.assert_has_calls( + [ + call( + "Checking if existing upgrade is running or queued for '8.8.8.8'" + ), + call("Querying RSU data for '8.8.8.8'"), + call("Adding '8.8.8.8' to the firmware manager upgrade queue"), + ] + ) + mock_logging.error.assert_not_called() + + upgrade_scheduler.upgrade_queue = deque([]) + + +# firmware_upgrade_completed tests + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +def test_firmware_upgrade_completed_missing_rsu_ip(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {} + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.firmware_upgrade_completed() + + mock_flask_jsonify.assert_called_with( + {"error": "Missing 'rsu_ip' parameter"} + ) + assert code == 400 + + # Assert logging + mock_logging.info.assert_not_called() + mock_logging.error.assert_not_called() + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +def test_firmware_upgrade_completed_unknown_process(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = { + "rsu_ip": "8.8.8.8", + "status": "success", + } + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.firmware_upgrade_completed() + + mock_flask_jsonify.assert_called_with( + { + "error": "Specified device is not actively being upgraded or was already completed" + } + ) + assert code == 400 + + # Assert logging + mock_logging.info.assert_not_called() + mock_logging.error.assert_not_called() + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {"8.8.8.8": fmv.upgrade_info}, +) +def test_firmware_upgrade_completed_missing_status(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.firmware_upgrade_completed() + + mock_flask_jsonify.assert_called_with( + {"error": "Missing 'status' parameter"} + ) + assert code == 400 + + # Assert logging + mock_logging.info.assert_not_called() + mock_logging.error.assert_not_called() + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {"8.8.8.8": fmv.upgrade_info}, +) +def test_firmware_upgrade_completed_illegal_status(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8", "status": "frog"} + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.firmware_upgrade_completed() + + mock_flask_jsonify.assert_called_with( + { + "error": "Wrong value for 'status' parameter - must be either 'success' or 'fail'" + } + ) + assert code == 400 + + # Assert logging + mock_logging.info.assert_not_called() + mock_logging.error.assert_not_called() + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {"8.8.8.8": fmv.upgrade_info}, +) +def test_firmware_upgrade_completed_fail_status(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8", "status": "fail"} + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.firmware_upgrade_completed() + + assert "8.8.8.8" not in upgrade_scheduler.active_upgrades + mock_flask_jsonify.assert_called_with( + {"message": "Firmware upgrade successfully marked as complete"} + ) + assert code == 204 + + # Assert logging + mock_logging.info.assert_called_with( + "Marking firmware upgrade as complete for '8.8.8.8'" + ) + mock_logging.error.assert_not_called() + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {"8.8.8.8": fmv.upgrade_info}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.write_db" +) +def test_firmware_upgrade_completed_success_status(mock_writedb, mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = { + "rsu_ip": "8.8.8.8", + "status": "success", + } + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.firmware_upgrade_completed() + + mock_writedb.assert_called_with( + "UPDATE public.rsus SET firmware_version=2 WHERE ipv4_address='8.8.8.8'" + ) + assert "8.8.8.8" not in upgrade_scheduler.active_upgrades + mock_flask_jsonify.assert_called_with( + {"message": "Firmware upgrade successfully marked as complete"} + ) + assert code == 204 + + # Assert logging + mock_logging.info.assert_called_with( + "Marking firmware upgrade as complete for '8.8.8.8'" + ) + mock_logging.error.assert_not_called() + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {"8.8.8.8": fmv.upgrade_info}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.write_db", + side_effect=Exception("Failure to query PostgreSQL"), +) +def test_firmware_upgrade_completed_success_status_exception( + mock_writedb, mock_logging +): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = { + "rsu_ip": "8.8.8.8", + "status": "success", + } + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.firmware_upgrade_completed() + + mock_writedb.assert_called_with( + "UPDATE public.rsus SET firmware_version=2 WHERE ipv4_address='8.8.8.8'" + ) + mock_flask_jsonify.assert_called_with( + { + "error": "Unexpected error occurred while querying the PostgreSQL database - firmware upgrade not marked as complete" + } + ) + assert code == 500 + + # Assert logging + mock_logging.info.assert_not_called() + mock_logging.error.assert_called_with( + "Encountered error of type while querying the PostgreSQL database: Failure to query PostgreSQL" + ) + + +# list_active_upgrades tests + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {"8.8.8.8": fmv.upgrade_info}, +) +def test_list_active_upgrades(mock_logging): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = { + "rsu_ip": "8.8.8.8", + "status": "success", + } + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.list_active_upgrades() + + expected_active_upgrades = { + "8.8.8.8": { + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", + } + } + mock_flask_jsonify.assert_called_with( + {"active_upgrades": expected_active_upgrades, "upgrade_queue": []} + ) + assert code == 200 + + # Assert logging + mock_logging.info.assert_not_called() + mock_logging.error.assert_not_called() + + +# check_for_upgrades tests + + +@patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_rsu_upgrade_data", + MagicMock(return_value=fmv.single_rsu_info), +) +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.requests.post", + side_effect=Exception("Exception during request"), +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_upgrade_limit" +) +def test_check_for_upgrades_exception(mock_upgrade_limit, mock_post, mock_logging): + mock_upgrade_limit.return_value = 5 + upgrade_scheduler.check_for_upgrades() + + # Assert firmware upgrade process was started with expected arguments + mock_post.assert_called_with( + "http://test-endpoint/run_firmware_upgrade", + json={ + "ipv4_address": "9.9.9.9", + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", + }, + ) + + # Assert the process reference is successfully tracked in the active_upgrades dictionary + assert "9.9.9.9" not in upgrade_scheduler.active_upgrades + mock_logging.info.assert_has_calls( + [ + call("Checking PostgreSQL DB for RSUs with new target firmware"), + call("Adding '9.9.9.9' to the firmware manager upgrade queue"), + call("Firmware upgrade successfully started for '9.9.9.9'"), + ] + ) + mock_logging.error.assert_called_with( + f"Encountered error of type {Exception} while starting automatic upgrade process for 9.9.9.9: Exception during request" + ) + + +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_rsu_upgrade_data", + MagicMock(return_value=fmv.multi_rsu_info), +) +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.start_tasks_from_queue" +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_upgrade_limit" +) +def test_check_for_upgrades(mock_upgrade_limit, mock_stfq, mock_logging): + mock_upgrade_limit.return_value = 5 + upgrade_scheduler.check_for_upgrades() + + # Assert firmware upgrade process was started with expected arguments + mock_stfq.assert_called_once_with() + + # Assert the process reference is successfully tracked in the active_upgrades dictionary + assert upgrade_scheduler.upgrade_queue[1] == "9.9.9.9" + mock_logging.info.assert_has_calls( + [ + call("Checking PostgreSQL DB for RSUs with new target firmware"), + call("Adding '8.8.8.8' to the firmware manager upgrade queue"), + call("Firmware upgrade successfully started for '8.8.8.8'"), + call("Adding '9.9.9.9' to the firmware manager upgrade queue"), + call("Firmware upgrade successfully started for '9.9.9.9'"), + ] + ) + mock_logging.info.assert_called_with( + "Firmware upgrade successfully started for '9.9.9.9'" + ) + + +# Other tests + + +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.serve") +def test_serve_rest_api(mock_serve): + upgrade_scheduler.serve_rest_api() + mock_serve.assert_called_with(upgrade_scheduler.app, host="0.0.0.0", port=8080) + + +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.BackgroundScheduler" +) +def test_init_background_task(mock_bgscheduler): + mock_bgscheduler_obj = mock_bgscheduler.return_value + + upgrade_scheduler.init_background_task() + + mock_bgscheduler.assert_called_with({"apscheduler.timezone": "UTC"}) + mock_bgscheduler_obj.add_job.assert_called_with( + upgrade_scheduler.check_for_upgrades, "cron", minute="0" + ) + mock_bgscheduler_obj.start.assert_called_with() + + +def test_get_upgrade_limit_no_env(): + limit = upgrade_scheduler.get_upgrade_limit() + assert limit == 1 + + +@patch.dict("os.environ", {"ACTIVE_UPGRADE_LIMIT": "5"}) +def test_get_upgrade_limit_with_env(): + limit = upgrade_scheduler.get_upgrade_limit() + assert limit == 5 + + +@patch.dict("os.environ", {"ACTIVE_UPGRADE_LIMIT": "bad_value"}) +def test_get_upgrade_limit_with_bad_env(): + with pytest.raises( + ValueError, + match="The environment variable 'ACTIVE_UPGRADE_LIMIT' must be an integer.", + ): + upgrade_scheduler.get_upgrade_limit() diff --git a/services/addons/tests/firmware_manager/test_firmware_manager_values.py b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler_values.py similarity index 100% rename from services/addons/tests/firmware_manager/test_firmware_manager_values.py rename to services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler_values.py diff --git a/services/pytest.ini b/services/pytest.ini index 463b4c36..48609a25 100644 --- a/services/pytest.ini +++ b/services/pytest.ini @@ -2,7 +2,8 @@ pythonpath = . addons/images/geo_msg_query addons/images/count_metric - addons/images/firmware_manager + addons/images/firmware_manager/upgrade_runner + addons/images/firmware_manager/upgrade_scheduler addons/images/iss_health_check addons/images/rsu_status_check addons/images/obu_ota_server From f417901ec2164144ff38909982f0fc1420c3e220 Mon Sep 17 00:00:00 2001 From: Drew Johnston <31270488+drewjj@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:11:09 -0600 Subject: [PATCH 4/7] Address pull request comments --- docker-compose.yml | 58 +++++++++---------- .../upgrade_scheduler/upgrade_scheduler.py | 4 ++ .../upgrade_runner/test_upgrade_runner.py | 2 +- .../test_upgrade_scheduler.py | 41 ++++++++++++- 4 files changed, 73 insertions(+), 32 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index eebc78d6..d9362828 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,35 +55,35 @@ services: max-size: '10m' max-file: '5' - # cvmanager_webapp: - # build: - # context: webapp - # dockerfile: Dockerfile - # args: - # API_URI: http://${WEBAPP_DOMAIN}:8081 - # MAPBOX_TOKEN: ${MAPBOX_TOKEN} - # KEYCLOAK_HOST_URL: http://${KEYCLOAK_DOMAIN}:8084/ - # COUNT_MESSAGE_TYPES: ${COUNTS_MSG_TYPES} - # VIEWER_MESSAGE_TYPES: ${VIEWER_MSG_TYPES} - # DOT_NAME: ${DOT_NAME} - # MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE} - # MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE} - # MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM} - # CVIZ_API_SERVER_URL: ${CVIZ_API_SERVER_URL} - # CVIZ_API_WS_URL: ${CVIZ_API_WS_URL} - # image: jpo_cvmanager_webapp:latest - # restart: always - # depends_on: - # cvmanager_keycloak: - # condition: service_healthy - # extra_hosts: - # ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} - # ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} - # ports: - # - '80:80' - # logging: - # options: - # max-size: '10m' + cvmanager_webapp: + build: + context: webapp + dockerfile: Dockerfile + args: + API_URI: http://${WEBAPP_DOMAIN}:8081 + MAPBOX_TOKEN: ${MAPBOX_TOKEN} + KEYCLOAK_HOST_URL: http://${KEYCLOAK_DOMAIN}:8084/ + COUNT_MESSAGE_TYPES: ${COUNTS_MSG_TYPES} + VIEWER_MESSAGE_TYPES: ${VIEWER_MSG_TYPES} + DOT_NAME: ${DOT_NAME} + MAPBOX_INIT_LATITUDE: ${MAPBOX_INIT_LATITUDE} + MAPBOX_INIT_LONGITUDE: ${MAPBOX_INIT_LONGITUDE} + MAPBOX_INIT_ZOOM: ${MAPBOX_INIT_ZOOM} + CVIZ_API_SERVER_URL: ${CVIZ_API_SERVER_URL} + CVIZ_API_WS_URL: ${CVIZ_API_WS_URL} + image: jpo_cvmanager_webapp:latest + restart: always + depends_on: + cvmanager_keycloak: + condition: service_healthy + extra_hosts: + ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} + ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} + ports: + - '80:80' + logging: + options: + max-size: '10m' cvmanager_postgres: image: postgis/postgis:15-master diff --git a/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py index 1993b571..140248c0 100644 --- a/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py +++ b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py @@ -78,6 +78,10 @@ def start_tasks_from_queue(): # Begin the firmware upgrade using the Upgrade Runner API upgrade_runner_endpoint = os.environ.get("UPGRADE_RUNNER_ENDPOINT", "UNDEFINED") + + if upgrade_runner_endpoint == "UNDEFINED": + raise Exception("The UPGRADE_RUNNER_ENDPOINT environment variable is undefined!") + response = requests.post(f"{upgrade_runner_endpoint}/run_firmware_upgrade", json=rsu_upgrade_info) # Remove redundant ipv4_address from rsu since it is the key for active_upgrades diff --git a/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner.py b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner.py index fff9b3e9..90ab999e 100644 --- a/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner.py +++ b/services/addons/tests/firmware_manager/upgrade_runner/test_upgrade_runner.py @@ -2,7 +2,7 @@ from subprocess import DEVNULL from addons.images.firmware_manager.upgrade_runner import upgrade_runner from werkzeug.exceptions import BadRequest -import services.addons.tests.firmware_manager.upgrade_runner.test_upgrade_runner_values as fmv +import addons.tests.firmware_manager.upgrade_runner.test_upgrade_runner_values as fmv # start_upgrade_task tests diff --git a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py index eeb35135..9239d31d 100644 --- a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py +++ b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py @@ -1,8 +1,8 @@ from unittest.mock import call, patch, MagicMock from collections import deque -import services.addons.tests.firmware_manager.upgrade_scheduler.test_upgrade_scheduler_values as fmv -import pytest from addons.images.firmware_manager.upgrade_scheduler import upgrade_scheduler +import addons.tests.firmware_manager.upgrade_scheduler.test_upgrade_scheduler_values as fmv +import pytest @patch( @@ -98,6 +98,43 @@ def test_start_tasks_from_queue_post_exception(mock_post, mock_logging): ) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.upgrade_queue", + deque(["8.8.8.8"]), +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.upgrade_queue_info", + { + "8.8.8.8": { + "ipv4_address": "8.8.8.8", + "manufacturer": "Commsignia", + "model": "ITS-RS4-M", + "ssh_username": "user", + "ssh_password": "psw", + "target_firmware_id": 2, + "target_firmware_version": "y20.39.0", + "install_package": "install_package.tar", + } + }, +) +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.requests.post" +) +def test_start_tasks_from_queue_no_env_var(mock_post, mock_logging): + upgrade_scheduler.start_tasks_from_queue() + + # Assert logging + mock_logging.info.assert_not_called() + mock_logging.error.assert_called_with( + f"Encountered error of type {Exception} while starting automatic upgrade process for 8.8.8.8: The UPGRADE_RUNNER_ENDPOINT environment variable is undefined!" + ) + + @patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", From dd128d0de378f2400b23c705af738d0b2a08c400 Mon Sep 17 00:00:00 2001 From: Michael7371 <40476797+Michael7371@users.noreply.github.com> Date: Sat, 12 Oct 2024 10:55:07 -0600 Subject: [PATCH 5/7] removing version from docker compose files. Updates to obu ota server to address the file check addition to gcs_download function --- docker-compose-addons.yml | 2 -- docker-compose-full-cm.yml | 1 - docker-compose-mongo.yml | 2 -- docker-compose-no-cm.yml | 1 - docker-compose-obu-ota-server.yml | 3 +-- docker-compose-webapp-deployment.yml | 1 - .../addons/images/obu_ota_server/obu_ota_server.py | 8 ++++++-- .../tests/obu_ota_server/test_obu_ota_server.py | 14 ++++++++++---- 8 files changed, 17 insertions(+), 15 deletions(-) diff --git a/docker-compose-addons.yml b/docker-compose-addons.yml index 2b925fa8..70d82fb7 100644 --- a/docker-compose-addons.yml +++ b/docker-compose-addons.yml @@ -1,5 +1,3 @@ -version: '3' - include: - docker-compose.yml diff --git a/docker-compose-full-cm.yml b/docker-compose-full-cm.yml index 7a0a68bd..dd9584d6 100644 --- a/docker-compose-full-cm.yml +++ b/docker-compose-full-cm.yml @@ -1,4 +1,3 @@ -version: '3' services: cvmanager_api: build: diff --git a/docker-compose-mongo.yml b/docker-compose-mongo.yml index 1292dfa2..b1aa6f4a 100644 --- a/docker-compose-mongo.yml +++ b/docker-compose-mongo.yml @@ -1,5 +1,3 @@ -version: '3' - include: - docker-compose.yml diff --git a/docker-compose-no-cm.yml b/docker-compose-no-cm.yml index 16d66d89..f18b6f2e 100644 --- a/docker-compose-no-cm.yml +++ b/docker-compose-no-cm.yml @@ -1,4 +1,3 @@ -version: '3.9' services: cvmanager_api: build: diff --git a/docker-compose-obu-ota-server.yml b/docker-compose-obu-ota-server.yml index 88854d9a..0cc87ef5 100644 --- a/docker-compose-obu-ota-server.yml +++ b/docker-compose-obu-ota-server.yml @@ -1,4 +1,3 @@ -version: '3' services: # OBU OTA Server and Nginx proxy services jpo_ota_backend: @@ -11,7 +10,7 @@ services: - 8085:8085 environment: SERVER_HOST: ${OBU_OTA_SERVER_HOST} - LOGGING_LEVEL: ${OBU_OTA_SERVER_LOGGING_LEVEL} + LOGGING_LEVEL: ${OBU_OTA_LOGGING_LEVEL} BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER} BLOB_STORAGE_BUCKET: ${OBU_OTA_BLOB_STORAGE_BUCKET} BLOB_STORAGE_PATH: ${OBU_OTA_BLOB_STORAGE_PATH} diff --git a/docker-compose-webapp-deployment.yml b/docker-compose-webapp-deployment.yml index 95539093..69436dba 100644 --- a/docker-compose-webapp-deployment.yml +++ b/docker-compose-webapp-deployment.yml @@ -1,7 +1,6 @@ # This file is used to build the webapp image for deployment. # The COUNTS_MSG_TYPES and DOT_NAME variables must be set in .env before building to populate # correctly in the deployed webapp as they are build-time variables. -version: '3' services: cvmanager_webapp: build: diff --git a/services/addons/images/obu_ota_server/obu_ota_server.py b/services/addons/images/obu_ota_server/obu_ota_server.py index 4a6669ac..4e65bf68 100644 --- a/services/addons/images/obu_ota_server/obu_ota_server.py +++ b/services/addons/images/obu_ota_server/obu_ota_server.py @@ -16,6 +16,8 @@ logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) security = HTTPBasic() +commsignia_file_ext = ".tar.sig" + def authenticate_user(credentials: HTTPBasicCredentials = Depends(security)) -> str: correct_username = os.getenv("OTA_USERNAME") @@ -43,7 +45,7 @@ async def read_root(request: Request): def get_firmware_list() -> list: blob_storage_provider = os.getenv("BLOB_STORAGE_PROVIDER", "DOCKER") files = [] - file_extension = ".tar.sig" + file_extension = commsignia_file_ext if blob_storage_provider.upper() == "DOCKER": files = glob.glob(f"/firmwares/*{file_extension}") elif blob_storage_provider.upper() == "GCP": @@ -75,7 +77,9 @@ def get_firmware(firmware_id: str, local_file_path: str) -> bool: # If configured to use GCP storage, download the firmware from GCP elif blob_storage_provider.upper() == "GCP": # Download blob will attempt to download the firmware and return True if successful - return gcs_utils.download_gcp_blob(firmware_id, local_file_path) + return gcs_utils.download_gcp_blob( + firmware_id, local_file_path, commsignia_file_ext + ) return True except Exception as e: logging.error(f"parse_range_header: Error getting firmware: {e}") diff --git a/services/addons/tests/obu_ota_server/test_obu_ota_server.py b/services/addons/tests/obu_ota_server/test_obu_ota_server.py index ea41e3cc..bb6365f8 100644 --- a/services/addons/tests/obu_ota_server/test_obu_ota_server.py +++ b/services/addons/tests/obu_ota_server/test_obu_ota_server.py @@ -94,13 +94,16 @@ def test_get_firmware_gcs_success( mock_os_path_exists.return_value = False mock_download_gcp_blob.return_value = True - firmware_id = "test_firmware_id" + firmware_file_ext = ".tar.sig" + firmware_id = "test_firmware_id" + firmware_file_ext local_file_path = "test_local_file_path" result = get_firmware(firmware_id, local_file_path) mock_os_getenv.assert_called_with("BLOB_STORAGE_PROVIDER", "DOCKER") mock_os_path_exists.assert_called_with(local_file_path) - mock_download_gcp_blob.assert_called_once_with(firmware_id, local_file_path) + mock_download_gcp_blob.assert_called_once_with( + firmware_id, local_file_path, firmware_file_ext + ) assert result == True @@ -115,13 +118,16 @@ def test_get_firmware_gcs_failure( mock_os_path_exists.return_value = False mock_download_gcp_blob.return_value = False - firmware_id = "test_firmware_id" + firmware_file_ext = ".tar.sig" + firmware_id = "test_firmware_id" + firmware_file_ext local_file_path = "test_local_file_path" result = get_firmware(firmware_id, local_file_path) mock_os_getenv.assert_called_with("BLOB_STORAGE_PROVIDER", "DOCKER") mock_os_path_exists.assert_called_with(local_file_path) - mock_download_gcp_blob.assert_called_once_with(firmware_id, local_file_path) + mock_download_gcp_blob.assert_called_once_with( + firmware_id, local_file_path, firmware_file_ext + ) assert result == False From c4de7c884003c43afc94a91e3fb8193555f871fc Mon Sep 17 00:00:00 2001 From: Drew Johnston <31270488+drewjj@users.noreply.github.com> Date: Fri, 25 Oct 2024 02:26:36 -0600 Subject: [PATCH 6/7] Update unit tests for the scheduler --- .../test_upgrade_scheduler.py | 277 +++++++++++++++++- 1 file changed, 267 insertions(+), 10 deletions(-) diff --git a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py index 9239d31d..a70ac6d5 100644 --- a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py +++ b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py @@ -309,6 +309,9 @@ def test_init_firmware_upgrade_already_running(mock_logging): mock_logging.error.assert_not_called() +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.was_latest_ping_successful_for_rsu" +) @patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", @@ -318,10 +321,13 @@ def test_init_firmware_upgrade_already_running(mock_logging): "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_rsu_upgrade_data", MagicMock(return_value=[]), ) -def test_init_firmware_upgrade_no_eligible_upgrade(mock_logging): +def test_init_firmware_upgrade_no_eligible_upgrade( + mock_logging, mock_was_latest_ping_successful_for_rsu +): mock_flask_request = MagicMock() mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} mock_flask_jsonify = MagicMock() + mock_was_latest_ping_successful_for_rsu.return_value = True with patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", mock_flask_request, @@ -351,6 +357,9 @@ def test_init_firmware_upgrade_no_eligible_upgrade(mock_logging): mock_logging.error.assert_not_called() +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.was_latest_ping_successful_for_rsu" +) @patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", @@ -363,7 +372,10 @@ def test_init_firmware_upgrade_no_eligible_upgrade(mock_logging): @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.start_tasks_from_queue" ) -def test_init_firmware_upgrade_success(mock_stfq, mock_logging): +def test_init_firmware_upgrade_success( + mock_stfq, mock_logging, mock_was_latest_ping_successful_for_rsu +): + mock_was_latest_ping_successful_for_rsu.return_value = True mock_flask_request = MagicMock() mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} mock_flask_jsonify = MagicMock() @@ -530,12 +542,27 @@ def test_firmware_upgrade_completed_illegal_status(mock_logging): mock_logging.error.assert_not_called() +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.reset_consecutive_failure_count_for_rsu" +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.increment_consecutive_failure_count_for_rsu" +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.is_rsu_at_max_retries_limit", + return_value=False, +) @patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", {"8.8.8.8": fmv.upgrade_info}, ) -def test_firmware_upgrade_completed_fail_status(mock_logging): +def test_firmware_upgrade_completed_fail_status( + mock_logging, + mock_is_rsu_at_max_retries_limit, + mock_increment_consecutive_failure_count_for_rsu, + mock_reset_consecutive_failure_count_for_rsu, +): mock_flask_request = MagicMock() mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8", "status": "fail"} mock_flask_jsonify = MagicMock() @@ -555,6 +582,10 @@ def test_firmware_upgrade_completed_fail_status(mock_logging): ) assert code == 204 + mock_is_rsu_at_max_retries_limit.asset_called_with("8.8.8.8") + mock_increment_consecutive_failure_count_for_rsu.assert_called_once() + mock_reset_consecutive_failure_count_for_rsu.assert_not_called() + # Assert logging mock_logging.info.assert_called_with( "Marking firmware upgrade as complete for '8.8.8.8'" @@ -562,6 +593,81 @@ def test_firmware_upgrade_completed_fail_status(mock_logging): mock_logging.error.assert_not_called() +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.reset_consecutive_failure_count_for_rsu" +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.write_db" +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.increment_consecutive_failure_count_for_rsu" +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.is_rsu_at_max_retries_limit", + return_value=True, +) +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {"8.8.8.8": fmv.upgrade_info}, +) +def test_firmware_upgrade_completed_fail_status_reached_max_retries( + mock_logging, + mock_is_rsu_at_max_retries_limit, + mock_increment_consecutive_failure_count_for_rsu, + mock_writedb, + mock_reset_consecutive_failure_count_for_rsu, +): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8", "status": "fail"} + mock_flask_jsonify = MagicMock() + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.firmware_upgrade_completed() + + assert "8.8.8.8" not in upgrade_scheduler.active_upgrades + mock_flask_jsonify.assert_called_with( + {"message": "Firmware upgrade successfully marked as complete"} + ) + assert code == 204 + + mock_increment_consecutive_failure_count_for_rsu.assert_called_once_with( + "8.8.8.8" + ) + mock_is_rsu_at_max_retries_limit.assert_called_with("8.8.8.8") + mock_writedb.assert_has_calls( + [ + call( + "UPDATE public.rsus SET target_firmware_version=firmware_version WHERE ipv4_address='8.8.8.8'" + ), + call( + "insert into max_retry_limit_reached_instances (rsu_id, reached_at, target_firmware_version) values ((select rsu_id from rsus where ipv4_address='8.8.8.8'), now(), (select firmware_id from firmware_images where name='y20.39.0'))" + ), + ] + ) + + mock_reset_consecutive_failure_count_for_rsu.assert_called_once_with( + "8.8.8.8" + ) + + # Assert logging + mock_logging.info.assert_called_with( + "Marking firmware upgrade as complete for '8.8.8.8'" + ) + mock_logging.error.assert_called_with( + "RSU 8.8.8.8 has reached the maximum number of upgrade retries. Setting target_firmware_version to firmware_version and resetting consecutive failures count." + ) + + +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.reset_consecutive_failure_count_for_rsu" +) @patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", @@ -570,7 +676,9 @@ def test_firmware_upgrade_completed_fail_status(mock_logging): @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.write_db" ) -def test_firmware_upgrade_completed_success_status(mock_writedb, mock_logging): +def test_firmware_upgrade_completed_success_status( + mock_writedb, mock_logging, mock_reset_consecutive_failure_count_for_rsu +): mock_flask_request = MagicMock() mock_flask_request.get_json.return_value = { "rsu_ip": "8.8.8.8", @@ -596,6 +704,8 @@ def test_firmware_upgrade_completed_success_status(mock_writedb, mock_logging): ) assert code == 204 + mock_reset_consecutive_failure_count_for_rsu.assert_called_with("8.8.8.8") + # Assert logging mock_logging.info.assert_called_with( "Marking firmware upgrade as complete for '8.8.8.8'" @@ -603,6 +713,9 @@ def test_firmware_upgrade_completed_success_status(mock_writedb, mock_logging): mock_logging.error.assert_not_called() +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.reset_consecutive_failure_count_for_rsu" +) @patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", @@ -613,7 +726,7 @@ def test_firmware_upgrade_completed_success_status(mock_writedb, mock_logging): side_effect=Exception("Failure to query PostgreSQL"), ) def test_firmware_upgrade_completed_success_status_exception( - mock_writedb, mock_logging + mock_writedb, mock_logging, mock_reset_consecutive_failure_count_for_rsu ): mock_flask_request = MagicMock() mock_flask_request.get_json.return_value = { @@ -696,6 +809,9 @@ def test_list_active_upgrades(mock_logging): @patch.dict("os.environ", {"UPGRADE_RUNNER_ENDPOINT": "http://test-endpoint"}) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.was_latest_ping_successful_for_rsu" +) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", {}, @@ -712,8 +828,11 @@ def test_list_active_upgrades(mock_logging): @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_upgrade_limit" ) -def test_check_for_upgrades_exception(mock_upgrade_limit, mock_post, mock_logging): +def test_check_for_upgrades_exception( + mock_upgrade_limit, mock_post, mock_logging, mock_was_latest_ping_successful_for_rsu +): mock_upgrade_limit.return_value = 5 + mock_was_latest_ping_successful_for_rsu.return_value = True upgrade_scheduler.check_for_upgrades() # Assert firmware upgrade process was started with expected arguments @@ -745,6 +864,9 @@ def test_check_for_upgrades_exception(mock_upgrade_limit, mock_post, mock_loggin ) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.was_latest_ping_successful_for_rsu" +) @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", {}, @@ -760,8 +882,11 @@ def test_check_for_upgrades_exception(mock_upgrade_limit, mock_post, mock_loggin @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_upgrade_limit" ) -def test_check_for_upgrades(mock_upgrade_limit, mock_stfq, mock_logging): +def test_check_for_upgrades( + mock_upgrade_limit, mock_stfq, mock_logging, mock_was_latest_ping_successful_for_rsu +): mock_upgrade_limit.return_value = 5 + mock_was_latest_ping_successful_for_rsu.return_value = True upgrade_scheduler.check_for_upgrades() # Assert firmware upgrade process was started with expected arguments @@ -778,14 +903,146 @@ def test_check_for_upgrades(mock_upgrade_limit, mock_stfq, mock_logging): call("Firmware upgrade successfully started for '9.9.9.9'"), ] ) - mock_logging.info.assert_called_with( - "Firmware upgrade successfully started for '9.9.9.9'" - ) # Other tests +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" +) +def test_was_latest_ping_successful_for_rsu(mock_query_db): + # prepare + rsu_ip = "8.8.8.8" + expected_query = "select result from ping where rsu_id=(select rsu_id from rsus where ipv4_address='8.8.8.8') order by timestamp desc limit 1" + mock_query_db.return_value = [(True,)] + + # execute + result = upgrade_scheduler.was_latest_ping_successful_for_rsu(rsu_ip) + + # verify + assert result == True + mock_query_db.assert_called_with(expected_query) + + +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" +) +def test_was_latest_ping_successful_for_rsu_NO_RESULTS(mock_query_db): + # prepare + rsu_ip = "8.8.8.8" + expected_query = "select result from ping where rsu_id=(select rsu_id from rsus where ipv4_address='8.8.8.8') order by timestamp desc limit 1" + mock_query_db.return_value = [] + + # execute + result = upgrade_scheduler.was_latest_ping_successful_for_rsu(rsu_ip) + + # verify + assert result == False + mock_query_db.assert_called_with(expected_query) + + +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.write_db" +) +def test_increment_consecutive_failure_count_for_rsu(mock_write_db): + # prepare + rsu_ip = "8.8.8.8" + expected_query = f"insert into consecutive_firmware_upgrade_failures (rsu_id, consecutive_failures) values ((select rsu_id from rsus where ipv4_address='{rsu_ip}'), 1) on conflict (rsu_id) do update set consecutive_failures=consecutive_firmware_upgrade_failures.consecutive_failures+1" + + # execute + upgrade_scheduler.increment_consecutive_failure_count_for_rsu(rsu_ip) + + # verify + mock_write_db.assert_called_with(expected_query) + + +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.write_db" +) +def test_reset_consecutive_failure_count_for_rsu(mock_write_db): + # prepare + rsu_ip = "8.8.8.8" + expected_query = f"insert into consecutive_firmware_upgrade_failures (rsu_id, consecutive_failures) values ((select rsu_id from rsus where ipv4_address='{rsu_ip}'), 0) on conflict (rsu_id) do update set consecutive_failures=0" + + # execute + upgrade_scheduler.reset_consecutive_failure_count_for_rsu(rsu_ip) + + # verify + mock_write_db.assert_called_with(expected_query) + + +@patch.dict("os.environ", {"FW_UPGRADE_MAX_RETRY_LIMIT": "3"}) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" +) +def test_is_rsu_at_max_retries_limit_TRUE(mock_query_db): + # prepare + rsu_ip = "8.8.8.8" + expected_query = "select consecutive_failures from consecutive_firmware_upgrade_failures where rsu_id=(select rsu_id from rsus where ipv4_address='8.8.8.8')" + mock_query_db.return_value = [(3,)] + + # execute + result = upgrade_scheduler.is_rsu_at_max_retries_limit(rsu_ip) + + # verify + assert result == True + mock_query_db.assert_called_with(expected_query) + + +@patch.dict("os.environ", {"FW_UPGRADE_MAX_RETRY_LIMIT": "3"}) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" +) +def test_is_rsu_at_max_retries_limit_FALSE(mock_query_db): + # prepare + rsu_ip = "8.8.8.8" + expected_query = "select consecutive_failures from consecutive_firmware_upgrade_failures where rsu_id=(select rsu_id from rsus where ipv4_address='8.8.8.8')" + mock_query_db.return_value = [(2,)] + + # execute + result = upgrade_scheduler.is_rsu_at_max_retries_limit(rsu_ip) + + # verify + assert result == False + mock_query_db.assert_called_with(expected_query) + + +@patch.dict("os.environ", {"FW_UPGRADE_MAX_RETRY_LIMIT": "3"}) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.query_db" +) +def test_is_rsu_at_max_retries_limit_NO_RESULTS(mock_query_db): + # prepare + rsu_ip = "8.8.8.8" + expected_query = "select consecutive_failures from consecutive_firmware_upgrade_failures where rsu_id=(select rsu_id from rsus where ipv4_address='8.8.8.8')" + mock_query_db.return_value = [] + + # execute + result = upgrade_scheduler.is_rsu_at_max_retries_limit(rsu_ip) + + # verify + assert result == False + mock_query_db.assert_called_with(expected_query) + + +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.pgquery.write_db" +) +def test_log_max_retries_reached_incident_for_rsu_to_postgres(mock_write_db): + # prepare + rsu_ip = "8.8.8.8" + expected_query = "insert into max_retry_limit_reached_instances (rsu_id, reached_at, target_firmware_version) values ((select rsu_id from rsus where ipv4_address='8.8.8.8'), now(), (select firmware_id from firmware_images where name='y20.39.0'))" + + # execute + upgrade_scheduler.log_max_retries_reached_incident_for_rsu_to_postgres( + rsu_ip, "y20.39.0" + ) + + # verify + mock_write_db.assert_called_with(expected_query) + + @patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.serve") def test_serve_rest_api(mock_serve): upgrade_scheduler.serve_rest_api() From 8ad52e73b3dd74de970b4a20b31eaafd6a4013d8 Mon Sep 17 00:00:00 2001 From: Drew Johnston <31270488+drewjj@users.noreply.github.com> Date: Fri, 25 Oct 2024 02:34:57 -0600 Subject: [PATCH 7/7] Add last test from the retry limit tests --- .../test_upgrade_scheduler.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py index a70ac6d5..825141f7 100644 --- a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py +++ b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py @@ -248,6 +248,53 @@ def test_start_tasks_from_queue_post_fail(mock_post, mock_logging): # init_firmware_upgrade tests +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades", + {}, +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.get_rsu_upgrade_data", + MagicMock(return_value=[]), +) +@patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.was_latest_ping_successful_for_rsu" +) +@patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") +def test_init_firmware_upgrade_rsu_not_reachable( + mock_logging, mock_was_latest_ping_successful_for_rsu +): + mock_flask_request = MagicMock() + mock_flask_request.get_json.return_value = {"rsu_ip": "8.8.8.8"} + mock_flask_jsonify = MagicMock() + mock_was_latest_ping_successful_for_rsu.return_value = False + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.request", + mock_flask_request, + ): + with patch( + "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.jsonify", + mock_flask_jsonify, + ): + message, code = upgrade_scheduler.init_firmware_upgrade() + + mock_flask_jsonify.assert_called_with( + { + "error": f"Firmware upgrade failed to start for '8.8.8.8': device is unreachable" + } + ) + assert code == 500 + + # Assert logging + mock_logging.info.assert_has_calls( + [ + call( + "Checking if existing upgrade is running or queued for '8.8.8.8'" + ) + ] + ) + mock_logging.error.assert_not_called() + + @patch("addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.logging") @patch( "addons.images.firmware_manager.upgrade_scheduler.upgrade_scheduler.active_upgrades",