Revert "Merge branch 'CDOT-CV:develop' into develop"

This reverts commit b483a69, reversing changes made to 74c29fb.
usdot-jpo-ode · Mar 25, 2024 · 7847621 · 7847621
1 parent 00aea6c
commit 7847621
Show file tree

Hide file tree

Showing 65 changed files with 613 additions and 1,571 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -130,7 +130,7 @@ jobs:
           sonar.python.coverage.reportPaths=$GITHUB_WORKSPACE/services/cov.xml
           sonar.python.version=3.12
           api.sonar.projectBaseDir=$GITHUB_WORKSPACE/services
-          api.sonar.sources=addons/images/bsm_query,addons/images/count_metric,addons/images/firmware_manager,addons/images/iss_health_check,addons/images/rsu_status_check,api/src,common/pgquery.py
+          api.sonar.sources=addons/images/bsm_query,addons/images/count_metric,addons/images/firmware_manager,addons/images/iss_health_check,addons/images/rsu_ping,api/src,common/pgquery.py
           api.sonar.tests=addons/tests,api/tests,common/tests
           webapp.sonar.projectBaseDir=$GITHUB_WORKSPACE/webapp
           webapp.sonar.sources=src

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -17,7 +17,7 @@
   },
   "python.envFile": "${workspaceFolder}/.env",
   "terminal.integrated.env.windows": {
-    "PYTHONPATH": "${workspaceFolder}/services;${workspaceFolder}/services/addons/images/bsm_query;${workspaceFolder}/services/addons/images/count_metric;${workspaceFolder}/services/addons/images/firmware_manager;${workspaceFolder}/services/addons/images/iss_health_check;${workspaceFolder}/services/addons/images/rsu_status_check;${workspaceFolder}/services/api/src;${workspaceFolder}/services/common"
+    "PYTHONPATH": "${workspaceFolder}/services;${workspaceFolder}/services/addons/images/bsm_query;${workspaceFolder}/services/addons/images/count_metric;${workspaceFolder}/services/addons/images/firmware_manager;${workspaceFolder}/services/addons/images/iss_health_check;${workspaceFolder}/services/addons/images/rsu_ping_fetch;${workspaceFolder}/services/api/src;${workspaceFolder}/services/common"
   },
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter",

diff --git a/docker-compose-addons.yml b/docker-compose-addons.yml
@@ -33,15 +33,17 @@ services:
         max-size: '10m'
         max-file: '5'
 
-  rsu_status_check:
+  jpo_rsu_ping_fetch:
     build:
       context: ./services
-      dockerfile: Dockerfile.rsu_status_check
-    image: rsu_status_check:latest
+      dockerfile: Dockerfile.rsu_ping_fetch
+    image: rsu_ping_fetch:latest
     restart: always
 
+    depends_on:
+      - cvmanager_postgres
     env_file:
-      - ./services/addons/images/rsu_status_check/.env
+      - ./services/addons/images/rsu_ping/.env
     logging:
       options:
         max-size: '10m'

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -118,15 +118,13 @@ services:
       KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN}
       KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
       WEBAPP_ORIGIN: http://${WEBAPP_DOMAIN}
-      WEBAPP_CM_ORIGIN: http://${WEBAPP_CM_DOMAIN}
       KC_HEALTH_ENABLED: true
       KC_DB: postgres
       KC_DB_URL: jdbc:postgresql://${PG_DB_HOST}/postgres?currentSchema=keycloak
       KC_DB_USERNAME: ${PG_DB_USER}
       KC_DB_PASSWORD: ${PG_DB_PASS}
       KC_HOSTNAME: ${KEYCLOAK_DOMAIN}
       KEYCLOAK_API_CLIENT_SECRET_KEY: ${KEYCLOAK_API_CLIENT_SECRET_KEY}
-      KEYCLOAK_CM_API_CLIENT_SECRET_KEY: ${KEYCLOAK_CM_API_CLIENT_SECRET_KEY}
       GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID}
       GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET}
     command:

diff --git a/resources/keycloak/azure-pipelines.yml b/resources/keycloak/azure-pipelines.yml
diff --git a/resources/kubernetes/cv-manager-postgres.yaml b/resources/kubernetes/cv-manager-postgres.yaml
@@ -227,7 +227,6 @@ data:
       snmp_credential_id integer NOT NULL DEFAULT nextval('snmp_credentials_snmp_credential_id_seq'::regclass),
       username character varying(128) COLLATE pg_catalog.default NOT NULL,
       password character varying(128) COLLATE pg_catalog.default NOT NULL,
-      encrypt_password character varying(128) COLLATE pg_catalog.default,
       nickname character varying(128) COLLATE pg_catalog.default NOT NULL,
       CONSTRAINT snmp_credentials_pkey PRIMARY KEY (snmp_credential_id),
       CONSTRAINT snmp_credentials_nickname UNIQUE (nickname)

diff --git a/resources/sql_scripts/CVManager_CreateTables.sql b/resources/sql_scripts/CVManager_CreateTables.sql
@@ -113,7 +113,6 @@ CREATE TABLE IF NOT EXISTS public.snmp_credentials
    snmp_credential_id integer NOT NULL DEFAULT nextval('snmp_credentials_snmp_credential_id_seq'::regclass),
    username character varying(128) COLLATE pg_catalog.default NOT NULL,
    password character varying(128) COLLATE pg_catalog.default NOT NULL,
-   encrypt_password character varying(128) COLLATE pg_catalog.default,
    nickname character varying(128) COLLATE pg_catalog.default NOT NULL,
    CONSTRAINT snmp_credentials_pkey PRIMARY KEY (snmp_credential_id),
    CONSTRAINT snmp_credentials_nickname UNIQUE (nickname)
@@ -362,43 +361,4 @@ CREATE TABLE IF NOT EXISTS public.scms_health
 		ON DELETE NO ACTION
 );
 
--- Create snmp_msgfwd_type table
-CREATE SEQUENCE public.snmp_msgfwd_type_id_seq
-   INCREMENT 1
-   START 1
-   MINVALUE 1
-   MAXVALUE 2147483647
-   CACHE 1;
-
-CREATE TABLE IF NOT EXISTS public.snmp_msgfwd_type
-(
-   snmp_msgfwd_type_id integer NOT NULL DEFAULT nextval('snmp_msgfwd_type_id_seq'::regclass),
-   name character varying(128) COLLATE pg_catalog.default NOT NULL,
-   CONSTRAINT snmp_msgfwd_type_pkey PRIMARY KEY (snmp_msgfwd_type_id),
-   CONSTRAINT snmp_msgfwd_type_name UNIQUE (name)
-);
-
--- Create snmp_msgfwd_config table
-CREATE TABLE IF NOT EXISTS public.snmp_msgfwd_config
-(
-   rsu_id integer NOT NULL,
-   msgfwd_type integer NOT NULL,
-   snmp_index integer NOT NULL,
-   message_type character varying(128) COLLATE pg_catalog.default NOT NULL,
-   dest_ipv4 inet NOT NULL,
-   dest_port integer NOT NULL,
-   start_datetime timestamp without time zone NOT NULL,
-   end_datetime timestamp without time zone NOT NULL,
-   active bit(1) NOT NULL,
-   CONSTRAINT snmp_msgfwd_config_pkey PRIMARY KEY (rsu_id, msgfwd_type, snmp_index),
-   CONSTRAINT fk_rsu_id FOREIGN KEY (rsu_id)
-		REFERENCES public.rsus (rsu_id) MATCH SIMPLE
-		ON UPDATE NO ACTION
-		ON DELETE NO ACTION,
-   CONSTRAINT fk_msgfwd_type FOREIGN KEY (msgfwd_type)
-		REFERENCES public.snmp_msgfwd_type (snmp_msgfwd_type_id) MATCH SIMPLE
-		ON UPDATE NO ACTION
-		ON DELETE NO ACTION
-);
-
 CREATE SCHEMA IF NOT EXISTS keycloak;
diff --git a/resources/sql_scripts/CVManager_SampleData.sql b/resources/sql_scripts/CVManager_SampleData.sql
@@ -20,8 +20,8 @@ INSERT INTO public.rsu_credentials(
 	VALUES ('username', 'password', 'cred1');
 
 INSERT INTO public.snmp_credentials(
-	username, password, encrypt_password, nickname)
-	VALUES ('username', 'password', 'encryption-pw', 'snmp1');
+	username, password, nickname)
+	VALUES ('username', 'password', 'snmp1');
 
 INSERT INTO public.snmp_versions(
 	version_code, nickname)
@@ -32,8 +32,8 @@ INSERT INTO public.snmp_versions(
 
 INSERT INTO public.rsus(
 	geography, milepost, ipv4_address, serial_number, iss_scms_id, primary_route, model, credential_id, snmp_credential_id, snmp_version_id, firmware_version, target_firmware_version)
-	VALUES (ST_GeomFromText('POINT(-105.014182 39.740422)'), 1, '10.0.0.180', 'E5672', 'E5672', 'I999', 1, 1, 1, 1, 1, 1), 
-	(ST_GeomFromText('POINT(-104.967723 39.918758)'), 2, '10.0.0.78', 'E5321', 'E5321', 'I999', 1, 1, 1, 2, 2, 2);
+	VALUES (ST_GeomFromText('POINT(-105.014182 39.740422)'), 1, '10.0.0.1', 'E5672', 'E5672', 'I999', 1, 1, 1, 1, 1, 1), 
+	(ST_GeomFromText('POINT(-104.980496 40.087737)'), 2, '10.0.0.2', 'E5321', 'E5321', 'I999', 1, 1, 1, 1, 2, 2);
 
 INSERT INTO public.organizations(
 	name)
@@ -56,6 +56,13 @@ INSERT INTO public.user_organization(
 	user_id, organization_id, role_id)
 	VALUES (1, 1, 1);
 
-INSERT INTO public.snmp_msgfwd_type(
-	name)
-	VALUES ('rsuDsrcFwd'), ('rsuReceivedMsg'), ('rsuXmitMsgFwding');
+INSERT INTO public.snmp_versions(
+	version_code, nickname)
+	VALUES ('4.1', '4.1');
+INSERT INTO public.snmp_versions(
+	version_code, nickname)
+	VALUES ('12.18', '12.18');
+
+ALTER TABLE public.rsus
+        ADD snmp_version_id integer NOT NULL
+    DEFAULT (1);
diff --git a/resources/sql_scripts/update_scripts/snmp_credentials_update.sql b/resources/sql_scripts/update_scripts/snmp_credentials_update.sql
diff --git a/resources/sql_scripts/update_scripts/snmp_msgfwd.sql b/resources/sql_scripts/update_scripts/snmp_msgfwd.sql
diff --git a/sample.env b/sample.env
@@ -4,7 +4,6 @@ DOCKER_HOST_IP=
 WEBAPP_HOST_IP=${DOCKER_HOST_IP}
 # Note if using WEBAPP_DOMAIN for the docker-compose-webapp-deployment.yml file you will need to include http:// or https://
 WEBAPP_DOMAIN=cvmanager.local.com
-WEBAPP_CM_DOMAIN=cimms.local.com
 KC_HOST_IP=${DOCKER_HOST_IP}
 
 # Firmware Manager connectivity in the format 'http://endpoint:port'
@@ -36,12 +35,10 @@ KEYCLOAK_ADMIN_PASSWORD=
 KEYCLOAK_REALM=cvmanager
 KEYCLOAK_API_CLIENT_ID=cvmanager-api
 KEYCLOAK_API_CLIENT_SECRET_KEY=
-KEYCLOAK_CM_API_CLIENT_SECRET_KEY=
 KEYCLOAK_LOGIN_THEME_NAME=sample_theme
 # Note if using KEYCLOAK_DOMAIN for the docker-compose-webapp-deployment.yml file you will need to include http:// or https://
 KEYCLOAK_DOMAIN=cvmanager.auth.com
 
-
 # GCP OAuth2.0 client ID for SSO authentication in keycloak - if not specified the google SSO will not be functional
 GOOGLE_CLIENT_ID=
 GOOGLE_CLIENT_SECRET=

diff --git a/services/Dockerfile.api b/services/Dockerfile.api
@@ -4,7 +4,7 @@ FROM python:3.12.0-slim
 RUN apt-get update
 RUN apt-get install -y snmpd
 RUN apt-get install -y snmp
-ADD resources/mibs/* /usr/share/snmp/mibs/
+ADD api/resources/mibs/* /usr/share/snmp/mibs/
 
 # Allow statements and log messages to immediately appear in the Knative logs
 ENV PYTHONUNBUFFERED True

diff --git a/services/Dockerfile.rsu_ping_fetch b/services/Dockerfile.rsu_ping_fetch
@@ -0,0 +1,17 @@
+FROM python:3.12.0-alpine3.18
+
+WORKDIR /home
+
+ADD addons/images/rsu_ping/crontab.rsu_ping_fetch ./crontab
+ADD addons/images/rsu_ping/requirements.txt .
+ADD addons/images/rsu_ping/rsu_ping_fetch.py .
+ADD addons/images/rsu_ping/purger.py .
+ADD common/*.py ./common/
+
+RUN pip3 install -r requirements.txt
+# fix the line endings from windows
+RUN dos2unix /home/crontab
+
+RUN crontab /home/crontab
+
+CMD ["crond", "-f"]
diff --git a/services/Dockerfile.rsu_pinger b/services/Dockerfile.rsu_pinger
@@ -0,0 +1,17 @@
+FROM python:3.12.0-alpine3.18
+
+WORKDIR /home
+
+ADD addons/images/rsu_ping/crontab.rsu_pinger ./crontab
+ADD addons/images/rsu_ping/requirements.txt .
+ADD addons/images/rsu_ping/rsu_pinger.py .
+ADD addons/images/rsu_ping/purger.py .
+ADD common/*.py ./common/
+
+RUN pip3 install -r requirements.txt
+# fix the line endings from windows
+RUN dos2unix /home/crontab
+
+RUN crontab /home/crontab
+
+CMD ["crond", "-f"]
diff --git a/services/Dockerfile.rsu_status_check b/services/Dockerfile.rsu_status_check
diff --git a/services/addons/images/rsu_ping/README.md b/services/addons/images/rsu_ping/README.md
@@ -0,0 +1,58 @@
+# RSU Ping Services
+
+## Table of Contents
+
+- [RSU Ping Services](#rsu-ping-services)
+  - [Table of Contents](#table-of-contents)
+  - [About ](#about-)
+  - [Requirements ](#requirements-)
+    - [rsu_ping_fetch](#rsu_ping_fetch)
+    - [rsu_pinger](#rsu_pinger)
+
+## About <a name = "about"></a>
+
+This directory contains two microservices that run within the CV Manager GKE Cluster. Both the 'rsu_ping_fetch' and 'rsu_pinger' applications populate the CV Manager PostGreSQL database's 'ping' table with the current online statuses of all RSUs recorded in the 'rsus' table. For 'rsu_ping_fetch', these statuses are retrieved directly from a [Zabbix server](https://www.zabbix.com/). For 'rsu_pinger', these statuses are generated by pinging every RSU in the 'rsus' table every 1 minute.
+
+If you have access to a Zabbix server that is tracking RSUs, it is recommended to use the 'rsu_ping_fetch' application. However, the 'rsu_pinger' is an effective alternative.
+
+Another feature both applications provide is a ping data purger that will remove stale ping data from the CV Manager PostgreSQL database to allow for high performance RSU ping queries. The amount of time a message needs to be in the database to be considered stale is configurable with the STALE_PERIOD environment variable. This purger will run once every 24 hours to check for stale ping data in the database.
+
+## Requirements <a name = "requirements"></a>
+
+### rsu_ping_fetch
+
+To properly run the rsu_ping_fetch microservice the following services are also required:
+
+- CV Manager PostgreSQL database with at least one RSU inserted into the 'rsus' table
+- Zabbix server with the REST API enabled
+- rsu_ping_fetch must be deployed in the same environment or K8s cluster as the PostgreSQL database
+- Network rules must be in place to allow proper routing between the rsu_ping_fetch microservice and the Zabbix server
+
+The rsu_ping_fetch microservice expects the following environment variables to be set:
+
+- ZABBIX_ENDPOINT - Zabbix API access endpoint.
+- ZABBIX_USER - Zabbix API access username.
+- ZABBIX_PASSWORD - Zabbix API access password.
+- DB_USER - PostgreSQL access username.
+- DB_PASS - PostgreSQL access password.
+- DB_NAME - PostgreSQL database name.
+- DB_HOST - PostgreSQL hostname, make sure to include port number.
+- STALE_PERIOD - Number of hours a ping log needs to be around in the PostgreSQL database to be considered stale.
+- LOGGING_LEVEL (optional, defaults to 'info')
+
+### rsu_pinger
+
+To properly run the rsu_pinger microservice the following services are also required:
+
+- CV Manager PostgreSQL database with at least one RSU inserted into the 'rsus' table
+- rsu_pinger must be deployed in the same environment or K8s cluster as the PostgreSQL database
+- Network rules must be in place to allow proper routing between the rsu_pinger microservice and deployed RSUs
+
+The rsu_pinger microservice expects the following environment variables to be set:
+
+- DB_USER - PostgreSQL access username.
+- DB_PASS - PostgreSQL access password.
+- DB_NAME - PostgreSQL database name.
+- DB_HOST - PostgreSQL hostname, make sure to include port number.
+- STALE_PERIOD - Number of hours a ping log needs to be around in the PostgreSQL database to be considered stale.
+- LOGGING_LEVEL (optional, defaults to 'info')
diff --git a/services/addons/images/rsu_ping/crontab.rsu_ping_fetch b/services/addons/images/rsu_ping/crontab.rsu_ping_fetch
@@ -0,0 +1,3 @@
+PYTHONPATH=/home
+*/1 * * * * /usr/local/bin/python3 /home/rsu_ping_fetch.py
+0 0 * * * /usr/local/bin/python3 /home/purger.py
diff --git a/...es/addons/images/rsu_status_check/crontab → ...addons/images/rsu_ping/crontab.rsu_pinger b/...es/addons/images/rsu_status_check/crontab → ...addons/images/rsu_ping/crontab.rsu_pinger
@@ -1,5 +1,3 @@
-PYTHONPATH=/home
-* * * * * /usr/local/bin/python3 /home/rsu_ping_fetch.py
-* * * * * /usr/local/bin/python3 /home/rsu_pinger.py
-0 0 * * * /usr/local/bin/python3 /home/purger.py
-0 */4 * * * /usr/local/bin/python3 /home/rsu_snmp_fetch.py
+PYTHONPATH=/home
+* * * * * /usr/local/bin/python3 /home/rsu_pinger.py
+0 0 * * * /usr/local/bin/python3 /home/purger.py