Revert "Ping Purger Integration, Dockerization, SNMP, UI Enhancements…

…, and Kubernetes Support"

.github/workflows/ci.yml

@@ -18,15 +18,15 @@ jobs:
           apt-get -y install python3-coverage python3-pip python3-pytest
 
       - name: Install Python Requirements
-        run: python3 -m pip install -r $GITHUB_WORKSPACE/services/requirements.txt
+        run: python3 -m pip install -r $GITHUB_WORKSPACE/api/requirements.txt
 
       - name: Run Tests and Generate Coverage
         continue-on-error: true
         run: |
           # Set PYTHONPATH and navigate to the tests directory
           export PYTHONPATH=$PYTHONPATH:/usr/lib/python3/dist-packages
-          cd $GITHUB_WORKSPACE/services
-
+          cd $GITHUB_WORKSPACE/api/tests/
+          
           # Run tests and generate coverage report
           python3 -m coverage run -m pytest
           python3 -m coverage xml --omit="/opt/*,/root/*,/tmp/*,/usr/*,/var/*,**/__init__.py"
@@ -35,12 +35,12 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: build_api
-          path: services/coverage.xml
+          path: api/tests/coverage.xml
 
       - name: Find coverage.xml
         shell: bash
         run: |
-          find "$GITHUB_WORKSPACE/services" -name "coverage.xml"
+          find "$GITHUB_WORKSPACE/api/tests" -name "coverage.xml"
 
   webapp:
     runs-on: ubuntu-latest
@@ -61,7 +61,7 @@ jobs:
           npm install -g nodemon
           npm init -y
           npm install --force
-
+          
           # Run tests with coverage and suppress failures
           npm test -- --coverage || true
 
@@ -89,7 +89,7 @@ jobs:
         continue-on-error: true
         with:
           name: build_api
-          path: services/coverage.xml
+          path: api/tests/coverage.xml
 
       - name: Download Code Coverage Results
         uses: actions/download-artifact@v3
@@ -124,11 +124,11 @@ jobs:
           sonar.projectBaseDir=$GITHUB_WORKSPACE
           sonar.projectKey=usdot-jpo-ode_jpo-cvmanager
           sonar.projectName=jpo-cvmanager
-          sonar.python.coverage.reportPaths=$GITHUB_WORKSPACE/services/coverage.xml
+          sonar.python.coverage.reportPaths=$GITHUB_WORKSPACE/api/tests/coverage.xml
           sonar.python.version=3.9
-          api.sonar.projectBaseDir=$GITHUB_WORKSPACE/services
-          api.sonar.sources=addons/images/bsm_query,addons/images/count_metric,addons/images/iss_health_check,addons/images/rsu_ping_fetch,api/src,common/pgquery.py
-          api.sonar.tests=addons/tests,api/tests,common/tests
+          api.sonar.projectBaseDir=$GITHUB_WORKSPACE/api
+          api.sonar.sources=src
+          api.sonar.tests=tests
           webapp.sonar.projectBaseDir=$GITHUB_WORKSPACE/webapp
           webapp.sonar.sources=src
           EOF

.gitignore

@@ -1,5 +1,4 @@
 *.pyc
-*.pyc.*
 .env
 .coverage
 cov.xml

.vscode/launch.json

@@ -1,28 +1,28 @@
-{
-  // Use IntelliSense to learn about possible attributes.
-  // Hover to view descriptions of existing attributes.
-  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "command": "npm start",
-      "name": "Launch web app",
-      "request": "launch",
-      "cwd": "${workspaceFolder}/webapp",
-      "type": "node-terminal"
-    },
-    {
-      "name": "Python: Flask",
-      "type": "python",
-      "request": "launch",
-      "stopOnEntry": false,
-      "program": "${workspaceRoot}/services/api/src/main.py",
-      "env": {
-        "FLASK_APP": "${workspaceRoot}/services/api/src/main.py"
-      },
-      "args": ["run"],
-      "envFile": "${workspaceRoot}/services/api/.env",
-      "preLaunchTask": "build-python-services"
-    }
-  ]
-}
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "command": "npm start",
+      "name": "Launch web app",
+      "request": "launch",
+      "cwd": "${workspaceFolder}/webapp",
+      "type": "node-terminal"
+    },
+    {
+      "name": "Python: Flask",
+      "type": "python",
+      "request": "launch",
+      "stopOnEntry": false,
+      "program": "${workspaceRoot}/api/src/main.py",
+      "env": {
+        "FLASK_APP": "${workspaceRoot}/api/src/main.py"
+      },
+      "args": ["run"],
+      "envFile": "${workspaceRoot}/api/.env",
+      "preLaunchTask": "build-python-api"
+    }
+  ]
+}

.vscode/settings.json

@@ -10,6 +10,5 @@
   "editor.formatOnPaste": false,
   "prettier.useEditorConfig": false,
   "prettier.useTabs": false,
-  "prettier.configPath": ".vscode/.prettierrc.json",
-  "python.testing.pytestArgs": ["services"]
+  "prettier.configPath": ".vscode/.prettierrc.json"
 }

.vscode/tasks.json

@@ -2,17 +2,24 @@
   "version": "2.0.0",
   "tasks": [
     {
-      "label": "build-python-services",
+      "label": "build-python-api",
       "type": "shell",
-      "command": "${command:python.interpreterPath} -m pip install -r ./services/requirements.txt"
+      "command": "${command:python.interpreterPath} -m pip install -r ./api/requirements.txt"
     },
     {
-      "label": "run-services-tests-and-coverage",
+      "label": "build-python-addons",
       "type": "shell",
-      "options": {
-        "cwd": "${workspaceFolder}/services/"
-      },
-      "command": "${command:python.interpreterPath} -m pytest -v --cov-report xml:cov.xml --cov ."
+      "command": "${command:python.interpreterPath} -m pip install -r ./addons/tests/requirements.txt"
+    },
+    {
+      "label": "run-api-tests-and-coverage",
+      "type": "shell",
+      "command": "${command:python.interpreterPath} -m pytest -v --cov-report xml:cov.xml --cov . ./api/tests/"
+    },
+    {
+      "label": "run-addons-tests-and-coverage",
+      "type": "shell",
+      "command": "${command:python.interpreterPath} -m pytest -v --cov-report xml:cov.xml --cov . ./addons/tests/"
     }
   ]
 }

addons/images/bsm_query/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3-slim
+
+WORKDIR /home
+
+ADD requirements.txt .
+
+RUN pip3 install -r requirements.txt
+
+ADD bsm_query.py .
+
+CMD ["/home/bsm_query.py"]
+ENTRYPOINT ["python3"]

addons/images/bsm_query/requirements.txt

@@ -0,0 +1,4 @@
+DateTime==5.1
+pymongo==4.3.3
+python-dateutil==2.8.2
+pytest-cov==4.0.0

addons/images/count_metric/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3-slim
+
+WORKDIR /home
+
+ADD requirements.txt .
+
+RUN pip3 install -r requirements.txt
+
+ADD driver.py .
+ADD kafka_counter.py .
+ADD pgquery_rsu.py .
+
+CMD ["/home/driver.py"]
+ENTRYPOINT ["python3"]

services/addons/images/count_metric/driver.py → addons/images/count_metric/driver.py

@@ -2,7 +2,7 @@
 import copy
 import threading
 import logging
-import common.pgquery as pgquery
+import pgquery_rsu
 
 from kafka_counter import KafkaMessageCounter
 
@@ -12,54 +12,44 @@
 rsu_location_dict = {}
 rsu_count_dict = {}
 
-# Query for RSU data from CV Manager PostgreSQL database
-def get_rsu_list():
-    result = []
-
-    # Execute the query and fetch all results
-    query = "SELECT to_jsonb(row) FROM (SELECT ipv4_address, primary_route FROM public.rsus ORDER BY ipv4_address) as row"
-    data = pgquery.query_db(query)
-
-    logging.debug("Parsing results...")
-    for row in data:
-        row = dict(row[0]) 
-        result.append(row)
-
-    return result
 
 # Create template dictionaries for RSU roads and counts using HTTP JSON data
-def populateRsuDict():
-    rsu_list = get_rsu_list()
-    for rsu in rsu_list:
-        rsu_ip = rsu['ipv4_address']
-        p_route = rsu['primary_route']
+def populateRsuDict(rsu_data):
+    for rsu in rsu_data:
+        rsuip = rsu["ipAddress"]
+        proute = rsu["primaryRoute"]
 
-        rsu_location_dict[rsu_ip] = p_route
+        rsu_location_dict[rsuip] = proute
         # Add IP to dict if the road exists in the dict already
-        if p_route in rsu_count_dict:
-            rsu_count_dict[p_route][rsu_ip] = 0
+        if proute in rsu_count_dict:
+            rsu_count_dict[proute][rsuip] = 0
         else:
-            rsu_count_dict[p_route] = {rsu_ip: 0}
+            rsu_count_dict[proute] = {rsuip: 0}
 
     rsu_count_dict["Unknown"] = {}
 
 
 def run():
     # Pull list of message types to run counts for from environment variable
-    messageTypesString = os.getenv("MESSAGE_TYPES", "")
-    if messageTypesString == "":
+    messageTypesString = os.getenv("MESSAGE_TYPES")
+    if messageTypesString is None:
         logging.error("MESSAGE_TYPES environment variable not set! Exiting.")
         exit("MESSAGE_TYPES environment variable not set! Exiting.")
     message_types = [
         msgtype.strip().lower() for msgtype in messageTypesString.split(",")
     ]
 
     # Configure logging based on ENV var or use default if not set
-    log_level = os.getenv("LOGGING_LEVEL", "INFO")
+    log_level = (
+        "INFO" if "LOGGING_LEVEL" not in os.environ else os.environ["LOGGING_LEVEL"]
+    )
     logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level)
 
+    rsu_data = pgquery_rsu.get_rsu_data()
+
+    logging.debug(f"RSU_Data received: {rsu_data}")
     logging.debug("Creating RSU and count dictionaries...")
-    populateRsuDict()
+    populateRsuDict(rsu_data)
 
     logging.info("Creating Data-In Kafka count threads...")
     # Start the Kafka counters on their own threads

services/addons/images/count_metric/kafka_counter.py → addons/images/count_metric/kafka_counter.py

@@ -35,7 +35,7 @@ def __init__(
         self.rsu_count_dict = rsu_count_dict
         self.rsu_count_dict_zero = rsu_count_dict_zero
         self.type = type
-        self.count_window = 60  # minutes
+        self.count_window = 5  # minutes
         if os.getenv("DESTINATION_DB") == "BIGQUERY":
             self.bq_client = bigquery.Client()
         elif os.getenv("DESTINATION_DB") == "MONGODB":
@@ -184,25 +184,14 @@ def process_message(self, message):
             )
 
     def listen_for_message_and_process(self, topic, bootstrap_server):
-        logging.debug(f"{self.thread_id}: Listening for messages on Kafka topic {topic}...")
-
-        if os.getenv('KAFKA_TYPE', '') == 'CONFLUENT':
-            username = os.getenv('CONFLUENT_KEY')
-            password = os.getenv('CONFLUENT_SECRET')
-            consumer = KafkaConsumer(topic, 
-                group_id=f'{self.thread_id}-counter', 
-                bootstrap_servers=bootstrap_server,
-                sasl_plain_username=username,
-                sasl_plain_password=password,
-                sasl_mechanism='PLAIN',
-                security_protocol='SASL_SSL')
-        else:
-            consumer = KafkaConsumer(
-                topic,
-                group_id=f"{self.thread_id}-counter",
-                bootstrap_servers=bootstrap_server,
-            )
-
+        logging.debug(
+            f"{self.thread_id}: Listening for messages on Kafka topic {topic}..."
+        )
+        consumer = KafkaConsumer(
+            topic,
+            group_id=f"{self.thread_id}-counter",
+            bootstrap_servers=bootstrap_server,
+        )
         for msg in consumer:
             self.process_message(msg)
         logging.warning(
@@ -233,7 +222,7 @@ def start_counter(self):
         # Setup scheduler for async metric uploads
         scheduler = BackgroundScheduler({"apscheduler.timezone": "UTC"})
         scheduler.add_job(
-            self.push_metrics, "cron", hour='*'
+            self.push_metrics, "cron", minute=f"*/{str(self.count_window)}"
         )
         scheduler.start()