57 update sql schema and stored procedures for latest internet draft (#109)

* moved amp-sql to ion

* updated sql to latest internet drafts

* fixed errors in stored procedures

* renamed amp-sql to refdb-sql

* cleaned up repo, removed unused files

* Backporting ION compiler compat fix from apl-fy24

* Aligning workflow conditions with dev branch

* Bumping CI action versions for compatibility

* removed old code

* Moving SQL containers into normal integration test configs

* Updates for working SQL DDL

---------

Co-authored-by: David Linko <[email protected]>
Co-authored-by: Brian Sipos <[email protected]>

Author: 3 people

integration-test-ion/Containerfile

@@ -105,14 +105,14 @@ COPY --chmod=755 systemd/service_is_running.sh /usr/local/bin/service_is_running
 
 # Test tools
 RUN apt-get update && apt-get install -y \
-    curl git tshark
+    curl git tshark postgresql-client
 RUN pip3 install --break-system-packages git+https://github.com/JHUAPL-DTNMA/dtnma-ace.git@apl-fy24
 RUN git clone --branch apl-fy24 https://github.com/JHUAPL-DTNMA/dtnma-adms.git /usr/local/share/ace/adms
 
 # REFDA and REFDM to test
 RUN apt-get update && apt-get install -y \
     cmake ninja-build ruby pkg-config \
-    flex libfl-dev bison libpcre2-dev civetweb libcivetweb-dev libssl-dev libcjson-dev libsystemd-dev
+    flex libfl-dev bison libpcre2-dev libpq-dev civetweb libcivetweb-dev libssl-dev libcjson-dev libsystemd-dev
 COPY deps /usr/src/nm/deps
 COPY cmake /usr/src/nm/cmake
 COPY src /usr/src/nm/src
@@ -121,6 +121,7 @@ RUN ls -lt /usr/src/nm/
 RUN cd /usr/src/nm && \
     cmake -S . -B build/default \
       -DCMAKE_BUILD_TYPE=Debug \
+      -DTRANSPORT_UNIX_SOCKET=OFF \
       -DTRANSPORT_ION_BP=ON \
       -DBUILD_TESTING=OFF \
       -DBUILD_DOCS_API=OFF -DBUILD_DOCS_MAN=OFF \

integration-test-ion/compose.yml

@@ -1,4 +1,21 @@
+volumes:
+  postgres-data:
+    # temporary storage
+
 services:
+  postgres:
+    build:
+      context: ../refdb-sql
+      dockerfile: Containerfile
+    hostname: postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_DB: ${DB_NAME}
+    volumes:
+      - "postgres-data:/var/lib/postgresql/data"
+
   manager:
     hostname: manager
     build:
@@ -7,7 +24,9 @@ services:
       target: manager
     ports:
       - "8089:8089/tcp"
-    depends_on: {}
+    depends_on:
+      postgres:
+        condition: service_healthy
     privileged: true
     tty: true
     cap_add:
@@ -21,6 +40,10 @@ services:
       DTNMA_LOGLEVEL: debug
       ION_NODE_NUM: 1
       MGR_NODE_NUM: 1
+      DB_HOST: postgres
+      DB_USER: ${DB_USER}
+      DB_PASSWORD: ${DB_PASSWORD}
+      DB_NAME: ${DB_NAME}
       DUMPCAP_OUTFILE: /var/log/pcap/manager.pcap
       DUMPCAP_OPTS: -i lo -i eth0
 

integration-test-ion/run.sh

@@ -23,16 +23,20 @@ cd "${SELFDIR}"
 
 DOCKER=${DOCKER:-docker}
 
+export DB_NAME=refdm
+export DB_USER=refdm
+export DB_PASSWORD=notsecret
+
 if [ "$1" = "start" ]
 then
     export DOCKER_BUILDKIT=1
 
     ${DOCKER} compose build
-    ${DOCKER} compose up -d --force-recreate --remove-orphans
+    ${DOCKER} compose up --detach --force-recreate --remove-orphans
 elif [ "$1" = "stop" ]
 then
-    ${DOCKER} compose stop
-    ${DOCKER} compose rm -fv
+    ${DOCKER} compose down --rmi local --volumes
+    ${DOCKER} compose rm --force --volumes
 elif [ "$1" = "check" ]
 then
     ${DOCKER} compose ps

integration-test-socket/Containerfile

@@ -92,14 +92,14 @@ COPY --chmod=755 systemd/service_is_running.sh /usr/local/bin/service_is_running
 
 # Test tools
 RUN apt-get update && apt-get install -y \
-    curl git tshark
+    curl git tshark postgresql-client
 RUN pip3 install --break-system-packages git+https://github.com/JHUAPL-DTNMA/dtnma-ace.git@apl-fy24
 RUN git clone --branch apl-fy24 https://github.com/JHUAPL-DTNMA/dtnma-adms.git /usr/local/share/ace/adms
 
 # REFDA and REFDM to test
 RUN apt-get update && apt-get install -y \
     cmake ninja-build ruby pkg-config \
-    flex libfl-dev bison libpcre2-dev civetweb libcivetweb-dev libssl-dev libcjson-dev libsystemd-dev
+    flex libfl-dev bison libpcre2-dev libpq-dev civetweb libcivetweb-dev libssl-dev libcjson-dev libsystemd-dev
 COPY deps /usr/src/nm/deps
 COPY cmake /usr/src/nm/cmake
 COPY src /usr/src/nm/src
@@ -108,6 +108,7 @@ RUN ls -lt /usr/src/nm/
 RUN cd /usr/src/nm && \
     cmake -S . -B build/default \
       -DCMAKE_BUILD_TYPE=Debug \
+      -DTRANSPORT_UNIX_SOCKET=ON \
       -DTRANSPORT_ION_BP=OFF \
       -DBUILD_TESTING=OFF \
       -DBUILD_DOCS_API=OFF -DBUILD_DOCS_MAN=OFF \

integration-test-socket/compose.yml

@@ -1,11 +1,30 @@
+volumes:
+  postgres-data:
+    # temporary storage
+
 services:
+  postgres:
+    build:
+      context: ../refdb-sql
+      dockerfile: Containerfile
+    hostname: postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_DB: ${DB_NAME}
+    volumes:
+      - "postgres-data:/var/lib/postgresql/data"
+
   combined:
     build:
       context: ..
       dockerfile: integration-test-socket/Containerfile
     ports:
       - "8089:8089/tcp"
-    depends_on: {}
+    depends_on:
+      postgres:
+        condition: service_healthy
     privileged: true
     tty: true
     cap_add:
@@ -18,3 +37,7 @@ services:
       REFDA_EID: file:/var/run/refda.sock
       REFDA_SOCKET: /var/run/refda.sock
       REFDM_SOCKET: /var/run/refdm.sock
+      DB_HOST: postgres
+      DB_USER: ${DB_USER}
+      DB_PASSWORD: ${DB_PASSWORD}
+      DB_NAME: ${DB_NAME}

integration-test-socket/run.sh

@@ -23,16 +23,20 @@ cd "${SELFDIR}"
 
 DOCKER=${DOCKER:-docker}
 
+export DB_NAME=refdb
+export DB_USER=refdm
+export DB_PASSWORD=notsecret
+
 if [ "$1" = "start" ]
 then
     export DOCKER_BUILDKIT=1
 
     ${DOCKER} compose build
-    ${DOCKER} compose up -d --force-recreate --remove-orphans
+    ${DOCKER} compose up --detach --force-recreate --remove-orphans
 elif [ "$1" = "stop" ]
 then
-    ${DOCKER} compose stop
-    ${DOCKER} compose rm -fv
+    ${DOCKER} compose down --rmi local --volumes
+    ${DOCKER} compose rm --force --volumes
 elif [ "$1" = "check" ]
 then
     ${DOCKER} compose ps

refdb-sql/.gitignore

@@ -0,0 +1 @@
+refdb-sql/postgres/pgpass.conf

refdb-sql/Containerfile

@@ -0,0 +1,28 @@
+# Copyright (c) 2025 The Johns Hopkins University Applied Physics
+# Laboratory LLC.
+
+# This file is part of the Delay-Tolerant Networking Management
+# Architecture (DTNMA) Tools package.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This is a postgres stateful database with data definition startup SQL scripts
+FROM postgres:14
+
+COPY postgres/Database_Scripts/*/*.sql /docker-entrypoint-initdb.d/
+COPY postgres/Database_Scripts/*.sql /docker-entrypoint-initdb.d/
+
+# This is used for testing, it is easier to delete the amp_agent after inserting it using adm_amp_agent.sql instead of removing the script since other scripts are also relying on amp_agent
+# COPY postgres/Database_Scripts/Routines/amp_agent_delete.sql /docker-entrypoint-initdb.d/31-amp_agent_delete.sql
+
+HEALTHCHECK --start-period=10s --interval=10s --timeout=5s --retries=5 \
+    CMD pg_isready -U healthcheck