Merge remote-tracking branch 'origin/apl-fy24' into 78-initial-amm-rule-support

Author: justinethier

.github/workflows/build-test.yaml

@@ -134,6 +134,11 @@ jobs:
         run: ./item-test/run.sh
 
   integration-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        transport: ['socket', 'ion']
+    name: Integration Test (${{matrix.transport}})
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout repository
@@ -144,9 +149,9 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Start
-        run: ./integration-test-socket/run.sh start
+        run: ./integration-test-${{matrix.transport}}/run.sh start
       - name: Test
-        run: ./integration-test-socket/run.sh check
+        run: ./integration-test-${{matrix.transport}}/run.sh check
       - name: Stop
         if: always()
-        run: ./integration-test-socket/run.sh stop
+        run: ./integration-test-${{matrix.transport}}/run.sh stop

CMakeLists.txt

@@ -25,6 +25,8 @@ option(BUILD_DOCS_API "Enable API documentation building" OFF)
 option(BUILD_DOCS_MAN "Enable manpage building" OFF)
 option(BUILD_AGENT "Build the Agent library and executable" ON)
 option(BUILD_MANAGER "Build the Manager library and executable" ON)
+option(TRANSPORT_UNIX_SOCKET "Enable transport bindings for UNIX datagram sockets" ON)
+option(TRANSPORT_ION_BP "Enable transport bindings for ION BP" ON)
 option(ARI_TEXT_PARSE "Build ARI text-form parsing capability" ON)
 option(ENABLE_LUT_CACHE "Enable runtime lookup caching" ON)
 option(REFDM_UI_CLI "Enable text UI CLI for refdm" OFF)
@@ -181,27 +183,35 @@ pkg_search_module(PCRE libpcre2-8 IMPORTED_TARGET)
 message(STATUS "Found PCRE version ${PCRE_VERSION}")
 pkg_search_module(LIBSYSTEMD libsystemd IMPORTED_TARGET)
 message(STATUS "Found libsystemd version ${LIBSYSTEMD_VERSION}")
-find_package(ION)
 
+if(TRANSPORT_ION_BP)
+    find_package(ION REQUIRED)
+    message(STATUS "Found ION ${ION_FOUND}")
+endif(TRANSPORT_ION_BP)
 if(ARI_TEXT_PARSE)
-  find_package(FLEX REQUIRED)
-  find_package(BISON REQUIRED)
+    find_package(FLEX REQUIRED)
+    message(STATUS "Found FLEX version ${FLEX_VERSION}")
+    find_package(BISON REQUIRED)
+    message(STATUS "Found BISON version ${BISON_VERSION}")
 endif(ARI_TEXT_PARSE)
 
 if(BUILD_MANAGER)
-  # All these are optional
-  pkg_search_module(MYSQLCLIENT mysqlclient IMPORTED_TARGET)
-  find_package(PostgreSQL)
-  find_package(civetweb)
-  find_package(cJSON)
+    # All these are optional
+    pkg_search_module(MYSQLCLIENT mysqlclient IMPORTED_TARGET)
+    message(STATUS "Found MySQLClient version ${MYSQLCLIENT_VERSION}")
+    find_package(PostgreSQL)
+    message(STATUS "Found PostgreSQL version ${PostgreSQL_VERSION}")
+    find_package(civetweb)
+    message(STATUS "Found civetweb version ${civetweb_VERSION}")
+    find_package(cJSON)
 endif(BUILD_MANAGER)
 
 add_subdirectory(src)
 
 if(BUILD_TESTING)
-  add_subdirectory(test)
+    add_subdirectory(test)
 endif(BUILD_TESTING)
 
 if(BUILD_DOCS_API OR BUILD_DOCS_MAN)
-  add_subdirectory(docs)
+    add_subdirectory(docs)
 endif(BUILD_DOCS_API OR BUILD_DOCS_MAN)

cmake/FindION.cmake

@@ -1,3 +1,4 @@
+# Sets ION_FOUND based on finding all libraries
 # Imports targets for using ION libraries:
 #  ION::ICI for libici
 #  ION::BP for libbp
@@ -64,6 +65,8 @@ if(LTP_HEADER AND LTP_LIB)
     target_link_libraries(ION::LTP INTERFACE ION::ICI)
 endif()
 
-if (NOT ICI_HEADER OR NOT BP_HEADER OR NOT LTP_HEADER)
+if (ICI_HEADER AND BP_HEADER AND LTP_HEADER)
+    set(ION_FOUND true)
+else()
     set(ION_FOUND false)
-endif()
+endif()

deps.sh

@@ -19,6 +19,8 @@
 
 #
 # From a fresh checkout install local-sourced dependencies.
+# The environment DEPS_BUILD_ION being set to "0" will disable
+# the ION source from being built and installed.
 #
 set -e
 
@@ -30,7 +32,7 @@ BUILDDIR=${BUILDDIR:-${SELFDIR}/deps/build}
 echo "Building in ${BUILDDIR}"
 echo "Installing to ${DESTDIR}"
 
-if [ -z "false" -a ! -e ${DESTDIR}/usr/include/ion.h ]
+if [ "${DEPS_BUILD_ION:-1}" -ne 0 -a ! -e ${DESTDIR}/usr/include/ion.h ]
 then
   mkdir -p ${BUILDDIR}
   rsync --recursive ${DEPSDIR}/ion/ ${BUILDDIR}/ion/
@@ -40,9 +42,10 @@ then
   patch -p1 <${SELFDIR}/deps/ion-4.1.2-local-deliver.patch
   patch -p1 <${SELFDIR}/deps/ion-4.1.2-private-headers.patch
   autoreconf -vif
+  export CFLAGS="-std=gnu99"
   ./configure --prefix=/usr
-  make -j$(nproc) clean
   make -j$(nproc)
+  export -n CFLAGS
   make install DESTDIR=${DESTDIR}
   make -j$(nproc) clean
   popd
@@ -85,6 +88,7 @@ then
     -DCMAKE_BUILD_TYPE=Debug \
     -DCMAKE_INSTALL_PREFIX=${DESTDIR}${PREFIX}
   cmake --build ${BUILDDIR}/unity -v
+  export -n CFLAGS
   cmake --install ${BUILDDIR}/unity
   rm -rf ${BUILDDIR}/unity
   popd

testenv/Dockerfile integration-test-ion/Containerfile

@@ -15,7 +15,7 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 ##
-FROM ubuntu:22.04 AS systemd-base
+FROM ubuntu:24.04 AS systemd-base
 ENV DEBIAN_FRONTEND="noninteractive"
 
 # APL network configuration from
@@ -43,80 +43,116 @@ CMD [ "/sbin/init" ]
 # Testing utilities
 RUN apt-get update && apt-get install -y \
     net-tools iproute2 iputils-ping \
-    lsof iftop gdb valgrind socat
+    lsof iftop gdb valgrind xxd socat jq ruby && \
+    gem install cbor-diag
 
 
-FROM ubuntu:22.04 AS deps-local
+FROM ubuntu:24.04 AS deps-local
 
 RUN apt-get update && apt-get install -y \
     build-essential \
-    autoconf libtool
-COPY deps/ion /usr/src/nm/deps/ion
+    cmake autoconf libtool && \
+    echo "/usr/local/lib" >/etc/ld.so.conf.d/local.conf
+
+    COPY deps/ion /usr/src/nm/deps/ion
 COPY deps/ion*.patch /usr/src/nm/deps/
 RUN cd /usr/src/nm/deps/ion && \
     patch -p1 <../ion-4.1.2-remove-nm.patch && \
     patch -p1 <../ion-4.1.2-local-deliver.patch && \
     patch -p1 <../ion-4.1.2-private-headers.patch && \
     autoreconf -vif && \
+    export CFLAGS="-std=gnu99" && \
     ./configure && \
     make -j$(nproc) && \
     make install && \
     make -j$(nproc) clean
 
 COPY deps/QCBOR /usr/src/nm/deps/QCBOR
 RUN cd /usr/src/nm/deps/QCBOR && \
-    make -j$(nproc) && \
-    make install && \
-    make -j$(nproc) clean
+    cmake -S . -B build \
+      -DCMAKE_BUILD_TYPE=Debug \
+      -DBUILD_SHARED_LIBS=YES && \
+    cmake --build build && \
+    cmake --install build && \
+    ldconfig && \
+    rm -rf build
 
 COPY deps/mlib /usr/src/nm/deps/mlib
 RUN cd /usr/src/nm/deps/mlib && \
     make -j$(nproc) && \
     make install && \
+    ldconfig && \
     make -j$(nproc) clean
 
+COPY deps/timespec /usr/src/nm/deps/timespec
+COPY deps/timespec-CMakeLists.txt /usr/src/nm/deps/timespec/CMakeLists.txt
+RUN cd /usr/src/nm/deps/timespec && \
+    cmake -S . -B build \
+      -DCMAKE_BUILD_TYPE=Debug && \
+    cmake --build build && \
+    cmake --install build && \
+    ldconfig && \
+    rm -rf build
+
 
-FROM systemd-base
+FROM systemd-base AS testenv
 COPY --from=deps-local /usr/local /usr/local
 
 # Helper utilities
 RUN apt-get update && apt-get install -y \
-    python3 python3-pip \
-    gcc python3-dev libsystemd-dev pkg-config && \
-    pip3 install --upgrade pip && \
-    pip3 install systemd-python
-COPY --chmod=755 testenv/ion_nm_wrap.py /usr/local/bin/ion_nm_wrap
-COPY --chmod=755 testenv/ion_ping_peers.sh /usr/local/bin/ion_ping_peers
-COPY --chmod=755 testenv/service_is_running.sh /usr/local/bin/service_is_running
+    python3 python3-pip
+COPY --chmod=755 systemd/service_is_running.sh /usr/local/bin/service_is_running
 
 # Test tools
 RUN apt-get update && apt-get install -y \
-    curl git tshark && \
-    pip3 install git+https://github.com/NASA-AMMOS/anms-ace.git
+    curl git tshark postgresql-client
+RUN pip3 install --break-system-packages git+https://github.com/JHUAPL-DTNMA/dtnma-ace.git@apl-fy24
+RUN git clone --branch apl-fy24 https://github.com/JHUAPL-DTNMA/dtnma-adms.git /usr/local/share/ace/adms
 
-# Agent to test
+# REFDA and REFDM to test
 RUN apt-get update && apt-get install -y \
-    cmake ninja-build ruby \
-    flex libfl-dev bison civetweb libcivetweb-dev libssl-dev libcjson-dev
+    cmake ninja-build ruby pkg-config \
+    flex libfl-dev bison libpcre2-dev libpq-dev civetweb libcivetweb-dev libssl-dev libcjson-dev libsystemd-dev
 COPY deps /usr/src/nm/deps
 COPY cmake /usr/src/nm/cmake
 COPY src /usr/src/nm/src
-COPY test /usr/src/nm/test
 COPY CMakeLists.txt /usr/src/nm/
 RUN ls -lt /usr/src/nm/
 RUN cd /usr/src/nm && \
     cmake -S . -B build/default \
       -DCMAKE_BUILD_TYPE=Debug \
+      -DTRANSPORT_UNIX_SOCKET=OFF \
+      -DTRANSPORT_ION_BP=ON \
+      -DBUILD_TESTING=OFF \
+      -DBUILD_DOCS_API=OFF -DBUILD_DOCS_MAN=OFF \
       -G Ninja && \
     cmake --build build/default && \
-    cmake --install build/default
+    cmake --install build/default && \
+    ldconfig
+# keep build artifacts for debuginfo
+
+
+FROM testenv AS manager
+
+# Systemd services
+COPY integration-test-ion/tmpfiles.conf /etc/tmpfiles.d/ion.conf
+COPY --chmod=644 systemd/ion.service systemd/ref*-ion.service systemd/[email protected] systemd/dumpcap.service \
+    /usr/local/lib/systemd/system/
+RUN systemctl enable ion bpecho@4 refdm-ion refda-ion dumpcap && \
+    mkdir -p /var/run/ion
+
+# Runtime config for this container
+COPY integration-test-ion/node-*.rc /etc/ion/
+
+
+FROM testenv AS agent
 
 # Systemd services
-COPY testenv/tmpfiles.conf /etc/tmpfiles.d/ion.conf
-COPY --chmod=644 systemd/ion.service systemd/ion-nm-*.service systemd/[email protected] /usr/local/lib/systemd/system/
-RUN systemctl enable ion bpecho@4 ion-nm-mgr ion-nm-agent && \
+COPY integration-test-ion/tmpfiles.conf /etc/tmpfiles.d/ion.conf
+COPY --chmod=644 systemd/ion.service systemd/ref*-ion.service systemd/[email protected] systemd/dumpcap.service \
+    /usr/local/lib/systemd/system/
+RUN systemctl enable ion bpecho@4 refda-ion dumpcap && \
     mkdir -p /var/run/ion
 
 # Runtime config for this container
-COPY testenv/node-1.rc /etc/ion/
-COPY adms /usr/local/share/ace/adms
+COPY integration-test-ion/node-*.rc /etc/ion/

integration-test-ion/compose.yml

@@ -0,0 +1,73 @@
+volumes:
+  postgres-data:
+    # temporary storage
+
+services:
+  postgres:
+    build:
+      context: ../refdb-sql
+      dockerfile: Containerfile
+    hostname: postgres
+    restart: no
+    environment:
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_DB: ${DB_NAME}
+    volumes:
+      - "postgres-data:/var/lib/postgresql/data"
+
+  manager:
+    hostname: manager
+    build:
+      context: ..
+      dockerfile: integration-test-ion/Containerfile
+      target: manager
+    ports:
+      - "8089:8089/tcp"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    privileged: true
+    tty: true
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+      - SYS_NICE
+    volumes:
+      - /var/tmp/pcap:/var/log/pcap
+    environment:
+      container: docker
+      DTNMA_LOGLEVEL: debug
+      ION_NODE_NUM: 1
+      MGR_NODE_NUM: 1
+      DB_HOST: postgres
+      DB_USER: ${DB_USER}
+      DB_PASSWORD: ${DB_PASSWORD}
+      DB_NAME: ${DB_NAME}
+      DUMPCAP_OUTFILE: /var/log/pcap/manager.pcap
+      DUMPCAP_OPTS: -i lo -i eth0
+
+  agent1:
+    hostname: agent1
+    build:
+      context: ..
+      dockerfile: integration-test-ion/Containerfile
+      target: agent
+    depends_on:
+      manager:
+        condition: service_started
+    privileged: true
+    tty: true
+    cap_add:
+      - NET_ADMIN
+      - NET_RAW
+      - SYS_NICE
+    volumes:
+      - /var/tmp/pcap:/var/log/pcap
+    environment:
+      container: docker
+      DTNMA_LOGLEVEL: debug
+      ION_NODE_NUM: 2
+      MGR_NODE_NUM: 1
+      DUMPCAP_OUTFILE: /var/log/pcap/agent1.pcap
+      DUMPCAP_OPTS: -i lo -i eth0

testenv/ion_ping_peers.sh integration-test-ion/ion_ping_peers.sh

@@ -1,4 +1,4 @@
-# ION Configuration File for Node N1, ipn:1
+# ION Configuration File for Node N1, ipn:1, hostname manager
 ## begin ionadmin
 1 1 ''
 s
@@ -28,13 +28,15 @@ a protocol tcp 1400 100
 a induct udp 0.0.0.0:4556 udpcli
 
 # Outducts
-a outduct udp localhost:4556 udpclo
+a outduct udp manager:4556 udpclo
+a outduct udp agent1:4556 udpclo
 
 s
 ## end bpadmin
 
 ## begin ipnadmin
-a plan 1 udp/localhost:4556
+a plan 1 udp/manager:4556
+a plan 2 udp/agent1:4556
 
 ## end ipnadmin