Added container configuration system from libfabric-sarus into mater

.gitignore

@@ -0,0 +1,54 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# CMake
+CMakeLists.txt.user
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+Testing
+Makefile
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+_deps
+bin/
+configuration/
+volumes/
+containers/config/htpasswd
+
+benchmarks/warm_benchmarker
+benchmarks/parallel_invocations
+benchmarks/cold_benchmarker
+benchmarks/cpp_interface

config/executor_manager.json

@@ -1,16 +1,73 @@
 {
   "config": {
     "rdma_device": "",
-    "rdma_device_port": 10000,
+    "rdma_device_port": 10004,
     "resource_manager_address": "",
     "resource_manager_port": 0,
     "resource_manager_secret": 0
   },
   "executor": {
-    "use_docker": false,
+    "sandbox_type": "process",
     "repetitions": 100,
     "warmup_iters": 0,
     "pin_threads": false
-  }
+  },
+  "sandbox-configuration": [
+    {
+      "key": "sarus",
+      "value": {
+        "index": 1,
+        "data": {
+          "user": "mcopik",
+          "name": "spcleth/hpc-disagg:rfaas-executor-daint",
+          "devices": [
+            "/dev/kgni0", "dev/kdreg"
+          ],
+          "mounts": [
+            "/opt/cray",
+            "/tmp/drcc.sock",
+            "/etc/opt/cray/rdma-credentials",
+            "/etc/alternatives/cray-ugni",
+            "/etc/alternatives/cray-xpmem",
+            "/etc/alternatives/cray-alps",
+            "/etc/alternatives/cray-udreg",
+            "/etc/alternatives/cray-wlm_detect"
+          ],
+          "mount_filesystem": [
+            "/scratch/snx3000/{user}"
+          ],
+          "env": [
+            {
+              "key": "LD_LIBRARY_PATH",
+              "value": "/opt/cray/xpmem/default/lib64/;/opt/cray/udreg/default/lib64;/opt/cray/alps/default/lib64;/opt/cray/wlm_detect/default/lib64/"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "key": "docker",
+      "value": {
+        "index": 0,
+        "data": {
+          "image": "rfaas-registry/rfaas-base",
+          "network": "mynet",
+          "ip": "172.31.82.202",
+          "volume": "/home/ubuntu/rfaas/containers/opt",
+          "registry_ip": "172.31.82.200",
+          "registry_port": 5000
+        }
+      }
+    },
+    {
+      "key": "singularity",
+      "value": {
+        "index": 2,
+        "data": {
+          "container": "rfaas-container-singularity"
+        }
+      }
+    }
+  ]
 }
 

rdmalib/include/rdmalib/util.hpp

@@ -2,6 +2,8 @@
 #ifndef __RDMALIB_UTIL_HPP__
 #define __RDMALIB_UTIL_HPP__
 
+#include <cstring>
+#include <rdma/fi_errno.h>
 #include <spdlog/spdlog.h>
 
 namespace rdmalib { namespace impl {
@@ -73,6 +75,25 @@ namespace rdmalib { namespace impl {
     assert(ptr);
   }
 
+  template<typename StrType>
+  const char* to_cstr(const StrType & str)
+  {
+    return str.c_str();
+  }
+
+  // Code borrowed from StackOverflow https://stackoverflow.com/questions/2342162/stdstring-formatting-like-sprintf
+  template<typename ... Args>
+  std::string string_format( const std::string& format, Args... args )
+  {
+    int size_s = std::snprintf( nullptr, 0, format.c_str(), to_cstr(args)... ) + 1;
+    if(size_s <= 0)
+      return "";
+    auto size = static_cast<size_t>(size_s);
+    std::unique_ptr<char[]> buf(new char[size]);
+    std::snprintf(buf.get(), size, format.c_str(), to_cstr(args)...);
+    return std::string(buf.get(), buf.get() + size - 1);
+  }
+
 }}
 
 #endif

scripts/docker/build_base.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build -t rfaas-base:latest - < rfaas-base.Dockerfile

scripts/docker/init_docker.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# This script builds the rfaas-base docker container, pushes it to the
+# registry, sets up a docker volume, and generates example configuration 
+# json which should then be updated in config/executor_manager.json.
+# The script also configures a docker network for suitable use with
+# docker_rdma_sriov
+
+# NOTE: Run this script from repo root, and make sure the sriov docker plugin
+# is installed
+
+set -e
+
+if [ $# -lt 2 ]; then
+    echo "usage: ./init_docker.sh <REGISTRY IP> <REGISTRY PORT> <NETWORK MODE> [<NETWORK DEVICE> <SUBNET>]"
+    exit
+fi
+
+REG_IP=$1    # IP or name of the docker registry
+REG_PORT=$2  # Port of the docker registry
+NET_MODE=$3  # Docker networking mode -- sriov or host
+DEVICE=$4    # The RDMA adapter to use for networking
+SUBNET=$5    # Subnet for the docker network
+
+IMG_NAME=rfaas-base
+REG_IMG=$REG_IP:$REG_PORT/$IMG_NAME
+
+# Build the docker container, login and push to the registry
+docker build -t $IMG_NAME - < containers/rfaas-base.Dockerfile
+echo "built rfaas-base image"
+docker login $REG_IP:$REG_PORT
+echo "logged into docker daemon"
+
+if docker push $REG_IMG; then
+    echo "ERROR: make sure a docker registry is actually running on $REG_IP:$REG_PORT.
+    Start one with scripts/run_registry.sh"
+    exit
+else
+    echo "pushed rfaas-base image to $REG_IMG"
+fi
+
+# Set up docker network
+net_name=testnet
+if ["$NET_MODE" = "sriov"]; then
+    docker network create -d sriov --subnet=$SUBNET -o netdevice=$DEVICE $net_name
+elif ["$NET_MODE" = "host"]; then
+    net_name="host"
+else
+    echo "ERROR: invalid networking mode $NET_MODE. Valid options are sriov and host."
+    exit
+fi
+echo "set up docker network"
+
+# Configure volume
+volume=$(pwd)/volumes/rfaas-test/opt # Do not put a trailing slash
+mkdir -p $volume/bin
+cp bin/executor $volume/bin
+cp examples/libfunctions.so $volume
+
+# Print json to be updated
+config=$(jq -n --arg use_docker "true" \
+    --arg image "$REG_IMG" \
+    --arg network "$net_name" \
+    --arg ip "<ip of container (any available ip within $SUBNET)>" \
+    --arg volume $volume \
+    --arg registry_ip "$REG_IP" \
+    --arg registry_port "$REG_PORT" \
+    '{
+        "image": $image,
+        "network": $network,
+        "ip": $ip,
+        "volume": $volume,
+        "registry_ip": $registry_ip,
+        "registry_port": $registry_port
+    }'
+)
+
+echo "Update config/executor_manager.json with"
+echo "$config"
+

scripts/docker/push_image.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Push an image to the local docker registry
+
+if [ $# -lt 3 ]; then
+    echo "usage: ./push_image.sh <IMAGE NAME> <REGISTRY NAME or IP> <REGISTRY PORT>";
+    exit
+fi
+
+IMAGE=$1
+IP=$2
+PORT=$3
+docker push $IP:$PORT/$IMAGE

scripts/docker/registry.yaml

@@ -0,0 +1,20 @@
+version: "3"
+
+services:
+  registry:
+    image: registry:2
+    container_name: rfaas-registry
+    ports:
+      - "5000:5000"
+    environment:
+      REGISTRY_AUTH: htpasswd
+      REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
+      REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
+      REGISTRY_STORAGE_DELETE_ENABLED: "true"
+      #REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
+      #REGISTRY_HTTP_TLS_KEY: /certs/domain.unencrypted.key
+      #REGISTRY_HTTP_SECRET: supersecrettext
+    volumes:
+      - /home/ubuntu/rfaas/containers/registry:/var/lib/registry
+      - /home/ubuntu/rfaas/containers/config:/auth
+      #- /home/ubuntu/rfaas/containers/config/certs:/certs

scripts/docker/rfaas-base.Dockerfile

@@ -0,0 +1,14 @@
+# Container version should be same ubuntu version as where `executor`
+# was built (due to glibc versioning)
+
+FROM ubuntu:22.04
+
+#ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -y && apt-get upgrade -y \
+    && apt-get install -y \
+    libibverbs-dev librdmacm-dev
+
+RUN mkdir -p /opt/bin
+WORKDIR "/opt/bin"
+

scripts/docker/run_registry.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Run this script on the server where you want the docker registry to be hosted
+# Recommended to host the registry on the same server as the executor manager
+
+# NOTE: Run this script from the repo root
+
+PORT=5000
+NAME="rfaas-registry"
+
+set -e
+
+# Make password file if it doesn't already exist
+cfg=containers/config
+mkdir -p $cfg
+if [ -s $cfg/htpasswd ]; then
+    echo "htpasswd exists"
+else
+    sudo htpasswd -Bc $cfg/htpasswd $USER
+    echo "created htpasswd file"
+fi
+
+# Generate certs to use TLS (if they dont already exist)
+## if [ -s $cfg/certs/domain.key ]; then
+##     echo "using certs in $cfg/certs"
+## else
+##     mkdir -p $cfg/certs
+##     openssl genpkey -algorithm RSA -out $cfg/certs/domain.key -aes256
+## 
+##     openssl req -new               \
+##         -key $cfg/certs/domain.key \
+##         -out $cfg/certs/domain.csr \
+##         -addext 'subjectAltName = IP:172.31.82.200'
+## 
+##     openssl x509 -req -days 365        \
+##         -in $cfg/certs/domain.csr      \
+##         -signkey $cfg/certs/domain.key \
+##         -out $cfg/certs/domain.crt
+##     
+##     openssl rsa -in $cfg/certs/domain.key -out $cfg/certs/domain.unencrypted.key
+##     echo "generated certs in $cfg/certs"
+## fi
+
+# Start registry
+sudo docker-compose -f scripts/registry.yaml up -d
+echo "started docker registry $NAME on port $PORT"
+

server/executor/opts.cpp

@@ -30,9 +30,15 @@ namespace server {
       ("mgr-secret", "Use selected port", cxxopts::value<int>())
       ("mgr-buf-addr", "Use selected port", cxxopts::value<uint64_t>())
       ("mgr-buf-rkey", "Use selected port", cxxopts::value<uint32_t>())
+      ("h,help", "Print usage")
     ;
     auto parsed_options = options.parse(argc, argv);
 
+    if (parsed_options.count("help")) {
+      std::cout << options.help() << std::endl;
+      exit(0);
+    }
+
     Options result;
     result.address = parsed_options["address"].as<std::string>();
     result.port = parsed_options["port"].as<int>();