Merge branch 'master' into documentation

.github/workflows/testing.yml

@@ -255,6 +255,10 @@ jobs:
         run: bin/start_etcd
       - name: bin/test_etcd
         run: bin/test_etcd
+      - name: bin/setup_ca
+        run: bin/setup_ca sites/udmi_site_model
+      - name: bin/start_mosquitto
+        run: bin/start_mosquitto
       - name: bin/test_mosquitto
         run: bin/test_mosquitto
       - name: bin/test_udmis

bin/container

@@ -1,12 +1,16 @@
 #!/bin/bash -e
 
 UDMI_ROOT=$(dirname $0)/..
+source ${UDMI_ROOT}/etc/shell_common.sh
+
+if [[ $# -lt 2 ]]; then
+    usage target { prep, build, push, apply } [--no-check] [repo]
+fi
 
 target=$1
 cmd=$2
-shift 2 || usage target { prep, build, push, apply } [--no-check] [repo]
+shift 2
 
-source ${UDMI_ROOT}/etc/shell_common.sh
 cd ${UDMI_ROOT}/${target}
 
 git_branch=$(git rev-parse --abbrev-ref HEAD)
@@ -20,15 +24,11 @@ GCP_PROJECT=$(gcloud config get project)
 REPOSITORY=ghcr.io/${git_repo}
 TEMPLATES=$(cd etc; ls k8s_*.yaml)
 
-if [[ $1 == "--no-check" ]]; then
-    NOCHECK=1
-    shift
-fi
+NOCHECK=
+[[ ${1:-} == "--no-check" ]] && NOCHECK=1 && shift
 
-if [[ -n $1 ]]; then
-    PUSH_TAG=$1
-    shift
-fi
+PUSH_TAG=
+[[ -n ${1:-} ]] && PUSH_TAG=$1 && shift
 
 if [[ $# != 0 ]]; then
     echo unknown arguments: $*
@@ -50,7 +50,13 @@ if [[ -z $NOCHECK ]]; then
 fi
 
 echo Using GCP project $GCP_PROJECT
-echo Using udmi namespace $UDMI_NAMESPACE
+
+if [[ -n ${UDMI_NAMESPACE:-} ]]; then
+    echo Using udmi namespace $UDMI_NAMESPACE
+else
+    echo Using udmi namespace [none]
+    UDMI_NAMESPACE=
+fi
 
 current_user=$USER@$HOSTNAME
 

bin/keygen

@@ -1,29 +1,63 @@
 #!/bin/bash -e
 
+TYPES="{ CA, CERT, RS256, RS256_X509, ES256, ES256_X509 }"
+
 if [ "$#" != 2 ]; then
-    echo $0 [type] [out_dir] 
+    echo $0 type out_dir
+    echo Where type is one of: $TYPES
     false
 fi
 
 type=$1
 cd $2
+shift 2
+
+if [[ $(basename $PWD) == reflector ]]; then
+    echo Generating self-signed cert for server admin
+    CA_DIR=$PWD
+    SUBJ=server
+else
+    CA_DIR=../../reflector
+    SUBJ=client
+fi
 
-if [ $type == RS256 ]; then
+CA_CRT=$CA_DIR/ca.crt
+CA_SRL=$CA_DIR/ca.srl
+CA_KEY=$CA_DIR/rsa_private.pem
+OPTS_509=/tmp/x509opts.txt
+
+echo "subjectAltName=DNS:localhost, IP:127.0.0.1" > $OPTS_509
+x509opts="-days 3650 -extfile $OPTS_509"
+
+if [[ $type == CA ]]; then
+    CERT_SUBJ=/CN=ca
+    echo Generating CA and server certs in $PWD
+    openssl req -new -x509 -days 3650 -key $CA_KEY -out ca.crt -subj $CERT_SUBJ
+    openssl genrsa -out server.key 2048
+    openssl req -new -key server.key -out server.csr -subj /CN=$SUBJ
+    openssl x509 -req -in server.csr -CA ca.crt -CAkey $CA_KEY -CAcreateserial -out server.crt $x509opts
+    echo Created certificate authority keys in $PWD
+elif [ $type == CERT ]; then
+    echo Generating self-signed cert from CA defined in $(realpath $CA_DIR)
+    echo Generating cert for device keys in $PWD
+    openssl req -new -key rsa_private.pem -out rsa_private.csr -subj /CN=$SUBJ
+    openssl x509 -req -in rsa_private.csr -CA $CA_CRT -CAkey $CA_KEY -CAcreateserial -out rsa_private.crt $x509opts
+elif [ $type == RS256 ]; then
     openssl genrsa -out rsa_private.pem 2048
     openssl rsa -in rsa_private.pem -pubout -out rsa_public.pem
 elif [ $type == RS256_X509 ]; then
     openssl genrsa -out rsa_private.pem 2048
     openssl rsa -in rsa_private.pem -pubout -out rsa_public.pem
-    openssl req -x509 -nodes -newkey rsa:2048 -keyout rsa_private.pem -days 1000000 -out rsa_cert.pem -subj "/CN=unused"
+    openssl req -x509 -nodes -newkey rsa:2048 -keyout rsa_private.pem -days 1000000 -out rsa_cert.pem -subj /CN=unused
 elif [ $type == ES256 ]; then
     openssl ecparam -genkey -name prime256v1 -noout -out ec_private.pem
     openssl ec -in ec_private.pem -pubout -out ec_public.pem
 elif [ $type == ES256_X509 ]; then
     openssl ecparam -genkey -name prime256v1 -noout -out ec_private.pem
     openssl ec -in ec_private.pem -pubout -out ec_public.pem
-    openssl req -x509 -new -key ec_private.pem -out ec_cert.pem -days 1000000 -subj "/CN=unused"
+    openssl req -x509 -new -key ec_private.pem -out ec_cert.pem -days 1000000 -subj /CN=unused
 else
-    echo Unknown key type $type. Try one of { RS256, RS256_X509, ES256, ES256_X509 }
+    echo Unknown key type $type. Try one of $TYPES
     false
 fi
 
@@ -32,3 +66,5 @@ if [ -f rsa_private.pem ]; then
 else
     openssl pkcs8 -topk8 -inform PEM -outform DER -in ec_private.pem -nocrypt > ec_private.pkcs8
 fi
+
+echo Done with keygen.

bin/pubber

@@ -4,7 +4,7 @@ UDMI_ROOT=$(realpath $(dirname $0)/..)
 source $UDMI_ROOT/etc/shell_common.sh
 
 function cleanup {
-    if [[ -n $PUB_PID && -d /proc/$PUB_PID ]]; then
+    if [[ -n ${PUB_PID:-} && -d /proc/$PUB_PID ]]; then
         echo Killing pubber runner $PUB_PID
         kill $PUB_PID
     fi
@@ -46,7 +46,7 @@ else
     jq -r .device_endpoint $config_file > /tmp/endpoint.json
 fi
 
-if [[ -n $UDMI_REGISTRY_SUFFIX ]]; then
+if [[ -n ${UDMI_REGISTRY_SUFFIX:-} ]]; then
     echo Using registry suffix $UDMI_REGISTRY_SUFFIX
     registry_suffix=$UDMI_REGISTRY_SUFFIX
 else
@@ -61,6 +61,7 @@ $UDMI_ROOT/pubber/bin/build
 
 echo Running tools version $udmi_version
 
+options_json=
 for option in $*; do
     if [[ $option == *"="* ]]; then
         k=$(echo $option | cut -d'=' -f1)

bin/sequencer

@@ -176,7 +176,7 @@ validator/bin/build
 
 echo $JAVA_CMD
 result=0
-$JAVA_CMD 2>&1 | tee $SEQUENCER_LOG
+$JAVA_CMD 2>&1 | tee $SEQUENCER_LOG || true
 
 echo
 echo Extracting sequence results:
@@ -188,7 +188,7 @@ egrep 'RESULT |CPBLTY ' $SEQUENCER_LOG | fgrep NOTICE | sort -k 6 | \
 
 # Sort by test name (3th field)
 fgrep RESULT $site_model/out/devices/$device_id/RESULT.log | \
-    fgrep ' schemas ' | sort -k 3 | tee $SCHEMA_OUT
+    fgrep ' schemas ' | sort -k 3 | tee $SCHEMA_OUT || true
 more $site_model/out/devices/$device_id/RESULT.log $SCHEMA_OUT | cat
 
 bin/sequencer_report $site_model $device_id

bin/sequencer_cache

@@ -42,6 +42,12 @@ if [[ $# == 0 ]]; then
     usage
 fi
 
+update=
+archive=
+validate_saved=
+validate_archive=
+import_cache=
+
 while getopts "i:auxt" opt; do
     case $opt in
         a)
@@ -110,7 +116,6 @@ if [[ -n $validate_saved ]]; then
     ls $CACHE_ROOT/ > out/cached_sequences.out
     ls $TEST_ROOT/tests/ > out/device_sequences.out
     diff -u out/cached_sequences.out out/device_sequences.out || fail Unexpected difference with cached results.
-    echo Verification successful for $root_count tests.
 fi
 
 if [[ -n $archive ]]; then

bin/setup_ca

@@ -0,0 +1,27 @@
+#!/bin/bash -eu
+
+UDMI_ROOT=$(dirname $0)/..
+source $UDMI_ROOT/etc/shell_common.sh
+
+CERT_DIR=/etc/mosquitto/certs/
+sudo rm -f $CERT_DIR/*
+
+[[ $# != 1 ]] && fail $0 site_model
+
+site_model=$(realpath $1)
+
+cd $UDMI_ROOT
+
+bin/keygen CA $site_model/reflector
+bin/keygen CERT $site_model/reflector
+
+echo Importing server certs from $site_model...
+sudo cp $site_model/reflector/ca.* $CERT_DIR/
+sudo cp $site_model/reflector/rsa_private.* $CERT_DIR/
+sudo chgrp -R mosquitto $CERT_DIR
+sudo chmod 0640 $CERT_DIR/rsa_private.*
+ls -l $CERT_DIR
+
+
+
+

bin/start_mosquitto

@@ -1,50 +1,58 @@
-#!/bin/bash -e
+#!/bin/bash -eu
 
+UDMI_ROOT=$(dirname $0)/..
+source $UDMI_ROOT/etc/shell_common.sh
+
+GROUP=mosquitto
 ETC_DIR=/etc/mosquitto
-CONF_FILE=$ETC_DIR/mosquitto.conf
-UDMI_FILE=$ETC_DIR/confg.d/udmi.conf
+UDMI_FILE=$ETC_DIR/conf.d/udmi.conf
 PASS_FILE=$ETC_DIR/mosquitto.passwd
 DYN_FILE=$ETC_DIR/dynamic_security.json
+CERT_DIR=$ETC_DIR/certs
+CA_CERT=$CERT_DIR/ca.crt
 
 AUTH_USER=scrumptious
 AUTH_PASS=aardvark
 echo Configuring MQTT user: $AUTH_USER
 
-sudo sed -i 's/allow_anonymous true/allow_anonymous false/' $CONF_FILE
-sudo sed -i 's/#listener/listener/' $CONF_FILE
-fgrep $PASS_FILE $CONF_FILE || (echo password_file ${PASS_FILE} | sudo tee -a $CONF_FILE)
-
-if ! fgrep -q $DYN_FILE $CONF_FILE; then
+if [[ ! -f $UDMI_FILE ]]; then
+    echo Creating new $UDMI_FILE from template...
+    sudo cp etc/mosquitto_udmi.conf $UDMI_FILE
     PLUGIN_FILE=$(whereis -b mosquitto_dynamic_security.so | awk '{print $2}')
     ls -l "$PLUGIN_FILE"
-    echo Installing dynamic security plugin $PLUGIN_FILE
-    echo plugin $PLUGIN_FILE | sudo tee -a $CONF_FILE
-    echo plugin_opt_config_file $DYN_FILE | sudo tee -a $CONF_FILE
+    echo Configuring dynamic security plugin $PLUGIN_FILE
+    sudo sed -i "s%plugin dynsec%plugin $PLUGIN_FILE%" $UDMI_FILE
+    sudo sed -i "s%plugin_opt_config_file dynsec%plugin_opt_config_file $DYN_FILE%" $UDMI_FILE
 fi
 
 if [[ ! -f $DYN_FILE ]]; then
     echo Creating new $DYN_FILE
     sudo mosquitto_ctrl dynsec init $DYN_FILE $AUTH_USER $AUTH_PASS
-    sudo chgrp mosquitto $DYN_FILE
+    sudo chgrp $GROUP $DYN_FILE
     sudo chmod 0660 $DYN_FILE
 fi
 
-sudo chown mosquitto $ETC_DIR
-
-sudo touch $PASS_FILE
-sudo mosquitto_passwd -b ${PASS_FILE} ${AUTH_USER} ${AUTH_PASS}
+if [[ ! -f $PASS_FILE ]]; then
+    echo Creating $PASS_FILE
+    sudo touch $PASS_FILE
+    sudo chmod 0640 $PASS_FILE
+    sudo chgrp $GROUP $PASS_FILE
+    sudo mosquitto_passwd -b ${PASS_FILE} ${AUTH_USER} ${AUTH_PASS}
+fi
 
 sudo systemctl restart mosquitto
 
-CTRL_CONNECT="-u $AUTH_USER -P $AUTH_PASS"
-mosquitto_ctrl $CTRL_CONNECT dynsec createRole device
-mosquitto_ctrl $CTRL_CONNECT dynsec addRoleACL device subscribePattern '/#' allow
-mosquitto_ctrl $CTRL_CONNECT dynsec addRoleACL device publishClientSend '/#' allow
-mosquitto_ctrl $CTRL_CONNECT dynsec createRole service
-mosquitto_ctrl $CTRL_CONNECT dynsec addRoleACL service subscribePattern '/#' allow
-mosquitto_ctrl $CTRL_CONNECT dynsec addRoleACL service publishClientSend '/#' allow
+CONNECT_OPTS="-p 8883 -u $AUTH_USER -P $AUTH_PASS --cafile $CA_CERT --cert $CERT_DIR/rsa_private.crt --key $CERT_DIR/rsa_private.pem"
+MOSQUITTO_CTRL="sudo mosquitto_ctrl $CONNECT_OPTS dynsec"
+
+$MOSQUITTO_CTRL createRole device
+$MOSQUITTO_CTRL addRoleACL device subscribePattern '/#' allow
+$MOSQUITTO_CTRL addRoleACL device publishClientSend '/#' allow
+$MOSQUITTO_CTRL createRole service
+$MOSQUITTO_CTRL addRoleACL service subscribePattern '/#' allow
+$MOSQUITTO_CTRL addRoleACL service publishClientSend '/#' allow
 
-clients=$(mosquitto_ctrl -u ${AUTH_USER} -P ${AUTH_PASS} dynsec listClients)
+clients=$($MOSQUITTO_CTRL listClients)
 if [[ $clients =~ ${AUTH_USER} ]]; then
     echo Found expected client $AUTH_USER
 else

bin/test_itemized

@@ -11,7 +11,7 @@ if [[ $# != 1 && $# != 2 ]]; then
 fi
 PROJECT_ID=$1
 shift 1
-TARGET_INDEX=$1
+TARGET_INDEX=${1-}
 
 SITE_PATH=sites/udmi_site_model
 DEVICE_ID=AHU-1
@@ -30,10 +30,10 @@ mkdir -p out
 echo > $AUGMENT_OPTS
 
 echo "export TARGET_PROJECT=$PROJECT_ID"
-echo "export UDMI_REGISTRY_SUFFIX=$UDMI_REGISTRY_SUFFIX"
-echo "export UDMI_ALT_REGISTRY=$UDMI_ALT_REGISTRY"
-echo "export MATRIX_SHARD_COUNT=$MATRIX_SHARD_COUNT"
-echo "export MATRIX_SHARD_INDEX=$MATRIX_SHARD_INDEX"
+echo "export UDMI_REGISTRY_SUFFIX=${UDMI_REGISTRY_SUFFIX:-}"
+echo "export UDMI_ALT_REGISTRY=${UDMI_ALT_REGISTRY:-}"
+echo "export MATRIX_SHARD_COUNT=${MATRIX_SHARD_COUNT:-}"
+echo "export MATRIX_SHARD_INDEX=${MATRIX_SHARD_INDEX:-}"
 
 pubber/bin/build
 validator/bin/build
@@ -62,7 +62,7 @@ while read -u 7 action test_name pubber_opts; do
         false
     fi
 
-    if [[ -z $MATRIX_SHARD_COUNT || $((test_index % MATRIX_SHARD_COUNT)) == $MATRIX_SHARD_INDEX ]]; then
+    if [[ -z ${MATRIX_SHARD_COUNT:-} || $((test_index % MATRIX_SHARD_COUNT)) == $MATRIX_SHARD_INDEX ]]; then
         echo Executing $test_name shard test index $test_marker
     else
         echo Skipping $test_name non-shard test index $test_marker
@@ -132,7 +132,7 @@ sed -E -i $RESULTS_OUT \
     -e 's/(Pipeline type event error: While processing message ).*/\1REDACTED/' \
     -e 's/ 202[-0-9T:]+Z / TIMESTAMP /g'
 
-if [[ -z $TARGET_INDEX && -z $MATRIX_SHARD_COUNT ]]; then
+if [[ -z $TARGET_INDEX && -z ${MATRIX_SHARD_COUNT:-} ]]; then
     bin/test_itemcheck
 else
     echo Checking for correctness of completed tests in $GOLDEN_FILE...