Merge branch 'master' into configclean

.gencode_hash.txt

@@ -3,8 +3,8 @@
 c70a7137ee1a30b35c9e19ca9bb54916247f9eb2e50eab37591cd2ca10b5eddb  gencode/docs/config.html
 22bfb3fd7c0e73dac4cbd830aa7983714917ec6da813f6bf5768ce3aa9786fb2  gencode/docs/config_mapping.html
 754c02f24a929e3b5f95696e961ab5588163c3dbaab9b5411524f01cb1130029  gencode/docs/configuration_endpoint.html
-99598dba319190427a07c54a16e7796c45e1d1a7e7b5e61c35db1f9cf88f0b35  gencode/docs/configuration_execution.html
-e361a8d1649d9e595ae4a220c9241d49a0355b996fed211b85cf6a3ecbe2a723  gencode/docs/configuration_pod.html
+319898a2b8f302e7c15e079bc8435e9ac11f6eda27c34d07600e77ec404b6540  gencode/docs/configuration_execution.html
+3bc5d62ea07f817fc5cf84859724176fc3bff019e6e81ecfab0c35c39486fa5f  gencode/docs/configuration_pod.html
 156ba6bf85ad5a097a7eb7c36331717641184504f3f43db0bdc90d1abee28572  gencode/docs/configuration_pubber.html
 f24d15e549f1b143b50d6011f2f7fd03286e1cd128a3e25591195b25f8efc472  gencode/docs/data_template.html
 cf81ddb4ed910159a9825cf4b6b68042421fdb9e1b49afa4ea4eced40d9cfc53  gencode/docs/events.html
@@ -76,7 +76,7 @@ d3fdb2d8b485c1e61786dfde45cc8be08294339bb31a1b8f02180485789a3ab2  gencode/java/u
 a4fac9e07e930eec7735f67d78eab1e72881987ecc0475c41ce46ae5a9bc5ea7  gencode/java/udmi/schema/GatewayConfig.java
 ce79df57185ee108f6a60efb49c2ea80f249e678d3ccd4d91e702b653571c21b  gencode/java/udmi/schema/GatewayModel.java
 3ac2583e350075d5a07431c4208b87c9f02a5d1e3c651e3a67cf78f5c680ac52  gencode/java/udmi/schema/GatewayState.java
-b3e65d763d581839f8023f2a9b715a487f14360e9b2d718bb2e2733c03bdf7a1  gencode/java/udmi/schema/IotAccess.java
+d93bdad3176b2cc0d1a8664afbc6e86d16378cd4919d7ce4196fe03fd5d79460  gencode/java/udmi/schema/IotAccess.java
 b0d4bff14a65ebddc2dd253c996708a4cce99592e82978f057f32a6d9c7768a1  gencode/java/udmi/schema/Jwt.java
 b539bc4304e1e604e9c6aa0242fb1344de9b18a8c88e12fb5c6f0a2530419501  gencode/java/udmi/schema/Level.java
 f02320b5b493d2475df0ef60cd18e86e4bf91244ebfec6f7ab45a792e0d44f15  gencode/java/udmi/schema/LocalnetConfig.java
@@ -95,7 +95,7 @@ df64e4ddbf543ac70e7c2af9d3fbc20ffe3dff68c6718aa9ceadab7f64d3d171  gencode/java/u
 31dab2c11cfcfdd346b1fe83e92a86abc652de05d85634a22a2c5c3f17c741aa  gencode/java/udmi/schema/MonitoringMetric.java
 4e7fb33911cf11845d494a07502577647cb96e1196cb7b76553190d14b6ed099  gencode/java/udmi/schema/Operation.java
 5e1c5411fae4d7c47391ceb5d19ae864fcd484df75ac6b6db39fd2d12647dec8  gencode/java/udmi/schema/Physical_tag.java
-23955b795d528d7cdb43e16262f065aaa6ebd09b461baab14cc570194c2fc67a  gencode/java/udmi/schema/PodConfiguration.java
+9cc6f769b6da904d59e161f4809d9bee2d0e3dd137f863af2955c468905c23ad  gencode/java/udmi/schema/PodConfiguration.java
 6346d4ff373e01c1f2df420ae5ef970e6af2f347db82ee3574eb3ccb8b7ba648  gencode/java/udmi/schema/PointDiscovery.java
 468c2c95e9dcbaf4ff706fad7f168c14321eaa35b7fb190848ea8bfd86d981d7  gencode/java/udmi/schema/PointPointsetConfig.java
 0d8c1078c554b36546e6267fb356250b44b585a28b9e383df7c1b2e731289634  gencode/java/udmi/schema/PointPointsetEvents.java
@@ -158,7 +158,7 @@ b15c649657d0fb2a102a3e1ff9653c9002715d3b412d44c544a080c37b6a9811  gencode/python
 30b1809e364cb3f7070002bb4a9954b11b25543b099b4bbe450d280001e4de55  gencode/python/udmi/schema/config_udmi.py
 9b2567eba7e0d42fde35b42b9a0c632d45b7f356ebfd12ec83f2e4b478e50331  gencode/python/udmi/schema/configuration_endpoint.py
 ce2ff5646f2707cad62426b54f769eac763308e411ef1925aa54eae1274a8e75  gencode/python/udmi/schema/configuration_execution.py
-87affaacbb01a89b983b6307ef5eaf30b9ea673940ca8cd158068dc468dd0945  gencode/python/udmi/schema/configuration_pod.py
+8f65402ddb43815f4482b1c0c965d44c390dbe1561899b3d9df572458ceb97cc  gencode/python/udmi/schema/configuration_pod.py
 c61ee66daa7e632850bc2705370baa8c057e7a34792c6a29b9e4ca00c0eb195b  gencode/python/udmi/schema/configuration_pod_base.py
 ef61eea743cc2629893b18411636672cdfec0e209e58eb7918b33b43edab5196  gencode/python/udmi/schema/configuration_pod_bridge.py
 bed77c13436a192047a0dcdcaea7c5d7175e99a76c6c40409cce9e232ab5bc12  gencode/python/udmi/schema/configuration_pubber.py

.github/workflows/testing.yml

@@ -259,6 +259,9 @@ jobs:
         run: bin/test_mosquitto
       - name: bin/test_udmis
         run: bin/test_udmis
+      - name: mosquitto debug
+        if: ${{ !cancelled() }}
+        run: sudo cat /etc/mosquitto/dynamic_security.json
       - name: support bundle
         if: ${{ !cancelled() }}
         run: bin/support ${{ github.repository_owner }}_${{ github.job }}_

bin/provision

@@ -1,35 +1,52 @@
 #!/bin/bash -e
 
-if [ $# != 3 ]; then
-    echo $0 site_path project_id device_id
+if [ $# != 2 ]; then
+    echo $0 site_path project_spec
     false
 fi
 
-ROOT_DIR=$(dirname $0)/..
+UDMI_ROOT=$(realpath $(dirname $0)/..)
+source $UDMI_ROOT/etc/shell_common.sh
 
 site_path=$(realpath $1)
-project=$2
-device=$3
-shift 3
+project_spec=$2
+registry_id=$3
+shift 2
 
 config_file=$site_path/cloud_iot_config.json
-device_dir=$site_path/devices/$device 
-
-if [ ! -d $site_path/devices/$device ]; then
-    echo Device directory $device_dir not found.
-    false
+registry_id=$(jq -r .registry_id $config_file)
+key_file=$site_path/reflector/rsa_private.pkcs8
+
+partial=${project_spec#//}
+project_id=${partial#*/}
+project_id=${project_id%/*}
+protocol=${partial%%/*}
+namespace=${project_spec##*/}
+REFLECT_REGISTRY=UDMI-REFLECT
+REGISTRY_REGION=us-central1
+
+[[ -n $namespace ]] && namespace_prefix=${namespace}~
+use_registry=$namespace_prefix$registry_id
+
+echo Provisioning reflector protocol $protocol project $project_id registry $use_registry
+
+if [[ $protocol == mqtt ]]; then
+    echo Adding registry $use_registry
+    ETCD="$UDMI_ROOT/udmis/bin/etcdctl --endpoints localhost:2379"
+    registries=$($ETCD get --print-value-only /registries)
+    updated=${registries},$use_registry
+    updated=${updated#,}
+    $ETCD put /registries "$updated"
+    echo Updated registries to $updated
+
+    PASSWORD_FILE=/etc/mosquitto/test_mosquitto.passwd
+    USERNAME=$project_id/$namespace_prefix$REFLECT_REGISTRY/$use_registry
+    hash=$(sha256sum $key_file)
+    PASSWORD=${hash:0:8}
+    echo Provisioning hash-key $USERNAME $PASSWORD
+    sudo mosquitto_passwd -b ${PASSWORD_FILE} ${USERNAME} ${PASSWORD}
+    sudo systemctl restart mosquitto
+    echo Device password provisioned
+else
+    fail Unable to provision protocol $protocol
 fi
-
-device=AHU-1
-registry=registrar_test
-region=us-central1
-key_gen=RS256
-key_type=rsa-pem
-key_file=$device_dir/rsa_public.pem
-
-$ROOT_DIR/bin/keygen $key_gen $device_dir
-
-gcloud_opts="--device=$device --registry=$registry --region=$region --project=$project"
-yes | gcloud iot devices credentials clear $gcloud_opts
-gcloud iot devices credentials create $gcloud_opts --type=$key_type --path=$key_file
-

bin/run_tests

@@ -54,6 +54,7 @@ case "$1" in
     test_wrap bin/gencode check
     test_wrap pubber/bin/build check
     test_wrap validator/bin/build check
+    test_wrap udmis/bin/build check
     test_wrap bin/test_pylint
     ;;
   schema_tests)

bin/start_mosquitto

@@ -0,0 +1,54 @@
+#!/bin/bash -e
+
+ETC_DIR=/etc/mosquitto
+CONF_FILE=$ETC_DIR/mosquitto.conf
+UDMI_FILE=$ETC_DIR/confg.d/udmi.conf
+PASS_FILE=$ETC_DIR/mosquitto.passwd
+DYN_FILE=$ETC_DIR/dynamic_security.json
+
+AUTH_USER=scrumptious
+AUTH_PASS=aardvark
+echo Configuring MQTT user: $AUTH_USER
+
+sudo sed -i 's/allow_anonymous true/allow_anonymous false/' $CONF_FILE
+sudo sed -i 's/#listener/listener/' $CONF_FILE
+fgrep $PASS_FILE $CONF_FILE || (echo password_file ${PASS_FILE} | sudo tee -a $CONF_FILE)
+
+if ! fgrep -q $DYN_FILE $CONF_FILE; then
+    PLUGIN_FILE=$(whereis -b mosquitto_dynamic_security.so | awk '{print $2}')
+    ls -l "$PLUGIN_FILE"
+    echo Installing dynamic security plugin $PLUGIN_FILE
+    echo plugin $PLUGIN_FILE | sudo tee -a $CONF_FILE
+    echo plugin_opt_config_file $DYN_FILE | sudo tee -a $CONF_FILE
+fi
+
+if [[ ! -f $DYN_FILE ]]; then
+    echo Creating new $DYN_FILE
+    sudo mosquitto_ctrl dynsec init $DYN_FILE $AUTH_USER $AUTH_PASS
+    sudo chgrp mosquitto $DYN_FILE
+    sudo chmod 0660 $DYN_FILE
+fi
+
+sudo chown mosquitto $ETC_DIR
+
+sudo touch $PASS_FILE
+sudo mosquitto_passwd -b ${PASS_FILE} ${AUTH_USER} ${AUTH_PASS}
+
+sudo systemctl restart mosquitto
+
+CTRL_CONNECT="-u $AUTH_USER -P $AUTH_PASS"
+mosquitto_ctrl $CTRL_CONNECT dynsec createRole device
+mosquitto_ctrl $CTRL_CONNECT dynsec addRoleACL device subscribePattern '/#' allow
+mosquitto_ctrl $CTRL_CONNECT dynsec addRoleACL device publishClientSend '/#' allow
+mosquitto_ctrl $CTRL_CONNECT dynsec createRole service
+mosquitto_ctrl $CTRL_CONNECT dynsec addRoleACL service subscribePattern '/#' allow
+mosquitto_ctrl $CTRL_CONNECT dynsec addRoleACL service publishClientSend '/#' allow
+
+clients=$(mosquitto_ctrl -u ${AUTH_USER} -P ${AUTH_PASS} dynsec listClients)
+if [[ $clients =~ ${AUTH_USER} ]]; then
+    echo Found expected client $AUTH_USER
+else
+    fail Improper client: $clients
+fi
+
+echo use: systemctl status mosquitto

bin/test_mosquitto

@@ -8,38 +8,57 @@ if [[ $# != 0 ]]; then
     false
 fi
 
+AUTH_USER=scrumptious
+AUTH_PASS=aardvark
+MQTT_USER=rocket
+MQTT_PASS=monkey
+OTHR_USER=kiwi
+OTHR_PASS=possum
+
+bin/start_mosquitto
+
 site_path=sites/udmi_site_model
+registry_id=site_model
 device_id=AHU-1 # Static device for testing
 serial_no=$RANDOM
 
 site_config=$site_path/cloud_iot_config.json
 cloud_region=$(jq -r .cloud_region $site_config)
 registry_id=$(jq -r .registry_id $site_config)
+mqtt_id=$registry_id/testing
+othr_id=$registry_id/other
 
-mkdir -p out
-PASSWORD_FILE=/etc/mosquitto/test_mosquitto.passwd
-USERNAME=scrumptus
-PASSWORD=aardvark
-
-echo password_file ${PASSWORD_FILE} | sudo tee /etc/mosquitto/conf.d/udmi_test.conf
-sudo sed -i 's/allow_anonymous true/allow_anonymous false/' /etc/mosquitto/mosquitto.conf
-sudo sed -i 's/#listener/listener/' /etc/mosquitto/mosquitto.conf
-
-sudo touch ${PASSWORD_FILE}
-sudo mosquitto_passwd -b ${PASSWORD_FILE} ${USERNAME} ${PASSWORD}
-sudo chmod 666 ${PASSWORD_FILE}
+mosquitto_ctrl -u $AUTH_USER -P $AUTH_PASS dynsec deleteClient $MQTT_USER || true
+mosquitto_ctrl -u $AUTH_USER -P $AUTH_PASS dynsec createClient $MQTT_USER -p $MQTT_PASS -c $mqtt_id
+mosquitto_ctrl -u $AUTH_USER -P $AUTH_PASS dynsec addClientRole $MQTT_USER device
+mosquitto_ctrl -u $AUTH_USER -P $AUTH_PASS dynsec deleteClient $OTHR_USER || true
+mosquitto_ctrl -u $AUTH_USER -P $AUTH_PASS dynsec createClient $OTHR_USER -p $OTHR_PASS -c $othr_id
+mosquitto_ctrl -u $AUTH_USER -P $AUTH_PASS dynsec addClientRole $OTHR_USER service
 
-sudo systemctl restart mosquitto
-systemctl status mosquitto
 killall mosquitto_sub || true
 
-echo Subscribing to mqtt/test in the background...
-mosquitto_sub -F "%t %p" -t \# -u ${USERNAME} -P ${PASSWORD} > out/mosquitto.sub &
+topic=/mqtt/test
+
+echo Subscribing to all topics in the background...
+mosquitto_sub -F "%t %p" -t '#' -u ${MQTT_USER} -P ${MQTT_PASS} -i $mqtt_id > out/mosquitto.sub &
 
 sleep 1
 
 echo Publishing test message
-mosquitto_pub -t mqtt/test -u ${USERNAME} -P ${PASSWORD} -m "Hello MQTT"
+mosquitto_pub -t $topic -u ${OTHR_USER} -P ${OTHR_PASS} -i $othr_id -m "Hello Racket"
+
+sleep 1
+
+echo Checking received message
+fgrep Racket out/mosquitto.sub
+
+DEV_USER=nobody
+hash_pass=$(sha256sum < $site_path/devices/$device_id/rsa_private.pkcs8)
+dev_pass=${hash_pass:0:8}
+dev_id=$registry_id/$device_id
+mosquitto_ctrl -u $AUTH_USER -P $AUTH_PASS dynsec deleteClient $DEV_USER || true
+mosquitto_ctrl -u $AUTH_USER -P $AUTH_PASS dynsec createClient $DEV_USER -p $dev_pass -c $dev_id
+mosquitto_ctrl -u $AUTH_USER -P $AUTH_PASS dynsec addClientRole $DEV_USER device
 
 sleep 1
 
@@ -48,15 +67,15 @@ cat <<EOF > /tmp/pubber_config.json
     "endpoint": {
       "protocol": "mqtt",
       "transport": "tcp",
-      "client_id": "$registry_id/$device_id",
+      "client_id": "$dev_id",
       "hostname": "127.0.0.1",
-      "msg_prefix": "/$serial_no/$device_id",
+      "msg_prefix": "/$registry_id/$device_id",
       "config_sync_sec": -1,
       "port": 1883,
       "auth_provider": {
         "basic": {
-          "username": "$USERNAME",
-          "password": "$PASSWORD"
+          "username": "$DEV_USER",
+          "password": "$dev_pass"
         }
       }
     },
@@ -73,19 +92,26 @@ pubber/bin/build
 # Clean out the persistant data store to ensure a clean state each time.
 rm -rf sites/udmi_site_model/out/devices/$device_id/persistent_data.json
 
-echo Publishing empty config to /$serial_no/$device_id/config
-mosquitto_pub -r -t /$serial_no/$device_id/config -u ${USERNAME} -P ${PASSWORD} -m "{}"
+echo Publishing empty config to /$registry_id/$device_id/config
+mosquitto_pub -r -t /$registry_id/$device_id/config -u ${MQTT_USER} -P ${MQTT_PASS} -i $mqtt_id -m "{}"
 
 echo Running pubber for 10s...
 timeout 10s pubber/bin/run /tmp/pubber_config.json || true
 
+echo Killing background clients
+killall mosquitto_sub || true
+
 echo Received client logs:
 cat out/mosquitto.sub
 
 received_no=$(fgrep operational out/mosquitto.sub | head -n 1 | sed -E 's/^[^{]+//' | jq -r .system.serial_no)
 received_topic=$(fgrep operational out/mosquitto.sub | head -n 1 | awk '{ print $1 }')
+if [[ -z $received_topic ]]; then
+    echo No matching receive message found.
+    false
+fi
 
-expected_topic=/$serial_no/$device_id/state
+expected_topic=/$registry_id/$device_id/state
 if [[ $received_topic != $expected_topic ]]; then
     echo Unexpected received topic $received_topic != $expected_topic
     false

bin/test_udmis

@@ -4,7 +4,10 @@ ROOT=$(dirname $0)/..
 cd $ROOT
 
 echo Checking for properly configured mosquitto...
-fgrep -q scrumptus /etc/mosquitto/test_mosquitto.passwd || (echo mosquitto not configured; false)
+sudo fgrep -q scrumptious /etc/mosquitto/dynamic_security.json || (echo mosquitto dynamic security not configured; false)
+
+echo Checking that mqtt broker is running
+sudo netstat -nlpa | fgrep :1883
 
 # This is the local address of the previously configured MQTT broker
 export MQTT_TEST_BROKER=tcp://localhost:1883

bin/toolrun

@@ -13,5 +13,6 @@ $UDMI_ROOT/validator/bin/build
 
 JAVA_CLASS=com.google.daq.mqtt.util.Dispatcher
 
+echo udmi version $UDMI_TOOLS
 echo java -cp $UDMI_JAR $JAVA_CLASS $util_name "$@"
 java -cp $UDMI_JAR $JAVA_CLASS $util_name "$@" 2>&1 | tee $OUT_DIR/$util_name.log

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -207,6 +207,14 @@ public static Runnable ignoreValue(Object ignored) {
     };
   }
 
+  public static boolean isNotEmpty(String s) {
+    return !ofNullable(s).map(String::isEmpty).orElse(true);
+  }
+
+  public static boolean isNullOrNotEmpty(String value) {
+    return !ofNullable(value).map(String::isEmpty).orElse(false);
+  }
+
   public static Date toDate(Instant lastSeen) {
     return ifNotNullGet(lastSeen, Date::from);
   }

etc/shell_common.sh

@@ -23,7 +23,7 @@ function usage {
     echo usage: $0 $*
     false
 }
-    
+
 UDMI_JAR=$UDMI_ROOT/validator/build/libs/validator-1.0-SNAPSHOT-all.jar
 
 udmi_version=$(cd $UDMI_ROOT; git describe --dirty --always)