Merge branch 'master' into documentation

.gencode_hash.txt

@@ -14,7 +14,7 @@ c3bae844432d172033bc416e623fecee7608efd01d916b7eaee96252932c552a  gencode/docs/e
 eed6ae125d94cf1986de96c210b0937b9c7b199724839f43fce3b357b057f8be  gencode/docs/events_system.html
 e3683cb4561b6dda5703cb659fd6a8f12242725de389709a12eb45f146cbb295  gencode/docs/events_validation.html
 ecafb56114c92e873329f184471ae39c002a12f1495bd75b24df22846ebb35a3  gencode/docs/metadata.html
-2171042c880238399b60ff640b00d9ace46d313ef78cded9e7fe4339281a419c  gencode/docs/monitoring.html
+b110edfb73182782c7539adef417a970959432c92d4c9f4a0bf7e328c0e8427d  gencode/docs/monitoring.html
 86b62cb8e66ffb97d717fb8f1bf6a4f7d0a759cd15a8d040d5cc3451666f7c72  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
 16512c8cc60e4f1a0e01555f3fb6c8a49d3f2b609275a8dc9e62f38fa7b404ca  gencode/docs/query_cloud.html
@@ -61,7 +61,7 @@ bb4509f13e30ebc9964e213eabf3e36f0c7d73a54a052fea281a484ad84c3596  gencode/java/u
 fbe412e8c874139d828021a034cf6a6a6e1d35d01ccffe98ae35ac48187de753  gencode/java/udmi/schema/DiscoveryState.java
 1bb0e11e80df75dcedcbea6caf0625412e43ef483ad960daadcf496afa16f522  gencode/java/udmi/schema/EndpointConfiguration.java
 dc25e685886e11a741418be9191a478e13c0244647b5a0cac65d6c1e55055578  gencode/java/udmi/schema/Entry.java
-0f8d16a7bc8a243c02b6329570f4a792421138c7e70732073cef24fd43179d12  gencode/java/udmi/schema/Envelope.java
+c92438f417b28b0b1856751f55344aaebfdfe14802d90e2b4d1b4e987462709b  gencode/java/udmi/schema/Envelope.java
 9c2e8b3984e873cac3a801d94ef1ef3ecd32f64402a51a0f3292127b99b372d9  gencode/java/udmi/schema/Events.java
 684aa5160bdb62e94b414e2426d65c685b179609e1e6d63220f395d85623f045  gencode/java/udmi/schema/ExecutionConfiguration.java
 25815abe54d4671858f55bd5d911292bd50f122b80878064b014afcfcbd86ea7  gencode/java/udmi/schema/FamilyDiscovery.java

.github/workflows/testing.yml

@@ -250,7 +250,10 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
       - name: base setup
-        run: bin/run_tests install_dependencies
+        run: |
+          bin/setup_base
+          bin/clone_model
+          bin/registrar sites/udmi_site_model
       - name: bin/start_etcd
         run: bin/start_etcd
       - name: bin/test_etcd

bin/add_site

@@ -0,0 +1,31 @@
+#!/bin/bash -eu
+
+UDMI_ROOT=$(dirname $0)/..
+cd $UDMI_ROOT
+
+source $UDMI_ROOT/etc/shell_common.sh
+
+if [[ $# != 1 ]]; then
+    echo Usage: $0 site_model
+    false
+fi
+
+site_path=$1
+shift
+
+source $UDMI_ROOT/etc/mosquitto_ctrl.sh
+
+site_config=$site_path/cloud_iot_config.json
+registry_id=$(jq -r .registry_id $site_config)
+
+hash_pass=$(sha256sum < $site_path/reflector/rsa_private.pkcs8)
+SERV_PASS=${hash_pass:0:8}
+SERV_USER=/r/UDMI-REFLECT/d/$registry_id
+SERV_ID=$SERV_USER
+
+$MOSQUITTO_CTRL deleteClient $SERV_USER
+$MOSQUITTO_CTRL createClient $SERV_USER -p $SERV_PASS -c $SERV_ID
+$MOSQUITTO_CTRL addClientRole $SERV_USER service
+
+echo Registered sites:
+$MOSQUITTO_CTRL listClients | fgrep /r/UDMI-REFLECT/d/

bin/pubber

@@ -82,6 +82,7 @@ fi
 # Format of cmdline project_id is {//PROVIDER/}PROJECT[@BROKER] (optional PROVIDER and BROKER)
 # e.g. bos-testing-dev or //gcp/bos-testing-dev or [email protected]
 project_target=${project_id##*/}
+udmi_prefix=
 
 if [[ -z $project_id ]]; then
     true using config file based configuration

bin/start_influx

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+# https://lucassardois.medium.com/handling-iot-data-with-mqtt-telegraf-influxdb-and-grafana-5a431480217
+
+docker run --name influxdb \
+       -v $PWD/influxdb-data:/var/lib/influxdb \
+       influxdb
+
+docker run --rm --name telegraf \
+       -v $PWD/telegraf.conf:/etc/telegraf/telegraf.conf:ro \
+       telegraf
+
+docker run -d -p 3000:3000 --name grafana \
+       -v $PWD/grafana-data:/var/lib/grafana \
+       grafana/grafana

bin/start_mosquitto

@@ -3,17 +3,12 @@
 UDMI_ROOT=$(dirname $0)/..
 source $UDMI_ROOT/etc/shell_common.sh
 
+source $UDMI_ROOT/etc/mosquitto_ctrl.sh
+
 GROUP=mosquitto
-ETC_DIR=/etc/mosquitto
 UDMI_FILE=$ETC_DIR/conf.d/udmi.conf
 PASS_FILE=$ETC_DIR/mosquitto.passwd
 DYN_FILE=$ETC_DIR/dynamic_security.json
-CERT_DIR=$ETC_DIR/certs
-CA_CERT=$CERT_DIR/ca.crt
-
-AUTH_USER=scrumptious
-AUTH_PASS=aardvark
-echo Configuring MQTT user: $AUTH_USER
 
 if [[ ! -f $UDMI_FILE ]]; then
     echo Creating new $UDMI_FILE from template...
@@ -27,6 +22,7 @@ fi
 
 if [[ ! -f $DYN_FILE ]]; then
     echo Creating new $DYN_FILE
+    echo Configuring MQTT user: $AUTH_USER
     sudo mosquitto_ctrl dynsec init $DYN_FILE $AUTH_USER $AUTH_PASS
     sudo chgrp $GROUP $DYN_FILE
     sudo chmod 0660 $DYN_FILE
@@ -42,9 +38,6 @@ fi
 
 sudo systemctl restart mosquitto
 
-CONNECT_OPTS="-p 8883 -u $AUTH_USER -P $AUTH_PASS --cafile $CA_CERT --cert $CERT_DIR/rsa_private.crt --key $CERT_DIR/rsa_private.pem"
-MOSQUITTO_CTRL="sudo mosquitto_ctrl $CONNECT_OPTS dynsec"
-
 $MOSQUITTO_CTRL createRole device
 $MOSQUITTO_CTRL addRoleACL device subscribePattern '/r/+/d/+/#' allow
 $MOSQUITTO_CTRL addRoleACL device publishClientSend '/r/+/d/+/#' allow

bin/start_udmis

@@ -0,0 +1,5 @@
+#!/bin/bash -e
+
+rm -f /tmp/pod_ready.txt
+
+sudo ETCD_CLUSTER=localhost SSL_SECRETS_DIR=/etc/mosquitto/certs udmis/bin/run udmis/etc/local_pod.json

bin/test_mosquitto

@@ -10,12 +10,8 @@ if [[ $# != 0 ]]; then
     false
 fi
 
-ETC_DIR=/etc/mosquitto
-CERT_DIR=$ETC_DIR/certs
-CA_CERT=$CERT_DIR/ca.crt
+source $UDMI_ROOT/etc/mosquitto_ctrl.sh
 
-AUTH_USER=scrumptious
-AUTH_PASS=aardvark
 SERV_USER=rocket
 SERV_PASS=monkey
 CLNT_USER=kiwi
@@ -24,26 +20,24 @@ CLNT_PASS=possum
 bin/start_mosquitto
 
 site_path=sites/udmi_site_model
-registry_id=site_model
 device_id=AHU-1 # Static device for testing
 serial_no=$RANDOM
 CLIENT_DIR=$site_path/devices/$device_id
+DEV_CONF=$site_path/devices/$device_id/out/generated_config.json
 
 site_config=$site_path/cloud_iot_config.json
 cloud_region=$(jq -r .cloud_region $site_config)
 registry_id=$(jq -r .registry_id $site_config)
-SERV_ID=$registry_id/server
+
 CLNT_ID=$registry_id/client
+SERV_ID=$registry_id/server
 
 sudo chmod a+r $CERT_DIR/rsa_private.*
 bin/keygen CERT $CLIENT_DIR
 
-CTRL_OPTS="-u $AUTH_USER -P $AUTH_PASS --cafile $CA_CERT --cert $CERT_DIR/rsa_private.crt --key $CERT_DIR/rsa_private.pem"
 SERVER_OPTS="-i $SERV_ID -u $SERV_USER -P $SERV_PASS --cafile $CA_CERT --cert $CERT_DIR/rsa_private.crt --key $CERT_DIR/rsa_private.pem"
 CLIENT_OPTS="-i $CLNT_ID -u $CLNT_USER -P $CLNT_PASS --cafile $CA_CERT --cert $CLIENT_DIR/rsa_private.crt --key $CLIENT_DIR/rsa_private.pem"
 
-MOSQUITTO_CTRL="sudo mosquitto_ctrl $CTRL_OPTS dynsec"
-
 $MOSQUITTO_CTRL deleteClient $SERV_USER
 $MOSQUITTO_CTRL createClient $SERV_USER -p $SERV_PASS # No client_id to allow multiple backend connections.
 $MOSQUITTO_CTRL addClientRole $SERV_USER service
@@ -112,8 +106,8 @@ pubber/bin/build
 # Clean out the persistant data store to ensure a clean state each time.
 rm -rf sites/udmi_site_model/out/devices/$device_id/persistent_data.json
 
-echo Publishing empty config to /r/$registry_id/d/$device_id/config
-mosquitto_pub $CLIENT_OPTS -r -t /r/$registry_id/d/$device_id/config -m "{}"
+echo Publishing config to /r/$registry_id/d/$device_id/config
+mosquitto_pub $CLIENT_OPTS -r -t /r/$registry_id/d/$device_id/config -f $DEV_CONF
 
 echo Running pubber smoke check
 # TODO: Replace this with proper smoke check when backend is working.
@@ -143,5 +137,5 @@ expected_topic=/r/$registry_id/d/$device_id/state
 
 [[ $received_no != $serial_no ]] && fail Mismatched/missing serial no: $received_no != $serial_no
 
-echo export MQTT_TEST_BROKER=tcp://127.0.0.1:1883
+echo export MQTT_TEST_BROKER=tcp://localhost:8883
 echo Done with mosquitto pubber check.

common/src/main/java/com/google/udmi/util/CertManager.java

@@ -26,7 +26,6 @@
 import org.bouncycastle.openssl.PEMParser;
 import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
 import org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder;
-import udmi.schema.EndpointConfiguration;
 import udmi.schema.EndpointConfiguration.Transport;
 
 /**
@@ -43,6 +42,7 @@ public class CertManager {
   private static final String CA_CERT_ALIAS = "ca-certificate";
   private static final String CLIENT_CERT_ALIAS = "certificate";
   private static final String PRIVATE_KEY_ALIAS = "private-key";
+  public static final String CA_CERT_FILE = "ca.crt";
   private final File caCrtFile;
   private final File keyFile;
   private final File crtFile;
@@ -56,10 +56,10 @@ public class CertManager {
   /**
    * Create a new cert manager for the given site model and configuration.
    */
-  public CertManager(File caCrtFile, File clientDir, EndpointConfiguration endpoint,
+  public CertManager(File caCrtFile, File clientDir, Transport transport,
       String passString, Consumer<String> logging) {
     this.caCrtFile = caCrtFile;
-    isSsl = Transport.SSL.equals(endpoint.transport);
+    isSsl = Transport.SSL.equals(transport);
 
     if (isSsl) {
       crtFile = new File(clientDir, "rsa_private.crt");

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -260,6 +260,11 @@ private static JsonGenerator getPrettyPrinterGenerator(OutputStream outputStream
   }
 
   public static Map<String, Object> getSubMap(Map<String, Object> input, String field) {
+    //noinspection unchecked
+    return (Map<String, Object>) input.get(field);
+  }
+
+  public static Map<String, Object> getSubMapNull(Map<String, Object> input, String field) {
     //noinspection unchecked
     return ifNotNullGet(input, map -> (Map<String, Object>) map.get(field));
   }

common/src/main/java/com/google/udmi/util/JsonUtil.java

@@ -18,8 +18,10 @@
 import java.nio.file.Files;
 import java.time.Instant;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.function.Function;
 
 /**
  * Collection of utilities for working with json things.
@@ -28,6 +30,7 @@ public abstract class JsonUtil {
 
   public static final String JSON_EXT = "json";
   public static final String JSON_SUFFIX = ".json";
+  public static final String JSON_OBJECT_LEADER = "{";
   private static final ObjectMapper STRICT_MAPPER = new ObjectMapper()
       .enable(Feature.ALLOW_COMMENTS)
       .enable(SerializationFeature.INDENT_OUTPUT)
@@ -40,7 +43,6 @@ public abstract class JsonUtil {
       .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
   public static final ObjectMapper TERSE_MAPPER = OBJECT_MAPPER.copy()
       .disable(SerializationFeature.INDENT_OUTPUT);
-  public static final String JSON_OBJECT_LEADER = "{";
   private static final String JSON_STRING_LEADER = "\"";
 
   /**
@@ -130,11 +132,6 @@ public static <T> T fromStringStrict(Class<T> targetClass, String messageString)
     }
   }
 
-  @SuppressWarnings("unchecked")
-  public static Map<String, Object> getAsMap(Map<String, Object> map, String key) {
-    return (Map<String, Object>) map.get(key);
-  }
-
   /**
    * Get a Date object parsed from a string representation.
    *
@@ -418,19 +415,6 @@ public static Object toObject(String message) {
     return unquoteJson(message);
   }
 
-  /**
-   * Extract the underlying string representation from a JSON encoded message.
-   */
-  public static String unquoteJson(String message) {
-    if (message == null || message.isEmpty() || message.startsWith(JSON_OBJECT_LEADER)) {
-      return message;
-    }
-    if (message.startsWith(JSON_STRING_LEADER)) {
-      return message.substring(1, message.length() - 1);
-    }
-    throw new RuntimeException("Unrecognized JSON start encoding: " + message.charAt(0));
-  }
-
   /**
    * Convert the pojo to a mapped representation of strings only.
    *
@@ -455,6 +439,19 @@ public static Map<String, String> toStringMap(String message) {
     return map;
   }
 
+  /**
+   * Extract the underlying string representation from a JSON encoded message.
+   */
+  public static String unquoteJson(String message) {
+    if (message == null || message.isEmpty() || message.startsWith(JSON_OBJECT_LEADER)) {
+      return message;
+    }
+    if (message.startsWith(JSON_STRING_LEADER)) {
+      return message.substring(1, message.length() - 1);
+    }
+    throw new RuntimeException("Unrecognized JSON start encoding: " + message.charAt(0));
+  }
+
   /**
    * Write json representation to a file.
    *

etc/mosquitto_ctrl.sh

@@ -0,0 +1,11 @@
+# Common setup for running mosquitto_ctrl
+
+ETC_DIR=/etc/mosquitto
+CERT_DIR=$ETC_DIR/certs
+CA_CERT=$CERT_DIR/ca.crt
+
+AUTH_USER=scrumptious
+AUTH_PASS=aardvark
+
+CTRL_OPTS="-u $AUTH_USER -P $AUTH_PASS --cafile $CA_CERT --cert $CERT_DIR/rsa_private.crt --key $CERT_DIR/rsa_private.pem"
+MOSQUITTO_CTRL="sudo mosquitto_ctrl $CTRL_OPTS dynsec"

pubber/src/main/java/daq/pubber/Pubber.java

@@ -771,7 +771,8 @@ private void initializeMqtt() {
     checkState(deviceTarget == null, "mqttPublisher already defined");
     String keyPassword = sha256(ensureKeyBytes()).substring(0, 8);
     CertManager certManager = new CertManager(new File(siteModel.getReflectorDir(), CA_CRT),
-        siteModel.getDeviceDir(config.deviceId), config.endpoint, keyPassword, this::info);
+        siteModel.getDeviceDir(config.deviceId), config.endpoint.transport, keyPassword,
+        this::info);
     deviceTarget = new MqttDevice(config, this::publisherException, certManager);
     registerMessageHandlers();
     publishDirtyState();

schema/envelope.json

@@ -67,6 +67,7 @@
     "subType": {
       "enum": [
         "invalid",
+        "reflect",
         "events",
         "errors",
         "commands",

tests/schemas/envelope/errors1.out

@@ -2,5 +2,5 @@
   /deviceId: ECMA 262 regex "^[A-Z]{2,6}-[1-9][0-9]*$" does not match input string "fcu-1"
   /deviceNumId: ECMA 262 regex "^[0-9]+$" does not match input string "921302198324X"
   /deviceRegistryId: ECMA 262 regex "^[a-zA-Z][-a-zA-Z0-9._+~%]*[a-zA-Z0-9]$" does not match input string "test/registry"
-  /subType: instance value (5) not found in enum (possible values: ["invalid","events","errors","commands","config","state","query","reply","model"])
+  /subType: instance value (5) not found in enum (possible values: ["invalid","reflect","events","errors","commands","config","state","query","reply","model"])
   object has missing required properties (["projectId","subFolder"])

udmis/bin/run

@@ -40,7 +40,6 @@ elif [[ -n $gce_project ]]; then
     export GCP_PROJECT=$gce_project
 else
     echo No GCP_PROJECT, gcloud, or GCE project defined.
-    false
 fi
 
 export UDMI_PREFIX=

udmis/etc/dev_pod.json

@@ -1,3 +1,8 @@
 {
-  "include": "prod_pod.json"
+  "include": "prod_pod.json",
+  "flows": {
+    "distributor": {
+      "enabled": "false"
+    }
+  }
 }