Fix local mqtt execution of some itemized tests #907

.gencode_hash.txt

@@ -2,10 +2,10 @@
 158e714e54b2a6d0b90bfca00d178369b04508dc4fa84a979690a813ca2dd724  gencode/docs/commands_mapping.html
 c70a7137ee1a30b35c9e19ca9bb54916247f9eb2e50eab37591cd2ca10b5eddb  gencode/docs/config.html
 22bfb3fd7c0e73dac4cbd830aa7983714917ec6da813f6bf5768ce3aa9786fb2  gencode/docs/config_mapping.html
-754c02f24a929e3b5f95696e961ab5588163c3dbaab9b5411524f01cb1130029  gencode/docs/configuration_endpoint.html
-319898a2b8f302e7c15e079bc8435e9ac11f6eda27c34d07600e77ec404b6540  gencode/docs/configuration_execution.html
-3bc5d62ea07f817fc5cf84859724176fc3bff019e6e81ecfab0c35c39486fa5f  gencode/docs/configuration_pod.html
-156ba6bf85ad5a097a7eb7c36331717641184504f3f43db0bdc90d1abee28572  gencode/docs/configuration_pubber.html
+1ce30b7c554a0be09a9124707cb5602dce78753304cca17c06268ecb3337da51  gencode/docs/configuration_endpoint.html
+0d78506590fa4b3371a388e168786c91574a993ac47cb9e254e70136d68416ff  gencode/docs/configuration_execution.html
+6a097f1f87ab7b9a82e4d3aa6f6cedf69499a95742863a7f53bb5fdf53fd6ff0  gencode/docs/configuration_pod.html
+647229fdf5967f27019766244005e863fb52155d697235edeba9a84e5b6e6607  gencode/docs/configuration_pubber.html
 f24d15e549f1b143b50d6011f2f7fd03286e1cd128a3e25591195b25f8efc472  gencode/docs/data_template.html
 0c6efea81a5f6c2b4fa2b2b7c42c1bc8cbcaf56d90005d7944ab18a9ce31dfa6  gencode/docs/events.html
 402b4282634bbd48d940adf804eb26b507140cc0d8d6ee2a715f01ecece680b2  gencode/docs/events_discovery.html
@@ -15,7 +15,7 @@ eed6ae125d94cf1986de96c210b0937b9c7b199724839f43fce3b357b057f8be  gencode/docs/e
 e3683cb4561b6dda5703cb659fd6a8f12242725de389709a12eb45f146cbb295  gencode/docs/events_validation.html
 015d928b1c87291b56c274433bc8c316c8fec67cb490101e403a33adc0edd8d7  gencode/docs/metadata.html
 b110edfb73182782c7539adef417a970959432c92d4c9f4a0bf7e328c0e8427d  gencode/docs/monitoring.html
-86b62cb8e66ffb97d717fb8f1bf6a4f7d0a759cd15a8d040d5cc3451666f7c72  gencode/docs/persistent_device.html
+c8a40993b2810dffb9119469f83592461d817c7148072203f157b89922187e7c  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
 16512c8cc60e4f1a0e01555f3fb6c8a49d3f2b609275a8dc9e62f38fa7b404ca  gencode/docs/query_cloud.html
 6da0305491ae5cf843cec827c3ace97ba2297dca37886f83639d6128a5b00f1a  gencode/docs/readme.md
@@ -59,7 +59,7 @@ e262066714d6028f987f92206bc0aec7c9d306ddf4cfdcf901fda66b780054d4  gencode/java/u
 12a4680fc1353be51c1049e4c8774c704376a893263edb31988e9cf46b38a001  gencode/java/udmi/schema/DiscoveryEvents.java
 bb4509f13e30ebc9964e213eabf3e36f0c7d73a54a052fea281a484ad84c3596  gencode/java/udmi/schema/DiscoveryModel.java
 fbe412e8c874139d828021a034cf6a6a6e1d35d01ccffe98ae35ac48187de753  gencode/java/udmi/schema/DiscoveryState.java
-1bb0e11e80df75dcedcbea6caf0625412e43ef483ad960daadcf496afa16f522  gencode/java/udmi/schema/EndpointConfiguration.java
+7fc7f578b73bfb563d12c2d1d9ab5801664db211ff84c0d011fffc69288fabf0  gencode/java/udmi/schema/EndpointConfiguration.java
 dc25e685886e11a741418be9191a478e13c0244647b5a0cac65d6c1e55055578  gencode/java/udmi/schema/Entry.java
 c92438f417b28b0b1856751f55344aaebfdfe14802d90e2b4d1b4e987462709b  gencode/java/udmi/schema/Envelope.java
 9c2e8b3984e873cac3a801d94ef1ef3ecd32f64402a51a0f3292127b99b372d9  gencode/java/udmi/schema/Events.java
@@ -156,7 +156,7 @@ ac3facbd96f7cb2f7e387e7497d6a36af379a2687329571f250c5670f9933244  gencode/python
 7da3bdb37f338260d5f3829fa5fcbb9bbf9f146b514a68319c314a96c6b8ac12  gencode/python/udmi/schema/config_system.py
 b15c649657d0fb2a102a3e1ff9653c9002715d3b412d44c544a080c37b6a9811  gencode/python/udmi/schema/config_system_testing.py
 9bc72e152352ca5a966f203211fb3e87672e2e6975ba266c332e113f20ce0e3d  gencode/python/udmi/schema/config_udmi.py
-9b2567eba7e0d42fde35b42b9a0c632d45b7f356ebfd12ec83f2e4b478e50331  gencode/python/udmi/schema/configuration_endpoint.py
+cd72d3ebc324e2db939e4161a9df246adaeb2dabecc8e9c4305ebbc5984bf2a4  gencode/python/udmi/schema/configuration_endpoint.py
 ce2ff5646f2707cad62426b54f769eac763308e411ef1925aa54eae1274a8e75  gencode/python/udmi/schema/configuration_execution.py
 8f65402ddb43815f4482b1c0c965d44c390dbe1561899b3d9df572458ceb97cc  gencode/python/udmi/schema/configuration_pod.py
 c61ee66daa7e632850bc2705370baa8c057e7a34792c6a29b9e4ca00c0eb195b  gencode/python/udmi/schema/configuration_pod_base.py

.github/workflows/testing.yml

@@ -307,6 +307,10 @@ jobs:
           java-version: '17'
       - name: base setup
         run: bin/run_tests install_dependencies
+      - name: local setup
+        run: bin/start_local $TARGET_PROJECT
+      - name: regclean
+        run: bin/test_regclean $TARGET_PROJECT
       - name: bin/test_proxy
         if: ${{ env.MQTT_PROXY_HOST != '' }}
         run: bin/test_proxy $TARGET_PROJECT $MQTT_PROXY_HOST

bin/keygen

@@ -40,8 +40,10 @@ if [[ $type == CA ]]; then
 elif [ $type == CERT ]; then
     echo Generating self-signed cert from CA defined in $(realpath $CA_DIR)
     echo Generating cert for device keys in $PWD
-    openssl req -new -key rsa_private.pem -out rsa_private.csr -subj /CN=$SUBJ
-    openssl x509 -req -in rsa_private.csr -CA $CA_CRT -CAkey $CA_KEY -CAcreateserial -out rsa_private.crt $x509opts
+    private_key=$(ls *.pkcs8)
+    key_prefix=${private_key%%_*}
+    openssl req -new -key ${key_prefix}_private.pem -out ${key_prefix}_private.csr -subj /CN=$SUBJ
+    openssl x509 -req -in ${key_prefix}_private.csr -CA $CA_CRT -CAkey $CA_KEY -CAcreateserial -out ${key_prefix}_private.crt $x509opts
 elif [ $type == RS256 ]; then
     openssl genrsa -out rsa_private.pem 2048
     openssl rsa -in rsa_private.pem -pubout -out rsa_public.pem

bin/pubber

@@ -111,7 +111,7 @@ if [[ ! -f $ENDPOINT_TMP ]]; then
     registry_actual=${udmi_prefix}${registry_id}${registry_suffix}
     client_id=\"projects/$project_target/locations/$cloud_region/registries/$registry_actual/devices/$device_id\"
 fi
-msg_prefix=null
+topic_prefix=null
 auth_provider=null
 transport=null
 
@@ -121,12 +121,19 @@ elif [[ $iot_provider == gbos ]]; then
     broker_hostname=mqtt.bos.goog
 elif [[ $iot_provider == mqtt ]]; then
     broker_hostname=$project_target
-    client_id=\"/r/$registry_actual/d/$device_id\"
-    hash_pass=$(sha256sum < $site_model/devices/$device_id/rsa_private.pkcs8)
+    gateway_id=$(jq -r .gateway.gateway_id $site_model/devices/$device_id/metadata.json) || true
+    echo Attached to gateway $gateway_id
+    target_id=$device_id
+    [[ $gateway_id == null ]] || target_id=$gateway_id
+    echo Target is $target_id
+    client_id=\"/r/$registry_actual/d/$target_id\"
+    key_file=$site_model/devices/$target_id/*_private.pkcs8
+    echo Extracting hashed password from key file $key_file
+    hash_pass=$(sha256sum < $key_file)
     client_pass=${hash_pass:0:8}
-    msg_prefix=$client_id
     auth_provider="{ \"basic\": { \"username\": $client_id, \"password\": \"$client_pass\" }}"
     transport=\"ssl\"
+    topic_prefix=\"/r/$registry_actual/d/$device_id\"
 elif [[ -z $broker_hostname ]]; then
     broker_hostname=mqtt.googleapis.com
 fi
@@ -137,7 +144,7 @@ if [[ ! -f $ENDPOINT_TMP ]]; then
     "protocol": "mqtt",
     "transport": $transport,
     "client_id": $client_id,
-    "msg_prefix": $msg_prefix,
+    "topic_prefix": $topic_prefix,
     "auth_provider": $auth_provider,
     "hostname": "$broker_hostname"
 }

bin/start_udmis

@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash -e
 
 UDMI_ROOT=$(dirname $0)/..
 source $UDMI_ROOT/etc/shell_common.sh

bin/test_itemized

@@ -38,6 +38,11 @@ echo "export MATRIX_SHARD_INDEX=${MATRIX_SHARD_INDEX:-}"
 pubber/bin/build
 validator/bin/build
 
+for device_id in AHU-1 GAT-123; do
+    echo bin/keygen CERT $SITE_PATH/devices/$device_id
+    bin/keygen CERT $SITE_PATH/devices/$device_id || true
+done
+
 test_index=1
 while read -u 7 action test_name pubber_opts; do
     test_marker=$(printf %02d $test_index)

bin/test_mosquitto

@@ -67,7 +67,7 @@ cat <<EOF > /tmp/pubber_config.json
       "transport": "ssl",
       "client_id": "$dev_id",
       "hostname": "127.0.0.1",
-      "msg_prefix": "/r/$registry_id/d/$device_id",
+      "topic_prefix": "/r/$registry_id/d/$device_id",
       "config_sync_sec": -1,
       "port": 8883,
       "auth_provider": {

bin/test_redirect

@@ -66,7 +66,7 @@ cat <<EOF > out/endpoint.json
 {
   "protocol": "mqtt",
   "client_id": "projects/$project_id/locations/$cloud_region/registries/reconfigure/devices/$device_id",
-  "msg_prefix": "/devices/$device_id",
+  "topic_prefix": "/devices/$device_id",
   "hostname": "mqtt.googleapis.com",
   "auth_provider": {
     "jwt": {

bin/test_regclean

@@ -86,7 +86,7 @@ else
 fi
 
 client_id=\"projects/$project_id/locations/$cloud_region/registries/$registry_prefix$registry_id$registry_suffix/devices/$device_id\"
-msg_prefix=null
+topic_prefix=null
 auth_provider=null
 transport=null
 
@@ -99,7 +99,7 @@ elif [[ $iot_provider == \"mqtt\" ]]; then
     client_id=\"/r/$registry_prefix$registry_id$registry_suffix/d/$device_id\"
     hash_pass=$(sha256sum < $site_path/devices/$device_id/rsa_private.pkcs8)
     client_pass=${hash_pass:0:8}
-    msg_prefix=$client_id
+    topic_prefix=$client_id
     auth_provider="{ \"basic\": { \"username\": $client_id, \"password\": \"$client_pass\" }}"
     transport=\"ssl\"
 elif [[ -z $broker_hostname ]]; then
@@ -112,7 +112,7 @@ cat <<EOF > $pubber_config
       "protocol": "mqtt",
       "transport": $transport,
       "client_id": $client_id,
-      "msg_prefix": $msg_prefix,
+      "topic_prefix": $topic_prefix,
       "auth_provider": $auth_provider,
       "hostname": "$broker_hostname"
     },

bin/test_validator

@@ -21,6 +21,7 @@ echo "export UDMI_REGISTRY_SUFFIX=${UDMI_REGISTRY_SUFFIX:-}"
 echo "export UDMI_ALT_REGISTRY=${UDMI_ALT_REGISTRY:-}"
 
 site_path=sites/udmi_site_model
+iot_config=$site_path/cloud_iot_config.json
 output_file=$site_path/out/validation_report.json
 mkdir -p out
 
@@ -29,14 +30,17 @@ VALIDATOR_LOG=out/validator.log
 VALIDATOR_OUT=out/validator.out
 GOLDEN_OUT=etc/validator.out
 TRACE_DIR=out/validator_trace
-WAITING=10
+WAITING=20
 
 echo Killing running pubber/validator instances...
 ps ax | fgrep pubber | fgrep java | awk '{print $1}' | xargs -r kill || true
 ps ax | fgrep validator | fgrep java | awk '{print $1}' | xargs -r kill || true
 
 rm -rf $site_path/out/devices
 
+# Sanitize the site_path because some other testing scripts might have corrupted it...
+jq 'del(.udmi_namespace)' $iot_config | sponge $iot_config
+
 # Run explicitly so that it's not done in the background when the program is launched.
 validator/bin/build
 pubber/bin/build
@@ -57,14 +61,13 @@ vpid=$!
 echo Started validator pid $vpid
 
 echo Waiting for validator to startup...
-sleep 10
 
 for i in `seq 1 $WAITING`; do
     if fgrep "Entering message loop" $VALIDATOR_LOG; then
         break
     fi
     echo Waiting for validator startup $((WAITING - i))...
-    sleep 2
+    sleep 1
 done
 
 if [[ $i -eq $WAITING ]]; then
@@ -77,27 +80,32 @@ echo Checking reported cloud version info
 jq .cloud_version.udmi_ref $output_file
 
 function pubber_bg {
-    did=$1
+    device_id=$1
     shift
-    outfile=$PUBBER_LOG.$did
+    outfile=$PUBBER_LOG.$device_id
     serial_no=validator-$RANDOM
+
+    device_dir=$site_path/devices/$device_id
+
+    echo bin/keygen CERT $device_dir
+    bin/keygen CERT $device_dir || true
+
     echo Writing pubber output to $outfile, serial no $serial_no
-    cmd="bin/pubber $site_path $project_id $did $serial_no $@"
+    cmd="bin/pubber $site_path $project_id $device_id $serial_no $@"
     echo $cmd
 
     date > $outfile
     echo $cmd >> $outfile
     $cmd >> $outfile 2>&1 &
 
     # Give a little bit of time to settle before deterministic check
-    sleep 10
 
     for i in `seq 1 $WAITING`; do
         if fgrep "Connection complete" $outfile; then
             break
         fi
         echo Waiting for pubber startup $((WAITING - i))...
-        sleep 2
+        sleep 1
     done
 
     if [[ $i -eq $WAITING ]]; then

common/src/main/java/com/google/udmi/util/CertManager.java

@@ -62,8 +62,10 @@ public CertManager(File caCrtFile, File clientDir, Transport transport,
     isSsl = Transport.SSL.equals(transport);
 
     if (isSsl) {
-      crtFile = new File(clientDir, "rsa_private.crt");
-      keyFile = new File(clientDir, "rsa_private.pem");
+      File rsaCrtFile = new File(clientDir, "rsa_private.crt");
+      String prefix = rsaCrtFile.exists() ? "rsa" : "ec";
+      crtFile = new File(clientDir, prefix + "_private.crt");
+      keyFile = new File(clientDir, prefix + "_private.pem");
       this.password = passString.toCharArray();
       logging.accept("CA cert file: " + caCrtFile);
       logging.accept("Device cert file: " + crtFile);

common/src/main/java/com/google/udmi/util/Common.java

@@ -42,7 +42,7 @@ public abstract class Common {
   public static final String DEFAULT_REGION = "us-central1";
   public static final String DEVICE_ID_KEY = "deviceId";
   public static final String GATEWAY_ID_KEY = "gatewayId";
-  public static final String MESSAGE_SOURCE_PROPERTY_KEY = "source";
+  public static final String SOURCE_KEY = "source";
   public static final String NO_SITE = "--";
   public static final String GCP_REFLECT_KEY_PKCS8 = "reflector/rsa_private.pkcs8";
   public static final String CONDENSER_STRING = "^^";

common/src/main/java/com/google/udmi/util/MessageDowngrader.java

@@ -88,17 +88,16 @@ public Map<String, Object> downgrade(String targetVersion) {
   }
 
   private Map<String, Object> downgradeState(String targetVersion) {
-    final String version = convertVersion(targetVersion);
+    SchemaVersion schemaVersion = SchemaVersion.fromKey(convertVersion(targetVersion));
 
-    ObjectNode system = (ObjectNode) message.get("system");
-    if (version.equals(VERSION_1_4_0.key())) {
+    if (schemaVersion.equals(VERSION_1_4_0)) {
+      ObjectNode system = (ObjectNode) message.get("system");
       ifNotNullThen(system, map -> {
         JsonNode operation = map.remove("operation");
         ifNotNullThen(operation, src -> map.set("operational", src.get("operational")));
       });
-    } else {
-      throw new RuntimeException("Unknown target legacy version " + version);
     }
+
     return JsonUtil.asMap(message);
   }
 

common/src/main/java/com/google/udmi/util/MessageUpgrader.java

@@ -27,6 +27,7 @@ public class MessageUpgrader {
   public static final String STATE_SYSTEM_SCHEMA = "state_system";
   public static final String METADATA_SCHEMA = "metadata";
   private static final String TARGET_FORMAT = "%d.%d.%d";
+  private static final String RAW_GIT_VERSION = "git";
   private final ObjectNode message;
   private final JsonNode original;
   private final String schemaName;
@@ -55,6 +56,12 @@ public MessageUpgrader(String schemaName, JsonNode message) {
       if (parts.length >= 4) {
         throw new IllegalArgumentException("More than 3 version components");
       }
+
+      if (RAW_GIT_VERSION.equals(parts[0])) {
+        major = 0;
+        return;
+      }
+
       try {
         major = Integer.parseInt(parts[0]);
         minor = parts.length >= 2 ? Integer.parseInt(parts[1]) : -1;
@@ -92,6 +99,10 @@ public Object upgrade(boolean forceUpgrade) {
   }
 
   private Object upgradeRaw(boolean forceUpgrade) {
+    if (major == 0) {
+      return message;
+    }
+
     if (major != 1) {
       throw new IllegalArgumentException("Starting major version " + major);
     }

common/src/main/java/com/google/udmi/util/SiteModel.java

@@ -179,7 +179,7 @@ private static String getEndpointHostname(ExecutionConfiguration executionConfig
           region -> format(DEFAULT_CLEARBLADE_HOSTNAME_FORMAT, region),
           DEFAULT_CLEARBLADE_HOSTNAME);
       case GBOS -> DEFAULT_GBOS_HOSTNAME;
-      case IMPLICIT -> LOCALHOST_HOSTNAME;
+      case IMPLICIT, DYNAMIC -> LOCALHOST_HOSTNAME;
       default -> throw new RuntimeException("Unsupported iot_provider " + iotProvider);
     };
   }

etc/shell_common.sh

@@ -32,7 +32,11 @@ UDMI_ROOT=$(realpath $UDMI_ROOT)
 
 UDMI_JAR=$UDMI_ROOT/validator/build/libs/validator-1.0-SNAPSHOT-all.jar
 
-udmi_version=$(cd $UDMI_ROOT; git describe --dirty --always)
+udmi_version=$(cd $UDMI_ROOT; git describe --dirty) || true
+
+if [[ -z $udmi_version ]]; then
+    udmi_version=git-$(cd $UDMI_ROOT; git describe --dirty --always)
+fi
 
 export UDMI_ROOT
 export UDMI_TOOLS=$udmi_version