More validator fixes for spurious error messages (faucetsdn#984)

.gencode_hash.txt

@@ -15,7 +15,7 @@ eed6ae125d94cf1986de96c210b0937b9c7b199724839f43fce3b357b057f8be  gencode/docs/e
 e59e52110ec12d7b82351abfc28c31255c57e2af637adf3fa5cbb7e3b26b49fe  gencode/docs/events_udmi.html
 e3683cb4561b6dda5703cb659fd6a8f12242725de389709a12eb45f146cbb295  gencode/docs/events_validation.html
 3e02bab5ef46438c99f20049f30f8dd4428530b6036aa20f29a8a3964ac2b580  gencode/docs/metadata.html
-b110edfb73182782c7539adef417a970959432c92d4c9f4a0bf7e328c0e8427d  gencode/docs/monitoring.html
+76f3516867ce60ae3e65f483a5319786eef6f3c3781c2b310d860ec5edaec490  gencode/docs/monitoring.html
 c8a40993b2810dffb9119469f83592461d817c7148072203f157b89922187e7c  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
 16512c8cc60e4f1a0e01555f3fb6c8a49d3f2b609275a8dc9e62f38fa7b404ca  gencode/docs/query_cloud.html
@@ -65,7 +65,7 @@ a83286d885ec79ccae1eea966e3b7caf99558c2f26165f207707ad4c5178acb4  gencode/java/u
 7019b8a1522261a69d708e2e7725b8bc44510c5c80f5c056543af2b7728bfa42  gencode/java/udmi/schema/Electricity.java
 75e81e5769efb15f33e06b85392ca3da1d9885f63f343eef8444c747cffd63ae  gencode/java/udmi/schema/EndpointConfiguration.java
 d2e7afd6e1a9250480144c114ee8877afb9b0dd7048495fe96a821d9e6c80475  gencode/java/udmi/schema/Entry.java
-0348208ca4005ced15c87a8f10a3c03b7d8e490ec8bf78c720cff53728627994  gencode/java/udmi/schema/Envelope.java
+e0ca9b52b4cfb4be9d6c911a549a9f2f69f5f948f05a9e3a7b224c988f932ae3  gencode/java/udmi/schema/Envelope.java
 4163478ed17191461b0a65abd11c4ddf912cbc104f040a734efd290306fab7b7  gencode/java/udmi/schema/Events.java
 5f48b9b3e602a1fab2e8b2956ff932ad65a04681ac7392e8f1b7c5afb4279609  gencode/java/udmi/schema/ExecutionConfiguration.java
 40686ad8c86447e5f760b6503c26afb6c404950c4458272d9d3fb874ab8ee624  gencode/java/udmi/schema/FamilyDiscovery.java

common/src/main/java/com/google/udmi/util/JsonUtil.java

@@ -20,10 +20,8 @@
 import java.nio.file.Files;
 import java.time.Instant;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.TreeMap;
-import java.util.function.Function;
 
 /**
  * Collection of utilities for working with json things.
@@ -91,8 +89,8 @@ public static Map<String, Object> asMap(Object input) {
    * Convert a generic object to a specific class.
    *
    * @param targetClass result class
-   * @param message object to convert
-   * @param <T> class parameter
+   * @param message     object to convert
+   * @param <T>         class parameter
    * @return converted object
    */
   public static <T> T convertTo(Class<T> targetClass, Object message) {
@@ -104,8 +102,8 @@ public static <T> T convertTo(Class<T> targetClass, Object message) {
    * Convert a generic object to a specific class with strict field mappings.
    *
    * @param targetClass result class
-   * @param message object to convert
-   * @param <T> class parameter
+   * @param message     object to convert
+   * @param <T>         class parameter
    * @return converted object
    */
   public static <T> T convertToStrict(Class<T> targetClass, Object message) {
@@ -153,7 +151,7 @@ public static Date getDate(String timestamp) {
    * @return Instant object
    */
   public static Instant getInstant(String timestamp) {
-    String replaced = timestamp.replaceFirst("\\+0000$", "Z");
+    String replaced = ifNotNullGet(timestamp, raw -> raw.replaceFirst("\\+0000$", "Z"));
     return timestamp == null ? null : Instant.parse(replaced);
   }
 
@@ -216,8 +214,8 @@ public static String currentIsoMs() {
    * Load a file to given type.
    *
    * @param clazz class of result
-   * @param file file to load
-   * @param <T> type of result
+   * @param file  file to load
+   * @param <T>   type of result
    * @return loaded object
    */
   public static <T> T loadFile(Class<T> clazz, File file) {
@@ -232,8 +230,8 @@ public static <T> T loadFile(Class<T> clazz, File file) {
    * Load a file to given type, requiring that it exists.
    *
    * @param clazz class of result
-   * @param file path of file to load
-   * @param <T> type of result
+   * @param file  path of file to load
+   * @param <T>   type of result
    * @return loaded object
    */
   public static <T> T loadFileRequired(Class<T> clazz, String file) {
@@ -244,8 +242,8 @@ public static <T> T loadFileRequired(Class<T> clazz, String file) {
    * Load a file to given type, requiring that it exists.
    *
    * @param clazz class of result
-   * @param file file to load
-   * @param <T> type of result
+   * @param file  file to load
+   * @param <T>   type of result
    * @return loaded object
    */
   public static <T> T loadFileRequired(Class<T> clazz, File file) {
@@ -263,8 +261,8 @@ public static <T> T loadFileRequired(Class<T> clazz, File file) {
    * Load file with strict(er) error checking, and throw an exception if necessary.
    *
    * @param clazz class of result
-   * @param file file to load
-   * @param <T> type of result
+   * @param file  file to load
+   * @param <T>   type of result
    * @return converted object
    */
   public static <T> T loadFileStrict(Class<T> clazz, File file) {
@@ -283,8 +281,8 @@ public static <T> T loadFileStrictRequired(Class<T> clazz, String file) {
    * Load file with strict(er) error checking and required-to-exist file.
    *
    * @param clazz class of result
-   * @param file file to load
-   * @param <T> type of result
+   * @param file  file to load
+   * @param <T>   type of result
    * @return converted object
    */
   public static <T> T loadFileStrictRequired(Class<T> clazz, File file) {
@@ -475,7 +473,7 @@ public static String unquoteJson(String message) {
    * Write json representation to a file.
    *
    * @param theThing object to write
-   * @param file output file
+   * @param file     output file
    */
   public static void writeFile(Object theThing, File file) {
     try {

schema/envelope.json

@@ -80,9 +80,6 @@
     }
   },
   "required": [
-    "projectId",
-    "deviceRegistryId",
-    "deviceId",
-    "subFolder"
+    "deviceRegistryId"
   ]
 }

tests/schemas/envelope/empty.out

@@ -1,2 +1,2 @@
 1 schema violations found
-  object has missing required properties (["deviceId","deviceRegistryId","projectId","subFolder"])
+  object has missing required properties (["deviceRegistryId"])

tests/schemas/envelope/errors1.out

@@ -1,6 +1,5 @@
-5 schema violations found
+4 schema violations found
   /deviceId: ECMA 262 regex "^[A-Z]{2,6}-[1-9][0-9]*$" does not match input string "fcu-1"
   /deviceNumId: ECMA 262 regex "^[0-9]+$" does not match input string "921302198324X"
   /deviceRegistryId: ECMA 262 regex "^[a-zA-Z][-a-zA-Z0-9._+~%]*[a-zA-Z0-9]$" does not match input string "test/registry"
   /subType: instance value (5) not found in enum (possible values: ["invalid","reflect","events","errors","commands","config","state","query","reply","model"])
-  object has missing required properties (["projectId","subFolder"])

tests/schemas/envelope/errors2.out

@@ -1,4 +1,3 @@
-3 schema violations found
+2 schema violations found
   /deviceId: ECMA 262 regex "^[A-Z]{2,6}-[1-9][0-9]*$" does not match input string "FCUs_02_NW_12"
   /deviceNumId: ECMA 262 regex "^[0-9]+$" does not match input string "-9213923812"
-  object has missing required properties (["projectId","subFolder"])

validator/src/main/java/com/google/daq/mqtt/registrar/LocalDevice.java

@@ -612,7 +612,7 @@ public void writeErrors() {
     ErrorTree errorTree = getErrorTree(ignoreErrors);
     if (errorTree != null) {
       try (PrintStream printStream = new PrintStream(Files.newOutputStream(errorsFile.toPath()))) {
-        System.err.println("Updating " + errorsFile);
+        System.err.println("Updating errors " + errorsFile);
         errorTree.write(printStream);
       } catch (Exception e) {
         throw new RuntimeException("While writing " + errorsFile.getAbsolutePath(), e);

validator/src/main/java/com/google/daq/mqtt/util/FileDataSink.java

@@ -49,7 +49,7 @@ public String publish(String deviceId, String topic, String data) {
     if (outFile == null) {
       return null;
     }
-    System.err.println("Updating " + outFile.getAbsolutePath());
+    System.err.println("Updating validated " + outFile.getAbsolutePath());
     try (PrintWriter out = new PrintWriter(Files.newOutputStream(outFile.toPath()))) {
       out.println(data);
     } catch (Exception e) {
@@ -75,7 +75,11 @@ private File getOutputFile(String deviceId, String subType, String subFolder, St
   }
 
   private File getDeviceDir(String deviceId) {
-    return new File(outBaseDir, String.format("devices/%s", deviceId));
+    if (deviceId == null || deviceId.isEmpty()) {
+      return new File(outBaseDir, "registry");
+    } else {
+      return new File(outBaseDir, String.format("devices/%s", deviceId));
+    }
   }
 
   @SuppressWarnings("unchecked")

validator/src/main/java/com/google/daq/mqtt/validator/Validator.java

@@ -507,6 +507,10 @@ private Map<String, JsonSchema> getSchemaMap() {
     if (!schemaMap.containsKey(ENVELOPE_SCHEMA_ID)) {
       throw new RuntimeException("Missing schema for attribute validation: " + ENVELOPE_SCHEMA_ID);
     }
+
+    // Rename the metadata schema to model, which is how it's handled programmatically.
+    schemaMap.put("model", schemaMap.remove("metadata"));
+
     return schemaMap;
   }
 
@@ -634,7 +638,10 @@ private void validateMessage(JsonSchema schema, Object message) {
   private Instant getInstant(Object msgObject, Map<String, String> attributes) {
     if (msgObject instanceof Map) {
       Map<String, Object> mapped = mapCast(msgObject);
-      return JsonUtil.getInstant((String) mapped.get(TIMESTAMP_KEY));
+      String timestamp = (String) mapped.get(TIMESTAMP_KEY);
+      if (timestamp != null) {
+        return JsonUtil.getInstant(timestamp);
+      }
     }
     return JsonUtil.getInstant(attributes.get(PUBLISH_TIME_KEY));
   }