From 9234c457124f33c85e249ba5859b306dccb40ea5 Mon Sep 17 00:00:00 2001 From: Trevor Date: Wed, 15 May 2024 08:23:53 -0700 Subject: [PATCH 1/2] Add test function for listing devices in a site registry (#889) --- .../ClearBladeIotAccessProvider.xml | 19 +++++++++++++ .../access/ClearBladeIotAccessProvider.java | 27 ++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 udmis/.idea/runConfigurations/ClearBladeIotAccessProvider.xml diff --git a/udmis/.idea/runConfigurations/ClearBladeIotAccessProvider.xml b/udmis/.idea/runConfigurations/ClearBladeIotAccessProvider.xml new file mode 100644 index 0000000000..7a89db79f3 --- /dev/null +++ b/udmis/.idea/runConfigurations/ClearBladeIotAccessProvider.xml @@ -0,0 +1,19 @@ + + + + + + + \ No newline at end of file diff --git a/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java b/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java index b8a14efa15..ddcc257d3c 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java @@ -67,7 +67,9 @@ import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.udmi.util.JsonUtil; import java.util.AbstractMap.SimpleEntry; import java.util.Base64; import java.util.Date; @@ -95,7 +97,8 @@ public class ClearBladeIotAccessProvider extends IotAccessBase { public static final String REGISTRIES_FIELD_MASK = "id,name"; - private static final Set CLOUD_REGIONS = ImmutableSet.of("us-central1"); + public static final String DEFAULT_REGION = "us-central1"; + private static final Set CLOUD_REGIONS = ImmutableSet.of(DEFAULT_REGION); private static final String EMPTY_JSON = "{}"; private static final BiMap AUTH_TYPE_MAP = ImmutableBiMap.of( Key_format.RS_256, PublicKeyFormat.RSA_PEM, @@ -117,9 +120,31 @@ public class ClearBladeIotAccessProvider extends IotAccessBase { private static final String UDMI_STATE_TOPIC = "udmi_state"; // TODO: Make this not hardcoded. private static final String TOPIC_NAME_FORMAT = "projects/%s/topics/%s"; private static final CharSequence BOUND_TO_GATEWAY_MARKER = " it's associated "; + public static final String CONFIG_ENV = "CLEARBLADE_CONFIGURATION"; private final String projectId; private final DeviceManagerInterface deviceManager; + /** + * Core test function for listing the devices in a registry. + */ + public static void main(String[] args) { + requireNonNull(System.getenv(CONFIG_ENV), CONFIG_ENV + " not defined"); + IotAccess iotAccess = new IotAccess(); + if (args.length != 1) { + System.err.println("Usage: registry_id"); + return; + } + final String registryId = args[0]; + Map stringObjectMap = JsonUtil.loadMap(System.getenv(CONFIG_ENV)); + iotAccess.project_id = (String) requireNonNull(stringObjectMap.get("project")); + System.err.println("Extracted project from ClearBlade config file: " + iotAccess.project_id); + ClearBladeIotAccessProvider clearBladeIotAccessProvider = + new ClearBladeIotAccessProvider(iotAccess); + clearBladeIotAccessProvider.populateRegistryRegions(); + CloudModel cloudModel = clearBladeIotAccessProvider.listDevices(registryId); + System.err.printf("Found %d results%n", cloudModel.device_ids.size()); + } + /** * Create a new instance for interfacing with GCP IoT Core. */ From 3883d16c8b7e36bd78b680bd324cf9ef5e903b58 Mon Sep 17 00:00:00 2001 From: Trevor Date: Wed, 15 May 2024 10:00:41 -0700 Subject: [PATCH 2/2] Automatic longer timeout for large sites (#890) --- .../src/main/java/com/google/udmi/util/SiteModel.java | 2 +- .../com/google/daq/mqtt/util/IotReflectorClient.java | 11 +++++++++-- .../com/google/daq/mqtt/util/MessagePublisher.java | 3 ++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java index 0acc72dcb9..d8c8f23dbb 100644 --- a/common/src/main/java/com/google/udmi/util/SiteModel.java +++ b/common/src/main/java/com/google/udmi/util/SiteModel.java @@ -272,7 +272,7 @@ public EndpointConfiguration makeEndpointConfig(String iotProject, String device return makeEndpointConfig(iotProject, exeConfig, deviceId); } - private Set getDeviceIds() { + public Set getDeviceIds() { checkState(sitePath != null, "sitePath not defined"); File devicesFile = new File(new File(sitePath), "devices"); File[] files = Objects.requireNonNull(devicesFile.listFiles(), diff --git a/validator/src/main/java/com/google/daq/mqtt/util/IotReflectorClient.java b/validator/src/main/java/com/google/daq/mqtt/util/IotReflectorClient.java index 5e4f651711..f996105f37 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/IotReflectorClient.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/IotReflectorClient.java @@ -51,10 +51,12 @@ public class IotReflectorClient implements IotProvider { // Requires functions that support cloud device manager support. private static final String CONFIG_TOPIC_FORMAT = "%s/config"; private static final File ERROR_DIR = new File("out"); + public static final double SLOW_QUERY_THRESHOLD = 10000; private final com.google.bos.iot.core.proxy.IotReflectorClient messageClient; private final Map>> futures = new ConcurrentHashMap<>(); private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private final boolean isSlow; /** * Create a new client. @@ -67,6 +69,11 @@ public IotReflectorClient(ExecutionConfiguration executionConfiguration) { messageClient = new com.google.bos.iot.core.proxy.IotReflectorClient(executionConfiguration, Validator.TOOLS_FUNCTIONS_VERSION); executor.execute(this::processReplies); + isSlow = siteModel.getDeviceIds().size() > SLOW_QUERY_THRESHOLD; + if (isSlow) { + // TODO: Replace this with a dynamic mechanism that gets incremental progress from UDMIS. + System.err.println("Using very long list devices timeout because of large number of devices"); + } } @Override @@ -156,8 +163,8 @@ public Map fetchCloudModels(String forGatewayId) { @Nullable private CloudModel fetchCloudModel(String deviceId) { try { - Map message = transaction(deviceId, CLOUD_QUERY_TOPIC, EMPTY_MESSAGE, - QuerySpeed.ETERNITY); + QuerySpeed speed = isSlow ? QuerySpeed.ETERNITY : QuerySpeed.SLOW; + Map message = transaction(deviceId, CLOUD_QUERY_TOPIC, EMPTY_MESSAGE, speed); return convertTo(CloudModel.class, message); } catch (Exception e) { if (e.getMessage().contains("NOT_FOUND")) { diff --git a/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java b/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java index d42bbeaa2b..d62b642a45 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/MessagePublisher.java @@ -78,7 +78,8 @@ enum QuerySpeed { QUICK(1), SHORT(15), LONG(30), - ETERNITY(90); + SLOW(90), + ETERNITY(600); private final int seconds;