Automatic detection of HMC resources that no longer exist

Details:

* HMC resources that no longer exist are automatically removed from the
  exported metrics. (Issue #203)

Signed-off-by: Andreas Maier <[email protected]>

Makefile

@@ -212,7 +212,8 @@ pylint: develop_$(pymn).done
 check_reqs: develop_$(pymn).done
 	@echo "Makefile: Checking missing dependencies of the package"
 	pip-missing-reqs $(package_dir) --requirements-file=requirements.txt
-	pip-missing-reqs $(package_dir) --requirements-file=minimum-constraints.txt
+# TODO: Remove error ignore marker once zhmcclient 1.4.0 is released
+	-pip-missing-reqs $(package_dir) --requirements-file=minimum-constraints.txt
 	@echo "Makefile: Done checking missing dependencies of the package"
 ifeq ($(PLATFORM),Windows_native)
 # Reason for skipping on Windows is https://github.com/r1chardj0n3s/pip-check-reqs/issues/67

docs/changes.rst

@@ -32,6 +32,9 @@ Released: not yet
 
 **Enhancements:**
 
+* HMC resources that no longer exist are automatically removed from the
+  exported metrics. (Issue #203)
+
 **Cleanup:**
 
 **Known issues:**

docs/usage.rst

@@ -276,6 +276,10 @@ retrieved from the HMC, but they are exported to Prometheus in the same way:
   representations and can immediately return them without having to turn around
   for getting them from the HMC.
 
+  Resources that no longer exist on the HMC are automatically not exported
+  anymore. Resources that were created on the HMC since the exporter was
+  started are not detected.
+
 The exporter code is agnostic to the actual set of metrics supported by the HMC.
 A new metric exposed by the HMC metric service or a new property added to one of
 the auto-updated resources can immediately be supported by just adding it to

minimum-constraints.txt

@@ -71,7 +71,8 @@ wheel==0.33.5; python_version >= '3.8'
 
 # Direct dependencies for runtime (must be consistent with requirements.txt)
 
-zhmcclient==1.3.1
+# TODO: Use zhmcclient 1.4.0 once released, before releasing this project.
+# zhmcclient==1.4.0
 
 prometheus-client==0.9.0
 urllib3==1.25.9; python_version <= '3.9'

requirements.txt

@@ -6,8 +6,9 @@
 
 # Direct dependencies for runtime (must be consistent with minimum-constraints.txt)
 
-# git+https://github.com/zhmcclient/python-zhmcclient.git@stable_1.3#egg=zhmcclient
-zhmcclient>=1.3.1
+# TODO: Use zhmcclient 1.4.0 once released, before releasing this project.
+git+https://github.com/zhmcclient/python-zhmcclient.git@master#egg=zhmcclient
+# zhmcclient>=1.4.0
 
 prometheus-client>=0.9.0
 urllib3>=1.25.9; python_version <= '3.9'

zhmc_prometheus_exporter/zhmc_prometheus_exporter.py

@@ -660,10 +660,19 @@ def resource(self, uri, object_value):
             self._resources[uri] = _resource
         return _resource
 
+    def remove(self, uri):
+        """
+        Remove the resource with a specified URI from the cache, if present.
+        If not present, nothing happens.
+        """
+        try:
+            del self._resources[uri]
+        except KeyError:
+            pass
+
 
 def build_family_objects(metrics_object, yaml_metric_groups, yaml_metrics,
-                         metrics_filename, extra_labels,
-                         resource_cache=None):
+                         metrics_filename, extra_labels, resource_cache):
     """
     Go through all retrieved metrics and build the Prometheus Family objects.
 
@@ -687,11 +696,8 @@ def build_family_objects(metrics_object, yaml_metric_groups, yaml_metrics,
             continue  # Skip this metric group
 
         for object_value in metric_group_value.object_values:
-            if resource_cache:
-                resource = resource_cache.resource(
-                    object_value.resource_uri, object_value)
-            else:
-                resource = object_value.resource
+            resource = resource_cache.resource(
+                object_value.resource_uri, object_value)
             metric_values = object_value.metrics
 
             # Calculate the resource labels:
@@ -776,7 +782,7 @@ def build_family_objects(metrics_object, yaml_metric_groups, yaml_metrics,
 
 def build_family_objects_res(
         resources, yaml_metric_groups, yaml_metrics, metrics_filename,
-        extra_labels):
+        extra_labels, resource_cache):
     """
     Go through all auto-updated resources and build the Prometheus Family
     objects for them.
@@ -793,7 +799,28 @@ def build_family_objects_res(
     for metric_group, res_list in resources.items():
 
         yaml_metric_group = yaml_metric_groups[metric_group]
-        for resource in res_list:
+        for i, resource in enumerate(list(res_list)):
+            # Note: We use list() because resources that no longer exist will
+            # be removed from the original list, so this provides a stable
+            # iteration when items are removed from the original list.
+
+            if resource.ceased_existence:
+                try:
+                    res_str = resource.name
+                except zhmcclient.CeasedExistence:
+                    # For attribute 'name', the exception is only raised when
+                    # the name is not yet known locally.
+                    res_str = "with URI {}".format(resource.uri)
+                verbose2("Resource no longer exists on HMC: {} {}".
+                         format(resource.manager.class_name, res_str))
+                # Remove the resource from the list so it no longer show up
+                # in Prometheus data.
+                del res_list[i]
+                # Remove the resource from the resource cache. This does not
+                # influence what is shown in Prometheus data, but it is simply
+                # a cleanup.
+                resource_cache.remove(resource.uri)
+                continue
 
             # Calculate the resource labels:
             labels = dict(extra_labels)
@@ -1018,7 +1045,7 @@ def collect(self):
         family_objects.update(build_family_objects_res(
             self.resources, self.yaml_metric_groups,
             self.yaml_metrics, self.filename_metrics,
-            self.extra_labels))
+            self.extra_labels, self.resource_cache))
 
         log_exporter("Returning family objects")
         # Yield all family objects