Add logging for missing node_id in ray_nodes_idle_duration_ms_by_id a…

…nd set default value Signed-off-by: Mimi Liao <[email protected]>
ray-project · Nov 25, 2024 · c40c314 · c40c314
1 parent 06858c0
commit c40c314
Showing 1 changed file with 10 additions and 2 deletions.
diff --git a/python/ray/autoscaler/_private/monitor.py b/python/ray/autoscaler/_private/monitor.py
@@ -320,7 +320,13 @@ def update_load_metrics(self):
             else:
                 ip = resource_message.node_manager_address
 
-            idle_duration_ms = ray_nodes_idle_duration_ms_by_id[node_id]
+            idle_duration_ms = 0.0
+            if node_id in ray_nodes_idle_duration_ms_by_id:
+                idle_duration_ms = ray_nodes_idle_duration_ms_by_id[node_id]
+            else:
+                logger.warning(
+                    f"node_id {node_id} not found in ray_nodes_idle_duration_ms_by_id"
+                )
 
             self.load_metrics.update(
                 ip,
@@ -331,7 +337,9 @@ def update_load_metrics(self):
                 infeasible_bundles,
                 pending_placement_groups,
                 cluster_full,
-                time.time() - idle_duration_ms / 1000,  # last_used_time
+                time.time()
+                - idle_duration_ms
+                / 1000,  # node_last_used_time_s = now - idle_duration
             )
         if self.readonly_config:
             self.readonly_config["available_node_types"].update(mirror_node_types)