Support for utilization metrics (#153)

* Support for utilization metrics * Update ze/tracer_ze_helpers.include.c Co-authored-by: Brice Videau <[email protected]> * Apply suggestions from code review Co-authored-by: Brice Videau <[email protected]> * sysman-metrics updated * Update ze/tracer_ze_helpers.include.c * tracepoint_enabled check added * Apply suggestions from code review Co-authored-by: Brice Videau <[email protected]> * Suggestions incorporated * Separation of initialization * Apply suggestions from code review Suggestions added Co-authored-by: Brice Videau <[email protected]> * Support for utilization metrics * Update ze/tracer_ze_helpers.include.c Co-authored-by: Brice Videau <[email protected]> * Apply suggestions from code review Co-authored-by: Brice Videau <[email protected]> * sysman-metrics updated * Update ze/tracer_ze_helpers.include.c * tracepoint_enabled check added * Apply suggestions from code review Co-authored-by: Brice Videau <[email protected]> * Suggestions incorporated * Separation of initialization * Apply suggestions from code review Suggestions added Co-authored-by: Brice Videau <[email protected]> * subDevice Handled * Multi-driver support * EngineProps and Subdevice Handled * Index consistancy maintianed * Templated --------- Co-authored-by: sbekele <[email protected]> Co-authored-by: Brice Videau <[email protected]> Co-authored-by: Thomas Applencourt <[email protected]>
argonne-lcf · Nov 1, 2023 · 09e6096 · 09e6096
1 parent 5bcdf54
commit 09e6096
Show file tree

Hide file tree

Showing 11 changed files with 623 additions and 135 deletions.
diff --git a/utils/xprof_utils.cpp b/utils/xprof_utils.cpp
@@ -118,6 +118,102 @@ bt_message* create_frequency_message(const char* hostname, const process_id_t pr
      return message;
 }
 
+bt_message* create_computeEU_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id,
+                                     const uintptr_t hDevice, const uint32_t subDevice, const float activeTime, const uint64_t ts,
+                                     bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend) {
+
+     /* Message creation */
+     bt_message *message = bt_message_event_create(
+                             message_iterator, event_class, stream);
+
+     /* event */
+     bt_event *downstream_event = bt_message_event_borrow_event(message);
+
+     /* Common context */
+     bt_field *context_field = bt_event_borrow_common_context_field(downstream_event);
+
+     // Hostname
+     bt_field *hostname_msg_field = bt_field_structure_borrow_member_field_by_index(context_field,0);
+     bt_field_string_set_value(hostname_msg_field, hostname);
+     // pid
+     bt_field *vpid_field = bt_field_structure_borrow_member_field_by_index(context_field,1);
+     bt_field_integer_signed_set_value(vpid_field, process_id);
+     // vid
+     bt_field *vtid_field = bt_field_structure_borrow_member_field_by_index(context_field,2);
+     bt_field_integer_signed_set_value(vtid_field, thread_id);
+     // ts
+     bt_field *ts_field = bt_field_structure_borrow_member_field_by_index(context_field,3);
+     bt_field_integer_signed_set_value(ts_field, ts);
+     // backend
+     bt_field *backend_field = bt_field_structure_borrow_member_field_by_index(context_field,4);
+     bt_field_integer_signed_set_value(backend_field, backend);
+
+     /* Payload */
+     bt_field *payload_field = bt_event_borrow_payload_field(downstream_event);
+
+     // did
+     bt_field *device_id_field = bt_field_structure_borrow_member_field_by_index(payload_field,0);
+     bt_field_integer_unsigned_set_value(device_id_field, hDevice);
+
+     //subDevice 
+     bt_field *subDevice_field = bt_field_structure_borrow_member_field_by_index(payload_field,1);
+     bt_field_integer_unsigned_set_value(subDevice_field, subDevice);
+
+     //activeTime
+     bt_field *activeTime_field = bt_field_structure_borrow_member_field_by_index(payload_field,2);
+     bt_field_real_single_precision_set_value(activeTime_field, activeTime);
+
+     return message;
+}
+
+bt_message* create_copyEU_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id,
+                                     const uintptr_t hDevice, const uint32_t subDevice, const float activeTime, const uint64_t ts,
+                                     bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend) {
+
+     /* Message creation */
+     bt_message *message = bt_message_event_create(
+                             message_iterator, event_class, stream);
+
+     /* event */
+     bt_event *downstream_event = bt_message_event_borrow_event(message);
+
+     /* Common context */
+     bt_field *context_field = bt_event_borrow_common_context_field(downstream_event);
+
+     // Hostname
+     bt_field *hostname_msg_field = bt_field_structure_borrow_member_field_by_index(context_field,0);
+     bt_field_string_set_value(hostname_msg_field, hostname);
+     // pid
+     bt_field *vpid_field = bt_field_structure_borrow_member_field_by_index(context_field,1);
+     bt_field_integer_signed_set_value(vpid_field, process_id);
+     // vid
+     bt_field *vtid_field = bt_field_structure_borrow_member_field_by_index(context_field,2);
+     bt_field_integer_signed_set_value(vtid_field, thread_id);
+     // ts
+     bt_field *ts_field = bt_field_structure_borrow_member_field_by_index(context_field,3);
+     bt_field_integer_signed_set_value(ts_field, ts);
+     // backend
+     bt_field *backend_field = bt_field_structure_borrow_member_field_by_index(context_field,4);
+     bt_field_integer_signed_set_value(backend_field, backend);
+
+     /* Payload */
+     bt_field *payload_field = bt_event_borrow_payload_field(downstream_event);
+
+     // did
+     bt_field *device_id_field = bt_field_structure_borrow_member_field_by_index(payload_field,0);
+     bt_field_integer_unsigned_set_value(device_id_field, hDevice);
+
+     // subDevice
+     bt_field *subDevice_field = bt_field_structure_borrow_member_field_by_index(payload_field,1);
+     bt_field_integer_unsigned_set_value(subDevice_field, subDevice);
+
+     //activeTime
+     bt_field *activeTime_field = bt_field_structure_borrow_member_field_by_index(payload_field,2);
+     bt_field_real_single_precision_set_value(activeTime_field, activeTime);
+
+     return message;
+}
+
 bt_message* create_host_message(const char* hostname, const process_id_t process_id, const thread_id_t thread_id, const char* name,
         const uint64_t ts, const uint64_t duration, const bool err,
         bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend) {

diff --git a/utils/xprof_utils.hpp b/utils/xprof_utils.hpp
@@ -46,7 +46,8 @@ typedef uintptr_t thread_id_t;
 typedef std::string hostname_t;
 typedef std::string thapi_function_name;
 typedef uintptr_t thapi_device_id;
-typedef uint32_t thapi_domain_id;
+typedef uint32_t thapi_domain_idx;
+typedef uint32_t thapi_sdevice_idx;
 
 // Represent a device and a sub device
 typedef std::tuple<thapi_device_id, thapi_device_id> dsd_t;
@@ -59,7 +60,8 @@ typedef std::tuple<hostname_t, process_id_t, thread_id_t, thapi_device_id, thapi
     hpt_device_function_name_t;
 typedef std::tuple<hostname_t, process_id_t, thapi_device_id> hp_device_t;
 typedef std::tuple<hostname_t, process_id_t, thapi_device_id, thapi_device_id> hp_dsd_t;
-typedef std::tuple<hostname_t, process_id_t, thapi_device_id, thapi_domain_id> hp_ddomain_t;
+typedef std::tuple<hostname_t, process_id_t, thapi_device_id, thapi_domain_idx> hp_ddomain_t;
+typedef std::tuple<hostname_t, process_id_t, thapi_device_id, thapi_sdevice_idx> hp_dsdev_t;
 typedef std::tuple<long, long> sd_t;
 typedef std::tuple<thread_id_t, thapi_function_name, long> tfn_ts_t;
 typedef std::tuple<thapi_function_name, long> fn_ts_t;
@@ -116,11 +118,18 @@ bt_message* create_power_message(const char* hostname, const process_id_t propro
                                  const uintptr_t hDevice, const uint32_t domain, const uint64_t power, const uint64_t ts,
                                  bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend = BACKEND_UNKNOWN);
 
-
 bt_message* create_frequency_message(const char* hostname, const process_id_t proprocess_id, const thread_id_t thread_id,
                                      const uintptr_t hDevice, const uint32_t domain, const uint64_t ts, const uint64_t frequency,
                                      bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend = BACKEND_UNKNOWN);
 
+bt_message* create_computeEU_message(const char* hostname, const process_id_t proprocess_id, const thread_id_t thread_id,
+                                     const uintptr_t hDevice, const uint32_t subDevice, const float activeTime, const uint64_t ts,
+                                     bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend = BACKEND_UNKNOWN);
+
+bt_message* create_copyEU_message(const char* hostname, const process_id_t proprocess_id, const thread_id_t thread_id,
+                                  const uintptr_t hDevice, const uint32_t subDevice, const float activeTime, const uint64_t ts,
+                                  bt_event_class *event_class, bt_self_message_iterator *message_iterator, bt_stream *stream, backend_t backend = BACKEND_UNKNOWN);
+
 bt_message *create_host_message(const char *hostname, const process_id_t, const thread_id_t,
                                 const char *name, const uint64_t ts, const uint64_t duration,
                                 const bool err, bt_event_class *, bt_self_message_iterator *,

diff --git a/xprof/btx_interval_model.yaml b/xprof/btx_interval_model.yaml
@@ -127,3 +127,39 @@
                 :type: integer_unsigned
                 :field_value_range: 64
                 :cast_type: uint64_t
+      - :name: lttng:computeEU
+        :payload_field_class:
+          :type: structure
+          :members:
+            - :name: did
+              :field_class:
+                :type: integer_unsigned
+                :field_value_range: 64
+                :cast_type: uint64_t
+            - :name: subDevice
+              :field_class:
+                :type: integer_unsigned
+                :field_value_range: 32
+                :cast_type: uint32_t
+            - :name: activeTime
+              :field_class:
+                :type: single
+                :cast_type: float
+      - :name: lttng:copyEU
+        :payload_field_class:
+          :type: structure
+          :members:
+            - :name: did
+              :field_class:
+                :type: integer_unsigned
+                :field_value_range: 64
+                :cast_type: uint64_t
+            - :name: subDevice
+              :field_class:
+                :type: integer_unsigned
+                :field_value_range: 32
+                :cast_type: uint32_t
+            - :name: activeTime
+              :field_class:
+                :type: single
+                :cast_type: float
diff --git a/xprof/btx_timeline.cpp b/xprof/btx_timeline.cpp
@@ -30,11 +30,13 @@ struct timeline_dispatch_s {
   std::unordered_map<hp_device_t, perfetto_uuid_t> hp_device2countertracks;
   std::unordered_map<hp_ddomain_t, perfetto_uuid_t> hp_ddomain2frqtracks;
   std::unordered_map<hp_ddomain_t, perfetto_uuid_t> hp_ddomain2pwrtracks;
-
+  std::unordered_map<hp_dsdev_t, perfetto_uuid_t> hp_dsdev2cpetracks;
+  std::unordered_map<hp_dsdev_t, perfetto_uuid_t> hp_dsdev2cpytracks;
+
   perfetto_pruned::Trace trace;
 };
 using timeline_dispatch_t = struct timeline_dispatch_s;
-
+using uuid_getter_t = perfetto_uuid_t(*)(timeline_dispatch_t*, std::string, uint64_t, uintptr_t, uint32_t);
 static perfetto_uuid_t gen_perfetto_uuid() {
   // Start at one, Look like UUID 0 is special
   static std::atomic<perfetto_uuid_t> uuid{1};
@@ -55,7 +57,7 @@ static perfetto_uuid_t get_parent_counter_track_uuid(timeline_dispatch_t *dispat
 
   // Create packet with track descriptor
   auto *packet = dispatch->trace.add_packet();
-  packet->set_trusted_packet_sequence_id(10000);
+  packet->set_trusted_packet_sequence_id(TRUSTED_PACKED_SEQUENCE_ID);
   packet->set_timestamp(0);
   // TODO: check if this is required
   packet->set_previous_packet_dropped(true);
@@ -72,7 +74,7 @@ static perfetto_uuid_t get_parent_counter_track_uuid(timeline_dispatch_t *dispat
 
 static perfetto_uuid_t get_counter_track_uuuid(timeline_dispatch_t *dispatch,
                                                std::unordered_map<hp_ddomain_t, perfetto_uuid_t> &counter_tracks, const std::string track_name,
-                                               std::string hostname, uint64_t process_id, thapi_device_id did, thapi_domain_id domain) {
+                                              std::string hostname, uint64_t process_id, thapi_device_id did, thapi_domain_idx domain) {
   perfetto_uuid_t hp_dev_uuid = 0;
   auto [it, inserted] = counter_tracks.insert({{hostname, process_id, did, domain}, hp_dev_uuid});
   auto &potential_uuid = it->second;
@@ -87,7 +89,7 @@ static perfetto_uuid_t get_counter_track_uuuid(timeline_dispatch_t *dispatch,
   // Create new track
   auto *packet = dispatch->trace.add_packet();
   packet->set_timestamp(0);
-  packet->set_trusted_packet_sequence_id(10000);
+  packet->set_trusted_packet_sequence_id(TRUSTED_PACKED_SEQUENCE_ID);
   auto *track_descriptor = packet->mutable_track_descriptor();
   track_descriptor->set_uuid(hp_dev_uuid);
   track_descriptor->set_parent_uuid(hp_uuid);
@@ -97,42 +99,68 @@ static perfetto_uuid_t get_counter_track_uuuid(timeline_dispatch_t *dispatch,
   track_descriptor->mutable_counter();
   return hp_dev_uuid;
 }
+
 static perfetto_uuid_t get_frequency_track_uuuid(timeline_dispatch_t *dispatch, std::string hostname,
-                                                 uint64_t process_id, thapi_device_id did, thapi_domain_id domain) {
-  return get_counter_track_uuuid(dispatch, dispatch->hp_ddomain2frqtracks, "GPU Frequency", hostname, process_id, did, domain);
+                                                 uint64_t process_id, thapi_device_id did, thapi_domain_idx domain) {
+  return get_counter_track_uuuid(dispatch, dispatch->hp_ddomain2frqtracks, " GPU Frequency", hostname, process_id, did, domain);
 }
 static perfetto_uuid_t get_power_track_uuuid(timeline_dispatch_t *dispatch, std::string hostname,
-                                             uint64_t process_id, thapi_device_id did, thapi_device_id domain) {
-  return get_counter_track_uuuid(dispatch, dispatch->hp_ddomain2pwrtracks, " GPU Power", hostname, process_id, did, domain);
+                                             uint64_t process_id, thapi_device_id did, thapi_domain_idx domain) {
+  //Extra leading space in the name field to make GPU Power the first track 
+  return get_counter_track_uuuid(dispatch, dispatch->hp_ddomain2pwrtracks, "  GPU Power", hostname, process_id, did, domain);
 }
 
-static void add_event_frequency(timeline_dispatch_t *dispatch, std::string hostname,
-                                uint64_t process_id, uint64_t thread_id, uintptr_t did,
-                                uint32_t domain, uint64_t timestamp, uint64_t frequency) {
-
-  perfetto_uuid_t track_uuid = get_frequency_track_uuuid(dispatch, hostname, process_id, did, domain);
+static perfetto_uuid_t get_computeEU_track_uuuid(timeline_dispatch_t *dispatch, std::string hostname,
+                                                 uint64_t process_id, thapi_device_id did, thapi_sdevice_idx subDevice) {
+  return get_counter_track_uuuid(dispatch, dispatch->hp_dsdev2cpetracks, "ComputeEngine (%)", hostname, process_id, did, subDevice);
+}
+
+static perfetto_uuid_t get_copyEU_track_uuuid(timeline_dispatch_t *dispatch, std::string hostname,
+                                              uint64_t process_id, thapi_device_id did, thapi_sdevice_idx subDevice) {
+  return get_counter_track_uuuid(dispatch, dispatch->hp_dsdev2cpytracks, "CopyEngine (%)", hostname, process_id, did, subDevice);
+}
+
+static void add_event_DTelemetry(timeline_dispatch_t *dispatch, std::string hostname, uint64_t process_id,
+                                 uint64_t thread_id, uintptr_t did, uint32_t subDevice, uint64_t timestamp,
+                                 float value, uuid_getter_t uuid_getter, const std::string& eventName) {
+  perfetto_uuid_t track_uuid = uuid_getter(dispatch, hostname, process_id, did, subDevice);
   auto *packet = dispatch->trace.add_packet();
-  packet->set_trusted_packet_sequence_id(10000);
+  packet->set_trusted_packet_sequence_id(TRUSTED_PACKED_SEQUENCE_ID);
   packet->set_timestamp(timestamp);
   auto *track_event = packet->mutable_track_event();
   track_event->set_type(perfetto_pruned::TrackEvent::TYPE_COUNTER);
   track_event->set_track_uuid(track_uuid);
-  track_event->set_name("Frequency");
-  track_event->set_counter_value(frequency);
+  track_event->set_name(eventName);
+  track_event->set_double_counter_value(value);
+}
+
+static void add_event_frequency(timeline_dispatch_t *dispatch, std::string hostname,
+                                uint64_t process_id, uint64_t thread_id, uintptr_t did,
+                                uint32_t domain, uint64_t timestamp, float frequency) {
+  add_event_DTelemetry(dispatch, hostname, process_id, thread_id, did, domain,
+                       timestamp, frequency, get_frequency_track_uuuid, "Frequency");
 }
 
 static void add_event_power(timeline_dispatch_t *dispatch, std::string hostname,
                             uint64_t process_id, uint64_t thread_id, uintptr_t did,
-                            uint32_t domain, uint64_t timestamp, uint64_t power) {
-  perfetto_uuid_t track_uuid = get_power_track_uuuid(dispatch, hostname, process_id, did, domain);
-  auto *packet = dispatch->trace.add_packet();
-  packet->set_trusted_packet_sequence_id(10000);
-  packet->set_timestamp(timestamp);
-  auto *track_event = packet->mutable_track_event();
-  track_event->set_type(perfetto_pruned::TrackEvent::TYPE_COUNTER);
-  track_event->set_track_uuid(track_uuid);
-  track_event->set_name("Power");
-  track_event->set_counter_value(power);
+                            uint32_t domain, uint64_t timestamp, float power)
+{
+  add_event_DTelemetry(dispatch, hostname, process_id, thread_id, did, domain,
+                       timestamp, power, get_power_track_uuuid, "Power");
+}
+
+static void add_event_computeEU(timeline_dispatch_t *dispatch, std::string hostname,
+                                uint64_t process_id, uint64_t thread_id, uintptr_t did,
+                                uint32_t subDevice, uint64_t timestamp, float activeTime) {
+  add_event_DTelemetry(dispatch, hostname, process_id, thread_id, did, subDevice,
+                       timestamp, activeTime, get_computeEU_track_uuuid, "ComputeEngine");
+}
+
+static void add_event_copyEU(timeline_dispatch_t *dispatch, std::string hostname,
+                             uint64_t process_id, uint64_t thread_id, uintptr_t did,
+                             uint32_t subDevice, uint64_t timestamp, float activeTime) {
+  add_event_DTelemetry(dispatch, hostname, process_id, thread_id, did, subDevice,
+                       timestamp, activeTime, get_copyEU_track_uuuid, "CopyEngine");
 }
 
 static void add_event_begin(timeline_dispatch_t *dispatch, perfetto_uuid_t uuid, timestamp_t begin,
@@ -351,11 +379,27 @@ static void power_usr_callback(void *btx_handle, void *usr_data, const char *hos
   add_event_power(dispatch, hostname, vpid, vtid, did, domain, ts, power);
 }
 
+static void computeEU_usr_callback(void *btx_handle, void *usr_data, const char *hostname,
+                                   int64_t vpid, uint64_t vtid, int64_t ts, int64_t backend,
+                                   uint64_t did, uint32_t subDevice, float activeTime) {
+  auto *dispatch = static_cast<timeline_dispatch_t *>(usr_data);
+  add_event_computeEU(dispatch, hostname, vpid, vtid, did, subDevice, ts, activeTime);
+}
+
+static void copyEU_usr_callback(void *btx_handle, void *usr_data, const char *hostname,
+                                int64_t vpid, uint64_t vtid, int64_t ts, int64_t backend,
+                                uint64_t did, uint32_t subDevice, float activeTime) {
+  auto *dispatch = static_cast<timeline_dispatch_t *>(usr_data);
+  add_event_copyEU(dispatch, hostname, vpid, vtid, did, subDevice, ts, activeTime);
+}
+
 void btx_register_usr_callbacks(void *btx_handle) {
   btx_register_callbacks_lttng_host(btx_handle, &host_usr_callback);
   btx_register_callbacks_lttng_device(btx_handle, &device_usr_callback);
   btx_register_callbacks_lttng_frequency(btx_handle, &frequency_usr_callback);
   btx_register_callbacks_lttng_power(btx_handle, &power_usr_callback);
+  btx_register_callbacks_lttng_computeEU(btx_handle, &computeEU_usr_callback);
+  btx_register_callbacks_lttng_copyEU(btx_handle, &copyEU_usr_callback);
   btx_register_callbacks_initialize_component(btx_handle, &btx_initialize_component_callback);
   btx_register_callbacks_finalize_component(btx_handle, &btx_finalize_component_callback);
 }
diff --git a/xprof/interval.c.erb b/xprof/interval.c.erb
@@ -99,6 +99,8 @@ bt_component_class_initialize_method_status <%= namespace %>_dispatch_initialize
     dispatch->device_name_event_class = create_lttng_device_name_event_class_message(trace_class, stream_class);
     dispatch->frequency_event_class = create_lttng_frequency_event_class_message(trace_class, stream_class);
     dispatch->power_event_class = create_lttng_power_event_class_message(trace_class, stream_class);
+    dispatch->computeEU_event_class = create_lttng_computeEU_event_class_message(trace_class, stream_class);
+    dispatch->copyEU_event_class = create_lttng_copyEU_event_class_message(trace_class, stream_class);
 
     /* Create a default trace from (instance of `trace_class`) */
     bt_trace *trace = bt_trace_create(trace_class);