From 19dc60a33fa1871e0a3c5a2f281a90dd28338ec6 Mon Sep 17 00:00:00 2001
From: David Boehme <boehme3@llnl.gov>
Date: Mon, 10 Jun 2024 16:51:22 -0700
Subject: [PATCH] Add communication region annotations

---
 include/caliper/cali.h                        | 17 +++++++++++++++++
 include/caliper/cali_macros.h                 | 11 +++++++++++
 scripts/radiuss-spack-configs                 |  2 +-
 scripts/uberenv                               |  2 +-
 src/caliper/api.cpp                           |  3 +++
 src/caliper/cali.cpp                          | 15 +++++++++++++++
 test/ci_app_tests/ci_test_mpi_before_cali.cpp |  4 ++--
 7 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/include/caliper/cali.h b/include/caliper/cali.h
index eb3d8cd5b..9db883f5f 100644
--- a/include/caliper/cali.h
+++ b/include/caliper/cali.h
@@ -390,6 +390,23 @@ cali_begin_phase(const char* name);
 void
 cali_end_phase(const char* name);
 
+/**
+ * \brief Begin communication region \a name
+ *
+ * A communication region can be used to mark communication operations (e.g.,
+ * MPI calls) that belong to a single communication pattern. They can be used
+ * to summarize communication pattern statistics. Otherwise they behave
+ * identical to regular Caliper regions.
+ */
+void
+cali_begin_comm_region(const char* name);
+
+/**
+ * \brief End phase region \a name
+ */
+void
+cali_end_comm_region(const char* name);
+
 /**
  * \brief Begin region where the value for \a attr is `true` on the blackboard.
  */
diff --git a/include/caliper/cali_macros.h b/include/caliper/cali_macros.h
index ad28caa48..4a44bc394 100644
--- a/include/caliper/cali_macros.h
+++ b/include/caliper/cali_macros.h
@@ -217,6 +217,17 @@ extern cali_id_t cali_annotation_attr_id;
 /// \brief Mark end of a phase region
 #define CALI_MARK_PHASE_END(name) \
     cali_end_phase(name)
+
+/// \brief Mark begin of a communication region
+///
+/// A communication region marks communication operations (e.g., MPI calls)
+/// that belong to a single communication pattern.
+#define CALI_MARK_COMM_REGION_BEGIN(name) \
+    cali_begin_comm_region(name)
+
+/// \brief Mark end of a communication region
+#define CALI_MARK_COMM_REGION_END(name) \
+    cali_end_comm_region(name)
 /**
  * \} (group)
  */
diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs
index 037ee6716..56c30b4ac 160000
--- a/scripts/radiuss-spack-configs
+++ b/scripts/radiuss-spack-configs
@@ -1 +1 @@
-Subproject commit 037ee671622bd3ec2ff955ced356b59dc96fec58
+Subproject commit 56c30b4ac50b49240ef4c8ef5441456f02c550f8
diff --git a/scripts/uberenv b/scripts/uberenv
index 205672b8b..0d00dc8e1 160000
--- a/scripts/uberenv
+++ b/scripts/uberenv
@@ -1 +1 @@
-Subproject commit 205672b8b2520d7dc69acefe8738960cd5db0937
+Subproject commit 0d00dc8e19a889ba07ae433590b87533c4b5b3da
diff --git a/src/caliper/api.cpp b/src/caliper/api.cpp
index c8802d9d9..e3261db0a 100644
--- a/src/caliper/api.cpp
+++ b/src/caliper/api.cpp
@@ -39,6 +39,7 @@ namespace cali
     Attribute region_attr;
     Attribute phase_attr;
     Attribute loop_attr;
+    Attribute comm_region_attr;
 
     void init_attribute_classes(Caliper* c) {
         class_aggregatable_attr =
@@ -65,6 +66,8 @@ namespace cali
             c->create_attribute("region", CALI_TYPE_STRING, CALI_ATTR_NESTED);
         phase_attr =
             c->create_attribute("phase", CALI_TYPE_STRING, CALI_ATTR_NESTED | CALI_ATTR_LEVEL_4);
+        comm_region_attr =
+            c->create_attribute("comm.region", CALI_TYPE_STRING, CALI_ATTR_NESTED | CALI_ATTR_LEVEL_1);
 
         cali_region_attr_id = region_attr.id();
         cali_phase_attr_id = phase_attr.id();
diff --git a/src/caliper/cali.cpp b/src/caliper/cali.cpp
index 0e9db8bab..7d198380b 100644
--- a/src/caliper/cali.cpp
+++ b/src/caliper/cali.cpp
@@ -32,6 +32,7 @@ namespace cali
 
 extern Attribute region_attr;
 extern Attribute phase_attr;
+extern Attribute comm_region_attr;
 
 }
 
@@ -404,6 +405,20 @@ cali_end_phase(const char* name)
     c.end_with_value_check(cali::phase_attr, Variant(name));
 }
 
+void
+cali_begin_comm_region(const char* name)
+{
+    Caliper c;
+    c.begin(cali::comm_region_attr, Variant(name));
+}
+
+void
+cali_end_comm_region(const char* name)
+{
+    Caliper c;
+    c.end_with_value_check(cali::comm_region_attr, Variant(name));
+}
+
 void
 cali_begin(cali_id_t attr_id)
 {
diff --git a/test/ci_app_tests/ci_test_mpi_before_cali.cpp b/test/ci_app_tests/ci_test_mpi_before_cali.cpp
index e9c16ff45..4bba64e2c 100644
--- a/test/ci_app_tests/ci_test_mpi_before_cali.cpp
+++ b/test/ci_app_tests/ci_test_mpi_before_cali.cpp
@@ -40,10 +40,10 @@ int main(int argc, char* argv[])
         MPI_Bcast(&val, 1, MPI_INT, 0, MPI_COMM_WORLD);
 
         int in  = val, out;
-
+        CALI_MARK_COMM_REGION_BEGIN("reduction");
         MPI_Reduce(&in, &out, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
-
         MPI_Barrier(MPI_COMM_WORLD);
+        CALI_MARK_COMM_REGION_END("reduction");
     }
 
     mgr.flush();