From 437de9e5aa726f8f313681c64a14a7434bf42661 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Fri, 20 Oct 2023 18:06:56 -0700
Subject: [PATCH 01/76] Change file structure

---
 core/federated/RTI/CMakeLists.txt                  | 2 +-
 core/federated/networks/CMakeLists.txt             | 5 +++++
 core/federated/{ => networks}/net_util.c           | 0
 include/core/federated/{ => networks}/net_common.h | 0
 include/core/federated/{ => networks}/net_util.h   | 0
 5 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 core/federated/networks/CMakeLists.txt
 rename core/federated/{ => networks}/net_util.c (100%)
 rename include/core/federated/{ => networks}/net_common.h (100%)
 rename include/core/federated/{ => networks}/net_util.h (100%)

diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt
index c43dd70f3..666f062b0 100644
--- a/core/federated/RTI/CMakeLists.txt
+++ b/core/federated/RTI/CMakeLists.txt
@@ -68,7 +68,7 @@ add_executable(
     ${CoreLib}/platform/lf_unix_clock_support.c
     ${CoreLib}/utils/util.c
     ${CoreLib}/tag.c
-    ${CoreLib}/federated/net_util.c
+    ${CoreLib}/federated/networks/net_util.c
     ${CoreLib}/utils/pqueue.c
     message_record/message_record.c
 )
diff --git a/core/federated/networks/CMakeLists.txt b/core/federated/networks/CMakeLists.txt
new file mode 100644
index 000000000..0e2bfe4d4
--- /dev/null
+++ b/core/federated/networks/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(LF_NETWORK_FILES net_util.c)
+list(APPEND INFO_SOURCES ${LF_NETWORK_FILES})
+
+list(TRANSFORM LF_NETWORK_FILES PREPEND networks/)
+target_sources(core PRIVATE ${LF_NETWORK_FILES})
diff --git a/core/federated/net_util.c b/core/federated/networks/net_util.c
similarity index 100%
rename from core/federated/net_util.c
rename to core/federated/networks/net_util.c
diff --git a/include/core/federated/net_common.h b/include/core/federated/networks/net_common.h
similarity index 100%
rename from include/core/federated/net_common.h
rename to include/core/federated/networks/net_common.h
diff --git a/include/core/federated/net_util.h b/include/core/federated/networks/net_util.h
similarity index 100%
rename from include/core/federated/net_util.h
rename to include/core/federated/networks/net_util.h

From 8b60b2399e9d5e6b0c6c672a3d34f19c57225608 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Fri, 20 Oct 2023 18:16:08 -0700
Subject: [PATCH 02/76] Change cmakes

---
 CMakeLists.txt                    | 1 +
 core/CMakeLists.txt               | 1 +
 core/federated/RTI/CMakeLists.txt | 1 +
 3 files changed, 3 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c675a3f53..c36a5b86c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,6 +32,7 @@ set(PlatformLib platform)
 include_directories(${CMAKE_SOURCE_DIR}/include)
 include_directories(${CMAKE_SOURCE_DIR}/include/core)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/federated)
+include_directories(${CMAKE_SOURCE_DIR}/include/core/federated/networks)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/modal_models)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/platform)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/threaded)
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index a9026392c..0804bdbf8 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -46,6 +46,7 @@ message(STATUS "Including the following sources: " ${PRINTABLE_SOURCE_LIST})
 target_include_directories(core PUBLIC ../include)
 target_include_directories(core PUBLIC ../include/core)
 target_include_directories(core PUBLIC ../include/core/federated)
+target_include_directories(core PUBLIC ../include/core/federated/networks)
 target_include_directories(core PUBLIC ../include/core/platform)
 target_include_directories(core PUBLIC ../include/core/modal_models)
 target_include_directories(core PUBLIC ../include/core/threaded)
diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt
index 666f062b0..466e194b5 100644
--- a/core/federated/RTI/CMakeLists.txt
+++ b/core/federated/RTI/CMakeLists.txt
@@ -52,6 +52,7 @@ endif()
 set(IncludeDir ../../../include/core)
 include_directories(${IncludeDir})
 include_directories(${IncludeDir}/federated)
+include_directories(${IncludeDir}/federated/networks)
 include_directories(${IncludeDir}/modal_models)
 include_directories(${IncludeDir}/platform)
 include_directories(${IncludeDir}/utils)

From e19d12acf4144e6886a1967d7628b049ffb6e91e Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Fri, 20 Oct 2023 18:39:22 -0700
Subject: [PATCH 03/76] Fix head

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 1f7391f92..633fedf7e 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-master
+e9249a300ddfeb98d129589912dbb2a9e6d27164

From 4dad39665a0fc5be355218f8c1e4db0b7996b6ca Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 12:32:19 -0700
Subject: [PATCH 04/76] Fix net_util.h pointing path

---
 include/core/federated/networks/net_util.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/core/federated/networks/net_util.h b/include/core/federated/networks/net_util.h
index cc621115d..5c6bcb966 100644
--- a/include/core/federated/networks/net_util.h
+++ b/include/core/federated/networks/net_util.h
@@ -48,8 +48,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <sys/types.h>
 #include <stdbool.h>
 
-#include "../platform.h"
-#include "../tag.h"
+#include "../../platform.h"
+#include "../../tag.h"
 
 #define HOST_LITTLE_ENDIAN 1
 #define HOST_BIG_ENDIAN 2

From aaddd50e4fca2f39ad74d6dfb5ba641e1e132786 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 13:49:33 -0700
Subject: [PATCH 05/76] Fix networks to network

---
 CMakeLists.txt                                            | 2 +-
 core/CMakeLists.txt                                       | 2 +-
 core/federated/RTI/CMakeLists.txt                         | 4 ++--
 core/federated/{networks => network}/CMakeLists.txt       | 2 +-
 core/federated/{networks => network}/net_util.c           | 0
 include/core/federated/{networks => network}/net_common.h | 0
 include/core/federated/{networks => network}/net_util.h   | 0
 7 files changed, 5 insertions(+), 5 deletions(-)
 rename core/federated/{networks => network}/CMakeLists.txt (71%)
 rename core/federated/{networks => network}/net_util.c (100%)
 rename include/core/federated/{networks => network}/net_common.h (100%)
 rename include/core/federated/{networks => network}/net_util.h (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c36a5b86c..695950ec4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,7 +32,7 @@ set(PlatformLib platform)
 include_directories(${CMAKE_SOURCE_DIR}/include)
 include_directories(${CMAKE_SOURCE_DIR}/include/core)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/federated)
-include_directories(${CMAKE_SOURCE_DIR}/include/core/federated/networks)
+include_directories(${CMAKE_SOURCE_DIR}/include/core/federated/network)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/modal_models)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/platform)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/threaded)
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 0804bdbf8..0a6d1060c 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -46,7 +46,7 @@ message(STATUS "Including the following sources: " ${PRINTABLE_SOURCE_LIST})
 target_include_directories(core PUBLIC ../include)
 target_include_directories(core PUBLIC ../include/core)
 target_include_directories(core PUBLIC ../include/core/federated)
-target_include_directories(core PUBLIC ../include/core/federated/networks)
+target_include_directories(core PUBLIC ../include/core/federated/network)
 target_include_directories(core PUBLIC ../include/core/platform)
 target_include_directories(core PUBLIC ../include/core/modal_models)
 target_include_directories(core PUBLIC ../include/core/threaded)
diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt
index 466e194b5..24ed4998f 100644
--- a/core/federated/RTI/CMakeLists.txt
+++ b/core/federated/RTI/CMakeLists.txt
@@ -52,7 +52,7 @@ endif()
 set(IncludeDir ../../../include/core)
 include_directories(${IncludeDir})
 include_directories(${IncludeDir}/federated)
-include_directories(${IncludeDir}/federated/networks)
+include_directories(${IncludeDir}/federated/network)
 include_directories(${IncludeDir}/modal_models)
 include_directories(${IncludeDir}/platform)
 include_directories(${IncludeDir}/utils)
@@ -69,7 +69,7 @@ add_executable(
     ${CoreLib}/platform/lf_unix_clock_support.c
     ${CoreLib}/utils/util.c
     ${CoreLib}/tag.c
-    ${CoreLib}/federated/networks/net_util.c
+    ${CoreLib}/federated/network/net_util.c
     ${CoreLib}/utils/pqueue.c
     message_record/message_record.c
 )
diff --git a/core/federated/networks/CMakeLists.txt b/core/federated/network/CMakeLists.txt
similarity index 71%
rename from core/federated/networks/CMakeLists.txt
rename to core/federated/network/CMakeLists.txt
index 0e2bfe4d4..cbf58d1e9 100644
--- a/core/federated/networks/CMakeLists.txt
+++ b/core/federated/network/CMakeLists.txt
@@ -1,5 +1,5 @@
 set(LF_NETWORK_FILES net_util.c)
 list(APPEND INFO_SOURCES ${LF_NETWORK_FILES})
 
-list(TRANSFORM LF_NETWORK_FILES PREPEND networks/)
+list(TRANSFORM LF_NETWORK_FILES PREPEND network/)
 target_sources(core PRIVATE ${LF_NETWORK_FILES})
diff --git a/core/federated/networks/net_util.c b/core/federated/network/net_util.c
similarity index 100%
rename from core/federated/networks/net_util.c
rename to core/federated/network/net_util.c
diff --git a/include/core/federated/networks/net_common.h b/include/core/federated/network/net_common.h
similarity index 100%
rename from include/core/federated/networks/net_common.h
rename to include/core/federated/network/net_common.h
diff --git a/include/core/federated/networks/net_util.h b/include/core/federated/network/net_util.h
similarity index 100%
rename from include/core/federated/networks/net_util.h
rename to include/core/federated/network/net_util.h

From 01e4b7559f1ec8522ffb652e33f0284e96b87cc9 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 14:18:06 -0700
Subject: [PATCH 06/76] Fix CMake

---
 core/federated/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/federated/CMakeLists.txt b/core/federated/CMakeLists.txt
index d5af17c2b..c0e32abde 100644
--- a/core/federated/CMakeLists.txt
+++ b/core/federated/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(FEDERATED_SOURCES clock-sync.c federate.c net_util.c)
+set(FEDERATED_SOURCES clock-sync.c federate.c)
 list(APPEND INFO_SOURCES ${FEDERATED_SOURCES})
 
 list(TRANSFORM FEDERATED_SOURCES PREPEND federated/)

From 7af1b7583fa656eb92c475a6de37b8aab8db3108 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 14:19:40 -0700
Subject: [PATCH 07/76] Update reactor-c

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 633fedf7e..8bc40f57b 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-e9249a300ddfeb98d129589912dbb2a9e6d27164
+7a24b9baf36f30ccdce31bfb731bf7d602cb41b1

From 2a1b745a85c44efb0fd5a1d27ad21a35a9fe848b Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 14:31:35 -0700
Subject: [PATCH 08/76] Add core/CMakeLists.txt to include the
 network/CMakeLists.txt

---
 core/CMakeLists.txt                   | 2 ++
 core/federated/network/CMakeLists.txt | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 0a6d1060c..af79f7b60 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -18,6 +18,7 @@ add_library(core ${GENERAL_SOURCES})
 # Add sources for either threaded or unthreaded runtime
 if (DEFINED FEDERATED)
     include(federated/CMakeLists.txt)
+    include(federated/network/CMakeLists.txt)
 endif()
 
 # Add sources for either threaded or unthreaded runtime
@@ -39,6 +40,7 @@ include(utils/CMakeLists.txt)
 include(modal_models/CMakeLists.txt)
 include(platform/CMakeLists.txt)
 
+
 # Print sources used for compilation
 list(JOIN INFO_SOURCES ", " PRINTABLE_SOURCE_LIST)
 message(STATUS "Including the following sources: " ${PRINTABLE_SOURCE_LIST})
diff --git a/core/federated/network/CMakeLists.txt b/core/federated/network/CMakeLists.txt
index cbf58d1e9..82795b7bb 100644
--- a/core/federated/network/CMakeLists.txt
+++ b/core/federated/network/CMakeLists.txt
@@ -1,5 +1,5 @@
 set(LF_NETWORK_FILES net_util.c)
 list(APPEND INFO_SOURCES ${LF_NETWORK_FILES})
 
-list(TRANSFORM LF_NETWORK_FILES PREPEND network/)
+list(TRANSFORM LF_NETWORK_FILES PREPEND federated/network/)
 target_sources(core PRIVATE ${LF_NETWORK_FILES})

From 9b2225c6d6e1606f4c1002295e700fb724257bff Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 14:32:17 -0700
Subject: [PATCH 09/76] Fix ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 8bc40f57b..adb9ac2dd 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-7a24b9baf36f30ccdce31bfb731bf7d602cb41b1
+35231ece18b0322d0e5c9b582f7c80fef1d4f5a2

From 0538726ac6092bb4d4c1c5cafd6ea98a3c418c03 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 16:05:48 -0700
Subject: [PATCH 10/76] Update ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index adb9ac2dd..59825e701 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-35231ece18b0322d0e5c9b582f7c80fef1d4f5a2
+0276aa6c9527fd08b93b849237a5cb65b2f76625

From 7be746548df477b8b5b5261f44ab1f4fec26004c Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@eecs.berkeley.edu>
Date: Sat, 28 Oct 2023 12:31:31 -0700
Subject: [PATCH 11/76] Point lingua-franca-ref.txt to master to see whether we
 can get tests to pass.

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index a5b97cc10..1f7391f92 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-enclaves2
+master

From 2807e1dd042d6fa82d79611206ba4967e9ef1bbe Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sat, 28 Oct 2023 12:33:32 -0700
Subject: [PATCH 12/76] Put current fed/enclave into visited array

---
 core/federated/RTI/rti_common.c | 1 +
 core/federated/RTI/rti_remote.c | 1 +
 2 files changed, 2 insertions(+)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index 4c55cdb16..ec6ff3fef 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -115,6 +115,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
     // To handle cycles, need to create a boolean array to keep
     // track of which upstream enclave have been visited.
     bool *visited = (bool *)calloc(rti_common->number_of_scheduling_nodes, sizeof(bool)); // Initializes to 0.
+    visited[e->id] = true;
 
     // Find the tag of the earliest possible incoming message from
     // upstream enclaves.
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index ae69421ec..d140a612f 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -261,6 +261,7 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag) {
             // To handle cycles, need to create a boolean array to keep
             // track of which upstream federates have been visited.
             bool* visited = (bool*)calloc(rti_remote->base.number_of_scheduling_nodes, sizeof(bool)); // Initializes to 0.
+            visited[upstream->enclave.id] = true;
 
             // Find the (transitive) next event tag upstream.
             tag_t upstream_next_event = transitive_next_event(

From fa8bbc3d0b3dabf5cb31a560719b25e79cce1502 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@eecs.berkeley.edu>
Date: Sat, 28 Oct 2023 16:55:56 -0700
Subject: [PATCH 13/76] Update lingua-franca-ref.txt to point to enclaves3
 branch.

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 1f7391f92..3443e2c23 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-master
+enclaves3

From c6343fe3415c1ac7597b811e8bbc3d6363de0bc4 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sat, 28 Oct 2023 15:27:14 -0700
Subject: [PATCH 14/76] Fix function template

---
 core/federated/RTI/rti_remote.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index 31a574640..b3249ec30 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -506,7 +506,7 @@ int process_args(int argc, const char* argv[]);
 /**
  * Initialize the _RTI instance.
  */
-void initialize_RTI();
+void initialize_RTI(rti_remote_t *rti);
 
 #endif // RTI_REMOTE_H
 #endif // STANDALONE_RTI
\ No newline at end of file

From d336937312524c21dbddb5ee839c6f77744bdffd Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sat, 28 Oct 2023 15:27:34 -0700
Subject: [PATCH 15/76] Acquire global mutex, not enclave-local one

---
 core/federated/RTI/rti_common.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index ec6ff3fef..ec7bd411e 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -10,6 +10,8 @@
 #if defined STANDALONE_RTI || defined LF_ENCLAVES
 #include "rti_common.h"
 
+extern lf_mutex_t rti_mutex;
+
 /**
  * Local reference to rti_common_t instance.
  */
@@ -52,7 +54,7 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) {
 void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
     // FIXME: Consolidate this message with NET to get NMR (Next Message Request).
     // Careful with handling startup and shutdown.
-    lf_mutex_lock(rti_common->mutex);
+    lf_mutex_lock(&rti_mutex);
 
     enclave->completed = completed;
 
@@ -70,7 +72,7 @@ void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
         free(visited);
     }
 
-    lf_mutex_unlock(rti_common->mutex);
+    lf_mutex_unlock(&rti_mutex);
 }
 
 tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {

From fc7b300a6122452f5dce851910905b440aa4adf2 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Mon, 30 Oct 2023 07:30:00 -0700
Subject: [PATCH 16/76] Fixing transitive next event

---
 core/federated/RTI/rti_common.c | 181 ++++++++++++++------------------
 core/federated/RTI/rti_common.h |  54 +++++-----
 core/federated/RTI/rti_remote.c |  30 +++---
 core/tag.c                      |  12 ++-
 include/core/tag.h              |   9 ++
 5 files changed, 140 insertions(+), 146 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index ec7bd411e..f4f019424 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -75,6 +75,40 @@ void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
     lf_mutex_unlock(&rti_mutex);
 }
 
+tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) {
+    // First, we need to find the shortest path (minimum delay) path to each upstream node
+    // and then find the minimum of the node's recorded NET plus the minimum path delay.
+    // The following array will have the minimum path delay.
+    // It needs to be initialized to FOREVER_TAG, except for the element corresponding to
+    // this fed/enclave, which is initialized to (0,0).
+    tag_t *path_delays = (tag_t *)malloc(rti_common->number_of_scheduling_nodes * sizeof(tag_t));
+    for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
+        path_delays[i] = FOREVER_TAG;
+    }
+    shortest_path_upstream(e, NULL, path_delays);
+
+    // Next, find the tag of the earliest possible incoming message from upstream enclaves or
+    // federates, which will be the smallest upstream NET plus the least delay.
+    // This could be NEVER_TAG if the RTI has not seen a NET from some upstream node.
+    tag_t t_d = FOREVER_TAG;
+    for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
+        if (path_delays[i].time < FOREVER) {
+            // Node i is upstream of e. Note that it could be that i == e.
+            scheduling_node_t* upstream = rti_common->scheduling_nodes[i];
+            tag_t earliest_tag_from_upstream = lf_tag_add(upstream->next_event, path_delays[i]);
+            LF_PRINT_DEBUG("RTI: Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".",
+                    e->id,
+                    upstream->id,
+                    earliest_tag_from_upstream.time - start_time, earliest_tag_from_upstream.microstep);
+            if (lf_tag_compare(earliest_tag_from_upstream, t_d) < 0) {
+                t_d = earliest_tag_from_upstream;
+            }
+        }
+    }
+    free(path_delays);
+    return t_d;
+}
+
 tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
     tag_advance_grant_t result = {.tag = NEVER_TAG, .is_provisional = false};
 
@@ -84,7 +118,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
     for (int j = 0; j < e->num_upstream; j++) {
         scheduling_node_t *upstream = rti_common->scheduling_nodes[e->upstream[j]];
 
-        // Ignore this enclave if it no longer connected.
+        // Ignore this enclave/federate if it is not connected.
         if (upstream->state == NOT_CONNECTED) continue;
 
         // Adjust by the "after" delay.
@@ -111,63 +145,16 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
     // If all (transitive) upstream scheduling_nodes of the enclave
     // have earliest event tags such that the
     // enclave can now advance its tag, then send it a TAG message.
-    // Find the earliest event time of each such upstream enclave,
-    // adjusted by delays on the connections.
+    // Find the tag of the earliest event that may be later received from an upstream enclave
+    // or federate (which includes any after delays on the connections).
+    tag_t t_d = earliest_future_incoming_message_tag(e);
 
-    // To handle cycles, need to create a boolean array to keep
-    // track of which upstream enclave have been visited.
-    bool *visited = (bool *)calloc(rti_common->number_of_scheduling_nodes, sizeof(bool)); // Initializes to 0.
-    visited[e->id] = true;
-
-    // Find the tag of the earliest possible incoming message from
-    // upstream enclaves.
-    tag_t t_d_nonzero_delay = FOREVER_TAG;
-    // The tag of the earliest possible incoming message from a zero-delay connection.
-    // Delayed connections are not guarded from STP violations by the MLAA; this property is
-    // acceptable because delayed connections impose no deadlock risk and in some cases (startup)
-    // this property is necessary to avoid deadlocks. However, it requires some special care here
-    // when potentially sending a PTAG because we must not send a PTAG for a tag at which data may
-    // still be received over nonzero-delay connections.
-    tag_t t_d_zero_delay = FOREVER_TAG;
     LF_PRINT_DEBUG("NOTE: FOREVER is displayed as " PRINTF_TAG " and NEVER as " PRINTF_TAG,
                    FOREVER_TAG.time - start_time, FOREVER_TAG.microstep,
                    NEVER_TAG.time - start_time, 0);
 
-    for (int j = 0; j < e->num_upstream; j++) {
-        scheduling_node_t *upstream = rti_common->scheduling_nodes[e->upstream[j]];
-
-        // Ignore this enclave if it is no longer connected.
-        if (upstream->state == NOT_CONNECTED) continue;
-
-        // Find the (transitive) next event tag upstream.
-        tag_t upstream_next_event = transitive_next_event(
-                upstream, upstream->next_event, visited);
-
-        LF_PRINT_DEBUG("RTI: Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".",
-                e->id,
-                upstream->id,
-                upstream_next_event.time - start_time, upstream_next_event.microstep);
-
-        // Adjust by the "after" delay.
-        // Note that "no delay" is encoded as NEVER,
-        // whereas one microstep delay is encoded as 0LL.
-        tag_t candidate = lf_delay_strict(upstream_next_event, e->upstream_delay[j]);
-
-        if (e->upstream_delay[j] == NEVER) {
-            if (lf_tag_compare(candidate, t_d_zero_delay) < 0) {
-                t_d_zero_delay = candidate;
-            }
-        } else {
-            if (lf_tag_compare(candidate, t_d_nonzero_delay) < 0) {
-                t_d_nonzero_delay = candidate;
-            }
-        }
-    }
-    free(visited);
-    tag_t t_d = (lf_tag_compare(t_d_zero_delay, t_d_nonzero_delay) < 0) ? t_d_zero_delay : t_d_nonzero_delay;
-
-    LF_PRINT_LOG("RTI: Earliest next event upstream has tag " PRINTF_TAG ".",
-            t_d.time - start_time, t_d.microstep);
+    LF_PRINT_LOG("RTI: Earliest next event upstream of node %d has tag " PRINTF_TAG ".",
+            e->id, t_d.time - start_time, t_d.microstep);
 
     if (
         lf_tag_compare(t_d, e->next_event) > 0       // The enclave has something to do.
@@ -176,27 +163,27 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
                                                                       // PTAGs).
         && lf_tag_compare(t_d, e->last_granted) > 0  // The grant is not redundant.
     ) {
-        // All upstream scheduling_nodes have events with a larger tag than fed, so it is safe to send a TAG.
-        LF_PRINT_LOG("Earliest upstream message time for fed/encl %d is " PRINTF_TAG
-                "(adjusted by after delay). Granting tag advance for " PRINTF_TAG,
+        // No upstream node can send events that will be received with a tag less than or equal to
+        // e->next_event, so it is safe to send a TAG.
+        LF_PRINT_LOG("RTI: Earliest upstream message time for fed/encl %d is " PRINTF_TAG
+                "(adjusted by after delay). Granting tag advance (TAG) for " PRINTF_TAG,
                 e->id,
                 t_d.time - lf_time_start(), t_d.microstep,
                 e->next_event.time - lf_time_start(),
                 e->next_event.microstep);
         result.tag = e->next_event;
     } else if (
-        lf_tag_compare(t_d_zero_delay, e->next_event) == 0      // The enclave has something to do.
-        && lf_tag_compare(t_d_zero_delay, t_d_nonzero_delay) < 0  // The statuses of nonzero-delay connections are known at tag t_d_zero_delay
-        && lf_tag_compare(t_d_zero_delay, e->last_provisionally_granted) > 0  // The grant is not redundant.
-        && lf_tag_compare(t_d_zero_delay, e->last_granted) > 0  // The grant is not redundant.
+        lf_tag_compare(t_d, e->next_event) == 0      // The enclave has something to do/
+        && lf_tag_compare(t_d, e->last_provisionally_granted) > 0  // The grant is not redundant.
+        && lf_tag_compare(t_d, e->last_granted) > 0  // The grant is not redundant.
     ) {
-        // Some upstream scheduling_nodes has an event that has the same tag as fed's next event, so we can only provisionally
-        // grant a TAG (via a PTAG).
-        LF_PRINT_LOG("Earliest upstream message time for fed/encl %d is " PRINTF_TAG
-            " (adjusted by after delay). Granting provisional tag advance.",
+        // Some upstream node may send an event that has the same tag as this node's next event,
+        // so we can only grant a PTAG.
+        LF_PRINT_LOG("RTI: Earliest upstream message time for fed/encl %d is " PRINTF_TAG
+            " (adjusted by after delay). Granting provisional tag advance (PTAG).",
             e->id,
-            t_d_zero_delay.time - start_time, t_d_zero_delay.microstep);
-        result.tag = t_d_zero_delay;
+            t_d.time - start_time, t_d.microstep);
+        result.tag = t_d;
         result.is_provisional = true;
     }
     return result;
@@ -230,7 +217,7 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne
     }
     // Check downstream scheduling_nodes to see whether they should now be granted a TAG.
     // To handle cycles, need to create a boolean array to keep
-    // track of which upstream scheduling_nodes have been visited.
+    // track of which downstream scheduling_nodes have been visited.
     bool *visited = (bool *)calloc(rti_common->number_of_scheduling_nodes, sizeof(bool)); // Initializes to 0.
     notify_downstream_advance_grant_if_safe(e, visited);
     free(visited);
@@ -248,44 +235,34 @@ void notify_advance_grant_if_safe(scheduling_node_t* e) {
     }
 }
 
-tag_t transitive_next_event(scheduling_node_t* e, tag_t candidate, bool visited[]) {
-    if (visited[e->id] || e->state == NOT_CONNECTED) {
-        // Enclave has stopped executing or we have visited it before.
-        // No point in checking upstream scheduling_nodes.
-        return candidate;
-    }
-
-    visited[e->id] = true;
-    tag_t result = e->next_event;
-
-    // If the candidate is less than this enclave's next_event, use the candidate.
-    if (lf_tag_compare(candidate, result) < 0) {
-        result = candidate;
+void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermediate, tag_t path_delays[]) {
+    // On first call, intermediate will be NULL, so the path delay is initialized to zero.
+    tag_t delay_from_intermediate_so_far = ZERO_TAG;
+    if (intermediate == NULL) {
+        intermediate = end;
+    } else {
+        // Not the first call, so intermediate is upstream of end.
+        delay_from_intermediate_so_far = path_delays[intermediate->id];
     }
-
-    // The result cannot be earlier than the start time.
-    if (result.time < start_time) {
-        // Earliest next event cannot be before the start time.
-        result = (tag_t){.time = start_time, .microstep = 0u};
+    if (intermediate->state == NOT_CONNECTED) {
+        // Enclave or federate is not connected.
+        // No point in checking upstream scheduling_nodes.
+        return;
     }
-
-    // Check upstream scheduling_nodes to see whether any of them might send
-    // an event that would result in an earlier next event.
-    for (int i = 0; i < e->num_upstream; i++) {
-        tag_t upstream_result = transitive_next_event(
-            rti_common->scheduling_nodes[e->upstream[i]], result, visited);
-
-        // Add the "after" delay of the connection to the result.
-        upstream_result = lf_delay_tag(upstream_result, e->upstream_delay[i]);
-
-        // If the adjusted event time is less than the result so far, update the result.
-        if (lf_tag_compare(upstream_result, result) < 0) {
-            result = upstream_result;
+    // Check nodes upstream of intermediate (or end on first call).
+    for (int i = 0; i < intermediate->num_upstream; i++) {
+        // Add connection delay to path delay so far.
+        tag_t path_delay = lf_delay_tag(delay_from_intermediate_so_far, intermediate->upstream_delay[i]);
+        // If the path delay is less than the so-far recorded path delay from upstream, update upstream.
+        if (lf_tag_compare(path_delay, path_delays[intermediate->upstream[i]]) < 0) {
+            path_delays[intermediate->upstream[i]] = path_delay;
+            // Since the path delay to upstream has changed, recursively update those upstream of it.
+            // Do not do this, however, if the upstream node is the end node because this means we have
+            // completed a cycle.
+            if (end->id != intermediate->upstream[i]) {
+                shortest_path_upstream(end, rti_common->scheduling_nodes[intermediate->upstream[i]], path_delays);
+            }
         }
     }
-    if (lf_tag_compare(result, e->completed) < 0) {
-        result = e->completed;
-    }
-    return result;
 }
 #endif
diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h
index 104b46a74..a38ee5143 100644
--- a/core/federated/RTI/rti_common.h
+++ b/core/federated/RTI/rti_common.h
@@ -163,7 +163,7 @@ void notify_tag_advance_grant(scheduling_node_t* e, tag_t tag);
 void notify_advance_grant_if_safe(scheduling_node_t* e);
 
 /**
- * Nontify a provisional tag advance grant (PTAG) message to the specified enclave.
+ * Notify a provisional tag advance grant (PTAG) message to the specified enclave.
  * Do not notify it if a previously sent PTAG or TAG was greater or equal.
  *
  * This function will keep a record of this PTAG in the federate's last_provisionally_granted
@@ -171,8 +171,6 @@ void notify_advance_grant_if_safe(scheduling_node_t* e);
  *
  * This function assumes that the caller holds the mutex lock.
  *
- * FIXME: This needs two implementations, one for enclaves and one for federates.
- *
  * @param e The enclave.
  * @param tag The tag to grant.
  */
@@ -224,29 +222,37 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e);
 void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t next_event_tag);
 
 /**
- * Find the earliest tag at which the specified federate may
- * experience its next event. This is the least next event tag (NET)
- * of the specified federate and (transitively) upstream federates
- * (with delays of the connections added). For upstream federates,
- * we assume (conservatively) that federate upstream of those
- * may also send an event. The result will never be less than
- * the completion time of the federate (which may be NEVER,
- * if the federate has not yet completed a logical time).
- *
- * FIXME: This could be made less conservative by building
- * at code generation time a causality interface table indicating
- * which outputs can be triggered by which inputs. For now, we
- * assume any output can be triggered by any input.
+ * Given a node (enclave or federate), find the tag of the earliest possible incoming
+ * message from upstream enclaves or federates, which will be the smallest upstream NET
+ * plus the least delay. This could be NEVER_TAG if the RTI has not seen a NET from some
+ * upstream node.
+ * @param e The target node.
+ * @return The earliest possible incoming message tag.
+ */
+tag_t earliest_future_incoming_message_tag(scheduling_node_t* e);
+
+/**
+ * Given a node (enclave or federate), find the shortest path (least total delay)
+ * from each upstream node to the node given by `end`.  The result is written into
+ * the `path_delay` array, with one value for each node (federate or enclave) in the system.
+ * An entry of FOREVER_TAG means that the node is not upstream of `end`.
+ * An entry of (0,0) means that the node is upstream and that there are no after delays
+ * on the path to `end`.  Otherwise, the entry is the sum of the after delays on the
+ * shortest path, where the sum is calculated using lf_delay_tag().
+ * 
+ * This function calls itself recursively. On the first call,`path_delay` should be an
+ * array whose size matches the number of nodes in the system.  Each entry in the array
+ * should be FOREVER_TAG. On that first call, `intermediate` should be NULL.
+ * 
+ * If the resulting entry for `end` remains FOREVER_TAG, then there is no cycle
+ * back from the outputs of `end` to itself. Otherwise, the value of the entry will
+ * be the minimum delay among all paths back to itself.
  *
- * @param e The enclave.
- * @param candidate A candidate tag (for the first invocation,
- *  this should be fed->next_event).
- * @param visited An array of booleans indicating which federates
- *  have been visited (for the first invocation, this should be
- *  an array of falses of size _RTI.number_of_federates).
- * @return The earliest next event tag of the enclave e.
+ * @param end The target node.
+ * @param intermediate Current intermediate node (NULL initially).
+ * @param path_delays An array in which to put the results.
  */
-tag_t transitive_next_event(scheduling_node_t *e, tag_t candidate, bool visited[]);
+void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermediate, tag_t path_delays[]);
 
 #endif // RTI_COMMON_H
 #endif // STANDALONE_RTI || LF_ENCLAVES
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index d140a612f..104492da2 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -250,31 +250,25 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag) {
         // Send PTAG to all upstream federates, if they have not had
         // a later or equal PTAG or TAG sent previously and if their transitive
         // NET is greater than or equal to the tag.
+        // This is needed to stimulate absent messages from upstream and break deadlocks.
         // NOTE: This could later be replaced with a TNET mechanism once
         // we have an available encoding of causality interfaces.
         // That might be more efficient.
+        // NOTE: This is not needed for enclaves because zero-delay loops are prohibited.
+        // It's only needed for federates, which is why this is implemented here.
         for (int j = 0; j < e->num_upstream; j++) {
-            federate_info_t* upstream = GET_FED_INFO(e->upstream[j]);
+            scheduling_node_t* upstream = rti_remote->base.scheduling_nodes[e->upstream[j]];
 
             // Ignore this federate if it has resigned.
-            if (upstream->enclave.state == NOT_CONNECTED) continue;
-            // To handle cycles, need to create a boolean array to keep
-            // track of which upstream federates have been visited.
-            bool* visited = (bool*)calloc(rti_remote->base.number_of_scheduling_nodes, sizeof(bool)); // Initializes to 0.
-            visited[upstream->enclave.id] = true;
-
-            // Find the (transitive) next event tag upstream.
-            tag_t upstream_next_event = transitive_next_event(
-                    &(upstream->enclave), upstream->enclave.next_event, visited);
-            free(visited);
-            // If these tags are equal, then
-            // a TAG or PTAG should have already been granted,
-            // in which case, another will not be sent. But it
-            // may not have been already granted.
-            if (lf_tag_compare(upstream_next_event, tag) >= 0) {
-                notify_provisional_tag_advance_grant(&(upstream->enclave), tag);
-            }
+            if (upstream->state == NOT_CONNECTED) continue;
+
+            tag_t earliest = earliest_future_incoming_message_tag(upstream);
 
+            // If these tags are equal, then a TAG or PTAG should have already been granted,
+            // in which case, another will not be sent. But it may not have been already granted.
+            if (lf_tag_compare(earliest, tag) >= 0) {
+                notify_provisional_tag_advance_grant(upstream, tag);
+            }
         }
     }
 }
diff --git a/core/tag.c b/core/tag.c
index aabeab630..809449f24 100644
--- a/core/tag.c
+++ b/core/tag.c
@@ -116,14 +116,22 @@ tag_t lf_tag(void *env) {
     return ((environment_t *)env)->current_tag;
 }
 
+tag_t lf_tag_add(tag_t a, tag_t b) {
+    if (a.time == NEVER || b.time == NEVER) return NEVER_TAG;
+    if (a.time == FOREVER || b.time == FOREVER) return FOREVER_TAG;
+    tag_t result = {.time = a.time + b.time, .microstep = a.microstep + b.microstep};
+    if (result.microstep < a.microstep) return FOREVER_TAG;
+    if (result.time < a.time && b.time > 0) return FOREVER_TAG;
+    if (result.time > a.time && b.time < 0) return NEVER_TAG;
+    return result;
+}
+
 int lf_tag_compare(tag_t tag1, tag_t tag2) {
     if (tag1.time < tag2.time) {
-        LF_PRINT_DEBUG(PRINTF_TIME " < " PRINTF_TIME, tag1.time, tag2.time);
         return -1;
     } else if (tag1.time > tag2.time) {
         return 1;
     } else if (tag1.microstep < tag2.microstep) {
-        LF_PRINT_DEBUG(PRINTF_TIME " and microstep < " PRINTF_TIME, tag1.time, tag2.time);
         return -1;
     } else if (tag1.microstep > tag2.microstep) {
         return 1;
diff --git a/include/core/tag.h b/include/core/tag.h
index b598cce6a..0a480a40b 100644
--- a/include/core/tag.h
+++ b/include/core/tag.h
@@ -40,6 +40,7 @@
 #define FOREVER_TAG (tag_t) { .time = FOREVER, .microstep = FOREVER_MICROSTEP }
 // Need a separate initializer expression to comply with some C compilers
 #define FOREVER_TAG_INITIALIZER { FOREVER,  FOREVER_MICROSTEP }
+#define ZERO_TAG (tag_t) { .time = 0LL, .microstep = 0u }
 
 // Convenience for converting times
 #define BILLION 1000000000LL
@@ -82,6 +83,14 @@ typedef struct {
  */
 tag_t lf_tag(void* env);
 
+/**
+ * Add two tags.  If either tag has has NEVER or FOREVER in its time field, then
+ * return NEVER_TAG or FOREVER_TAG, respectively. Also return NEVER_TAG or FOREVER_TAG
+ * if the result underflows or overflows when adding the times.
+ * If the microstep overflows, also return FOREVER_TAG.
+ */
+tag_t lf_tag_add(tag_t a, tag_t b);
+
 /**
  * Compare two tags. Return -1 if the first is less than
  * the second, 0 if they are equal, and +1 if the first is

From a53ad1e9f194d2092248ede864a701563b07318a Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Mon, 30 Oct 2023 08:44:30 -0700
Subject: [PATCH 17/76] Fix lingua-franca-ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 59825e701..b31da30ef 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-0276aa6c9527fd08b93b849237a5cb65b2f76625
+81819f2df9addee1ca86d33bd699826eff4c0ed9

From 1e44a45af9d5b3ca1a2d49ecb5204926a5ed49ba Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 31 Oct 2023 03:51:35 -0700
Subject: [PATCH 18/76] Reverted mutex change

---
 core/federated/RTI/rti_common.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index f4f019424..d0129ff46 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -10,8 +10,6 @@
 #if defined STANDALONE_RTI || defined LF_ENCLAVES
 #include "rti_common.h"
 
-extern lf_mutex_t rti_mutex;
-
 /**
  * Local reference to rti_common_t instance.
  */
@@ -54,7 +52,7 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) {
 void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
     // FIXME: Consolidate this message with NET to get NMR (Next Message Request).
     // Careful with handling startup and shutdown.
-    lf_mutex_lock(&rti_mutex);
+    lf_mutex_lock(rti_common->mutex);
 
     enclave->completed = completed;
 
@@ -72,7 +70,7 @@ void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
         free(visited);
     }
 
-    lf_mutex_unlock(&rti_mutex);
+    lf_mutex_unlock(rti_common->mutex);
 }
 
 tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) {

From 15dbee7b949d43a113319bf4224197a6dda72028 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 31 Oct 2023 04:27:26 -0700
Subject: [PATCH 19/76] Added comment about precomputing shortest path

---
 core/federated/RTI/rti_common.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index d0129ff46..50fd9bc5f 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -242,6 +242,9 @@ void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermedi
         // Not the first call, so intermediate is upstream of end.
         delay_from_intermediate_so_far = path_delays[intermediate->id];
     }
+    // NOTE: Except for this one check, the information calculated here is completely static
+    // and could be precomputed at compile time.  But doing so would then cause transient
+    // federates in feedback loops to potentially cause deadlocks by being absent.
     if (intermediate->state == NOT_CONNECTED) {
         // Enclave or federate is not connected.
         // No point in checking upstream scheduling_nodes.

From f5f7a5ebc3f342902a8788c8379cc835b9d46dea Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 31 Oct 2023 04:41:51 -0700
Subject: [PATCH 20/76] Removed comments from .c file and tuned comments

---
 core/utils/util.c         | 82 ---------------------------------------
 include/core/utils/util.h | 27 +++++--------
 2 files changed, 9 insertions(+), 100 deletions(-)

diff --git a/core/utils/util.c b/core/utils/util.c
index d6bde95a7..9a428c79f 100644
--- a/core/utils/util.c
+++ b/core/utils/util.c
@@ -48,12 +48,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 /** Number of nanoseconds to sleep before retrying a socket read. */
 #define SOCKET_READ_RETRY_INTERVAL 1000000
 
-/**
- * The ID of this federate. For a non-federated execution, this will
- * be -1.  For a federated execution, it will be assigned when the generated function
- * _lf_initialize_trigger_objects() is called.
- * @see xtext/org.icyphy.linguafranca/src/org/icyphy/generator/CGenerator.xtend.
- */
 int _lf_my_fed_id = -1;
 
 /**
@@ -65,9 +59,6 @@ print_message_function_t* print_message_function = NULL;
 /** The level of messages to redirect to print_message_function. */
 int print_message_level = -1;
 
-/**
- * Return the federate ID or -1 if this program is not part of a federation.
- */
 int lf_fed_id() {
 	return _lf_my_fed_id;
 }
@@ -129,14 +120,6 @@ void _lf_message_print(
 	}
 }
 
-/**
- * Report an informational message on stdout with
- * a newline appended at the end.
- * If this execution is federated, then
- * the message will be prefaced by "Federate n: ",
- * where n is the federate ID.
- * The arguments are just like printf().
- */
 void lf_print(const char* format, ...) {
 	va_list args;
     va_start (args, format);
@@ -144,21 +127,10 @@ void lf_print(const char* format, ...) {
 	va_end (args);
 }
 
-/**
- * varargs alternative of "lf_print"
- */
 void lf_vprint(const char* format, va_list args) {
     _lf_message_print(0, "", format, args, LOG_LEVEL_INFO);
 }
 
-/**
- * Report an log message on stdout with the prefix
- * "LOG: " and a newline appended
- * at the end. If this execution is federated, then
- * the message will be prefaced by "Federate n: ",
- * where n is the federate ID.
- * The arguments are just like printf().
- */
 void lf_print_log(const char* format, ...) {
 	va_list args;
     va_start (args, format);
@@ -166,22 +138,10 @@ void lf_print_log(const char* format, ...) {
 	va_end (args);
 }
 
-/**
- * varargs alternative of "lf_print_log"
- */
 void lf_vprint_log(const char* format, va_list args) {
     _lf_message_print(0, "LOG: ", format, args, LOG_LEVEL_LOG);
 }
 
-
-/**
- * Report an debug message on stdout with the prefix
- * "DEBUG: " and a newline appended
- * at the end. If this execution is federated, then
- * the message will be prefaced by "Federate n: ",
- * where n is the federate ID.
- * The arguments are just like printf().
- */
 void lf_print_debug(const char* format, ...) {
     va_list args;
     va_start (args, format);
@@ -189,17 +149,10 @@ void lf_print_debug(const char* format, ...) {
     va_end (args);
 }
 
-/**
- * varargs alternative of "lf_print_debug"
- */
 void lf_vprint_debug(const char* format, va_list args) {
     _lf_message_print(0, "DEBUG: ", format, args, LOG_LEVEL_DEBUG);
 }
 
-/**
- * Report an error with the prefix "ERROR: " and a newline appended
- * at the end.  The arguments are just like printf().
- */
 void lf_print_error(const char* format, ...) {
     va_list args;
     va_start (args, format);
@@ -207,17 +160,10 @@ void lf_print_error(const char* format, ...) {
     va_end (args);
 }
 
-/**
- * varargs alternative of "lf_print_error"
- */
 void lf_vprint_error(const char* format, va_list args) {
     _lf_message_print(1, "ERROR: ", format, args, LOG_LEVEL_ERROR);
 }
 
-/**
- * Report a warning with the prefix "WARNING: " and a newline appended
- * at the end.  The arguments are just like printf().
- */
 void lf_print_warning(const char* format, ...) {
     va_list args;
     va_start (args, format);
@@ -225,18 +171,10 @@ void lf_print_warning(const char* format, ...) {
     va_end (args);
 }
 
-/**
- * varargs alternative of "lf_print_warning"
- */
 void lf_vprint_warning(const char* format, va_list args) {
     _lf_message_print(1, "WARNING: ", format, args, LOG_LEVEL_WARNING);
 }
 
-/**
- * Report an error with the prefix "ERROR: " and a newline appended
- * at the end, then exit with the failure code EXIT_FAILURE.
- * The arguments are just like printf().
- */
 void lf_print_error_and_exit(const char* format, ...) {
     va_list args;
     va_start (args, format);
@@ -245,11 +183,6 @@ void lf_print_error_and_exit(const char* format, ...) {
     exit(EXIT_FAILURE);
 }
 
-/**
- * Report an error with the prefix "ERROR: " and a newline appended
- * at the end, then exit with the failure code EXIT_FAILURE.
- * The arguments are just like printf().
- */
 void lf_assert(bool condition, const char* format, ...) {
 	if (!condition) {
     	va_list args;
@@ -260,25 +193,10 @@ void lf_assert(bool condition, const char* format, ...) {
 	}
 }
 
-/**
- * varargs alternative of "lf_print_error_and_exit"
- */
 void lf_vprint_error_and_exit(const char* format, va_list args) {
     _lf_message_print(1, "FATAL ERROR: ", format, args, LOG_LEVEL_ERROR);
 }
 
-/**
- * Register a function to display messages. After calling this,
- * all messages passed to the above print functions will be
- * printed using the specified function rather than printf
- * if their log level is greater than the specified level.
- * The level should be one of LOG_LEVEL_ERROR, LOG_LEVEL_WARNING,
- * LOG_LEVEL_INFO, LOG_LEVEL_LOG, or LOG_LEVEL_DEBUG.
- *
- * @param function The print message function or NULL to revert
- *  to using printf.
- * @param log_level The level of messages to redirect.
- */
 void lf_register_print_function(print_message_function_t* function, int log_level) {
     print_message_function = function;
     print_message_level = log_level;
diff --git a/include/core/utils/util.h b/include/core/utils/util.h
index debab132f..75a03e7dc 100644
--- a/include/core/utils/util.h
+++ b/include/core/utils/util.h
@@ -124,12 +124,9 @@ extern int _lf_my_fed_id;
 int lf_fed_id(void);
 
 /**
- * Report an informational message on stdout with
- * a newline appended at the end.
- * If this execution is federated, then
- * the message will be prefaced by "Federate n: ",
- * where n is the federate ID.
- * The arguments are just like printf().
+ * Report an informational message on stdout with a newline appended at the end.
+ * If this execution is federated, then the message will be prefaced by identifying
+ * information for the federate. The arguments are just like printf().
  */
 void lf_print(const char* format, ...) ATTRIBUTE_FORMAT_PRINTF(1, 2);
 
@@ -139,12 +136,9 @@ void lf_print(const char* format, ...) ATTRIBUTE_FORMAT_PRINTF(1, 2);
 void lf_vprint(const char* format, va_list args)  ATTRIBUTE_FORMAT_PRINTF(1, 0);
 
 /**
- * Report an log message on stdout with the prefix
- * "LOG: " and a newline appended
- * at the end. If this execution is federated, then
- * the message will be prefaced by "Federate n: ",
- * where n is the federate ID.
- * The arguments are just like printf().
+ * Report an log message on stdout with the prefix "LOG: " and a newline appended
+ * at the end. If this execution is federated, then the message will be prefaced by
+ * identifying information for the federate. The arguments are just like printf().
  */
 void lf_print_log(const char* format, ...) ATTRIBUTE_FORMAT_PRINTF(1, 2);
 
@@ -176,12 +170,9 @@ void lf_vprint_log(const char* format, va_list args) ATTRIBUTE_FORMAT_PRINTF(1,
                 } } while (0)
 
 /**
- * Report an debug message on stdout with the prefix
- * "DEBUG: " and a newline appended
- * at the end. If this execution is federated, then
- * the message will be prefaced by "Federate n: ",
- * where n is the federate ID.
- * The arguments are just like printf().
+ * Report an debug message on stdout with the prefix "DEBUG: " and a newline appended
+ * at the end. If this execution is federated, then the message will be prefaced by
+ * identifying information for the federate. The arguments are just like printf().
  */
 void lf_print_debug(const char* format, ...) ATTRIBUTE_FORMAT_PRINTF(1, 2);
 

From fceeb9d82b0e6c2452a5d56c76f09c3fcae81e41 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 31 Oct 2023 05:06:14 -0700
Subject: [PATCH 21/76] Print federate name for lf_print

---
 core/utils/util.c | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/core/utils/util.c b/core/utils/util.c
index 9a428c79f..71e920699 100644
--- a/core/utils/util.c
+++ b/core/utils/util.c
@@ -32,6 +32,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include "util.h"
+#include "environment.h"
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -98,14 +99,25 @@ void _lf_message_print(
 		// If we make multiple calls to printf(), then the results could be
 		// interleaved between threads.
 		// vprintf() is a version that takes an arg list rather than multiple args.
-		size_t length = strlen(prefix) + strlen(format) + 32;
-		char* message = (char*) malloc(length + 1);
+		char* message;
 		if (_lf_my_fed_id < 0) {
+			size_t length = strlen(prefix) + strlen(format) + 32;
+			message = (char*) malloc(length + 1);
 			snprintf(message, length, "%s%s\n",
 					prefix, format);
 		} else {
-			snprintf(message, length, "Federate %d: %s%s\n",
-					_lf_my_fed_id, prefix, format);
+			// Get the federate name from the top-level environment, which by convention is the first.
+			environment_t *envs;
+			_lf_get_environments(&envs);
+
+			size_t length = strlen(prefix) + strlen(format) + +strlen(envs->name) + 32;
+			message = (char*) malloc(length + 1);
+			char* name = envs->name;
+			// If the name has prefix "federate__", strip that out.
+			if (strncmp(name, "federate__", 10) == 0) name += 10;
+
+			snprintf(message, length, "Fed %d (%s): %s%s\n",
+					_lf_my_fed_id, name, prefix, format);
 		}
 		if (print_message_function == NULL) {
 			if (is_error) {

From 4648bff1d95b1328fea4f7f2bf32dc98857b9b00 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 31 Oct 2023 05:06:33 -0700
Subject: [PATCH 22/76] Moved _lf_get_environments to environment.h

---
 include/core/environment.h | 8 ++++++++
 include/core/reactor.h     | 9 ---------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/include/core/environment.h b/include/core/environment.h
index c18272c5d..4a1cf558f 100644
--- a/include/core/environment.h
+++ b/include/core/environment.h
@@ -154,4 +154,12 @@ void environment_init_tags(
     environment_t *env, instant_t start_time, interval_t duration
 );
 
+/**
+ * @brief Will update the argument to point to the beginning of the array of environments in this program
+ * @note Is code-generated by the compiler
+ * @param envs A double pointer which will be dereferenced and modified
+ * @return int The number of environments in the array
+ */
+int _lf_get_environments(environment_t **envs);
+
 #endif
diff --git a/include/core/reactor.h b/include/core/reactor.h
index 76189d67a..82f5e463d 100644
--- a/include/core/reactor.h
+++ b/include/core/reactor.h
@@ -555,15 +555,6 @@ trigger_handle_t _lf_schedule_copy(lf_action_base_t* action, interval_t offset,
 // See reactor.h for doc.
 int _lf_fd_send_stop_request_to_rti(tag_t stop_tag);
 
-/**
- * @brief Will update the argument to point to the beginning of the array of environments in this program
- * @note Is code-generated by the compiler
- * @param envs A double pointer which will be dereferenced and modified
- * @return int The number of environments in the array
- */
-int _lf_get_environments(environment_t **envs);
-
-
 /**
  * @brief Will create and initialize the required number of environments for the program
  * @note Will be code generated by the compiler

From dfb515ee119acc45c3d51c8ea2966d514a84f381 Mon Sep 17 00:00:00 2001
From: Erling Rennemo Jellum <erling.r.jellum@ntnu.no>
Date: Tue, 31 Oct 2023 16:20:40 +0100
Subject: [PATCH 23/76] Implement _lf_get_environments stub for unit tests

---
 test/src_gen_stub.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/src_gen_stub.c b/test/src_gen_stub.c
index 6ac14c84e..5b43d103b 100644
--- a/test/src_gen_stub.c
+++ b/test/src_gen_stub.c
@@ -1,8 +1,15 @@
 #include <stdbool.h>
 #include "tag.h"
+#include "environment.h"
+
+environment_t _env;
 
 void _lf_initialize_trigger_objects() {}
 void terminate_execution() {}
 void _lf_set_default_command_line_options() {}
 void _lf_initialize_watchdog_mutexes() {}
 void logical_tag_complete(tag_t tag_to_send) {}
+int _lf_get_environments(environment_t ** envs) {
+  *envs = &_env;
+  return 1;
+}
\ No newline at end of file

From 7548cfca2460ba9c694fa50ff54fd7ac01e04236 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 31 Oct 2023 10:03:38 -0700
Subject: [PATCH 24/76] Removed not too helpful debug message

---
 core/tag.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/tag.c b/core/tag.c
index 809449f24..4a8a8957f 100644
--- a/core/tag.c
+++ b/core/tag.c
@@ -164,7 +164,6 @@ tag_t lf_delay_tag(tag_t tag, interval_t interval) {
 tag_t lf_delay_strict(tag_t tag, interval_t interval) {
     tag_t result = lf_delay_tag(tag, interval);
     if (interval != 0 && interval != NEVER && interval != FOREVER && result.time != NEVER && result.time != FOREVER) {
-        LF_PRINT_DEBUG("interval=%lld, result time=%lld", (long long) interval, (long long) result.time);
         result.time -= 1;
         result.microstep = UINT_MAX;
     }

From fbf995077bfe2b5ef7624c157675f5332900620b Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 31 Oct 2023 10:03:54 -0700
Subject: [PATCH 25/76] Enabled compilation for standalone RTI

---
 core/utils/util.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/core/utils/util.c b/core/utils/util.c
index 71e920699..0e54e7c21 100644
--- a/core/utils/util.c
+++ b/core/utils/util.c
@@ -32,7 +32,11 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include "util.h"
+
+#ifndef STANDALONE_RTI
 #include "environment.h"
+#endif
+
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -106,18 +110,24 @@ void _lf_message_print(
 			snprintf(message, length, "%s%s\n",
 					prefix, format);
 		} else {
+#if defined STANDALONE_RTI
+			size_t length = strlen(prefix) + strlen(format) + 37;
+			message = (char*) malloc(length + 1);
+			snprintf(message, length, "RTI: %s%s\n",
+					prefix, format);
+#else
 			// Get the federate name from the top-level environment, which by convention is the first.
 			environment_t *envs;
 			_lf_get_environments(&envs);
-
-			size_t length = strlen(prefix) + strlen(format) + +strlen(envs->name) + 32;
-			message = (char*) malloc(length + 1);
 			char* name = envs->name;
+			size_t length = strlen(prefix) + strlen(format) + +strlen(name) + 32;
+			message = (char*) malloc(length + 1);
 			// If the name has prefix "federate__", strip that out.
 			if (strncmp(name, "federate__", 10) == 0) name += 10;
 
 			snprintf(message, length, "Fed %d (%s): %s%s\n",
 					_lf_my_fed_id, name, prefix, format);
+#endif // STANDALONE_RTI
 		}
 		if (print_message_function == NULL) {
 			if (is_error) {

From 4805fc920b063dd3c15d058457843b5167d02f65 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Fri, 20 Oct 2023 18:06:56 -0700
Subject: [PATCH 26/76] Change file structure

---
 core/federated/RTI/CMakeLists.txt                  | 2 +-
 core/federated/networks/CMakeLists.txt             | 5 +++++
 core/federated/{ => networks}/net_util.c           | 0
 include/core/federated/{ => networks}/net_common.h | 0
 include/core/federated/{ => networks}/net_util.h   | 0
 5 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 core/federated/networks/CMakeLists.txt
 rename core/federated/{ => networks}/net_util.c (100%)
 rename include/core/federated/{ => networks}/net_common.h (100%)
 rename include/core/federated/{ => networks}/net_util.h (100%)

diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt
index c43dd70f3..666f062b0 100644
--- a/core/federated/RTI/CMakeLists.txt
+++ b/core/federated/RTI/CMakeLists.txt
@@ -68,7 +68,7 @@ add_executable(
     ${CoreLib}/platform/lf_unix_clock_support.c
     ${CoreLib}/utils/util.c
     ${CoreLib}/tag.c
-    ${CoreLib}/federated/net_util.c
+    ${CoreLib}/federated/networks/net_util.c
     ${CoreLib}/utils/pqueue.c
     message_record/message_record.c
 )
diff --git a/core/federated/networks/CMakeLists.txt b/core/federated/networks/CMakeLists.txt
new file mode 100644
index 000000000..0e2bfe4d4
--- /dev/null
+++ b/core/federated/networks/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(LF_NETWORK_FILES net_util.c)
+list(APPEND INFO_SOURCES ${LF_NETWORK_FILES})
+
+list(TRANSFORM LF_NETWORK_FILES PREPEND networks/)
+target_sources(core PRIVATE ${LF_NETWORK_FILES})
diff --git a/core/federated/net_util.c b/core/federated/networks/net_util.c
similarity index 100%
rename from core/federated/net_util.c
rename to core/federated/networks/net_util.c
diff --git a/include/core/federated/net_common.h b/include/core/federated/networks/net_common.h
similarity index 100%
rename from include/core/federated/net_common.h
rename to include/core/federated/networks/net_common.h
diff --git a/include/core/federated/net_util.h b/include/core/federated/networks/net_util.h
similarity index 100%
rename from include/core/federated/net_util.h
rename to include/core/federated/networks/net_util.h

From a06fafbdfe2a0ea80bcc05dec643fa0c42e4b295 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Fri, 20 Oct 2023 18:16:08 -0700
Subject: [PATCH 27/76] Change cmakes

---
 CMakeLists.txt                    | 1 +
 core/CMakeLists.txt               | 1 +
 core/federated/RTI/CMakeLists.txt | 1 +
 3 files changed, 3 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c675a3f53..c36a5b86c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,6 +32,7 @@ set(PlatformLib platform)
 include_directories(${CMAKE_SOURCE_DIR}/include)
 include_directories(${CMAKE_SOURCE_DIR}/include/core)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/federated)
+include_directories(${CMAKE_SOURCE_DIR}/include/core/federated/networks)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/modal_models)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/platform)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/threaded)
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index a9026392c..0804bdbf8 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -46,6 +46,7 @@ message(STATUS "Including the following sources: " ${PRINTABLE_SOURCE_LIST})
 target_include_directories(core PUBLIC ../include)
 target_include_directories(core PUBLIC ../include/core)
 target_include_directories(core PUBLIC ../include/core/federated)
+target_include_directories(core PUBLIC ../include/core/federated/networks)
 target_include_directories(core PUBLIC ../include/core/platform)
 target_include_directories(core PUBLIC ../include/core/modal_models)
 target_include_directories(core PUBLIC ../include/core/threaded)
diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt
index 666f062b0..466e194b5 100644
--- a/core/federated/RTI/CMakeLists.txt
+++ b/core/federated/RTI/CMakeLists.txt
@@ -52,6 +52,7 @@ endif()
 set(IncludeDir ../../../include/core)
 include_directories(${IncludeDir})
 include_directories(${IncludeDir}/federated)
+include_directories(${IncludeDir}/federated/networks)
 include_directories(${IncludeDir}/modal_models)
 include_directories(${IncludeDir}/platform)
 include_directories(${IncludeDir}/utils)

From 49e4bddf3822a5dd7f384d4c1e70dfbe72d60a36 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Fri, 20 Oct 2023 18:39:22 -0700
Subject: [PATCH 28/76] Fix head

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 1f7391f92..633fedf7e 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-master
+e9249a300ddfeb98d129589912dbb2a9e6d27164

From 57efaed119b81a70ca5a2acb0ca3f3aad362a981 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 12:32:19 -0700
Subject: [PATCH 29/76] Fix net_util.h pointing path

---
 include/core/federated/networks/net_util.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/core/federated/networks/net_util.h b/include/core/federated/networks/net_util.h
index cc621115d..5c6bcb966 100644
--- a/include/core/federated/networks/net_util.h
+++ b/include/core/federated/networks/net_util.h
@@ -48,8 +48,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <sys/types.h>
 #include <stdbool.h>
 
-#include "../platform.h"
-#include "../tag.h"
+#include "../../platform.h"
+#include "../../tag.h"
 
 #define HOST_LITTLE_ENDIAN 1
 #define HOST_BIG_ENDIAN 2

From 78e92576095c4560396329264d593ca0f995e5fc Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 13:49:33 -0700
Subject: [PATCH 30/76] Fix networks to network

---
 CMakeLists.txt                                            | 2 +-
 core/CMakeLists.txt                                       | 2 +-
 core/federated/RTI/CMakeLists.txt                         | 4 ++--
 core/federated/{networks => network}/CMakeLists.txt       | 2 +-
 core/federated/{networks => network}/net_util.c           | 0
 include/core/federated/{networks => network}/net_common.h | 0
 include/core/federated/{networks => network}/net_util.h   | 0
 7 files changed, 5 insertions(+), 5 deletions(-)
 rename core/federated/{networks => network}/CMakeLists.txt (71%)
 rename core/federated/{networks => network}/net_util.c (100%)
 rename include/core/federated/{networks => network}/net_common.h (100%)
 rename include/core/federated/{networks => network}/net_util.h (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c36a5b86c..695950ec4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,7 +32,7 @@ set(PlatformLib platform)
 include_directories(${CMAKE_SOURCE_DIR}/include)
 include_directories(${CMAKE_SOURCE_DIR}/include/core)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/federated)
-include_directories(${CMAKE_SOURCE_DIR}/include/core/federated/networks)
+include_directories(${CMAKE_SOURCE_DIR}/include/core/federated/network)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/modal_models)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/platform)
 include_directories(${CMAKE_SOURCE_DIR}/include/core/threaded)
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 0804bdbf8..0a6d1060c 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -46,7 +46,7 @@ message(STATUS "Including the following sources: " ${PRINTABLE_SOURCE_LIST})
 target_include_directories(core PUBLIC ../include)
 target_include_directories(core PUBLIC ../include/core)
 target_include_directories(core PUBLIC ../include/core/federated)
-target_include_directories(core PUBLIC ../include/core/federated/networks)
+target_include_directories(core PUBLIC ../include/core/federated/network)
 target_include_directories(core PUBLIC ../include/core/platform)
 target_include_directories(core PUBLIC ../include/core/modal_models)
 target_include_directories(core PUBLIC ../include/core/threaded)
diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt
index 466e194b5..24ed4998f 100644
--- a/core/federated/RTI/CMakeLists.txt
+++ b/core/federated/RTI/CMakeLists.txt
@@ -52,7 +52,7 @@ endif()
 set(IncludeDir ../../../include/core)
 include_directories(${IncludeDir})
 include_directories(${IncludeDir}/federated)
-include_directories(${IncludeDir}/federated/networks)
+include_directories(${IncludeDir}/federated/network)
 include_directories(${IncludeDir}/modal_models)
 include_directories(${IncludeDir}/platform)
 include_directories(${IncludeDir}/utils)
@@ -69,7 +69,7 @@ add_executable(
     ${CoreLib}/platform/lf_unix_clock_support.c
     ${CoreLib}/utils/util.c
     ${CoreLib}/tag.c
-    ${CoreLib}/federated/networks/net_util.c
+    ${CoreLib}/federated/network/net_util.c
     ${CoreLib}/utils/pqueue.c
     message_record/message_record.c
 )
diff --git a/core/federated/networks/CMakeLists.txt b/core/federated/network/CMakeLists.txt
similarity index 71%
rename from core/federated/networks/CMakeLists.txt
rename to core/federated/network/CMakeLists.txt
index 0e2bfe4d4..cbf58d1e9 100644
--- a/core/federated/networks/CMakeLists.txt
+++ b/core/federated/network/CMakeLists.txt
@@ -1,5 +1,5 @@
 set(LF_NETWORK_FILES net_util.c)
 list(APPEND INFO_SOURCES ${LF_NETWORK_FILES})
 
-list(TRANSFORM LF_NETWORK_FILES PREPEND networks/)
+list(TRANSFORM LF_NETWORK_FILES PREPEND network/)
 target_sources(core PRIVATE ${LF_NETWORK_FILES})
diff --git a/core/federated/networks/net_util.c b/core/federated/network/net_util.c
similarity index 100%
rename from core/federated/networks/net_util.c
rename to core/federated/network/net_util.c
diff --git a/include/core/federated/networks/net_common.h b/include/core/federated/network/net_common.h
similarity index 100%
rename from include/core/federated/networks/net_common.h
rename to include/core/federated/network/net_common.h
diff --git a/include/core/federated/networks/net_util.h b/include/core/federated/network/net_util.h
similarity index 100%
rename from include/core/federated/networks/net_util.h
rename to include/core/federated/network/net_util.h

From def23281cd6daa31a773e1221d673f4d1c643a8b Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 14:18:06 -0700
Subject: [PATCH 31/76] Fix CMake

---
 core/federated/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/federated/CMakeLists.txt b/core/federated/CMakeLists.txt
index d5af17c2b..c0e32abde 100644
--- a/core/federated/CMakeLists.txt
+++ b/core/federated/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(FEDERATED_SOURCES clock-sync.c federate.c net_util.c)
+set(FEDERATED_SOURCES clock-sync.c federate.c)
 list(APPEND INFO_SOURCES ${FEDERATED_SOURCES})
 
 list(TRANSFORM FEDERATED_SOURCES PREPEND federated/)

From d987d0c44c96ab272a5d7737adf5cdaae0f3997b Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 14:19:40 -0700
Subject: [PATCH 32/76] Update reactor-c

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 633fedf7e..8bc40f57b 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-e9249a300ddfeb98d129589912dbb2a9e6d27164
+7a24b9baf36f30ccdce31bfb731bf7d602cb41b1

From ad92df5fd61adaa42d3d5479f347390cbd7c754d Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 14:31:35 -0700
Subject: [PATCH 33/76] Add core/CMakeLists.txt to include the
 network/CMakeLists.txt

---
 core/CMakeLists.txt                   | 2 ++
 core/federated/network/CMakeLists.txt | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 0a6d1060c..af79f7b60 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -18,6 +18,7 @@ add_library(core ${GENERAL_SOURCES})
 # Add sources for either threaded or unthreaded runtime
 if (DEFINED FEDERATED)
     include(federated/CMakeLists.txt)
+    include(federated/network/CMakeLists.txt)
 endif()
 
 # Add sources for either threaded or unthreaded runtime
@@ -39,6 +40,7 @@ include(utils/CMakeLists.txt)
 include(modal_models/CMakeLists.txt)
 include(platform/CMakeLists.txt)
 
+
 # Print sources used for compilation
 list(JOIN INFO_SOURCES ", " PRINTABLE_SOURCE_LIST)
 message(STATUS "Including the following sources: " ${PRINTABLE_SOURCE_LIST})
diff --git a/core/federated/network/CMakeLists.txt b/core/federated/network/CMakeLists.txt
index cbf58d1e9..82795b7bb 100644
--- a/core/federated/network/CMakeLists.txt
+++ b/core/federated/network/CMakeLists.txt
@@ -1,5 +1,5 @@
 set(LF_NETWORK_FILES net_util.c)
 list(APPEND INFO_SOURCES ${LF_NETWORK_FILES})
 
-list(TRANSFORM LF_NETWORK_FILES PREPEND network/)
+list(TRANSFORM LF_NETWORK_FILES PREPEND federated/network/)
 target_sources(core PRIVATE ${LF_NETWORK_FILES})

From 4a7a112511436539883002f0b789fc2b2c117613 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 14:32:17 -0700
Subject: [PATCH 34/76] Fix ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 8bc40f57b..adb9ac2dd 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-7a24b9baf36f30ccdce31bfb731bf7d602cb41b1
+35231ece18b0322d0e5c9b582f7c80fef1d4f5a2

From 1befad0370159b2cac09ce6fe4b8ee281c142435 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 25 Oct 2023 16:05:48 -0700
Subject: [PATCH 35/76] Update ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index adb9ac2dd..59825e701 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-35231ece18b0322d0e5c9b582f7c80fef1d4f5a2
+0276aa6c9527fd08b93b849237a5cb65b2f76625

From f42708b71ec69006cb4fcb9e998fe5ae40d14b46 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Mon, 30 Oct 2023 08:44:30 -0700
Subject: [PATCH 36/76] Fix lingua-franca-ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 59825e701..b31da30ef 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-0276aa6c9527fd08b93b849237a5cb65b2f76625
+81819f2df9addee1ca86d33bd699826eff4c0ed9

From 6ce33644895e94e36e51b24e8ee3f36c3e03465a Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sun, 5 Nov 2023 09:00:52 -0800
Subject: [PATCH 37/76] Typo

---
 include/core/federated/federate.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h
index aca041e5b..880408ec6 100644
--- a/include/core/federated/federate.h
+++ b/include/core/federated/federate.h
@@ -363,7 +363,7 @@ void stall_advance_level_federation(environment_t* env, size_t level);
 /**
  * @brief Update the max level allowed to advance (MLAA).
  * If the specified tag is greater than the current_tag of the top-level environment
- * (or equal an is_provisional is false), then set the MLAA to MAX_INT and return.
+ * (or equal and is_provisional is false), then set the MLAA to MAX_INT and return.
  * This removes any barriers on execution at the current tag due to network inputs.
  * Otherwise, set the MLAA to the minimum level over all (non-physical) network input ports
  * where the status of the input port is not known at that current_tag.

From fd98f3aa9c39360839d904f933bb4256017e9602 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sun, 5 Nov 2023 09:01:06 -0800
Subject: [PATCH 38/76] Handle cycles better

---
 core/federated/RTI/rti_common.c | 4 +++-
 core/federated/RTI/rti_common.h | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index 50fd9bc5f..36bf192d5 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -83,6 +83,7 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) {
     for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
         path_delays[i] = FOREVER_TAG;
     }
+    path_delays[e->id] = ZERO_TAG;
     shortest_path_upstream(e, NULL, path_delays);
 
     // Next, find the tag of the earliest possible incoming message from upstream enclaves or
@@ -90,7 +91,7 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) {
     // This could be NEVER_TAG if the RTI has not seen a NET from some upstream node.
     tag_t t_d = FOREVER_TAG;
     for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
-        if (path_delays[i].time < FOREVER) {
+        if (e->id != i && path_delays[i].time < FOREVER) {
             // Node i is upstream of e. Note that it could be that i == e.
             scheduling_node_t* upstream = rti_common->scheduling_nodes[i];
             tag_t earliest_tag_from_upstream = lf_tag_add(upstream->next_event, path_delays[i]);
@@ -171,6 +172,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
                 e->next_event.microstep);
         result.tag = e->next_event;
     } else if (
+        // FIXME: if e->is_in_zero_delay_cycle
         lf_tag_compare(t_d, e->next_event) == 0      // The enclave has something to do/
         && lf_tag_compare(t_d, e->last_provisionally_granted) > 0  // The grant is not redundant.
         && lf_tag_compare(t_d, e->last_granted) > 0  // The grant is not redundant.
diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h
index a38ee5143..1b06838cf 100644
--- a/core/federated/RTI/rti_common.h
+++ b/core/federated/RTI/rti_common.h
@@ -242,7 +242,8 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e);
  * 
  * This function calls itself recursively. On the first call,`path_delay` should be an
  * array whose size matches the number of nodes in the system.  Each entry in the array
- * should be FOREVER_TAG. On that first call, `intermediate` should be NULL.
+ * should be FOREVER_TAG except the node for which we finding the shortest path, which
+ * should have an entry (0,0). On that first call, `intermediate` should be NULL.
  * 
  * If the resulting entry for `end` remains FOREVER_TAG, then there is no cycle
  * back from the outputs of `end` to itself. Otherwise, the value of the entry will

From 1a054364d19c9ece05a20b964cf782e893f6f567 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sun, 5 Nov 2023 16:14:42 -0800
Subject: [PATCH 39/76] Malloc memory only for array sizes > 0

---
 core/federated/RTI/rti_remote.c | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 104492da2..4cece2064 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1260,14 +1260,22 @@ int receive_connection_information(int socket_id, uint16_t fed_id) {
             fed_id);
 
         // Allocate memory for the upstream and downstream pointers
-        fed->enclave.upstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_upstream);
-        fed->enclave.downstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_downstream);
-
-        // Allocate memory for the upstream delay pointers
-        fed->enclave.upstream_delay =
-            (interval_t*)malloc(
-                sizeof(interval_t) * fed->enclave.num_upstream
-            );
+        if (fed->enclave.num_upstream > 0) {
+            fed->enclave.upstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_upstream);
+            // Allocate memory for the upstream delay pointers
+            fed->enclave.upstream_delay =
+                (interval_t*)malloc(
+                    sizeof(interval_t) * fed->enclave.num_upstream
+                );
+        } else {
+            fed->enclave.upstream = (int*)NULL;
+            fed->enclave.upstream_delay = (interval_t*)NULL;
+        }
+        if (fed->enclave.num_downstream > 0) {
+            fed->enclave.downstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_downstream);
+        } else {
+            fed->enclave.downstream = (int*)
+        }
 
         size_t connections_info_body_size = ((sizeof(uint16_t) + sizeof(int64_t)) *
             fed->enclave.num_upstream) + (sizeof(uint16_t) * fed->enclave.num_downstream);

From f7c80af74437684e8d59d69fdb27aa650ad76d33 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sun, 5 Nov 2023 17:46:26 -0800
Subject: [PATCH 40/76] Free dynamically allocated memory

---
 core/federated/RTI/main.c        |  1 +
 core/federated/RTI/rti_common.c  |  1 +
 core/federated/RTI/rti_common.h  |  5 +++++
 core/federated/RTI/rti_local.c   | 12 +++++++++++-
 core/federated/RTI/rti_local.h   |  6 +++++-
 core/federated/RTI/rti_remote.c  | 13 ++++++++++++-
 core/threaded/reactor_threaded.c |  3 +++
 7 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c
index 42aa56839..d051edcf6 100644
--- a/core/federated/RTI/main.c
+++ b/core/federated/RTI/main.c
@@ -284,6 +284,7 @@ int main(int argc, const char* argv[]) {
 
     int socket_descriptor = start_rti_server(rti.user_specified_port);
     wait_for_federates(socket_descriptor);
+    free_scheduling_nodes(rti.base.scheduling_nodes, rti.base.number_of_scheduling_nodes);
     lf_print("RTI is exiting.");
     return 0;
 }
diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index 36bf192d5..c042d25db 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -268,4 +268,5 @@ void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermedi
         }
     }
 }
+
 #endif
diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h
index 1b06838cf..b1e6c77e2 100644
--- a/core/federated/RTI/rti_common.h
+++ b/core/federated/RTI/rti_common.h
@@ -255,5 +255,10 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e);
  */
 void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermediate, tag_t path_delays[]);
 
+/**
+ * Free dynamically allocated memory on the scheduling nodes and the scheduling node array itself.
+ */
+void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number_of_scheduling_nodes);
+
 #endif // RTI_COMMON_H
 #endif // STANDALONE_RTI || LF_ENCLAVES
diff --git a/core/federated/RTI/rti_local.c b/core/federated/RTI/rti_local.c
index c70e8bfe5..83c9daf2c 100644
--- a/core/federated/RTI/rti_local.c
+++ b/core/federated/RTI/rti_local.c
@@ -60,7 +60,7 @@ static rti_local_t * rti_local;
 lf_mutex_t rti_mutex;
 
 void initialize_local_rti(environment_t *envs, int num_envs) {
-    rti_local = malloc(sizeof(rti_local_t));
+    rti_local = (rti_local_t*)malloc(sizeof(rti_local_t));
     if (rti_local == NULL) lf_print_error_and_exit("Out of memory");
 
     initialize_rti_common(&rti_local->base);
@@ -85,6 +85,11 @@ void initialize_local_rti(environment_t *envs, int num_envs) {
     }
 }
 
+void free_local_rti() {
+    free_scheduling_nodes(rti_local->base.scheduling_nodes, rti_local->base.number_of_scheduling_nodes);
+    free(rti_local);
+}
+
 void initialize_enclave_info(enclave_info_t* enclave, int idx, environment_t * env) {
     initialize_scheduling_node(&enclave->base, idx);
 
@@ -203,4 +208,9 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag) {
     LF_PRINT_LOG("RTI: enclave %u callback with PTAG " PRINTF_TAG " ",
         e->id, tag.time - lf_time_start(), tag.microstep);
 }
+
+void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number_of_scheduling_nodes) {
+    // Nothing to do here.
+}
+
 #endif //LF_ENCLAVES
diff --git a/core/federated/RTI/rti_local.h b/core/federated/RTI/rti_local.h
index 30be259a7..93b36c656 100644
--- a/core/federated/RTI/rti_local.h
+++ b/core/federated/RTI/rti_local.h
@@ -29,10 +29,14 @@ typedef struct {
 
 /**
  * @brief Dynamically create and initialize the local RTI.
- * 
  */
 void initialize_local_rti(environment_t* envs, int num_envs);
 
+/**
+ * @brief Free memory associated with the local the RTI and the local RTI iself.
+ */
+void free_local_rti();
+
 /**
  * @brief Initialize the enclave object.
  * 
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 4cece2064..a58cec380 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1274,7 +1274,7 @@ int receive_connection_information(int socket_id, uint16_t fed_id) {
         if (fed->enclave.num_downstream > 0) {
             fed->enclave.downstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_downstream);
         } else {
-            fed->enclave.downstream = (int*)
+            fed->enclave.downstream = (int*)NULL;
         }
 
         size_t connections_info_body_size = ((sizeof(uint16_t) + sizeof(int64_t)) *
@@ -1649,4 +1649,15 @@ void initialize_RTI(rti_remote_t *rti){
     rti_remote->base.tracing_enabled = false;
     rti_remote->stop_in_progress = false;
 }
+
+void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number_of_scheduling_nodes) {
+    for (uint16_t i = 0; i < number_of_scheduling_nodes; i++) {
+        // FIXME: Gives error freeing memory not allocated!!!!
+        scheduling_node_t* node = scheduling_nodes[i];
+        if (node->upstream != NULL) free(node->upstream);
+        if (node->downstream != NULL) free(node->downstream);
+    }
+    free(scheduling_nodes);
+}
+
 #endif // STANDALONE_RTI
diff --git a/core/threaded/reactor_threaded.c b/core/threaded/reactor_threaded.c
index fd92c0ab0..b5febb2d1 100644
--- a/core/threaded/reactor_threaded.c
+++ b/core/threaded/reactor_threaded.c
@@ -1216,6 +1216,9 @@ int lf_reactor_c_main(int argc, const char* argv[]) {
             LF_PRINT_LOG("---- All worker threads exited successfully.");
         }
     }
+#if defined LF_ENCLAVES
+    free_local_rti();
+#endif
     return 0;
 }   
 

From 158260bf8b2724d8af2c0b8e36fafc64883ef07d Mon Sep 17 00:00:00 2001
From: erlingrj <erling.jellum@gmail.com>
Date: Wed, 8 Nov 2023 19:18:31 +0100
Subject: [PATCH 41/76] add _locked version of stop_trace to be called from
 within critical section

---
 core/reactor_common.c | 2 +-
 core/trace.c          | 6 +++++-
 include/core/trace.h  | 2 ++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/core/reactor_common.c b/core/reactor_common.c
index 5422d1453..eb31c761e 100644
--- a/core/reactor_common.c
+++ b/core/reactor_common.c
@@ -1749,7 +1749,7 @@ void termination(void) {
             continue;
         }
         // Stop any tracing, if it is running.
-        stop_trace(env->trace);
+        stop_trace_locked(env->trace);
 
         _lf_start_time_step(env);
 
diff --git a/core/trace.c b/core/trace.c
index 66eb1a853..8a45f7493 100644
--- a/core/trace.c
+++ b/core/trace.c
@@ -459,6 +459,11 @@ void tracepoint_reaction_deadline_missed(trace_t* trace, reaction_t *reaction, i
 
 void stop_trace(trace_t* trace) {
     lf_critical_section_enter(trace->env);
+    stop_trace_locked(trace);
+    lf_critical_section_exit(trace->env);
+}
+
+void stop_trace_locked(trace_t* trace) {
     if (trace->_lf_trace_stop) {
         // Trace was already stopped. Nothing to do.
         return;
@@ -480,7 +485,6 @@ void stop_trace(trace_t* trace) {
     fclose(trace->_lf_trace_file);
     trace->_lf_trace_file = NULL;
     LF_PRINT_DEBUG("Stopped tracing.");
-    lf_critical_section_exit(trace->env);
 }
 
 ////////////////////////////////////////////////////////////
diff --git a/include/core/trace.h b/include/core/trace.h
index 6e22332ff..598c669bf 100644
--- a/include/core/trace.h
+++ b/include/core/trace.h
@@ -435,6 +435,7 @@ void tracepoint_reaction_deadline_missed(trace_t* trace, reaction_t *reaction, i
  * close the files.
  */
 void stop_trace(trace_t* trace);
+void stop_trace_locked(trace_t* trace);
 
 ////////////////////////////////////////////////////////////
 //// For federated execution
@@ -540,6 +541,7 @@ typedef struct trace_t trace_t;
 
 #define start_trace(...)
 #define stop_trace(...)
+#define stop_trace_locked(...)
 #define trace_new(...) NULL
 #define trace_free(...)
 

From 4e9c1a285ff280bd8a2edc88cd56a4fb02999f36 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Wed, 8 Nov 2023 11:35:57 -0800
Subject: [PATCH 42/76] Protect next_reaction_level against overflow

---
 core/threaded/reactor_threaded.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/threaded/reactor_threaded.c b/core/threaded/reactor_threaded.c
index f3227dd93..b3b20819f 100644
--- a/core/threaded/reactor_threaded.c
+++ b/core/threaded/reactor_threaded.c
@@ -924,8 +924,9 @@ void try_advance_level(environment_t* env, volatile size_t* next_reaction_level)
     #ifdef FEDERATED
     stall_advance_level_federation(env, *next_reaction_level);
     #endif
-    *next_reaction_level += 1;
+    if (*next_reaction_level < SIZE_MAX) *next_reaction_level += 1;
 }
+
 /**
  * The main looping logic of each LF worker thread.
  * This function assumes the caller holds the mutex lock.

From c2707610d8dcbb5112fa6a565f3d4d38fbe19f95 Mon Sep 17 00:00:00 2001
From: Erling Rennemo Jellum <erling.r.jellum@ntnu.no>
Date: Fri, 10 Nov 2023 09:51:11 +0100
Subject: [PATCH 43/76] Update lf-ref

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 3443e2c23..7e9223177 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-enclaves3
+enclaves4

From 6d67ceb2ecf0c8ed362580301f586f3c6d952488 Mon Sep 17 00:00:00 2001
From: Erling Rennemo Jellum <erling.r.jellum@ntnu.no>
Date: Fri, 10 Nov 2023 11:28:24 +0100
Subject: [PATCH 44/76] Fix superflous newline in logging

---
 core/federated/RTI/main.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c
index d051edcf6..9380179d0 100644
--- a/core/federated/RTI/main.c
+++ b/core/federated/RTI/main.c
@@ -194,7 +194,7 @@ int process_args(int argc, const char* argv[]) {
                 return 0;
             }
             rti.base.number_of_scheduling_nodes = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines
-            lf_print("RTI: Number of federates: %d\n", rti.base.number_of_scheduling_nodes);
+            lf_print("RTI: Number of federates: %d", rti.base.number_of_scheduling_nodes);
         } else if (strcmp(argv[i], "-p") == 0 || strcmp(argv[i], "--port") == 0) {
             if (argc < i + 2) {
                 lf_print_error(

From f9e787bf8832b5747a1fdc0305dbae6d3b7662ff Mon Sep 17 00:00:00 2001
From: Erling Rennemo Jellum <erling.r.jellum@ntnu.no>
Date: Fri, 10 Nov 2023 11:29:12 +0100
Subject: [PATCH 45/76] RTI: Add zero-delay-cycle detection

---
 core/federated/RTI/rti_common.c | 51 ++++++++++++++++++++++++++++++++-
 core/federated/RTI/rti_common.h |  4 +++
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index c042d25db..fe234b40b 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -46,7 +46,7 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) {
     e->downstream = NULL;
     e->num_downstream = 0;
     e->mode = REALTIME;
-
+    e->is_in_zero_delay_cycle = false;
 }
 
 void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
@@ -269,4 +269,53 @@ void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermedi
     }
 }
 
+// Cycle-detection algorithm adapted from: https://rohithv63.medium.com/graph-algorithm-cycle-detection-in-directed-graph-using-dfs-939512865fd6
+static bool _find_cycles(int v, bool visited[], bool recStack[], scheduling_node_t** nodes, int num_nodes) {
+    if (recStack[v]) {
+        return true;
+    }
+    if (visited[v]) {
+        return false;
+    }
+    
+    visited[v] = true;
+    recStack[v] = true;
+
+    scheduling_node_t *temp = nodes[v];
+    for (int i = 0; i<temp->num_upstream; i++) {
+        if (temp->upstream_delay[i] != NEVER) { // NEVER means zero-delay
+            continue;
+        }
+    
+        if (_find_cycles(i, visited, recStack, nodes, num_nodes)) {
+            return true;
+        }
+    }
+    recStack[v] = false; // Remove the node from the current path
+    return false;
+}
+
+void find_cycles(scheduling_node_t** nodes, int num_nodes) {
+    bool* visited = (bool*)malloc(num_nodes * sizeof(bool));
+    bool* recStack = (bool*)malloc(num_nodes * sizeof(bool));
+
+    for (int i = 0; i < num_nodes; i++) {
+        visited[i] = 0;
+        recStack[i] = 0;
+    }
+
+    for (int i = 0; i < num_nodes; i++) {
+        if (!visited[i]) {
+            if (_find_cycles(i, visited, recStack, nodes, num_nodes)) {
+                LF_PRINT_LOG("Node %i part of zero-delay cycle", i);
+                nodes[i]->is_in_zero_delay_cycle = true;
+            } else {
+            }
+        }
+    }
+
+    free(visited);
+    free(recStack);
+}
+
 #endif
diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h
index 3e87a1a1e..38ec74011 100644
--- a/core/federated/RTI/rti_common.h
+++ b/core/federated/RTI/rti_common.h
@@ -59,6 +59,7 @@ typedef struct scheduling_node_t {
     int* downstream;                    // Array of downstream scheduling node ids.
     int num_downstream;                 // Size of the array of downstream scheduling nodes.
     execution_mode_t mode;              // FAST or REALTIME.
+    bool is_in_zero_delay_cycle;         // This scheduling node is part of a zero-delay cycle
 } scheduling_node_t;
 
 /**
@@ -257,5 +258,8 @@ void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermedi
  */
 void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number_of_scheduling_nodes);
 
+
+void find_cycles(scheduling_node_t** nodes, int num_nodes);
+
 #endif // RTI_COMMON_H
 #endif // STANDALONE_RTI || LF_ENCLAVES

From 65f491ce2091bdbd9008696458749ac24af6b614 Mon Sep 17 00:00:00 2001
From: Erling Rennemo Jellum <erling.r.jellum@ntnu.no>
Date: Fri, 10 Nov 2023 11:29:42 +0100
Subject: [PATCH 46/76] RTI: Only give a PTAG if there is a zero-delay cycle

---
 core/federated/RTI/rti_common.c | 33 +++++++++++++++++++++------------
 core/federated/RTI/rti_remote.c |  3 +++
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index fe234b40b..cb18bff0f 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -155,12 +155,20 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
     LF_PRINT_LOG("RTI: Earliest next event upstream of node %d has tag " PRINTF_TAG ".",
             e->id, t_d.time - start_time, t_d.microstep);
 
-    if (
-        lf_tag_compare(t_d, e->next_event) > 0       // The enclave has something to do.
+    // Given an EIMT there are three possible scenarios.
+    //  1) The EIMT is greater than the NET we want to advance to.
+    //  2) The EIMT is greater, however, the federate is part of a zero-delay cycle (ZDC)
+    //  3) The EIMT is equal to the NET.
+    //  4) The EIMT is less than the NET
+    // In (1) we can give a TAG to NET. In (2) and (3) we can give a PTAG.
+
+    if ( // Scenario (1) above
+        lf_tag_compare(t_d, e->next_event) > 0                      // EIMT greater than NET
         && lf_tag_compare(t_d, e->last_provisionally_granted) >= 0  // The grant is not redundant
                                                                       // (equal is important to override any previous
                                                                       // PTAGs).
-        && lf_tag_compare(t_d, e->last_granted) > 0  // The grant is not redundant.
+        && lf_tag_compare(t_d, e->last_granted) > 0                   // The grant is not redundant.
+        && !e->is_in_zero_delay_cycle                               // The node is not part of a ZDC
     ) {
         // No upstream node can send events that will be received with a tag less than or equal to
         // e->next_event, so it is safe to send a TAG.
@@ -171,19 +179,20 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
                 e->next_event.time - lf_time_start(),
                 e->next_event.microstep);
         result.tag = e->next_event;
-    } else if (
-        // FIXME: if e->is_in_zero_delay_cycle
-        lf_tag_compare(t_d, e->next_event) == 0      // The enclave has something to do/
-        && lf_tag_compare(t_d, e->last_provisionally_granted) > 0  // The grant is not redundant.
-        && lf_tag_compare(t_d, e->last_granted) > 0  // The grant is not redundant.
-    ) {
+    } else if( // Scenario (2) or (3) above
+        lf_tag_compare(t_d, e->next_event) >= 0                      // EIMT greater than or equal to NET
+        && lf_tag_compare(t_d, e->last_provisionally_granted) >= 0  // The grant is not redundant
+        && lf_tag_compare(t_d, e->last_granted) > 0                 // The grant is not redundant.
+    ) { 
         // Some upstream node may send an event that has the same tag as this node's next event,
         // so we can only grant a PTAG.
         LF_PRINT_LOG("RTI: Earliest upstream message time for fed/encl %d is " PRINTF_TAG
-            " (adjusted by after delay). Granting provisional tag advance (PTAG).",
+            " (adjusted by after delay). Granting provisional tag advance (PTAG) for " PRINTF_TAG,
             e->id,
-            t_d.time - start_time, t_d.microstep);
-        result.tag = t_d;
+            t_d.time - start_time, t_d.microstep,
+            e->next_event.time - lf_time_start(),
+            e->next_event.microstep);
+        result.tag = e->next_event;
         result.is_provisional = true;
     }
     return result;
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index a58cec380..613394389 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1577,6 +1577,9 @@ void wait_for_federates(int socket_descriptor) {
     // Wait for connections from federates and create a thread for each.
     connect_to_federates(socket_descriptor);
 
+    // Do cycle detection
+    find_cycles(rti_remote->base.scheduling_nodes, rti_remote->base.number_of_scheduling_nodes);
+
     // All federates have connected.
     lf_print("RTI: All expected federates have connected. Starting execution.");
 

From a192cf97c3aad08b968cfd15dc8fb8c8efc46399 Mon Sep 17 00:00:00 2001
From: Erling Rennemo Jellum <erling.r.jellum@ntnu.no>
Date: Fri, 10 Nov 2023 11:46:39 +0100
Subject: [PATCH 47/76] RTI: Print out ZDC detected by the RTI

---
 core/federated/RTI/rti_common.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index cb18bff0f..8f5e9546d 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -312,16 +312,22 @@ void find_cycles(scheduling_node_t** nodes, int num_nodes) {
         visited[i] = 0;
         recStack[i] = 0;
     }
-
+    bool first=true;
     for (int i = 0; i < num_nodes; i++) {
         if (!visited[i]) {
             if (_find_cycles(i, visited, recStack, nodes, num_nodes)) {
-                LF_PRINT_LOG("Node %i part of zero-delay cycle", i);
+                if (first) {
+                    printf("RTI: Nodes part of a zero-delay-cycle:");
+                    first = false;
+                }
+                printf(" %i,", i);
                 nodes[i]->is_in_zero_delay_cycle = true;
-            } else {
             }
         }
     }
+    if (!first) {
+        printf("\n");
+    }
 
     free(visited);
     free(recStack);

From 301dac63de250accf9a3712cca77dab886ee7be8 Mon Sep 17 00:00:00 2001
From: Erling Rennemo Jellum <erling.r.jellum@ntnu.no>
Date: Fri, 10 Nov 2023 11:51:43 +0100
Subject: [PATCH 48/76] Improve documentation

---
 core/federated/RTI/rti_common.c | 16 ++++++++++------
 core/federated/RTI/rti_common.h | 11 +++++++++--
 core/federated/RTI/rti_remote.c |  2 +-
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index 8f5e9546d..fa4a665e5 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -278,8 +278,12 @@ void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermedi
     }
 }
 
-// Cycle-detection algorithm adapted from: https://rohithv63.medium.com/graph-algorithm-cycle-detection-in-directed-graph-using-dfs-939512865fd6
-static bool _find_cycles(int v, bool visited[], bool recStack[], scheduling_node_t** nodes, int num_nodes) {
+/**
+ * Cycle-detection algorithm based on Depth-first search. adapted from: 
+ * https://rohithv63.medium.com/graph-algorithm-cycle-detection-in-directed-graph-using-dfs-939512865fd6
+ * 
+ */
+static bool _find_zero_delay_cycles(int v, bool visited[], bool recStack[], scheduling_node_t** nodes, int num_nodes) {
     if (recStack[v]) {
         return true;
     }
@@ -296,7 +300,7 @@ static bool _find_cycles(int v, bool visited[], bool recStack[], scheduling_node
             continue;
         }
     
-        if (_find_cycles(i, visited, recStack, nodes, num_nodes)) {
+        if (_find_zero_delay_cycles(i, visited, recStack, nodes, num_nodes)) {
             return true;
         }
     }
@@ -304,7 +308,7 @@ static bool _find_cycles(int v, bool visited[], bool recStack[], scheduling_node
     return false;
 }
 
-void find_cycles(scheduling_node_t** nodes, int num_nodes) {
+void find_zero_delay_cycles(scheduling_node_t** nodes, int num_nodes) {
     bool* visited = (bool*)malloc(num_nodes * sizeof(bool));
     bool* recStack = (bool*)malloc(num_nodes * sizeof(bool));
 
@@ -312,10 +316,10 @@ void find_cycles(scheduling_node_t** nodes, int num_nodes) {
         visited[i] = 0;
         recStack[i] = 0;
     }
-    bool first=true;
+    bool first=true; // For printing the ZDC on a single line
     for (int i = 0; i < num_nodes; i++) {
         if (!visited[i]) {
-            if (_find_cycles(i, visited, recStack, nodes, num_nodes)) {
+            if (_find_zero_delay_cycles(i, visited, recStack, nodes, num_nodes)) {
                 if (first) {
                     printf("RTI: Nodes part of a zero-delay-cycle:");
                     first = false;
diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h
index 38ec74011..6ef6ae476 100644
--- a/core/federated/RTI/rti_common.h
+++ b/core/federated/RTI/rti_common.h
@@ -258,8 +258,15 @@ void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermedi
  */
 void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number_of_scheduling_nodes);
 
-
-void find_cycles(scheduling_node_t** nodes, int num_nodes);
+/**
+ * @brief Search the directed graph of nodes and find all nodes that are part of
+ * a zero-delay cycle (ZDC). These nodes are marked by setting the `is_part_of_zero_delay_cycle`
+ * field of the scheduling_node_t struct.
+ * 
+ * @param nodes An array of scheduling node pointers
+ * @param num_nodes The length of the array
+ */
+void find_zero_delay_cycles(scheduling_node_t** nodes, int num_nodes);
 
 #endif // RTI_COMMON_H
 #endif // STANDALONE_RTI || LF_ENCLAVES
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 613394389..efd77f9e3 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1578,7 +1578,7 @@ void wait_for_federates(int socket_descriptor) {
     connect_to_federates(socket_descriptor);
 
     // Do cycle detection
-    find_cycles(rti_remote->base.scheduling_nodes, rti_remote->base.number_of_scheduling_nodes);
+    find_zero_delay_cycles(rti_remote->base.scheduling_nodes, rti_remote->base.number_of_scheduling_nodes);
 
     // All federates have connected.
     lf_print("RTI: All expected federates have connected. Starting execution.");

From b3e2251a7322d6d7ee15902ee4ef87fab18c6b23 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Fri, 10 Nov 2023 10:32:53 -0800
Subject: [PATCH 49/76] Comments only

---
 core/federated/RTI/rti_local.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/core/federated/RTI/rti_local.h b/core/federated/RTI/rti_local.h
index 93b36c656..41b05053e 100644
--- a/core/federated/RTI/rti_local.h
+++ b/core/federated/RTI/rti_local.h
@@ -45,7 +45,8 @@ void free_local_rti();
 void initialize_enclave_info(enclave_info_t* enclave, int idx, environment_t *env);
 
 /**
- * @brief This function call may block. A call to this function serves two purposes. 
+ * @brief Notify the local RTI of a next event tag (NET).
+ * This function call may block. A call to this function serves two purposes. 
  * 1) It is a promise that, unless receiving events from other enclaves, this
  * enclave will not produce any event until the next_event_tag (NET) argument.
  * 2) It is a request for permission to advance the logical tag of the enclave
@@ -54,6 +55,8 @@ void initialize_enclave_info(enclave_info_t* enclave, int idx, environment_t *en
  * This function call will block until the enclave has been granted a TAG,
  * which might not be the tag requested.
  * 
+ * This function assumes the mutex lock of enclave's environment is held.
+ * 
  * @param enclave The enclave requesting to advance to the NET.
  * @param next_event_tag The tag of the next event in the enclave
  * @return tag_t A tag which the enclave can safely advance its time to. It 

From 7fe502ac412a13fcd8f418f9cb85cbbb33e7bbd6 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Mon, 13 Nov 2023 09:29:34 -0700
Subject: [PATCH 50/76] Update lingua-franca-ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index b31da30ef..0cf11b8bf 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-81819f2df9addee1ca86d33bd699826eff4c0ed9
+b5e73d091cd9275fc5bbdc42c936d67a934512ec

From 9ff9ebe95bc51e1c72da02b4450f4d2dcc425eb0 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 14 Nov 2023 07:35:03 +0100
Subject: [PATCH 51/76] Make lf_delay_tag exactly match the theory

---
 core/tag.c | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/core/tag.c b/core/tag.c
index 54fade355..91f05bfa3 100644
--- a/core/tag.c
+++ b/core/tag.c
@@ -142,7 +142,8 @@ int lf_tag_compare(tag_t tag1, tag_t tag2) {
 
 tag_t lf_delay_tag(tag_t tag, interval_t interval) {
     if (tag.time == NEVER || interval < 0LL) return tag;
-    if (tag.time >= FOREVER - interval) return tag;
+    // Note that overflow in C is undefined for signed variables.
+    if (tag.time >= FOREVER - interval) return FOREVER_TAG; // Overflow.
     tag_t result = tag;
     if (interval == 0LL) {
         // Note that unsigned variables will wrap on overflow.
@@ -150,12 +151,7 @@ tag_t lf_delay_tag(tag_t tag, interval_t interval) {
         // microsteps.
         result.microstep++;
     } else {
-        // Note that overflow in C is undefined for signed variables.
-        if (FOREVER - interval < result.time) {
-            result.time = FOREVER;
-        } else {
-            result.time += interval;
-        }
+        result.time += interval;
         result.microstep = 0;
     }
     return result;

From 2d2757f4e6fed509fd8179b97236a9a1ce3d9e6f Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Fri, 10 Nov 2023 17:32:07 -0800
Subject: [PATCH 52/76] Refactor pqueue part 1

---
 core/federated/RTI/CMakeLists.txt             |   1 +
 core/utils/CMakeLists.txt                     |   2 +-
 core/utils/{pqueue.c => pqueue_base.c}        | 102 +-----------------
 .../core/utils/{pqueue.h => pqueue_base.h}    |  28 ++---
 4 files changed, 9 insertions(+), 124 deletions(-)
 rename core/utils/{pqueue.c => pqueue_base.c} (81%)
 rename include/core/utils/{pqueue.h => pqueue_base.h} (90%)

diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt
index 9208f6df0..087d39c23 100644
--- a/core/federated/RTI/CMakeLists.txt
+++ b/core/federated/RTI/CMakeLists.txt
@@ -69,6 +69,7 @@ add_executable(
     ${CoreLib}/utils/util.c
     ${CoreLib}/tag.c
     ${CoreLib}/federated/net_util.c
+    ${CoreLib}/utils/pqueue_base.c
     ${CoreLib}/utils/pqueue.c
     message_record/message_record.c
 )
diff --git a/core/utils/CMakeLists.txt b/core/utils/CMakeLists.txt
index 41e96ff50..9cf3b5905 100644
--- a/core/utils/CMakeLists.txt
+++ b/core/utils/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(UTIL_SOURCES vector.c pqueue.c util.c semaphore.c)
+set(UTIL_SOURCES vector.c pqueue_base.c pqueue.c util.c semaphore.c)
 
 list(APPEND INFO_SOURCES ${UTIL_SOURCES})
 
diff --git a/core/utils/pqueue.c b/core/utils/pqueue_base.c
similarity index 81%
rename from core/utils/pqueue.c
rename to core/utils/pqueue_base.c
index aa1f1bd15..d19a14d61 100644
--- a/core/utils/pqueue.c
+++ b/core/utils/pqueue_base.c
@@ -37,10 +37,8 @@
 #include <assert.h>
 #include <stdbool.h>
 
-#include "platform.h"
-#include "pqueue.h"
+#include "pqueue_base.h"
 #include "util.h"
-#include "lf_types.h"
 
 #define LF_LEFT(i)   ((i) << 1)
 #define LF_RIGHT(i)  (((i) << 1) + 1)
@@ -375,101 +373,3 @@ static int subtree_is_valid(pqueue_t *q, int pos) {
 int pqueue_is_valid(pqueue_t *q) {
     return subtree_is_valid(q, 1);
 }
-
-// ********** Priority Queue Support Start
-
-/**
- * Return whether the first and second argument are given in reverse order.
- */
-int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that) {
-    return (thiz > that);
-}
-
-/**
- * Return false (0) regardless of reaction order.
- */
-int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that) {
-    return false;
-}
-
-/**
- * Return whether or not the given events have matching triggers.
- */
-int event_matches(void* next, void* curr) {
-    return (((event_t*)next)->trigger == ((event_t*)curr)->trigger);
-}
-
-/**
- * Return whether or not the given reaction_t pointers
- * point to the same struct.
- */
-int reaction_matches(void* next, void* curr) {
-    return (next == curr);
-}
-
-/**
- * Report a priority equal to the time of the given event.
- * Used for sorting pointers to event_t structs in the event queue.
- */
-pqueue_pri_t get_event_time(void *a) {
-    return (pqueue_pri_t)(((event_t*) a)->time);
-}
-
-/**
- * Report a priority equal to the index of the given reaction.
- * Used for sorting pointers to reaction_t structs in the
- * blocked and executing queues.
- */
-pqueue_pri_t get_reaction_index(void *a) {
-    return ((reaction_t*) a)->index;
-}
-
-/**
- * Return the given event's position in the queue.
- */
-size_t get_event_position(void *a) {
-    return ((event_t*) a)->pos;
-}
-
-/**
- * Return the given reaction's position in the queue.
- */
-size_t get_reaction_position(void *a) {
-    return ((reaction_t*) a)->pos;
-}
-
-/**
- * Set the given event's position in the queue.
- */
-void set_event_position(void *a, size_t pos) {
-    ((event_t*) a)->pos = pos;
-}
-
-/**
- * Return the given reaction's position in the queue.
- */
-void set_reaction_position(void *a, size_t pos) {
-    ((reaction_t*) a)->pos = pos;
-}
-
-/**
- * Print some information about the given reaction.
- *
- * DEBUG function only.
- */
-void print_reaction(void *reaction) {
-    reaction_t *r = (reaction_t*)reaction;
-    LF_PRINT_DEBUG("%s: chain_id:%llu, index: %llx, reaction: %p",
-            r->name, r->chain_id, r->index, r);
-}
-
-/**
- * Print some information about the given event.
- *
- * DEBUG function only.
- */
-void print_event(void *event) {
-    event_t *e = (event_t*)event;
-    LF_PRINT_DEBUG("time: " PRINTF_TIME ", trigger: %p, token: %p",
-            e->time, e->trigger, e->token);
-}
diff --git a/include/core/utils/pqueue.h b/include/core/utils/pqueue_base.h
similarity index 90%
rename from include/core/utils/pqueue.h
rename to include/core/utils/pqueue_base.h
index 5c3e7fe2b..e6a0bbe0c 100644
--- a/include/core/utils/pqueue.h
+++ b/include/core/utils/pqueue_base.h
@@ -21,6 +21,7 @@
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * 
  * Modified by Marten Lohstroh (May, 2019).
  * Changes:
  * - Require implementation of a pqueue_eq_elem_f function to determine
@@ -28,18 +29,15 @@
  * - The provided pqueue_eq_elem_f implementation is used to test and
  *   search for equal elements present in the queue; and
  * - Removed capability to reassign priorities.
- */
-
-/**
- * @file  pqueue.h
- * @brief Priority Queue function declarations
+ *
+ * @brief Priority Queue function declarations used as a base for Lingua Franca priority queues.
  *
  * @{
  */
 
 
-#ifndef PQUEUE_H
-#define PQUEUE_H
+#ifndef PQUEUE_BASE_H
+#define PQUEUE_BASE_H
 
 #include <stddef.h>
 
@@ -212,19 +210,5 @@ pqueue_dump(pqueue_t *q,
  */
 int pqueue_is_valid(pqueue_t *q);
 
-// ********** Priority Queue Support Start
-int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
-int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that);
-int event_matches(void* next, void* curr);
-int reaction_matches(void* next, void* curr);
-pqueue_pri_t get_event_time(void *a);
-pqueue_pri_t get_reaction_index(void *a);
-size_t get_event_position(void *a);
-size_t get_reaction_position(void *a);
-void set_event_position(void *a, size_t pos);
-void set_reaction_position(void *a, size_t pos);
-void print_reaction(void *reaction);
-void print_event(void *event);
-
-#endif /* PQUEUE_H */
+#endif /* PQUEUE_BASE_H */
 /** @} */

From 515e6aea9bc569fdce340a050f8f9527fc4996e0 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Fri, 10 Nov 2023 17:33:02 -0800
Subject: [PATCH 53/76] Refactor pqueue part 2

---
 core/utils/pqueue.c         | 74 ++++++++++++++++++++++++++++++
 include/core/utils/pqueue.h | 91 +++++++++++++++++++++++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 core/utils/pqueue.c
 create mode 100644 include/core/utils/pqueue.h

diff --git a/core/utils/pqueue.c b/core/utils/pqueue.c
new file mode 100644
index 000000000..6c02f7b65
--- /dev/null
+++ b/core/utils/pqueue.c
@@ -0,0 +1,74 @@
+/**
+ * @file pqueue.c
+ * @author Marten Lohstroh
+ * @author Edward A. Lee
+ * @copyright (c) 2020-2023, The University of California at Berkeley.
+ * License: <a href="https://github.com/lf-lang/reactor-c/blob/main/LICENSE.md">BSD 2-clause</a>
+ * 
+ * @brief Priority queue definitions for the event queue and reaction queue.
+ */
+
+// FIXME: Trim
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <stdbool.h>
+
+#include "platform.h"
+#include "pqueue.h"
+#include "util.h"
+#include "lf_types.h"
+
+
+int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that) {
+    return (thiz > that);
+}
+
+int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that) {
+    return 0;
+}
+
+int event_matches(void* event1, void* event2) {
+    return (((event_t*)event1)->trigger == ((event_t*)event2)->trigger);
+}
+
+int reaction_matches(void* a, void* b) {
+    return (a == b);
+}
+
+pqueue_pri_t get_event_time(void *event) {
+    return (pqueue_pri_t)(((event_t*) event)->time);
+}
+
+pqueue_pri_t get_reaction_index(void *reaction) {
+    return ((reaction_t*) reaction)->index;
+}
+
+size_t get_event_position(void *event) {
+    return ((event_t*) event)->pos;
+}
+
+size_t get_reaction_position(void *reaction) {
+    return ((reaction_t*) reaction)->pos;
+}
+
+void set_event_position(void *event, size_t pos) {
+    ((event_t*) event)->pos = pos;
+}
+
+void set_reaction_position(void *reaction, size_t pos) {
+    ((reaction_t*) reaction)->pos = pos;
+}
+
+void print_reaction(void *reaction) {
+    reaction_t *r = (reaction_t*)reaction;
+    LF_PRINT_DEBUG("%s: chain_id: %llu, index: %llx, reaction: %p",
+            r->name, r->chain_id, r->index, r);
+}
+
+void print_event(void *event) {
+    event_t *e = (event_t*)event;
+    LF_PRINT_DEBUG("time: " PRINTF_TIME ", trigger: %p, token: %p",
+            e->time, e->trigger, e->token);
+}
diff --git a/include/core/utils/pqueue.h b/include/core/utils/pqueue.h
new file mode 100644
index 000000000..b6ca9bca5
--- /dev/null
+++ b/include/core/utils/pqueue.h
@@ -0,0 +1,91 @@
+/**
+ * @file pqueue.h
+ * @author Marten Lohstroh
+ * @author Edward A. Lee
+ * @copyright (c) 2020-2023, The University of California at Berkeley.
+ * License: <a href="https://github.com/lf-lang/reactor-c/blob/main/LICENSE.md">BSD 2-clause</a>
+ * 
+ * @brief Priority queue declarations for the event queue and reaction queue.
+ */
+
+#ifndef PQUEUE_H
+#define PQUEUE_H
+
+#include "pqueue_base.h"
+
+/**
+ * Return 1 if the first argument is greater than the second and zero otherwise.
+ */
+int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
+
+/**
+ * Return 0 regardless of argument order.
+ */
+int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that);
+
+/**
+ * Return 1 if the two events have the same trigger.
+ * @param event1 A pointer to an event_t.
+ * @param event2 A pointer to an event_t.
+ */
+int event_matches(void* event1, void* event2);
+
+/**
+ * Return 1 if the two arguments are identical pointers.
+ */
+int reaction_matches(void* a, void* b);
+
+/**
+ * Report a priority equal to the time of the given event.
+ * This is used for sorting pointers to event_t structs in the event queue.
+ * @param a A pointer to an event_t.
+ */
+pqueue_pri_t get_event_time(void *event);
+
+/**
+ * Report a priority equal to the index of the given reaction.
+ * Used for sorting pointers to reaction_t structs in the
+ * blocked and executing queues.
+ * @param reaction A pointer to a reaction_t.
+ */
+pqueue_pri_t get_reaction_index(void *reaction_t);
+
+/**
+ * Return the given event's position in the queue.
+ * @param event A pointer to an event_t.
+ */
+size_t get_event_position(void *event);
+
+/**
+ * Return the given reaction's position in the queue.
+ * @param reaction A pointer to a reaction_t.
+ */
+size_t get_reaction_position(void *reaction);
+
+/**
+ * Set the given event's position in the queue.
+ * @param event A pointer to an event_t
+ * @param pos The position.
+ */
+void set_event_position(void *event, size_t pos);
+
+/**
+ * Set the given reaction's position in the queue.
+ * @param event A pointer to a reaction_t.
+ * @param pos The position.
+ */
+void set_reaction_position(void *reaction, size_t pos);
+
+/**
+ * Print some information about the given reaction.
+ * This only prints something if logging is set to DEBUG.
+ */
+void print_reaction(void *reaction);
+
+/**
+ * Print some information about the given event.
+ * This only prints something if logging is set to DEBUG.
+ */
+void print_event(void *event);
+
+#endif /* PQUEUE_H */

From 544e0999be5f93fbd366b0801836005442b9fdd5 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Fri, 10 Nov 2023 18:00:05 -0800
Subject: [PATCH 54/76] Docs and lf-ref

---
 core/utils/pqueue_base.c    | 18 ------------------
 docs/README.md              |  8 +++-----
 include/core/utils/pqueue.h |  8 ++++++++
 lingua-franca-ref.txt       |  2 +-
 4 files changed, 12 insertions(+), 24 deletions(-)

diff --git a/core/utils/pqueue_base.c b/core/utils/pqueue_base.c
index d19a14d61..9e6ddecbb 100644
--- a/core/utils/pqueue_base.c
+++ b/core/utils/pqueue_base.c
@@ -44,10 +44,6 @@
 #define LF_RIGHT(i)  (((i) << 1) + 1)
 #define LF_PARENT(i) ((i) >> 1)
 
-/**
- * Find an element in the queue that matches the given element up to
- * and including the given maximum priority.
- */
 void* find_equal(pqueue_t *q, void *e, int pos, pqueue_pri_t max) {
     if (pos < 0) {
         lf_print_error_and_exit("find_equal() called with a negative pos index.");
@@ -80,11 +76,6 @@ void* find_equal(pqueue_t *q, void *e, int pos, pqueue_pri_t max) {
     return NULL;
 }
 
-/**
- * Find an element in the queue that matches the given element up to
- * but not including the given maximum priority. The matching element
- * has to _also_ have the same priority.
- */
 void* find_equal_same_priority(pqueue_t *q, void *e, int pos) {
     if (pos < 0) {
         lf_print_error_and_exit("find_equal_same_priority() called with a negative pos index.");
@@ -268,14 +259,6 @@ void* pqueue_pop(pqueue_t *q) {
     return head;
 }
 
-/**
- * @brief Empty 'src' into 'dest'.
- *
- * As an optimization, this function might swap 'src' and 'dest'.
- *
- * @param dest The queue to fill up
- * @param src  The queue to empty
- */
 void pqueue_empty_into(pqueue_t** dest, pqueue_t** src) {
     assert(src);
     assert(dest);
@@ -337,7 +320,6 @@ void pqueue_print(pqueue_t *q, pqueue_print_entry_f print) {
     pqueue_free(dup);
 }
 
-
 static int subtree_is_valid(pqueue_t *q, int pos) {
     if (pos < 0) {
         lf_print_error_and_exit("subtree_is_valid() called with a negative pos index.");
diff --git a/docs/README.md b/docs/README.md
index 3ece865a8..2174ca664 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -17,11 +17,9 @@ To build the doc files locally in your clone of the reactor-c repo, we use sphin
 
 - Install `python3`, `pip3` and `doxygen`
 - Install the required Python modules:
-  - `pip3 install sphinx`
-  - `pip3 install sphinx_sitemap`
-  - `pip3 install sphinx-rtd-theme`
-  - `pip3 install breathe`
-  - `pip3 install exhale`
+```
+  pip3 install sphinx sphinx_sitemap sphinx-rtd-theme breathe exhale
+```
 
 ### Build Documentation Files
 
diff --git a/include/core/utils/pqueue.h b/include/core/utils/pqueue.h
index b6ca9bca5..edfd4968c 100644
--- a/include/core/utils/pqueue.h
+++ b/include/core/utils/pqueue.h
@@ -15,11 +15,15 @@
 
 /**
  * Return 1 if the first argument is greater than the second and zero otherwise.
+ * @param thiz First argument.
+ * @param that Second argument.
  */
 int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
 
 /**
  * Return 0 regardless of argument order.
+ * @param thiz First argument.
+ * @param that Second argument.
  */
 int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that);
 
@@ -32,6 +36,8 @@ int event_matches(void* event1, void* event2);
 
 /**
  * Return 1 if the two arguments are identical pointers.
+ * @param a First argument.
+ * @param b Second argument.
  */
 int reaction_matches(void* a, void* b);
 
@@ -79,12 +85,14 @@ void set_reaction_position(void *reaction, size_t pos);
 /**
  * Print some information about the given reaction.
  * This only prints something if logging is set to DEBUG.
+ * @param reaction A pointer to a reaction_t.
  */
 void print_reaction(void *reaction);
 
 /**
  * Print some information about the given event.
  * This only prints something if logging is set to DEBUG.
+ * @param event A pointer to an event_t.
  */
 void print_event(void *event);
 
diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 7e9223177..c4e40568f 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-enclaves4
+c-enclaves

From aaad588bbd4ef60725cb18d26afb469af7620669 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sat, 11 Nov 2023 12:09:36 -0800
Subject: [PATCH 55/76] First step at a pqueue with tag sorting

---
 core/federated/RTI/CMakeLists.txt |  1 +
 core/tag.c                        |  3 -
 core/utils/CMakeLists.txt         |  2 +-
 core/utils/pqueue_tag.c           | 92 +++++++++++++++++++++++++++++++
 include/core/utils/pqueue_base.h  | 23 ++++----
 include/core/utils/pqueue_tag.h   | 50 +++++++++++++++++
 test/general/utils/pqueue_test.c  | 15 +++++
 7 files changed, 172 insertions(+), 14 deletions(-)
 create mode 100644 core/utils/pqueue_tag.c
 create mode 100644 include/core/utils/pqueue_tag.h
 create mode 100644 test/general/utils/pqueue_test.c

diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt
index 087d39c23..fc33b47bc 100644
--- a/core/federated/RTI/CMakeLists.txt
+++ b/core/federated/RTI/CMakeLists.txt
@@ -70,6 +70,7 @@ add_executable(
     ${CoreLib}/tag.c
     ${CoreLib}/federated/net_util.c
     ${CoreLib}/utils/pqueue_base.c
+    ${CoreLib}/utils/pqueue_tag.c
     ${CoreLib}/utils/pqueue.c
     message_record/message_record.c
 )
diff --git a/core/tag.c b/core/tag.c
index 91f05bfa3..eee10699f 100644
--- a/core/tag.c
+++ b/core/tag.c
@@ -171,9 +171,6 @@ instant_t lf_time_logical(void *env) {
     return ((environment_t *) env)->current_tag.time;
 }
 
-/**
- * Return the elapsed logical time in nanoseconds since the start of execution.
- */
 interval_t lf_time_logical_elapsed(void *env) {
     return lf_time_logical(env) - start_time;
 }
diff --git a/core/utils/CMakeLists.txt b/core/utils/CMakeLists.txt
index 9cf3b5905..fea1e5468 100644
--- a/core/utils/CMakeLists.txt
+++ b/core/utils/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(UTIL_SOURCES vector.c pqueue_base.c pqueue.c util.c semaphore.c)
+set(UTIL_SOURCES vector.c pqueue_base.c pqueue_tag.c pqueue.c util.c semaphore.c)
 
 list(APPEND INFO_SOURCES ${UTIL_SOURCES})
 
diff --git a/core/utils/pqueue_tag.c b/core/utils/pqueue_tag.c
new file mode 100644
index 000000000..e94457c5e
--- /dev/null
+++ b/core/utils/pqueue_tag.c
@@ -0,0 +1,92 @@
+/**
+ * @file pqueue_tag.c
+ * @author Byeonggil Jun
+ * @author Edward A. Lee
+ * @copyright (c) 2023, The University of California at Berkeley
+ * License in [BSD 2-clause](https://github.com/lf-lang/reactor-c/blob/main/LICENSE.md)
+ * 
+ * @brief Priority queue that uses tags for sorting.
+ */
+
+#include "pqueue_tag.h"
+#include "util.h"     // For lf_print
+#include "platform.h" // For PRINTF_TAG
+
+//////////////////
+// Local functions, not intended for use outside this file.
+
+/**
+ * @brief Callback function to get the priority of an element.
+ * Return the pointer argument cast to pqueue_pri_t because the
+ * element is also the priority. This function is of type pqueue_get_pri_f.
+ * @param element A pointer to a pqueue_tag_element_t, cast to void*.
+ */
+static pqueue_pri_t pqueue_tag_get_priority(void *element) {
+    return (pqueue_pri_t) element;
+}
+
+/**
+ * @brief Callback comparison function for the tag-based priority queue.
+ * Return -1 if the first argument is less than second, 0 if they are equal,
+ * and +1 otherwise. This function is of type pqueue_cmp_pri_f.
+ * @param priority1 A pointer to a pqueue_tag_element_t, cast to pqueue_pri_t.
+ * @param priority2 A pointer to a pqueue_tag_element_t, cast to pqueue_pri_t.
+*/
+static int pqueue_tag_compare(pqueue_pri_t priority1, pqueue_pri_t priority2) {
+    return lf_tag_compare(((pqueue_tag_element_t*) priority1)->tag, ((pqueue_tag_element_t*) priority2)->tag);
+}
+
+/**
+ * @brief Callback function to determine whether two elements are equivalent.
+ * Return 1 if the tags contained by given elements are identical, 0 otherwise.
+ * This function is of type pqueue_eq_elem_f.
+ * @param element1 A pointer to a pqueue_tag_element_t, cast to void*.
+ * @param element2 A pointer to a pqueue_tag_element_t, cast to void*.
+ */
+static int pqueue_tag_matches(void* element1, void* element2) {
+    return lf_tag_compare(((pqueue_tag_element_t*) element1)->tag, ((pqueue_tag_element_t*) element2)->tag) == 0;
+}
+
+/**
+ * @brief Callback function to return the position of an element.
+ * This function is of type pqueue_get_pos_f.
+ * @param element A pointer to a pqueue_tag_element_t, cast to void*.
+ */
+static size_t pqueue_tag_get_position(void *element) {
+    return ((pqueue_tag_element_t*)element)->pos;
+}
+
+/**
+ * @brief Callback function to set the position of an element.
+ * This function is of type pqueue_set_pos_f.
+ * @param element A pointer to a pqueue_tag_element_t, cast to void*.
+ * @param pos The position.
+ */
+static void pqueue_tag_set_position(void *element, size_t pos) {
+    ((pqueue_tag_element_t*)element)->pos = pos;
+}
+
+/**
+ * @brief Callback function to print information about an element.
+ * This is a function of type pqueue_print_entry_f.
+ * @param element A pointer to a pqueue_tag_element_t, cast to void*.
+ */
+static void pqueue_tag_print(void *element) {
+    tag_t tag = ((pqueue_tag_element_t*) element)->tag;
+    lf_print("Element with tag " PRINTF_TAG " and payload %p.\n",
+           tag.time, tag.microstep, ((pqueue_tag_element_t*) element)->payload);
+}
+
+//////////////////
+// Functions defined in pqueue_tag.h.
+
+pqueue_tag_t* pqueue_tag_init(size_t initial_size) {
+    return (pqueue_tag_t*) pqueue_init(
+            initial_size,
+            pqueue_tag_compare,
+            pqueue_tag_get_priority,
+            pqueue_tag_get_position,
+            pqueue_tag_set_position,
+            pqueue_tag_matches,
+            pqueue_tag_print);
+}
diff --git a/include/core/utils/pqueue_base.h b/include/core/utils/pqueue_base.h
index e6a0bbe0c..f7d98b85e 100644
--- a/include/core/utils/pqueue_base.h
+++ b/include/core/utils/pqueue_base.h
@@ -41,23 +41,28 @@
 
 #include <stddef.h>
 
-/** priority data type */
+/** Priority data type. */
 typedef unsigned long long pqueue_pri_t;
 
-/** callback functions to get/set/compare the priority of an element */
+/** Callback to get the priority of an element. */
 typedef pqueue_pri_t (*pqueue_get_pri_f)(void *a);
-typedef void (*pqueue_set_pri_f)(void *a, pqueue_pri_t pri);
+
+/** Callback to compare two priorities. */
 typedef int (*pqueue_cmp_pri_f)(pqueue_pri_t next, pqueue_pri_t curr);
+
+/** Callback to determine whether two elements are equivalent. */
 typedef int (*pqueue_eq_elem_f)(void* next, void* curr);
 
-/** callback functions to get/set the position of an element */
+/** Callback functions to get the position of an element. */
 typedef size_t (*pqueue_get_pos_f)(void *a);
+
+/** Callback functions to set the position of an element. */
 typedef void (*pqueue_set_pos_f)(void *a, size_t pos);
 
-/** debug callback function to print a entry */
+/** Debug callback function to print a entry. */
 typedef void (*pqueue_print_entry_f)(void *a);
 
-/** the priority queue handle */
+/** The priority queue handle. */
 typedef struct pqueue_t
 {
     size_t size;                /**< number of elements in this queue plus 1 */
@@ -73,7 +78,7 @@ typedef struct pqueue_t
 } pqueue_t;
 
 /**
- * initialize the queue
+ * @brief Allocate and initialize a priority queue.
  *
  * @param n the initial estimate of the number of queue items for which memory
  *     should be preallocated
@@ -84,7 +89,7 @@ typedef struct pqueue_t
  * @param getpos the callback function to get the current element's position
  * @param setpos the callback function to set the current element's position
  *
- * @return the handle or NULL for insufficent memory
+ * @return The handle or NULL for insufficent memory.
  */
 pqueue_t *
 pqueue_init(size_t n,
@@ -102,7 +107,6 @@ pqueue_init(size_t n,
  */
 void pqueue_free(pqueue_t *q);
 
-
 /**
  * return the size of the queue.
  * @param q the queue
@@ -128,7 +132,6 @@ pqueue_change_priority(pqueue_t *q,
                        pqueue_pri_t new_pri,
                        void *d);
 
-
 /**
  * Pop the highest-ranking item from the queue.
  * @param q the queue
diff --git a/include/core/utils/pqueue_tag.h b/include/core/utils/pqueue_tag.h
new file mode 100644
index 000000000..070198fd2
--- /dev/null
+++ b/include/core/utils/pqueue_tag.h
@@ -0,0 +1,50 @@
+/**
+ * @file tag_pqueue.h
+ * @author Byeonggil Jun
+ * @author Edward A. Lee
+ * @copyright (c) 2023, The University of California at Berkeley
+ * License in [BSD 2-clause](https://github.com/lf-lang/reactor-c/blob/main/LICENSE.md)
+ * @brief Priority queue that uses tags for sorting.
+ * 
+ * This file extends the pqueue infrastructure with support for queues that are sorted
+ * by tag instead of by a long long.  Elements in this queue are structs of type
+ * tagged_element_t, which can carry a payload that is a pointer (or an int, assuming
+ * that an int has no more bits than a pointer). What you put onto the
+ * queue is a pointer to a tagged_element_t struct. That pointer is cast to pqueue_pri_t,
+ * an alias for long long.
+ */
+
+#ifndef PQUEUE_TAG_H
+#define PQUEUE_TAG_H
+
+#include "pqueue_base.h"
+#include "tag.h"
+
+/**
+ * @brief An element in the priority queue, sorted by tag.
+ * The payload can be pointer to anything or an int, assuming that an int
+ * can be safely cast to a pointer.  The payload may also go unused if you
+ * want just a priority queue with tags only.  In this design, a pointer to
+ * this struct is priority (can be cast to pqueue_pri_t) and also an element
+ * in the queue (can be cast to void*).
+ */
+typedef struct {
+    tag_t tag;
+    size_t pos;   // Needed by any pqueue element.
+    void* payload;
+} pqueue_tag_element_t;
+
+/**
+ * Type of a priority queue using tags.
+ */
+typedef pqueue_t pqueue_tag_t;
+
+/**
+ * @brief Create a priority queue sorted by tags.
+ * The elements of the priority queue will be of type pqueue_tag_element_t.
+ * The caller should call pqueue_tag_free() when finished with the queue.
+ * @return A dynamically allocated priority queue or NULL if memory allocation fails.
+ */
+pqueue_tag_t* pqueue_tag_init(size_t initial_size);
+
+#endif // PQUEUE_TAG_H
\ No newline at end of file
diff --git a/test/general/utils/pqueue_test.c b/test/general/utils/pqueue_test.c
new file mode 100644
index 000000000..c0ebea285
--- /dev/null
+++ b/test/general/utils/pqueue_test.c
@@ -0,0 +1,15 @@
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+#include "pqueue_tag.h"
+#include "tag.h"
+
+static void trivial(void) {
+    // Create an event queue.
+    pqueue_t* q = pqueue_tag_init(1);
+    assert(q != NULL);
+}
+
+int main(int argc, char *argv[]) {
+    trivial();
+}

From 0406ee7db8ac909ea8eacdf36fe10786bca2ea02 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sat, 11 Nov 2023 17:49:23 -0800
Subject: [PATCH 56/76] Implemented and tested priority queue with tag ordering

---
 core/utils/pqueue_base.c         |  10 ++-
 core/utils/pqueue_tag.c          |  57 +++++++++++++++--
 include/core/utils/pqueue_base.h |  16 ++---
 include/core/utils/pqueue_tag.h  | 106 +++++++++++++++++++++++++++----
 test/general/utils/pqueue_test.c |  58 ++++++++++++++++-
 5 files changed, 214 insertions(+), 33 deletions(-)

diff --git a/core/utils/pqueue_base.c b/core/utils/pqueue_base.c
index 9e6ddecbb..9bba5289e 100644
--- a/core/utils/pqueue_base.c
+++ b/core/utils/pqueue_base.c
@@ -314,9 +314,13 @@ void pqueue_print(pqueue_t *q, pqueue_print_entry_f print) {
 
     memcpy(dup->d, q->d, (q->size * sizeof(void *)));
 
-    while ((e = pqueue_pop(dup)))
-        print(e);
-
+    while ((e = pqueue_pop(dup))) {
+        if (print == NULL) {
+            q->prt(e);
+        } else {
+            print(e);
+        }
+    }
     pqueue_free(dup);
 }
 
diff --git a/core/utils/pqueue_tag.c b/core/utils/pqueue_tag.c
index e94457c5e..24bc298ec 100644
--- a/core/utils/pqueue_tag.c
+++ b/core/utils/pqueue_tag.c
@@ -8,6 +8,8 @@
  * @brief Priority queue that uses tags for sorting.
  */
 
+#include <stdlib.h>
+
 #include "pqueue_tag.h"
 #include "util.h"     // For lf_print
 #include "platform.h" // For PRINTF_TAG
@@ -27,13 +29,13 @@ static pqueue_pri_t pqueue_tag_get_priority(void *element) {
 
 /**
  * @brief Callback comparison function for the tag-based priority queue.
- * Return -1 if the first argument is less than second, 0 if they are equal,
- * and +1 otherwise. This function is of type pqueue_cmp_pri_f.
+ * Return 0 if the first argument is less than second and 1 otherwise.
+ * This function is of type pqueue_cmp_pri_f.
  * @param priority1 A pointer to a pqueue_tag_element_t, cast to pqueue_pri_t.
  * @param priority2 A pointer to a pqueue_tag_element_t, cast to pqueue_pri_t.
 */
 static int pqueue_tag_compare(pqueue_pri_t priority1, pqueue_pri_t priority2) {
-    return lf_tag_compare(((pqueue_tag_element_t*) priority1)->tag, ((pqueue_tag_element_t*) priority2)->tag);
+    return (lf_tag_compare(((pqueue_tag_element_t*) priority1)->tag, ((pqueue_tag_element_t*) priority2)->tag) >= 0);
 }
 
 /**
@@ -71,10 +73,9 @@ static void pqueue_tag_set_position(void *element, size_t pos) {
  * This is a function of type pqueue_print_entry_f.
  * @param element A pointer to a pqueue_tag_element_t, cast to void*.
  */
-static void pqueue_tag_print(void *element) {
+static void pqueue_tag_print_element(void *element) {
     tag_t tag = ((pqueue_tag_element_t*) element)->tag;
-    lf_print("Element with tag " PRINTF_TAG " and payload %p.\n",
-           tag.time, tag.microstep, ((pqueue_tag_element_t*) element)->payload);
+    lf_print("Element with tag " PRINTF_TAG ".", tag.time, tag.microstep);
 }
 
 //////////////////
@@ -88,5 +89,47 @@ pqueue_tag_t* pqueue_tag_init(size_t initial_size) {
             pqueue_tag_get_position,
             pqueue_tag_set_position,
             pqueue_tag_matches,
-            pqueue_tag_print);
+            pqueue_tag_print_element);
+}
+
+void pqueue_tag_free(pqueue_tag_t *q) {
+    for (int i = 1; i < q->size ;i++) {
+        if (((pqueue_tag_element_t*)q->d[i])->is_dynamic) {
+            free(q->d[i]);
+        }
+    }
+    pqueue_free((pqueue_t*)q);
+}
+
+size_t pqueue_tag_size(pqueue_tag_t *q) {
+    return pqueue_size((pqueue_t*)q);
+}
+
+int pqueue_tag_insert(pqueue_tag_t* q, pqueue_tag_element_t* d) {
+    return pqueue_insert((pqueue_t*)q, (void*)d);
+}
+
+int pqueue_tag_insert_tag(pqueue_tag_t* q, tag_t t) {
+    pqueue_tag_element_t* d = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
+    d->is_dynamic = 1;
+    d->tag = t;
+    return pqueue_tag_insert(q, d);
+}
+
+pqueue_tag_element_t* pqueue_tag_pop(pqueue_tag_t* q) {
+    return (pqueue_tag_element_t*)pqueue_pop((pqueue_t*)q);
+}
+
+tag_t pqueue_tag_pop_tag(pqueue_tag_t* q) {
+    pqueue_tag_element_t* element = (pqueue_tag_element_t*)pqueue_tag_pop(q);
+    if (element == NULL) return FOREVER_TAG;
+    else return element->tag;
+}
+
+int pqueue_tag_remove(pqueue_tag_t* q, pqueue_tag_element_t* e) {
+    return pqueue_remove((pqueue_t*) q, (void*) e);
+}
+
+pqueue_tag_element_t* pqueue_tag_peek(pqueue_tag_t* q) {
+    return (pqueue_tag_element_t*) pqueue_peek((pqueue_t*)q);
 }
diff --git a/include/core/utils/pqueue_base.h b/include/core/utils/pqueue_base.h
index f7d98b85e..210cc0eec 100644
--- a/include/core/utils/pqueue_base.h
+++ b/include/core/utils/pqueue_base.h
@@ -88,6 +88,8 @@ typedef struct pqueue_t
  * @param getpri the callback function to run to set a score to an element
  * @param getpos the callback function to get the current element's position
  * @param setpos the callback function to set the current element's position
+ * @param eqelem the callback function to check equivalence of entries
+ * @param prt the callback function to print an element
  *
  * @return The handle or NULL for insufficent memory.
  */
@@ -100,7 +102,6 @@ pqueue_init(size_t n,
             pqueue_eq_elem_f eqelem,
             pqueue_print_entry_f prt);
 
-
 /**
  * free all memory used by the queue
  * @param q the queue
@@ -183,16 +184,13 @@ int pqueue_remove(pqueue_t *q, void *e);
  */
 void *pqueue_peek(pqueue_t *q);
 
+
 /**
- * Print the queue.
- * @internal
- * DEBUG function only
- * @param q the queue
- * @param the callback function to print the entry
+ * Print the contents of the queue.
+ * @param q The queue.
+ * @param print The callback function to print the entry or NULL to use the default.
  */
-void
-pqueue_print(pqueue_t *q,
-             pqueue_print_entry_f print);
+void pqueue_print(pqueue_t *q, pqueue_print_entry_f print);
 
 /**
  * Dump the queue and it's internal structure.
diff --git a/include/core/utils/pqueue_tag.h b/include/core/utils/pqueue_tag.h
index 070198fd2..2133af8b6 100644
--- a/include/core/utils/pqueue_tag.h
+++ b/include/core/utils/pqueue_tag.h
@@ -8,10 +8,9 @@
  * 
  * This file extends the pqueue infrastructure with support for queues that are sorted
  * by tag instead of by a long long.  Elements in this queue are structs of type
- * tagged_element_t, which can carry a payload that is a pointer (or an int, assuming
- * that an int has no more bits than a pointer). What you put onto the
- * queue is a pointer to a tagged_element_t struct. That pointer is cast to pqueue_pri_t,
- * an alias for long long.
+ * pqueue_tag_element_t or a derived struct, as explained below. What you put onto the
+ * queue is a pointer to a tagged_element_t struct. That pointer, when cast to pqueue_pri_t,
+ * an alias for long long, also serves as the "priority" for the queue.
  */
 
 #ifndef PQUEUE_TAG_H
@@ -21,21 +20,39 @@
 #include "tag.h"
 
 /**
- * @brief An element in the priority queue, sorted by tag.
- * The payload can be pointer to anything or an int, assuming that an int
- * can be safely cast to a pointer.  The payload may also go unused if you
- * want just a priority queue with tags only.  In this design, a pointer to
- * this struct is priority (can be cast to pqueue_pri_t) and also an element
- * in the queue (can be cast to void*).
+ * @brief The type for an element in a priority queue that is sorted by tag.
+ * 
+ * In this design, a pointer to this struct is also a "priority" (it can be
+ * cast to pqueue_pri_t).  The actual priority is the tag field of the struct,
+ * in that the queue is sorted from least tag to largest.
+ * 
+ * If your struct is dynamically allocated using malloc or calloc, and you
+ * would like the memory freed when the queue is freed, then set the is_dynamic
+ * field to a non-zero value.
+ * 
+ * To customize the element you put onto the queue, you can create your
+ * own element struct type by simply declaring the first field to be
+ * a pqueue_tag_element_t.  For example, if you want an element of the
+ * queue to include a pointer to your own payload, you can declare the
+ * following struct type:
+ * <pre>
+ *     typedef struct {
+ *         pqueue_tag_element_t base;
+ *         my_type* my_payload;
+ *     } my_element_type_t;
+ * </pre>
+ * When inserting your struct into the queue, simply cast your pointer
+ * to (pqueue_tag_element_t*).  When accessing your struct from the queue,
+ * simply cast the result to (my_element_type_t*);
  */
 typedef struct {
     tag_t tag;
-    size_t pos;   // Needed by any pqueue element.
-    void* payload;
+    size_t pos;       // Needed by any pqueue element.
+    int is_dynamic;   // Non-zero to free this struct when the queue is freed.
 } pqueue_tag_element_t;
 
 /**
- * Type of a priority queue using tags.
+ * Type of a priority queue sorted by tags.
  */
 typedef pqueue_t pqueue_tag_t;
 
@@ -47,4 +64,67 @@ typedef pqueue_t pqueue_tag_t;
  */
 pqueue_tag_t* pqueue_tag_init(size_t initial_size);
 
+/**
+ * Free all memory used by the queue including any elements that are marked is_dynamic.
+ * @param q The queue.
+ */
+void pqueue_tag_free(pqueue_tag_t *q);
+
+/**
+ * Return the size of the queue.
+ * @param q The queue.
+ */
+size_t pqueue_tag_size(pqueue_tag_t *q);
+
+/**
+ * @brief Insert a tag into the queue.
+ * This automatically allocates memory for the element in the queue
+ * and ensures that if the element is still on the queue when pqueue_tag_free
+ * is called, that memory will be freed.
+ * @param q The queue.
+ * @param t The tag to insert.
+ * @return 0 on success
+ */
+int pqueue_tag_insert_tag(pqueue_tag_t* q, tag_t t);
+
+/**
+ * @brief Pop the least-tag element from the queue and return its tag.
+ * If the queue is empty, return FOREVER_TAG.
+ * @param q The queue.
+ * @return NULL on error, otherwise the entry
+ */
+tag_t pqueue_tag_pop_tag(pqueue_tag_t* q);
+
+/**
+ * Insert an element into the queue.
+ * @param q The queue.
+ * @param e The element to insert.
+ * @return 0 on success
+ */
+int pqueue_tag_insert(pqueue_tag_t* q, pqueue_tag_element_t* d);
+
+/**
+ * @brief Pop the least-tag element from the queue.
+ * If the entry was dynamically allocated, then it is now up to the caller
+ * to ensure that it is freed. It will not be freed by pqueue_tag_free.
+ * @param q The queue.
+ * @return NULL on error, otherwise the entry
+ */
+pqueue_tag_element_t* pqueue_tag_pop(pqueue_tag_t* q);
+
+/**
+ * Remove an item from the queue.
+ * @param q The queue.
+ * @param e The entry to remove.
+ * @return 0 on success
+ */
+int pqueue_tag_remove(pqueue_tag_t* q, pqueue_tag_element_t* e);
+
+/**
+ * Access highest-ranking item without removing it.
+ * @param q The queue.
+ * @return NULL on error, otherwise the entry.
+ */
+pqueue_tag_element_t* pqueue_tag_peek(pqueue_tag_t* q);
+
 #endif // PQUEUE_TAG_H
\ No newline at end of file
diff --git a/test/general/utils/pqueue_test.c b/test/general/utils/pqueue_test.c
index c0ebea285..1cd33a385 100644
--- a/test/general/utils/pqueue_test.c
+++ b/test/general/utils/pqueue_test.c
@@ -6,10 +6,66 @@
 
 static void trivial(void) {
     // Create an event queue.
-    pqueue_t* q = pqueue_tag_init(1);
+    pqueue_tag_t* q = pqueue_tag_init(1);
     assert(q != NULL);
+    assert(pqueue_is_valid((pqueue_t*)q));
+    pqueue_print((pqueue_t*)q, NULL);
+    pqueue_tag_free(q);
+}
+
+static void insert_on_queue(pqueue_tag_t* q) {
+    tag_t t1 = {.time = USEC(3), .microstep = 0};
+    tag_t t2 = {.time = USEC(2), .microstep = 1};
+    tag_t t3 = {.time = USEC(2), .microstep = 0};
+    tag_t t4 = {.time = USEC(1), .microstep = 2};
+    assert(!pqueue_tag_insert_tag(q, t1));
+    assert(!pqueue_tag_insert_tag(q, t2));
+    assert(!pqueue_tag_insert_tag(q, t3));
+    assert(!pqueue_tag_insert_tag(q, t4));
+    printf("======== Contents of the queue:\n");
+    pqueue_print((pqueue_t*)q, NULL);
+    assert(pqueue_tag_size(q) == 4);
+}
+
+static void pop_from_queue(pqueue_tag_t* q) {
+    tag_t t1_back = pqueue_tag_pop_tag(q);
+    assert(t1_back.time == USEC(1));
+    assert(t1_back.microstep == 2);
+    tag_t t2_back = pqueue_tag_pop_tag(q);
+    assert(t2_back.time == USEC(2));
+    assert(t2_back.microstep == 0);
+    tag_t t3_back = pqueue_tag_pop_tag(q);
+    assert(t3_back.time == USEC(2));
+    assert(t3_back.microstep == 1);
+    tag_t t4_back = pqueue_tag_pop_tag(q);
+    assert(t4_back.time == USEC(3));
+    assert(t4_back.microstep == 0);
+}
+
+static void pop_empty(pqueue_tag_t* q) {
+    assert(pqueue_tag_size(q) == 0);
+    assert(pqueue_tag_pop(q) == NULL);
+}
+
+static void remove_from_queue(pqueue_tag_t* q) {
+    pqueue_tag_element_t e1 = {.tag = {.time = USEC(3), .microstep = 0}, .pos = 0, .is_dynamic = 0};
+    pqueue_tag_element_t e2 = {.tag = {.time = USEC(2), .microstep = 0}, .pos = 0, .is_dynamic = 0};
+    assert(pqueue_tag_insert(q, &e1) == 0);
+    assert(pqueue_tag_insert(q, &e2) == 0);
+    assert(pqueue_tag_remove(q, &e1) == 0);
+    assert(pqueue_tag_peek(q) == &e2);
+    assert(pqueue_tag_size(q) == 1);
 }
 
 int main(int argc, char *argv[]) {
     trivial();
+    // Create an event queue.
+    pqueue_tag_t* q = pqueue_tag_init(2);
+
+    insert_on_queue(q);
+    pop_from_queue(q);
+    pop_empty(q);
+    remove_from_queue(q);
+
+    pqueue_tag_free(q);
 }

From 899b5e89eb9d8e3c936080486663eeb310a1d04e Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sun, 12 Nov 2023 08:08:20 -0800
Subject: [PATCH 57/76] Free element on popping a tag

---
 core/utils/pqueue_tag.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/core/utils/pqueue_tag.c b/core/utils/pqueue_tag.c
index 24bc298ec..44039848b 100644
--- a/core/utils/pqueue_tag.c
+++ b/core/utils/pqueue_tag.c
@@ -94,7 +94,7 @@ pqueue_tag_t* pqueue_tag_init(size_t initial_size) {
 
 void pqueue_tag_free(pqueue_tag_t *q) {
     for (int i = 1; i < q->size ;i++) {
-        if (((pqueue_tag_element_t*)q->d[i])->is_dynamic) {
+        if (q->d[i] != NULL && ((pqueue_tag_element_t*)q->d[i])->is_dynamic) {
             free(q->d[i]);
         }
     }
@@ -123,7 +123,11 @@ pqueue_tag_element_t* pqueue_tag_pop(pqueue_tag_t* q) {
 tag_t pqueue_tag_pop_tag(pqueue_tag_t* q) {
     pqueue_tag_element_t* element = (pqueue_tag_element_t*)pqueue_tag_pop(q);
     if (element == NULL) return FOREVER_TAG;
-    else return element->tag;
+    else {
+        tag_t result = element->tag;
+        if (element->is_dynamic) free(element);
+        return result;
+    }
 }
 
 int pqueue_tag_remove(pqueue_tag_t* q, pqueue_tag_element_t* e) {

From 21501bf321931f00fe7d011f8e3197922c654492 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sun, 12 Nov 2023 08:08:31 -0800
Subject: [PATCH 58/76] Fix memory error in unit test

---
 test/general/utils/pqueue_test.c | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/test/general/utils/pqueue_test.c b/test/general/utils/pqueue_test.c
index 1cd33a385..7b01e7a64 100644
--- a/test/general/utils/pqueue_test.c
+++ b/test/general/utils/pqueue_test.c
@@ -47,13 +47,11 @@ static void pop_empty(pqueue_tag_t* q) {
     assert(pqueue_tag_pop(q) == NULL);
 }
 
-static void remove_from_queue(pqueue_tag_t* q) {
-    pqueue_tag_element_t e1 = {.tag = {.time = USEC(3), .microstep = 0}, .pos = 0, .is_dynamic = 0};
-    pqueue_tag_element_t e2 = {.tag = {.time = USEC(2), .microstep = 0}, .pos = 0, .is_dynamic = 0};
-    assert(pqueue_tag_insert(q, &e1) == 0);
-    assert(pqueue_tag_insert(q, &e2) == 0);
-    assert(pqueue_tag_remove(q, &e1) == 0);
-    assert(pqueue_tag_peek(q) == &e2);
+static void remove_from_queue(pqueue_tag_t* q, pqueue_tag_element_t* e1, pqueue_tag_element_t* e2) {
+    assert(pqueue_tag_insert(q, e1) == 0);
+    assert(pqueue_tag_insert(q, e2) == 0);
+    assert(pqueue_tag_remove(q, e1) == 0);
+    assert(pqueue_tag_peek(q) == e2);
     assert(pqueue_tag_size(q) == 1);
 }
 
@@ -65,7 +63,11 @@ int main(int argc, char *argv[]) {
     insert_on_queue(q);
     pop_from_queue(q);
     pop_empty(q);
-    remove_from_queue(q);
+
+    pqueue_tag_element_t e1 = {.tag = {.time = USEC(3), .microstep = 0}, .pos = 0, .is_dynamic = 0};
+    pqueue_tag_element_t e2 = {.tag = {.time = USEC(2), .microstep = 0}, .pos = 0, .is_dynamic = 0};
+
+    remove_from_queue(q, &e1, &e2);
 
     pqueue_tag_free(q);
 }

From d0dfdb51766e081e51528278189917e663c6fb48 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 15 Nov 2023 09:46:01 +0900
Subject: [PATCH 59/76] Do not insert a duplicate tag into the tag-based
 priority queue

---
 core/utils/pqueue_tag.c          | 45 +++++++++++++++++++++++++++++++-
 test/general/utils/pqueue_test.c |  3 +++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/core/utils/pqueue_tag.c b/core/utils/pqueue_tag.c
index 44039848b..34ae4fdbc 100644
--- a/core/utils/pqueue_tag.c
+++ b/core/utils/pqueue_tag.c
@@ -14,6 +14,10 @@
 #include "util.h"     // For lf_print
 #include "platform.h" // For PRINTF_TAG
 
+#define LF_LEFT(i)   ((i) << 1)
+#define LF_RIGHT(i)  (((i) << 1) + 1)
+#define LF_PARENT(i) ((i) >> 1)
+
 //////////////////
 // Local functions, not intended for use outside this file.
 
@@ -35,7 +39,7 @@ static pqueue_pri_t pqueue_tag_get_priority(void *element) {
  * @param priority2 A pointer to a pqueue_tag_element_t, cast to pqueue_pri_t.
 */
 static int pqueue_tag_compare(pqueue_pri_t priority1, pqueue_pri_t priority2) {
-    return (lf_tag_compare(((pqueue_tag_element_t*) priority1)->tag, ((pqueue_tag_element_t*) priority2)->tag) >= 0);
+    return (lf_tag_compare(((pqueue_tag_element_t*) priority1)->tag, ((pqueue_tag_element_t*) priority2)->tag) > 0);
 }
 
 /**
@@ -78,6 +82,40 @@ static void pqueue_tag_print_element(void *element) {
     lf_print("Element with tag " PRINTF_TAG ".", tag.time, tag.microstep);
 }
 
+void* pqueue_tag_find_equal_same_priority(pqueue_t *q, void *e, int pos) {
+    if (pos < 0) {
+        lf_print_error_and_exit("find_equal_same_priority() called with a negative pos index.");
+    }
+
+    // Stop the recursion when we've reached the end of the
+    // queue. This has to be done before accessing the queue
+    // to avoid segmentation fault.
+    if (!q || (size_t)pos >= q->size) {
+        return NULL;
+    }
+
+    void* rval;
+    void* curr = q->d[pos];
+
+    // Stop the recursion once we've surpassed the priority of the element
+    // we're looking for.
+    if (!curr || q->cmppri(q->getpri(curr), q->getpri(e))) {
+        return NULL;
+    }
+
+    if (pqueue_tag_matches(q->getpri(curr), q->getpri(e)) && q->eqelem(curr, e)) {
+        return curr;
+    } else {
+        rval = pqueue_tag_find_equal_same_priority(q, e, LF_LEFT(pos));
+        if (rval) {
+            return rval;
+        } else {
+            return pqueue_tag_find_equal_same_priority(q, e, LF_RIGHT(pos));
+        }
+    }
+    return NULL;
+}
+
 //////////////////
 // Functions defined in pqueue_tag.h.
 
@@ -113,6 +151,11 @@ int pqueue_tag_insert_tag(pqueue_tag_t* q, tag_t t) {
     pqueue_tag_element_t* d = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
     d->is_dynamic = 1;
     d->tag = t;
+    if (pqueue_tag_find_equal_same_priority(q, d, 1) != NULL) {
+        LF_PRINT_LOG("The tag d " PRINTF_TAG " is already in the queue. Aborting.\n",
+        t.time, t.microstep);
+        return 1;
+    }
     return pqueue_tag_insert(q, d);
 }
 
diff --git a/test/general/utils/pqueue_test.c b/test/general/utils/pqueue_test.c
index 7b01e7a64..943d9d98e 100644
--- a/test/general/utils/pqueue_test.c
+++ b/test/general/utils/pqueue_test.c
@@ -22,6 +22,9 @@ static void insert_on_queue(pqueue_tag_t* q) {
     assert(!pqueue_tag_insert_tag(q, t2));
     assert(!pqueue_tag_insert_tag(q, t3));
     assert(!pqueue_tag_insert_tag(q, t4));
+
+    assert(pqueue_tag_insert_tag(q, t4));
+    assert(pqueue_tag_insert_tag(q, t1));
     printf("======== Contents of the queue:\n");
     pqueue_print((pqueue_t*)q, NULL);
     assert(pqueue_tag_size(q) == 4);

From e437092e398bb7137505cd4c057c8dcdd3f60d32 Mon Sep 17 00:00:00 2001
From: erling <erling.jellum@gmail.com>
Date: Mon, 13 Nov 2023 13:33:45 +0100
Subject: [PATCH 60/76] Update lingua-franca-ref.txt

---
 lingua-franca-ref.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index c4e40568f..c9fc081b7 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1,5 @@
+<<<<<<< HEAD
 c-enclaves
+=======
+master
+>>>>>>> 349cd589 (Update lingua-franca-ref.txt)

From 7dba9f3c7d7d83cf6b21ca4e79b78f4dbb681ade Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 15 Nov 2023 10:40:30 +0900
Subject: [PATCH 61/76] Add a function for searching the tag and a test for it

---
 core/utils/pqueue_tag.c          | 73 +++++++++++++++-----------------
 include/core/utils/pqueue_tag.h  |  8 ++++
 test/general/utils/pqueue_test.c | 38 +++++++++++++++--
 3 files changed, 77 insertions(+), 42 deletions(-)

diff --git a/core/utils/pqueue_tag.c b/core/utils/pqueue_tag.c
index 34ae4fdbc..c0b2c9851 100644
--- a/core/utils/pqueue_tag.c
+++ b/core/utils/pqueue_tag.c
@@ -82,40 +82,6 @@ static void pqueue_tag_print_element(void *element) {
     lf_print("Element with tag " PRINTF_TAG ".", tag.time, tag.microstep);
 }
 
-void* pqueue_tag_find_equal_same_priority(pqueue_t *q, void *e, int pos) {
-    if (pos < 0) {
-        lf_print_error_and_exit("find_equal_same_priority() called with a negative pos index.");
-    }
-
-    // Stop the recursion when we've reached the end of the
-    // queue. This has to be done before accessing the queue
-    // to avoid segmentation fault.
-    if (!q || (size_t)pos >= q->size) {
-        return NULL;
-    }
-
-    void* rval;
-    void* curr = q->d[pos];
-
-    // Stop the recursion once we've surpassed the priority of the element
-    // we're looking for.
-    if (!curr || q->cmppri(q->getpri(curr), q->getpri(e))) {
-        return NULL;
-    }
-
-    if (pqueue_tag_matches(q->getpri(curr), q->getpri(e)) && q->eqelem(curr, e)) {
-        return curr;
-    } else {
-        rval = pqueue_tag_find_equal_same_priority(q, e, LF_LEFT(pos));
-        if (rval) {
-            return rval;
-        } else {
-            return pqueue_tag_find_equal_same_priority(q, e, LF_RIGHT(pos));
-        }
-    }
-    return NULL;
-}
-
 //////////////////
 // Functions defined in pqueue_tag.h.
 
@@ -151,11 +117,6 @@ int pqueue_tag_insert_tag(pqueue_tag_t* q, tag_t t) {
     pqueue_tag_element_t* d = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
     d->is_dynamic = 1;
     d->tag = t;
-    if (pqueue_tag_find_equal_same_priority(q, d, 1) != NULL) {
-        LF_PRINT_LOG("The tag d " PRINTF_TAG " is already in the queue. Aborting.\n",
-        t.time, t.microstep);
-        return 1;
-    }
     return pqueue_tag_insert(q, d);
 }
 
@@ -173,6 +134,40 @@ tag_t pqueue_tag_pop_tag(pqueue_tag_t* q) {
     }
 }
 
+pqueue_tag_element_t* pqueue_tag_find_same_tag(pqueue_tag_t *q, void *e, int pos) {
+    if (pos < 0) {
+        lf_print_error_and_exit("find_equal_same_priority() called with a negative pos index.");
+    }
+
+    // Stop the recursion when we've reached the end of the
+    // queue. This has to be done before accessing the queue
+    // to avoid segmentation fault.
+    if (!q || (size_t)pos >= q->size) {
+        return NULL;
+    }
+
+    void* rval;
+    void* curr = q->d[pos];
+
+    // Stop the recursion once we've surpassed the priority of the element
+    // we're looking for.
+    if (!curr || q->cmppri(q->getpri(curr), q->getpri(e))) {
+        return NULL;
+    }
+
+    if (q->eqelem(curr, e)) {
+        return curr;
+    } else {
+        rval = pqueue_tag_find_same_tag(q, e, LF_LEFT(pos));
+        if (rval) {
+            return rval;
+        } else {
+            return pqueue_tag_find_same_tag(q, e, LF_RIGHT(pos));
+        }
+    }
+    return NULL;
+}
+
 int pqueue_tag_remove(pqueue_tag_t* q, pqueue_tag_element_t* e) {
     return pqueue_remove((pqueue_t*) q, (void*) e);
 }
diff --git a/include/core/utils/pqueue_tag.h b/include/core/utils/pqueue_tag.h
index 2133af8b6..eabf580b7 100644
--- a/include/core/utils/pqueue_tag.h
+++ b/include/core/utils/pqueue_tag.h
@@ -112,6 +112,14 @@ int pqueue_tag_insert(pqueue_tag_t* q, pqueue_tag_element_t* d);
  */
 pqueue_tag_element_t* pqueue_tag_pop(pqueue_tag_t* q);
 
+/**
+ * Find the highest-ranking item with the same tag.
+ * @param q the queue
+ * @param e the entry to compare against
+ * @return NULL if no matching event has been found, otherwise the entry
+ */
+pqueue_tag_element_t* pqueue_tag_find_same_tag(pqueue_tag_t *q, void *e, int pos);
+
 /**
  * Remove an item from the queue.
  * @param q The queue.
diff --git a/test/general/utils/pqueue_test.c b/test/general/utils/pqueue_test.c
index 943d9d98e..c00779d31 100644
--- a/test/general/utils/pqueue_test.c
+++ b/test/general/utils/pqueue_test.c
@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include <stdlib.h>
 #include <assert.h>
 #include <string.h>
 #include "pqueue_tag.h"
@@ -22,14 +23,44 @@ static void insert_on_queue(pqueue_tag_t* q) {
     assert(!pqueue_tag_insert_tag(q, t2));
     assert(!pqueue_tag_insert_tag(q, t3));
     assert(!pqueue_tag_insert_tag(q, t4));
-
-    assert(pqueue_tag_insert_tag(q, t4));
-    assert(pqueue_tag_insert_tag(q, t1));
     printf("======== Contents of the queue:\n");
     pqueue_print((pqueue_t*)q, NULL);
     assert(pqueue_tag_size(q) == 4);
 }
 
+static void find_from_queue(pqueue_tag_t* q) {
+    tag_t t1 = {.time = USEC(3), .microstep = 0};
+    tag_t t2 = {.time = USEC(2), .microstep = 1};
+    tag_t t3 = {.time = USEC(2), .microstep = 0};
+    tag_t t4 = {.time = USEC(1), .microstep = 2};
+    tag_t t5 = {.time = USEC(0), .microstep = 0};
+    tag_t t6 = {.time = USEC(3), .microstep = 2};
+    pqueue_tag_element_t* d_t1 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
+    d_t1->is_dynamic = 1;
+    d_t1->tag = t1;
+    pqueue_tag_element_t* d_t2 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
+    d_t2->is_dynamic = 1;
+    d_t2->tag = t2;
+    pqueue_tag_element_t* d_t3 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
+    d_t3->is_dynamic = 1;
+    d_t3->tag = t3;
+    pqueue_tag_element_t* d_t4 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
+    d_t4->is_dynamic = 1;
+    d_t4->tag = t4;
+    pqueue_tag_element_t* d_t5 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
+    d_t5->is_dynamic = 1;
+    d_t5->tag = t5;
+    pqueue_tag_element_t* d_t6 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
+    d_t6->is_dynamic = 1;
+    d_t6->tag = t6;
+    assert(pqueue_tag_find_same_tag(q, d_t1, 1) != NULL);
+    assert(pqueue_tag_find_same_tag(q, d_t2, 1) != NULL);
+    assert(pqueue_tag_find_same_tag(q, d_t3, 1) != NULL);
+    assert(pqueue_tag_find_same_tag(q, d_t4, 1) != NULL);
+    assert(pqueue_tag_find_same_tag(q, d_t5, 1) == NULL);
+    assert(pqueue_tag_find_same_tag(q, d_t6, 1) == NULL);
+}
+
 static void pop_from_queue(pqueue_tag_t* q) {
     tag_t t1_back = pqueue_tag_pop_tag(q);
     assert(t1_back.time == USEC(1));
@@ -64,6 +95,7 @@ int main(int argc, char *argv[]) {
     pqueue_tag_t* q = pqueue_tag_init(2);
 
     insert_on_queue(q);
+    find_from_queue(q);
     pop_from_queue(q);
     pop_empty(q);
 

From 1583eee308cf19c68dd827ca10721b4d606763b0 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 15 Nov 2023 10:44:41 +0900
Subject: [PATCH 62/76] Free dynamic memory in the tag-based pqueue unit test

---
 test/general/utils/pqueue_test.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/general/utils/pqueue_test.c b/test/general/utils/pqueue_test.c
index c00779d31..db8c6b883 100644
--- a/test/general/utils/pqueue_test.c
+++ b/test/general/utils/pqueue_test.c
@@ -59,6 +59,14 @@ static void find_from_queue(pqueue_tag_t* q) {
     assert(pqueue_tag_find_same_tag(q, d_t4, 1) != NULL);
     assert(pqueue_tag_find_same_tag(q, d_t5, 1) == NULL);
     assert(pqueue_tag_find_same_tag(q, d_t6, 1) == NULL);
+    
+    free(d_t1);
+    free(d_t2);
+    free(d_t3);
+    free(d_t4);
+    free(d_t5);
+    free(d_t6);
+
 }
 
 static void pop_from_queue(pqueue_tag_t* q) {

From 1101f0d51ddfce296357f789d255901f94991c10 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 15 Nov 2023 15:49:38 +0900
Subject: [PATCH 63/76] Let the searching function only needs a tag to find an
 element

---
 core/utils/pqueue_tag.c          | 74 ++++++++++++++++++--------------
 include/core/utils/pqueue_tag.h  |  6 +--
 test/general/utils/pqueue_test.c | 38 +++-------------
 3 files changed, 51 insertions(+), 67 deletions(-)

diff --git a/core/utils/pqueue_tag.c b/core/utils/pqueue_tag.c
index c0b2c9851..b6eefaa1d 100644
--- a/core/utils/pqueue_tag.c
+++ b/core/utils/pqueue_tag.c
@@ -72,6 +72,42 @@ static void pqueue_tag_set_position(void *element, size_t pos) {
     ((pqueue_tag_element_t*)element)->pos = pos;
 }
 
+pqueue_tag_element_t* find_equal_same_tag(pqueue_tag_t *q, void *e, int pos) {
+    if (pos < 0) {
+        lf_print_error_and_exit("find_equal_same_priority() called with a negative pos index.");
+    }
+
+    // Stop the recursion when we've reached the end of the
+    // queue. This has to be done before accessing the queue
+    // to avoid segmentation fault.
+    if (!q || (size_t)pos >= q->size) {
+        return NULL;
+    }
+
+    void* rval;
+    void* curr = q->d[pos];
+
+    // Stop the recursion once we've surpassed the priority of the element
+    // we're looking for.
+    if (!curr || pqueue_tag_compare(pqueue_tag_get_priority(curr), pqueue_tag_get_priority(e))) {
+        return NULL;
+    }
+
+    // Note that we cannot use the function find_equal_same_priority in pqueue_base.c because
+    // we cannot compare priorities using the "==" operator.
+    if (pqueue_tag_matches((void*)pqueue_tag_get_priority(curr), (void*)pqueue_tag_get_priority(e)) && q->eqelem(curr, e)) {
+        return curr;
+    } else {
+        rval = find_equal_same_tag(q, e, LF_LEFT(pos));
+        if (rval) {
+            return rval;
+        } else {
+            return find_equal_same_tag(q, e, LF_RIGHT(pos));
+        }
+    }
+    return NULL;
+}
+
 /**
  * @brief Callback function to print information about an element.
  * This is a function of type pqueue_print_entry_f.
@@ -134,38 +170,12 @@ tag_t pqueue_tag_pop_tag(pqueue_tag_t* q) {
     }
 }
 
-pqueue_tag_element_t* pqueue_tag_find_same_tag(pqueue_tag_t *q, void *e, int pos) {
-    if (pos < 0) {
-        lf_print_error_and_exit("find_equal_same_priority() called with a negative pos index.");
-    }
-
-    // Stop the recursion when we've reached the end of the
-    // queue. This has to be done before accessing the queue
-    // to avoid segmentation fault.
-    if (!q || (size_t)pos >= q->size) {
-        return NULL;
-    }
-
-    void* rval;
-    void* curr = q->d[pos];
-
-    // Stop the recursion once we've surpassed the priority of the element
-    // we're looking for.
-    if (!curr || q->cmppri(q->getpri(curr), q->getpri(e))) {
-        return NULL;
-    }
-
-    if (q->eqelem(curr, e)) {
-        return curr;
-    } else {
-        rval = pqueue_tag_find_same_tag(q, e, LF_LEFT(pos));
-        if (rval) {
-            return rval;
-        } else {
-            return pqueue_tag_find_same_tag(q, e, LF_RIGHT(pos));
-        }
-    }
-    return NULL;
+pqueue_tag_element_t* pqueue_tag_find_equal_same_tag(pqueue_tag_t *q, tag_t t) {
+    pqueue_tag_element_t* target_element = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
+    target_element->tag = t;
+    pqueue_tag_element_t* result = find_equal_same_tag(q, target_element, 1);
+    free(target_element);
+    return result;
 }
 
 int pqueue_tag_remove(pqueue_tag_t* q, pqueue_tag_element_t* e) {
diff --git a/include/core/utils/pqueue_tag.h b/include/core/utils/pqueue_tag.h
index eabf580b7..4653ae921 100644
--- a/include/core/utils/pqueue_tag.h
+++ b/include/core/utils/pqueue_tag.h
@@ -115,10 +115,10 @@ pqueue_tag_element_t* pqueue_tag_pop(pqueue_tag_t* q);
 /**
  * Find the highest-ranking item with the same tag.
  * @param q the queue
- * @param e the entry to compare against
- * @return NULL if no matching event has been found, otherwise the entry
+ * @param t the tag to compare against
+ * @return NULL if no matching tag has been found, otherwise the entry
  */
-pqueue_tag_element_t* pqueue_tag_find_same_tag(pqueue_tag_t *q, void *e, int pos);
+pqueue_tag_element_t* pqueue_tag_find_equal_same_tag(pqueue_tag_t *q, tag_t t);
 
 /**
  * Remove an item from the queue.
diff --git a/test/general/utils/pqueue_test.c b/test/general/utils/pqueue_test.c
index db8c6b883..d37c88273 100644
--- a/test/general/utils/pqueue_test.c
+++ b/test/general/utils/pqueue_test.c
@@ -35,38 +35,12 @@ static void find_from_queue(pqueue_tag_t* q) {
     tag_t t4 = {.time = USEC(1), .microstep = 2};
     tag_t t5 = {.time = USEC(0), .microstep = 0};
     tag_t t6 = {.time = USEC(3), .microstep = 2};
-    pqueue_tag_element_t* d_t1 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
-    d_t1->is_dynamic = 1;
-    d_t1->tag = t1;
-    pqueue_tag_element_t* d_t2 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
-    d_t2->is_dynamic = 1;
-    d_t2->tag = t2;
-    pqueue_tag_element_t* d_t3 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
-    d_t3->is_dynamic = 1;
-    d_t3->tag = t3;
-    pqueue_tag_element_t* d_t4 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
-    d_t4->is_dynamic = 1;
-    d_t4->tag = t4;
-    pqueue_tag_element_t* d_t5 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
-    d_t5->is_dynamic = 1;
-    d_t5->tag = t5;
-    pqueue_tag_element_t* d_t6 = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
-    d_t6->is_dynamic = 1;
-    d_t6->tag = t6;
-    assert(pqueue_tag_find_same_tag(q, d_t1, 1) != NULL);
-    assert(pqueue_tag_find_same_tag(q, d_t2, 1) != NULL);
-    assert(pqueue_tag_find_same_tag(q, d_t3, 1) != NULL);
-    assert(pqueue_tag_find_same_tag(q, d_t4, 1) != NULL);
-    assert(pqueue_tag_find_same_tag(q, d_t5, 1) == NULL);
-    assert(pqueue_tag_find_same_tag(q, d_t6, 1) == NULL);
-    
-    free(d_t1);
-    free(d_t2);
-    free(d_t3);
-    free(d_t4);
-    free(d_t5);
-    free(d_t6);
-
+    assert(pqueue_tag_find_equal_same_tag(q, t1) != NULL);
+    assert(pqueue_tag_find_equal_same_tag(q, t2) != NULL);
+    assert(pqueue_tag_find_equal_same_tag(q, t3) != NULL);
+    assert(pqueue_tag_find_equal_same_tag(q, t4) != NULL);
+    assert(pqueue_tag_find_equal_same_tag(q, t5) == NULL);
+    assert(pqueue_tag_find_equal_same_tag(q, t6) == NULL);
 }
 
 static void pop_from_queue(pqueue_tag_t* q) {

From cb989e561c90d71a3e1dc649e5ad59612c26cbc5 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 15 Nov 2023 22:09:10 +0900
Subject: [PATCH 64/76] Don't use malloc when finding a tag inside the
 pqueue_tag and add insert_tag_if_not_present

---
 core/utils/pqueue_tag.c          | 30 +++++++++++++++++-------------
 include/core/utils/pqueue_tag.h  | 13 ++++++++++++-
 test/general/utils/pqueue_test.c | 17 ++++++++++-------
 3 files changed, 39 insertions(+), 21 deletions(-)

diff --git a/core/utils/pqueue_tag.c b/core/utils/pqueue_tag.c
index b6eefaa1d..13654bb24 100644
--- a/core/utils/pqueue_tag.c
+++ b/core/utils/pqueue_tag.c
@@ -72,9 +72,9 @@ static void pqueue_tag_set_position(void *element, size_t pos) {
     ((pqueue_tag_element_t*)element)->pos = pos;
 }
 
-pqueue_tag_element_t* find_equal_same_tag(pqueue_tag_t *q, void *e, int pos) {
+pqueue_tag_element_t* find_same_tag(pqueue_tag_t *q, tag_t t, int pos) {
     if (pos < 0) {
-        lf_print_error_and_exit("find_equal_same_priority() called with a negative pos index.");
+        lf_print_error_and_exit("find_equal_same_tag() called with a negative pos index.");
     }
 
     // Stop the recursion when we've reached the end of the
@@ -85,24 +85,24 @@ pqueue_tag_element_t* find_equal_same_tag(pqueue_tag_t *q, void *e, int pos) {
     }
 
     void* rval;
-    void* curr = q->d[pos];
+    pqueue_tag_element_t* curr = (pqueue_tag_element_t*) q->d[pos];
 
     // Stop the recursion once we've surpassed the priority of the element
     // we're looking for.
-    if (!curr || pqueue_tag_compare(pqueue_tag_get_priority(curr), pqueue_tag_get_priority(e))) {
+    if (!curr || lf_tag_compare(curr->tag, t) == 1) {
         return NULL;
     }
 
     // Note that we cannot use the function find_equal_same_priority in pqueue_base.c because
     // we cannot compare priorities using the "==" operator.
-    if (pqueue_tag_matches((void*)pqueue_tag_get_priority(curr), (void*)pqueue_tag_get_priority(e)) && q->eqelem(curr, e)) {
+    if (lf_tag_compare(curr->tag, t) == 0) {
         return curr;
     } else {
-        rval = find_equal_same_tag(q, e, LF_LEFT(pos));
+        rval = find_same_tag(q, t, LF_LEFT(pos));
         if (rval) {
             return rval;
         } else {
-            return find_equal_same_tag(q, e, LF_RIGHT(pos));
+            return find_same_tag(q, t, LF_RIGHT(pos));
         }
     }
     return NULL;
@@ -156,6 +156,14 @@ int pqueue_tag_insert_tag(pqueue_tag_t* q, tag_t t) {
     return pqueue_tag_insert(q, d);
 }
 
+int pqueue_tag_insert_tag_if_not_present(pqueue_tag_t* q, tag_t t) {
+    if (find_same_tag(q, t, 1) == NULL) {
+        return pqueue_tag_insert_tag(q, t);
+    } else {
+        return 1;
+    }
+}
+
 pqueue_tag_element_t* pqueue_tag_pop(pqueue_tag_t* q) {
     return (pqueue_tag_element_t*)pqueue_pop((pqueue_t*)q);
 }
@@ -170,12 +178,8 @@ tag_t pqueue_tag_pop_tag(pqueue_tag_t* q) {
     }
 }
 
-pqueue_tag_element_t* pqueue_tag_find_equal_same_tag(pqueue_tag_t *q, tag_t t) {
-    pqueue_tag_element_t* target_element = (pqueue_tag_element_t*) malloc(sizeof(pqueue_tag_element_t));
-    target_element->tag = t;
-    pqueue_tag_element_t* result = find_equal_same_tag(q, target_element, 1);
-    free(target_element);
-    return result;
+pqueue_tag_element_t* pqueue_tag_find_same_tag(pqueue_tag_t *q, tag_t t) {
+    return find_same_tag(q, t, 1);
 }
 
 int pqueue_tag_remove(pqueue_tag_t* q, pqueue_tag_element_t* e) {
diff --git a/include/core/utils/pqueue_tag.h b/include/core/utils/pqueue_tag.h
index 4653ae921..7280899f9 100644
--- a/include/core/utils/pqueue_tag.h
+++ b/include/core/utils/pqueue_tag.h
@@ -87,6 +87,17 @@ size_t pqueue_tag_size(pqueue_tag_t *q);
  */
 int pqueue_tag_insert_tag(pqueue_tag_t* q, tag_t t);
 
+/**
+ * @brief Insert a tag into the queue if the tag is not in the queue.
+ * This automatically allocates memory for the element in the queue
+ * and ensures that if the element is still on the queue when pqueue_tag_free
+ * is called, that memory will be freed.
+ * @param q The queue.
+ * @param t The tag to insert.
+ * @return 0 on success
+ */
+int pqueue_tag_insert_tag_if_not_present(pqueue_tag_t* q, tag_t t);
+
 /**
  * @brief Pop the least-tag element from the queue and return its tag.
  * If the queue is empty, return FOREVER_TAG.
@@ -118,7 +129,7 @@ pqueue_tag_element_t* pqueue_tag_pop(pqueue_tag_t* q);
  * @param t the tag to compare against
  * @return NULL if no matching tag has been found, otherwise the entry
  */
-pqueue_tag_element_t* pqueue_tag_find_equal_same_tag(pqueue_tag_t *q, tag_t t);
+pqueue_tag_element_t* pqueue_tag_find_same_tag(pqueue_tag_t *q, tag_t t);
 
 /**
  * Remove an item from the queue.
diff --git a/test/general/utils/pqueue_test.c b/test/general/utils/pqueue_test.c
index d37c88273..09f6284de 100644
--- a/test/general/utils/pqueue_test.c
+++ b/test/general/utils/pqueue_test.c
@@ -22,7 +22,10 @@ static void insert_on_queue(pqueue_tag_t* q) {
     assert(!pqueue_tag_insert_tag(q, t1));
     assert(!pqueue_tag_insert_tag(q, t2));
     assert(!pqueue_tag_insert_tag(q, t3));
-    assert(!pqueue_tag_insert_tag(q, t4));
+
+    assert(!pqueue_tag_insert_tag_if_not_present(q, t4));
+    assert(pqueue_tag_insert_tag_if_not_present(q, t1));
+    assert(pqueue_tag_insert_tag_if_not_present(q, t4));
     printf("======== Contents of the queue:\n");
     pqueue_print((pqueue_t*)q, NULL);
     assert(pqueue_tag_size(q) == 4);
@@ -35,12 +38,12 @@ static void find_from_queue(pqueue_tag_t* q) {
     tag_t t4 = {.time = USEC(1), .microstep = 2};
     tag_t t5 = {.time = USEC(0), .microstep = 0};
     tag_t t6 = {.time = USEC(3), .microstep = 2};
-    assert(pqueue_tag_find_equal_same_tag(q, t1) != NULL);
-    assert(pqueue_tag_find_equal_same_tag(q, t2) != NULL);
-    assert(pqueue_tag_find_equal_same_tag(q, t3) != NULL);
-    assert(pqueue_tag_find_equal_same_tag(q, t4) != NULL);
-    assert(pqueue_tag_find_equal_same_tag(q, t5) == NULL);
-    assert(pqueue_tag_find_equal_same_tag(q, t6) == NULL);
+    assert(pqueue_tag_find_same_tag(q, t1) != NULL);
+    assert(pqueue_tag_find_same_tag(q, t2) != NULL);
+    assert(pqueue_tag_find_same_tag(q, t3) != NULL);
+    assert(pqueue_tag_find_same_tag(q, t4) != NULL);
+    assert(pqueue_tag_find_same_tag(q, t5) == NULL);
+    assert(pqueue_tag_find_same_tag(q, t6) == NULL);
 }
 
 static void pop_from_queue(pqueue_tag_t* q) {

From 4e6921d59996f61df0917e63ae7abccfae29c031 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sat, 18 Nov 2023 08:00:25 +0100
Subject: [PATCH 65/76] Removed unnecessary imports

---
 core/utils/pqueue.c | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/core/utils/pqueue.c b/core/utils/pqueue.c
index 6c02f7b65..f7fe4bb67 100644
--- a/core/utils/pqueue.c
+++ b/core/utils/pqueue.c
@@ -8,13 +8,6 @@
  * @brief Priority queue definitions for the event queue and reaction queue.
  */
 
-// FIXME: Trim
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
-#include <stdbool.h>
-
 #include "platform.h"
 #include "pqueue.h"
 #include "util.h"

From a3411eec3db50687017a6e364a09696a42a7ef23 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sat, 18 Nov 2023 09:58:42 +0100
Subject: [PATCH 66/76] Simplified tag insertion

---
 core/utils/pqueue_tag.c          | 56 ++++++-----------------------
 include/core/utils/pqueue_tag.h  | 61 +++++++++++++++++---------------
 test/general/utils/pqueue_test.c | 29 ++++++++++-----
 3 files changed, 63 insertions(+), 83 deletions(-)

diff --git a/core/utils/pqueue_tag.c b/core/utils/pqueue_tag.c
index 13654bb24..579926f99 100644
--- a/core/utils/pqueue_tag.c
+++ b/core/utils/pqueue_tag.c
@@ -14,10 +14,6 @@
 #include "util.h"     // For lf_print
 #include "platform.h" // For PRINTF_TAG
 
-#define LF_LEFT(i)   ((i) << 1)
-#define LF_RIGHT(i)  (((i) << 1) + 1)
-#define LF_PARENT(i) ((i) >> 1)
-
 //////////////////
 // Local functions, not intended for use outside this file.
 
@@ -72,42 +68,6 @@ static void pqueue_tag_set_position(void *element, size_t pos) {
     ((pqueue_tag_element_t*)element)->pos = pos;
 }
 
-pqueue_tag_element_t* find_same_tag(pqueue_tag_t *q, tag_t t, int pos) {
-    if (pos < 0) {
-        lf_print_error_and_exit("find_equal_same_tag() called with a negative pos index.");
-    }
-
-    // Stop the recursion when we've reached the end of the
-    // queue. This has to be done before accessing the queue
-    // to avoid segmentation fault.
-    if (!q || (size_t)pos >= q->size) {
-        return NULL;
-    }
-
-    void* rval;
-    pqueue_tag_element_t* curr = (pqueue_tag_element_t*) q->d[pos];
-
-    // Stop the recursion once we've surpassed the priority of the element
-    // we're looking for.
-    if (!curr || lf_tag_compare(curr->tag, t) == 1) {
-        return NULL;
-    }
-
-    // Note that we cannot use the function find_equal_same_priority in pqueue_base.c because
-    // we cannot compare priorities using the "==" operator.
-    if (lf_tag_compare(curr->tag, t) == 0) {
-        return curr;
-    } else {
-        rval = find_same_tag(q, t, LF_LEFT(pos));
-        if (rval) {
-            return rval;
-        } else {
-            return find_same_tag(q, t, LF_RIGHT(pos));
-        }
-    }
-    return NULL;
-}
-
 /**
  * @brief Callback function to print information about an element.
  * This is a function of type pqueue_print_entry_f.
@@ -156,8 +116,16 @@ int pqueue_tag_insert_tag(pqueue_tag_t* q, tag_t t) {
     return pqueue_tag_insert(q, d);
 }
 
-int pqueue_tag_insert_tag_if_not_present(pqueue_tag_t* q, tag_t t) {
-    if (find_same_tag(q, t, 1) == NULL) {
+pqueue_tag_element_t* pqueue_tag_find_with_tag(pqueue_tag_t *q, tag_t t) {
+    // Create elements on the stack. These elements are only needed during
+    // the duration of this function call, so putting them on the stack is OK.
+    pqueue_tag_element_t element = {.tag = t, .pos = 0, .is_dynamic = false};
+    pqueue_tag_element_t forever = {.tag = FOREVER_TAG, .pos = 0, .is_dynamic = false};
+    return pqueue_find_equal((pqueue_t*)q, (void*)&element, (pqueue_pri_t)&forever);
+}
+
+int pqueue_tag_insert_if_no_match(pqueue_tag_t* q, tag_t t) {
+    if (pqueue_tag_find_with_tag(q, t) == NULL) {
         return pqueue_tag_insert_tag(q, t);
     } else {
         return 1;
@@ -178,10 +146,6 @@ tag_t pqueue_tag_pop_tag(pqueue_tag_t* q) {
     }
 }
 
-pqueue_tag_element_t* pqueue_tag_find_same_tag(pqueue_tag_t *q, tag_t t) {
-    return find_same_tag(q, t, 1);
-}
-
 int pqueue_tag_remove(pqueue_tag_t* q, pqueue_tag_element_t* e) {
     return pqueue_remove((pqueue_t*) q, (void*) e);
 }
diff --git a/include/core/utils/pqueue_tag.h b/include/core/utils/pqueue_tag.h
index 7280899f9..aef72d507 100644
--- a/include/core/utils/pqueue_tag.h
+++ b/include/core/utils/pqueue_tag.h
@@ -28,13 +28,17 @@
  * 
  * If your struct is dynamically allocated using malloc or calloc, and you
  * would like the memory freed when the queue is freed, then set the is_dynamic
- * field to a non-zero value.
+ * field of the element to a non-zero value.
  * 
- * To customize the element you put onto the queue, you can create your
- * own element struct type by simply declaring the first field to be
- * a pqueue_tag_element_t.  For example, if you want an element of the
- * queue to include a pointer to your own payload, you can declare the
- * following struct type:
+ * For a priority queue that contains only tags with no payload, you can
+ * avoid creating the element struct by using the functions
+ * pqueue_tag_insert_tag, pqueue_tag_insert_if_no_match, and pqueue_tag_pop_tag.
+ * 
+ * To customize the element you put onto the queue, for example to carry
+ * a pyaload, you can create your own element struct type by simply declaring
+ * the first field to be a pqueue_tag_element_t.  For example, if you want an
+ * element of the queue to include a pointer to your own payload, you can 
+ * declare the following struct type:
  * <pre>
  *     typedef struct {
  *         pqueue_tag_element_t base;
@@ -76,11 +80,19 @@ void pqueue_tag_free(pqueue_tag_t *q);
  */
 size_t pqueue_tag_size(pqueue_tag_t *q);
 
+/**
+ * Insert an element into the queue.
+ * @param q The queue.
+ * @param e The element to insert.
+ * @return 0 on success
+ */
+int pqueue_tag_insert(pqueue_tag_t* q, pqueue_tag_element_t* d);
+
 /**
  * @brief Insert a tag into the queue.
- * This automatically allocates memory for the element in the queue
+ * This automatically creates a dynamically allocated element in the queue
  * and ensures that if the element is still on the queue when pqueue_tag_free
- * is called, that memory will be freed.
+ * is called, then that memory will be freed.
  * @param q The queue.
  * @param t The tag to insert.
  * @return 0 on success
@@ -88,32 +100,25 @@ size_t pqueue_tag_size(pqueue_tag_t *q);
 int pqueue_tag_insert_tag(pqueue_tag_t* q, tag_t t);
 
 /**
- * @brief Insert a tag into the queue if the tag is not in the queue.
- * This automatically allocates memory for the element in the queue
+ * @brief Insert a tag into the queue if the tag is not already in the queue.
+ * This automatically creates a dynamically allocated element in the queue
  * and ensures that if the element is still on the queue when pqueue_tag_free
- * is called, that memory will be freed.
+ * is called, then that memory will be freed.
  * @param q The queue.
  * @param t The tag to insert.
- * @return 0 on success
+ * @return 0 on success, 1 otherwise.
  */
-int pqueue_tag_insert_tag_if_not_present(pqueue_tag_t* q, tag_t t);
+int pqueue_tag_insert_if_no_match(pqueue_tag_t* q, tag_t t);
 
 /**
  * @brief Pop the least-tag element from the queue and return its tag.
- * If the queue is empty, return FOREVER_TAG.
+ * If the queue is empty, return FOREVER_TAG. This function handles freeing 
+ * the element struct if it was dynamically allocated.
  * @param q The queue.
  * @return NULL on error, otherwise the entry
  */
 tag_t pqueue_tag_pop_tag(pqueue_tag_t* q);
 
-/**
- * Insert an element into the queue.
- * @param q The queue.
- * @param e The element to insert.
- * @return 0 on success
- */
-int pqueue_tag_insert(pqueue_tag_t* q, pqueue_tag_element_t* d);
-
 /**
  * @brief Pop the least-tag element from the queue.
  * If the entry was dynamically allocated, then it is now up to the caller
@@ -124,12 +129,12 @@ int pqueue_tag_insert(pqueue_tag_t* q, pqueue_tag_element_t* d);
 pqueue_tag_element_t* pqueue_tag_pop(pqueue_tag_t* q);
 
 /**
- * Find the highest-ranking item with the same tag.
- * @param q the queue
- * @param t the tag to compare against
- * @return NULL if no matching tag has been found, otherwise the entry
+ * Return the first item with the specified tag or NULL if there is none.
+ * @param q The queue.
+ * @param t The tag.
+ * @return An entry with the specified tag or NULL if there isn't one.
  */
-pqueue_tag_element_t* pqueue_tag_find_same_tag(pqueue_tag_t *q, tag_t t);
+pqueue_tag_element_t* pqueue_tag_find_with_tag(pqueue_tag_t *q, tag_t t);
 
 /**
  * Remove an item from the queue.
@@ -146,4 +151,4 @@ int pqueue_tag_remove(pqueue_tag_t* q, pqueue_tag_element_t* e);
  */
 pqueue_tag_element_t* pqueue_tag_peek(pqueue_tag_t* q);
 
-#endif // PQUEUE_TAG_H
\ No newline at end of file
+#endif // PQUEUE_TAG_H
diff --git a/test/general/utils/pqueue_test.c b/test/general/utils/pqueue_test.c
index 09f6284de..f95492799 100644
--- a/test/general/utils/pqueue_test.c
+++ b/test/general/utils/pqueue_test.c
@@ -23,9 +23,9 @@ static void insert_on_queue(pqueue_tag_t* q) {
     assert(!pqueue_tag_insert_tag(q, t2));
     assert(!pqueue_tag_insert_tag(q, t3));
 
-    assert(!pqueue_tag_insert_tag_if_not_present(q, t4));
-    assert(pqueue_tag_insert_tag_if_not_present(q, t1));
-    assert(pqueue_tag_insert_tag_if_not_present(q, t4));
+    assert(!pqueue_tag_insert_if_no_match(q, t4));
+    assert(pqueue_tag_insert_if_no_match(q, t1));
+    assert(pqueue_tag_insert_if_no_match(q, t4));
     printf("======== Contents of the queue:\n");
     pqueue_print((pqueue_t*)q, NULL);
     assert(pqueue_tag_size(q) == 4);
@@ -38,12 +38,22 @@ static void find_from_queue(pqueue_tag_t* q) {
     tag_t t4 = {.time = USEC(1), .microstep = 2};
     tag_t t5 = {.time = USEC(0), .microstep = 0};
     tag_t t6 = {.time = USEC(3), .microstep = 2};
-    assert(pqueue_tag_find_same_tag(q, t1) != NULL);
-    assert(pqueue_tag_find_same_tag(q, t2) != NULL);
-    assert(pqueue_tag_find_same_tag(q, t3) != NULL);
-    assert(pqueue_tag_find_same_tag(q, t4) != NULL);
-    assert(pqueue_tag_find_same_tag(q, t5) == NULL);
-    assert(pqueue_tag_find_same_tag(q, t6) == NULL);
+    assert(pqueue_tag_find_with_tag(q, t1) != NULL);
+    assert(pqueue_tag_find_with_tag(q, t2) != NULL);
+    assert(pqueue_tag_find_with_tag(q, t3) != NULL);
+    assert(pqueue_tag_find_with_tag(q, t4) != NULL);
+    assert(pqueue_tag_find_with_tag(q, t5) == NULL);
+    assert(pqueue_tag_find_with_tag(q, t6) == NULL);
+}
+
+static void insert_if_no_match(pqueue_tag_t* q) {
+    int size = pqueue_tag_size(q);
+    tag_t t1 = {.time = USEC(3), .microstep = 0};
+    tag_t t4 = {.time = USEC(1), .microstep = 2};
+    // Return value is non-zero on failure to insert:
+    assert(pqueue_tag_insert_if_no_match(q, t1));
+    assert(pqueue_tag_insert_if_no_match(q, t4));
+    assert(size == pqueue_tag_size(q));
 }
 
 static void pop_from_queue(pqueue_tag_t* q) {
@@ -81,6 +91,7 @@ int main(int argc, char *argv[]) {
 
     insert_on_queue(q);
     find_from_queue(q);
+    insert_if_no_match(q);
     pop_from_queue(q);
     pop_empty(q);
 

From 2c856bd4adc6ed6cd68f911c3eb13917b2790dda Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Sat, 18 Nov 2023 23:29:13 +0100
Subject: [PATCH 67/76] Cache minimum upstream delays

---
 core/federated/RTI/rti_common.c | 160 ++++++++++++++++----------------
 core/federated/RTI/rti_common.h |  67 ++++++-------
 core/federated/RTI/rti_remote.c |   3 -
 3 files changed, 116 insertions(+), 114 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index fa4a665e5..4b50be9d0 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -33,6 +33,16 @@ void initialize_rti_common(rti_common_t * _rti_common) {
 //        Currently, all calls to tracepoint_from_federate() and 
 //        tracepoint_to_federate() are in rti_lib.c
 
+#define IS_IN_ZERO_DELAY_CYCLE 1
+#define IS_IN_CYCLE 2
+
+void invalidate_min_delays_upstream(scheduling_node_t* node) {
+    if(node->min_delays != NULL) free(node->min_delays);
+    node->min_delays = NULL;
+    node->num_min_delays = 0;
+    node->flags = IS_IN_ZERO_DELAY_CYCLE | IS_IN_CYCLE; // Pessimistic, in case it is accessed without updating.
+}
+
 void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) {
     e->id = id;
     e->completed = NEVER_TAG;
@@ -46,7 +56,7 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) {
     e->downstream = NULL;
     e->num_downstream = 0;
     e->mode = REALTIME;
-    e->is_in_zero_delay_cycle = false;
+    invalidate_min_delays_upstream(e);
 }
 
 void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
@@ -76,35 +86,25 @@ void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
 tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) {
     // First, we need to find the shortest path (minimum delay) path to each upstream node
     // and then find the minimum of the node's recorded NET plus the minimum path delay.
-    // The following array will have the minimum path delay.
-    // It needs to be initialized to FOREVER_TAG, except for the element corresponding to
-    // this fed/enclave, which is initialized to (0,0).
-    tag_t *path_delays = (tag_t *)malloc(rti_common->number_of_scheduling_nodes * sizeof(tag_t));
-    for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
-        path_delays[i] = FOREVER_TAG;
-    }
-    path_delays[e->id] = ZERO_TAG;
-    shortest_path_upstream(e, NULL, path_delays);
+    // Update the shortest paths, if necessary.
+    update_min_delays_upstream(e);
 
     // Next, find the tag of the earliest possible incoming message from upstream enclaves or
     // federates, which will be the smallest upstream NET plus the least delay.
     // This could be NEVER_TAG if the RTI has not seen a NET from some upstream node.
     tag_t t_d = FOREVER_TAG;
-    for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
-        if (e->id != i && path_delays[i].time < FOREVER) {
-            // Node i is upstream of e. Note that it could be that i == e.
-            scheduling_node_t* upstream = rti_common->scheduling_nodes[i];
-            tag_t earliest_tag_from_upstream = lf_tag_add(upstream->next_event, path_delays[i]);
-            LF_PRINT_DEBUG("RTI: Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".",
-                    e->id,
-                    upstream->id,
-                    earliest_tag_from_upstream.time - start_time, earliest_tag_from_upstream.microstep);
-            if (lf_tag_compare(earliest_tag_from_upstream, t_d) < 0) {
-                t_d = earliest_tag_from_upstream;
-            }
+    for (int i = 0; i < e->num_min_delays; i++) {
+        // Node e->min_delays[i].id is upstream of e with min delay e->min_delays[i].min_delay.
+        scheduling_node_t* upstream = rti_common->scheduling_nodes[e->min_delays[i].id];
+        tag_t earliest_tag_from_upstream = lf_tag_add(upstream->next_event, e->min_delays[i].min_delay);
+        LF_PRINT_DEBUG("RTI: Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".",
+                e->id,
+                upstream->id,
+                earliest_tag_from_upstream.time - start_time, earliest_tag_from_upstream.microstep);
+        if (lf_tag_compare(earliest_tag_from_upstream, t_d) < 0) {
+            t_d = earliest_tag_from_upstream;
         }
     }
-    free(path_delays);
     return t_d;
 }
 
@@ -168,7 +168,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
                                                                       // (equal is important to override any previous
                                                                       // PTAGs).
         && lf_tag_compare(t_d, e->last_granted) > 0                   // The grant is not redundant.
-        && !e->is_in_zero_delay_cycle                               // The node is not part of a ZDC
+        && !is_in_zero_delay_cycle(e)                               // The node is not part of a ZDC
     ) {
         // No upstream node can send events that will be received with a tag less than or equal to
         // e->next_event, so it is safe to send a TAG.
@@ -244,7 +244,9 @@ void notify_advance_grant_if_safe(scheduling_node_t* e) {
     }
 }
 
-void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermediate, tag_t path_delays[]) {
+// Local function used recursively to find minimum delays upstream.
+// Return in count the number of non-FOREVER_TAG entries in path_delays[].
+static void _update_min_delays_upstream(scheduling_node_t* end, scheduling_node_t* intermediate, tag_t path_delays[], size_t* count) {
     // On first call, intermediate will be NULL, so the path delay is initialized to zero.
     tag_t delay_from_intermediate_so_far = ZERO_TAG;
     if (intermediate == NULL) {
@@ -253,88 +255,88 @@ void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermedi
         // Not the first call, so intermediate is upstream of end.
         delay_from_intermediate_so_far = path_delays[intermediate->id];
     }
-    // NOTE: Except for this one check, the information calculated here is completely static
-    // and could be precomputed at compile time.  But doing so would then cause transient
-    // federates in feedback loops to potentially cause deadlocks by being absent.
     if (intermediate->state == NOT_CONNECTED) {
         // Enclave or federate is not connected.
         // No point in checking upstream scheduling_nodes.
         return;
     }
     // Check nodes upstream of intermediate (or end on first call).
+    // NOTE: It would be better to iterate through these sorted by minimum delay,
+    // but for most programs, the gain might be negligible since there are relatively few
+    // upstream nodes.
     for (int i = 0; i < intermediate->num_upstream; i++) {
         // Add connection delay to path delay so far.
         tag_t path_delay = lf_delay_tag(delay_from_intermediate_so_far, intermediate->upstream_delay[i]);
         // If the path delay is less than the so-far recorded path delay from upstream, update upstream.
         if (lf_tag_compare(path_delay, path_delays[intermediate->upstream[i]]) < 0) {
+            if (path_delays[intermediate->upstream[i]].time == FOREVER) {
+                // Found a finite path.
+                *count = *count + 1;
+            }
             path_delays[intermediate->upstream[i]] = path_delay;
             // Since the path delay to upstream has changed, recursively update those upstream of it.
             // Do not do this, however, if the upstream node is the end node because this means we have
             // completed a cycle.
             if (end->id != intermediate->upstream[i]) {
-                shortest_path_upstream(end, rti_common->scheduling_nodes[intermediate->upstream[i]], path_delays);
+                _update_min_delays_upstream(end, rti_common->scheduling_nodes[intermediate->upstream[i]], path_delays, count);
+            } else {
+                // Found a cycle.
+                end->flags = end->flags | IS_IN_CYCLE;
+                // Is it a zero-delay cycle?
+                if (lf_tag_compare(path_delay, ZERO_TAG) == 0 && intermediate->upstream_delay[i] < 0) {
+                    end->flags = end->flags | IS_IN_ZERO_DELAY_CYCLE;
+                }
             }
         }
     }
 }
 
-/**
- * Cycle-detection algorithm based on Depth-first search. adapted from: 
- * https://rohithv63.medium.com/graph-algorithm-cycle-detection-in-directed-graph-using-dfs-939512865fd6
- * 
- */
-static bool _find_zero_delay_cycles(int v, bool visited[], bool recStack[], scheduling_node_t** nodes, int num_nodes) {
-    if (recStack[v]) {
-        return true;
-    }
-    if (visited[v]) {
-        return false;
-    }
-    
-    visited[v] = true;
-    recStack[v] = true;
-
-    scheduling_node_t *temp = nodes[v];
-    for (int i = 0; i<temp->num_upstream; i++) {
-        if (temp->upstream_delay[i] != NEVER) { // NEVER means zero-delay
-            continue;
-        }
-    
-        if (_find_zero_delay_cycles(i, visited, recStack, nodes, num_nodes)) {
-            return true;
-        }
-    }
-    recStack[v] = false; // Remove the node from the current path
-    return false;
-}
+void update_min_delays_upstream(scheduling_node_t* node) {
+    // Check whether cached result is valid.
+    if (node->min_delays == NULL) {
 
-void find_zero_delay_cycles(scheduling_node_t** nodes, int num_nodes) {
-    bool* visited = (bool*)malloc(num_nodes * sizeof(bool));
-    bool* recStack = (bool*)malloc(num_nodes * sizeof(bool));
+        // This is not Dijkstra's algorithm, but rather one optimized for sparse upstream nodes.
+        // There must be a name for this algorithm.
 
-    for (int i = 0; i < num_nodes; i++) {
-        visited[i] = 0;
-        recStack[i] = 0;
-    }
-    bool first=true; // For printing the ZDC on a single line
-    for (int i = 0; i < num_nodes; i++) {
-        if (!visited[i]) {
-            if (_find_zero_delay_cycles(i, visited, recStack, nodes, num_nodes)) {
-                if (first) {
-                    printf("RTI: Nodes part of a zero-delay-cycle:");
-                    first = false;
+        // Array of results on the stack:
+        tag_t path_delays[rti_common->number_of_scheduling_nodes];
+        // This will be the number of non-FOREVER entries put into path_delays.
+        size_t count = 1;
+
+        for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
+            if (i == node->id) {
+                path_delays[i] = ZERO_TAG;
+            } else {
+                path_delays[i] = FOREVER_TAG;
+            }
+        }
+        _update_min_delays_upstream(node, NULL, path_delays, &count);
+
+        // Put the results onto the node's struct.
+        node->num_min_delays = count;
+        node->min_delays = (minimum_delay_t*)malloc(count * sizeof(minimum_delay_t));
+        int k = 0;
+        for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
+            if (lf_tag_compare(path_delays[i], FOREVER_TAG) < 0) {
+                // Node i is upstream.
+                if (k >= count) {
+                    lf_print_error_and_exit("Internal error! Count of upstream nodes %zu for node %d is wrong!", count, i);
                 }
-                printf(" %i,", i);
-                nodes[i]->is_in_zero_delay_cycle = true;
+                minimum_delay_t min_delay = {.id = i, .min_delay = path_delays[i]};
+                node->min_delays[k++] = min_delay;
             }
         }
     }
-    if (!first) {
-        printf("\n");
-    }
+}
 
-    free(visited);
-    free(recStack);
+bool is_in_zero_delay_cycle(scheduling_node_t* node) {
+    update_min_delays_upstream(node);
+    return node->flags & IS_IN_ZERO_DELAY_CYCLE;
+}
+
+bool is_in_cycle(scheduling_node_t* node) {
+    update_min_delays_upstream(node);
+    return node->flags & IS_IN_CYCLE;
 }
 
 #endif
diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h
index 546f68295..d71751a98 100644
--- a/core/federated/RTI/rti_common.h
+++ b/core/federated/RTI/rti_common.h
@@ -34,6 +34,12 @@ typedef enum scheduling_node_state_t {
     PENDING         // Waiting for upstream scheduling nodes.
 } scheduling_node_state_t;
 
+/** Struct for minimum delays from upstream nodes. */
+typedef struct minimum_delay_t {
+    int id;          // ID of the upstream node.
+    tag_t min_delay; // Minimum delay from upstream.
+} minimum_delay_t;
+
 /**
  * Information about the scheduling nodes coordinated by the RTI.
  * The abstract scheduling node could either be an enclave or a federate.
@@ -59,7 +65,9 @@ typedef struct scheduling_node_t {
     int* downstream;                    // Array of downstream scheduling node ids.
     int num_downstream;                 // Size of the array of downstream scheduling nodes.
     execution_mode_t mode;              // FAST or REALTIME.
-    bool is_in_zero_delay_cycle;         // This scheduling node is part of a zero-delay cycle
+    minimum_delay_t* min_delays;        // Array of minimum delays from upstream nodes, not including this node.
+    size_t num_min_delays;              // Size of min_delays array.
+    int flags;                          // Or of IS_IN_ZERO_DELAY_CYCLE, IS_IN_CYCLE
 } scheduling_node_t;
 
 /**
@@ -208,7 +216,6 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag);
  */
 tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e);
 
-
 /**
  * @brief Update the next event tag of an scheduling node.
  *
@@ -232,43 +239,39 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne
 tag_t earliest_future_incoming_message_tag(scheduling_node_t* e);
 
 /**
- * Given a node (enclave or federate), find the shortest path (least total delay)
- * from each upstream node to the node given by `end`.  The result is written into
- * the `path_delay` array, with one value for each node (federate or enclave) in the system.
- * An entry of FOREVER_TAG means that the node is not upstream of `end`.
- * An entry of (0,0) means that the node is upstream and that there are no after delays
- * on the path to `end`.  Otherwise, the entry is the sum of the after delays on the
- * shortest path, where the sum is calculated using lf_delay_tag().
- * 
- * This function calls itself recursively. On the first call,`path_delay` should be an
- * array whose size matches the number of nodes in the system.  Each entry in the array
- * should be FOREVER_TAG except the node for which we finding the shortest path, which
- * should have an entry (0,0). On that first call, `intermediate` should be NULL.
- * 
- * If the resulting entry for `end` remains FOREVER_TAG, then there is no cycle
- * back from the outputs of `end` to itself. Otherwise, the value of the entry will
- * be the minimum delay among all paths back to itself.
- *
- * @param end The target node.
- * @param intermediate Current intermediate node (NULL initially).
- * @param path_delays An array in which to put the results.
+ * Return true if the node is in a zero-delay cycle.
+ * @param node The node.
  */
-void shortest_path_upstream(scheduling_node_t* end, scheduling_node_t* intermediate, tag_t path_delays[]);
+bool is_in_zero_delay_cycle(scheduling_node_t* node);
 
 /**
- * Free dynamically allocated memory on the scheduling nodes and the scheduling node array itself.
+ * Return true if the node is in a cycle (possibly a zero-delay cycle).
+ * @param node The node.
  */
-void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number_of_scheduling_nodes);
+bool is_in_cycle(scheduling_node_t* node);
 
 /**
- * @brief Search the directed graph of nodes and find all nodes that are part of
- * a zero-delay cycle (ZDC). These nodes are marked by setting the `is_part_of_zero_delay_cycle`
- * field of the scheduling_node_t struct.
- * 
- * @param nodes An array of scheduling node pointers
- * @param num_nodes The length of the array
+ * For the given scheduling node (enclave or federate), if necessary, update the `min_delays`,
+ * `num_min_delays`, and the fields that indicate cycles.  These fields will be
+ * updated only if they have not been previously updated or if invalidate_min_delays_upstream
+ * has been called since they were last updated.
+ * @param node The node.
  */
-void find_zero_delay_cycles(scheduling_node_t** nodes, int num_nodes);
+void update_min_delays_upstream(scheduling_node_t* node);
+
+/**
+ * For the given scheduling node (enclave or federate), invalidate the `min_delays`,
+ * `num_min_delays`, and the fields that indicate cycles.
+ * This should be called whenever the structure of the connections upstream of the
+ * given node have changed.
+ * @param node The node.
+ */
+void invalidate_min_delays_upstream(scheduling_node_t* node);
+
+/**
+ * Free dynamically allocated memory on the scheduling nodes and the scheduling node array itself.
+ */
+void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number_of_scheduling_nodes);
 
 #endif // RTI_COMMON_H
 #endif // STANDALONE_RTI || LF_ENCLAVES
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index bf54d276a..2fce8b1bf 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1577,9 +1577,6 @@ void wait_for_federates(int socket_descriptor) {
     // Wait for connections from federates and create a thread for each.
     connect_to_federates(socket_descriptor);
 
-    // Do cycle detection
-    find_zero_delay_cycles(rti_remote->base.scheduling_nodes, rti_remote->base.number_of_scheduling_nodes);
-
     // All federates have connected.
     lf_print("RTI: All expected federates have connected. Starting execution.");
 

From ec4223a486012d2e95ba77b6102a532b4601a620 Mon Sep 17 00:00:00 2001
From: Jakio815 <dongha@asu.edu>
Date: Wed, 22 Nov 2023 09:10:22 -0700
Subject: [PATCH 68/76] Update lingua-franca-ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 1f7391f92..c81de7a34 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-master
+47b87a49ceece1f0065a9dbf9b8a06c9fcdfe01d

From fad885570e888f5fe70499819941648da2b14e73 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Thu, 23 Nov 2023 07:49:05 -0800
Subject: [PATCH 69/76] Send PTAG only if in ZDC

---
 core/federated/RTI/rti_common.c | 39 +++++++++++++++++++++------------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index 4b50be9d0..2151a6564 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -40,7 +40,7 @@ void invalidate_min_delays_upstream(scheduling_node_t* node) {
     if(node->min_delays != NULL) free(node->min_delays);
     node->min_delays = NULL;
     node->num_min_delays = 0;
-    node->flags = IS_IN_ZERO_DELAY_CYCLE | IS_IN_CYCLE; // Pessimistic, in case it is accessed without updating.
+    node->flags = 0; // All flags cleared because they get set lazily.
 }
 
 void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) {
@@ -96,6 +96,11 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) {
     for (int i = 0; i < e->num_min_delays; i++) {
         // Node e->min_delays[i].id is upstream of e with min delay e->min_delays[i].min_delay.
         scheduling_node_t* upstream = rti_common->scheduling_nodes[e->min_delays[i].id];
+        // If we haven't heard from the upstream node, then assume it can send an event at the start time.
+        if (lf_tag_compare(upstream->next_event, NEVER_TAG) == 0) {
+            tag_t start_tag = {.time = start_time, .microstep = 0};
+            upstream->next_event = start_tag;
+        }
         tag_t earliest_tag_from_upstream = lf_tag_add(upstream->next_event, e->min_delays[i].min_delay);
         LF_PRINT_DEBUG("RTI: Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".",
                 e->id,
@@ -155,12 +160,13 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
     LF_PRINT_LOG("RTI: Earliest next event upstream of node %d has tag " PRINTF_TAG ".",
             e->id, t_d.time - start_time, t_d.microstep);
 
-    // Given an EIMT there are three possible scenarios.
-    //  1) The EIMT is greater than the NET we want to advance to.
-    //  2) The EIMT is greater, however, the federate is part of a zero-delay cycle (ZDC)
-    //  3) The EIMT is equal to the NET.
+    // Given an EIMT (earliest incoming message tag) there are these possible scenarios:
+    //  1) The EIMT is greater than the NET we want to advance to. Grant a TAG.
+    //  2) The EIMT is equal to the NET and the federate is part of a zero-delay cycle (ZDC).
+    //  3) The EIMT is equal to the NET and the federate is not part of a ZDC.
     //  4) The EIMT is less than the NET
-    // In (1) we can give a TAG to NET. In (2) and (3) we can give a PTAG.
+    // In (1) we can give a TAG to NET. In (2) we can give a PTAG.
+    // In (3) and (4), we wait for further updates from upstream federates.
 
     if ( // Scenario (1) above
         lf_tag_compare(t_d, e->next_event) > 0                      // EIMT greater than NET
@@ -168,7 +174,6 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
                                                                       // (equal is important to override any previous
                                                                       // PTAGs).
         && lf_tag_compare(t_d, e->last_granted) > 0                   // The grant is not redundant.
-        && !is_in_zero_delay_cycle(e)                               // The node is not part of a ZDC
     ) {
         // No upstream node can send events that will be received with a tag less than or equal to
         // e->next_event, so it is safe to send a TAG.
@@ -180,7 +185,8 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
                 e->next_event.microstep);
         result.tag = e->next_event;
     } else if( // Scenario (2) or (3) above
-        lf_tag_compare(t_d, e->next_event) >= 0                      // EIMT greater than or equal to NET
+        lf_tag_compare(t_d, e->next_event) >= 0                     // EIMT greater than or equal to NET
+        && is_in_zero_delay_cycle(e)                                // The node is part of a ZDC
         && lf_tag_compare(t_d, e->last_provisionally_granted) >= 0  // The grant is not redundant
         && lf_tag_compare(t_d, e->last_granted) > 0                 // The grant is not redundant.
     ) { 
@@ -223,6 +229,9 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne
     // nor expect a reply. It just proceeds to advance time.
     if (e->num_upstream > 0) {
         notify_advance_grant_if_safe(e);
+    } else {
+        // Even though there was no grant, mark the tag as if there was.
+        e->last_granted = next_event_tag;
     }
     // Check downstream scheduling_nodes to see whether they should now be granted a TAG.
     // To handle cycles, need to create a boolean array to keep
@@ -285,6 +294,9 @@ static void _update_min_delays_upstream(scheduling_node_t* end, scheduling_node_
                 // Is it a zero-delay cycle?
                 if (lf_tag_compare(path_delay, ZERO_TAG) == 0 && intermediate->upstream_delay[i] < 0) {
                     end->flags = end->flags | IS_IN_ZERO_DELAY_CYCLE;
+                } else {
+                    // Clear the flag.
+                    end->flags = end->flags & ~IS_IN_ZERO_DELAY_CYCLE;
                 }
             }
         }
@@ -301,20 +313,17 @@ void update_min_delays_upstream(scheduling_node_t* node) {
         // Array of results on the stack:
         tag_t path_delays[rti_common->number_of_scheduling_nodes];
         // This will be the number of non-FOREVER entries put into path_delays.
-        size_t count = 1;
+        size_t count = 0;
 
         for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
-            if (i == node->id) {
-                path_delays[i] = ZERO_TAG;
-            } else {
-                path_delays[i] = FOREVER_TAG;
-            }
+            path_delays[i] = FOREVER_TAG;
         }
         _update_min_delays_upstream(node, NULL, path_delays, &count);
 
         // Put the results onto the node's struct.
         node->num_min_delays = count;
         node->min_delays = (minimum_delay_t*)malloc(count * sizeof(minimum_delay_t));
+        LF_PRINT_DEBUG("++++ Node %hu(is in ZDC: %d\n", node->id, node->flags & IS_IN_ZERO_DELAY_CYCLE);
         int k = 0;
         for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) {
             if (lf_tag_compare(path_delays[i], FOREVER_TAG) < 0) {
@@ -324,6 +333,8 @@ void update_min_delays_upstream(scheduling_node_t* node) {
                 }
                 minimum_delay_t min_delay = {.id = i, .min_delay = path_delays[i]};
                 node->min_delays[k++] = min_delay;
+                // N^2 debug statement could be a problem with large benchmarks.
+                // LF_PRINT_DEBUG("++++    Node %hu is upstream with delay" PRINTF_TAG "\n", i, path_delays[i].time, path_delays[i].microstep);
             }
         }
     }

From c25334fb3142dee32a2cab3a5c8d3dd25083f737 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Fri, 24 Nov 2023 07:57:06 -0800
Subject: [PATCH 70/76] Response to review

---
 core/federated/RTI/rti_common.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index 2151a6564..62ad80664 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -185,9 +185,9 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
                 e->next_event.microstep);
         result.tag = e->next_event;
     } else if( // Scenario (2) or (3) above
-        lf_tag_compare(t_d, e->next_event) >= 0                     // EIMT greater than or equal to NET
+        lf_tag_compare(t_d, e->next_event) == 0                     // EIMT equal to NET
         && is_in_zero_delay_cycle(e)                                // The node is part of a ZDC
-        && lf_tag_compare(t_d, e->last_provisionally_granted) >= 0  // The grant is not redundant
+        && lf_tag_compare(t_d, e->last_provisionally_granted) > 0   // The grant is not redundant
         && lf_tag_compare(t_d, e->last_granted) > 0                 // The grant is not redundant.
     ) { 
         // Some upstream node may send an event that has the same tag as this node's next event,

From 59e99c075b3d6baf6c745880da6564af4c4a9890 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 28 Nov 2023 11:20:14 -0800
Subject: [PATCH 71/76] Respond to review

---
 core/federated/RTI/rti_common.c | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index 62ad80664..befc404b0 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -57,6 +57,10 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) {
     e->num_downstream = 0;
     e->mode = REALTIME;
     invalidate_min_delays_upstream(e);
+
+    LF_PRINT_DEBUG("NOTE: FOREVER is displayed as " PRINTF_TAG " and NEVER as " PRINTF_TAG,
+                   FOREVER_TAG.time - start_time, FOREVER_TAG.microstep,
+                   NEVER_TAG.time - start_time, 0);
 }
 
 void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
@@ -153,10 +157,6 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) {
     // or federate (which includes any after delays on the connections).
     tag_t t_d = earliest_future_incoming_message_tag(e);
 
-    LF_PRINT_DEBUG("NOTE: FOREVER is displayed as " PRINTF_TAG " and NEVER as " PRINTF_TAG,
-                   FOREVER_TAG.time - start_time, FOREVER_TAG.microstep,
-                   NEVER_TAG.time - start_time, 0);
-
     LF_PRINT_LOG("RTI: Earliest next event upstream of node %d has tag " PRINTF_TAG ".",
             e->id, t_d.time - start_time, t_d.microstep);
 
@@ -241,7 +241,6 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne
     free(visited);
 }
 
-
 void notify_advance_grant_if_safe(scheduling_node_t* e) {
     tag_advance_grant_t grant = tag_advance_grant_if_safe(e);
     if (lf_tag_compare(grant.tag, NEVER_TAG) != 0) {

From bc3982cc27ca6a0674c3050aaa13d7397fb37431 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 28 Nov 2023 11:20:34 -0800
Subject: [PATCH 72/76] Fixed comment

---
 core/federated/federate.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index d49cd20a4..c57553464 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2067,8 +2067,8 @@ void spawn_staa_thread(){
  * If current_time is less than the specified PTAG, then this will
  * also insert into the event_q a dummy event with the specified tag.
  * This will ensure that the federate advances time to the specified
- * tag and, for centralized coordination, inserts blocking reactions
- * and null-message-sending output reactions at that tag.
+ * tag and, for centralized coordination, stimulates null-message-sending
+ * output reactions at that tag.
  *
  * @note This function is similar to handle_tag_advance_grant() except that
  *  it sets last_TAG_was_provisional to true and also it does not update the

From caeae15a71854e02883423e53b7521c49b57ae1e Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Tue, 28 Nov 2023 17:00:17 -0800
Subject: [PATCH 73/76] Report how start is displayed after start_time is
 computed

---
 core/federated/RTI/rti_common.c | 4 ----
 core/reactor.c                  | 5 +++++
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index befc404b0..a6554195e 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -57,10 +57,6 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) {
     e->num_downstream = 0;
     e->mode = REALTIME;
     invalidate_min_delays_upstream(e);
-
-    LF_PRINT_DEBUG("NOTE: FOREVER is displayed as " PRINTF_TAG " and NEVER as " PRINTF_TAG,
-                   FOREVER_TAG.time - start_time, FOREVER_TAG.microstep,
-                   NEVER_TAG.time - start_time, 0);
 }
 
 void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) {
diff --git a/core/reactor.c b/core/reactor.c
index e715514e9..ce95b057d 100644
--- a/core/reactor.c
+++ b/core/reactor.c
@@ -371,6 +371,11 @@ int lf_reactor_c_main(int argc, const char* argv[]) {
         initialize_global();
         // Set start time
         start_time = lf_time_physical();
+
+        LF_PRINT_DEBUG("NOTE: FOREVER is displayed as " PRINTF_TAG " and NEVER as " PRINTF_TAG,
+                FOREVER_TAG.time - start_time, FOREVER_TAG.microstep,
+                NEVER_TAG.time - start_time, 0);
+
         environment_init_tags(env, start_time, duration);
         // Start tracing if enalbed
         start_trace(env->trace);

From 8f4e526611db54623d942ed1847632ccf465cf5d Mon Sep 17 00:00:00 2001
From: Marten Lohstroh <marten@berkeley.edu>
Date: Fri, 1 Dec 2023 16:01:46 -0800
Subject: [PATCH 74/76] Update lingua-franca-ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index c81de7a34..258a8ab0b 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-47b87a49ceece1f0065a9dbf9b8a06c9fcdfe01d
+network-structure

From 1cd699d2cf6577e585aaea2ea11976df6fb67cf6 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@berkeley.edu>
Date: Fri, 1 Dec 2023 16:07:07 -0800
Subject: [PATCH 75/76] Comments only (forgot to commit to enclaves3 before
 merging)

---
 core/utils/util.c   | 4 ++++
 test/src_gen_stub.c | 7 +++++++
 2 files changed, 11 insertions(+)

diff --git a/core/utils/util.c b/core/utils/util.c
index df325b125..23daef364 100644
--- a/core/utils/util.c
+++ b/core/utils/util.c
@@ -53,6 +53,10 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 /** Number of nanoseconds to sleep before retrying a socket read. */
 #define SOCKET_READ_RETRY_INTERVAL 1000000
 
+/**
+ * The ID of this federate. For a non-federated execution, this will be -1. 
+ * For a federated execution, it will be assigned in the generated code.
+ */
 int _lf_my_fed_id = -1;
 
 /**
diff --git a/test/src_gen_stub.c b/test/src_gen_stub.c
index 5b43d103b..46f451096 100644
--- a/test/src_gen_stub.c
+++ b/test/src_gen_stub.c
@@ -2,6 +2,13 @@
 #include "tag.h"
 #include "environment.h"
 
+/**
+ * This file enables unit tests to run without there having been an actual code generation
+ * from a Lingua Franca program. It defines (mostly empty) functions that would normally be
+ * code generated. Of course, this strategy will only work for tests that do not actually
+ * need functional versions of these functions.
+ */
+
 environment_t _env;
 
 void _lf_initialize_trigger_objects() {}

From 182d64b58e90a64c0ace90a8d7350ef067296d59 Mon Sep 17 00:00:00 2001
From: "Edward A. Lee" <eal@eecs.berkeley.edu>
Date: Sat, 2 Dec 2023 07:34:41 -0800
Subject: [PATCH 76/76] Update lingua-franca-ref.txt to point to master

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 258a8ab0b..1f7391f92 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-network-structure
+master