From 7995d88330964908094d205658b45f10c7a96028 Mon Sep 17 00:00:00 2001
From: Byeonggil-Jun <junbg@hanyang.ac.kr>
Date: Wed, 8 Mar 2023 09:12:27 +0900
Subject: [PATCH 01/49] First commit for fowarding NET message to optimize RTI

---
 core/federated/RTI/rti_lib.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 0ad6975b6..4d1932954 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -566,7 +566,10 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
         next_event_tag = min_in_transit_tag;
     }
 
-    _RTI.federates[federate_id].next_event = next_event_tag;
+    if (lf_tag_compare(_RTI.federates[federate_id].next_event, next_event_tag) != 0) {
+        _RTI.federates[federate_id].next_event = next_event_tag;
+        //FIXME: notify upstream federates
+    }
 
     LF_PRINT_DEBUG(
        "RTI: Updated the recorded next event tag for federate %d to (%ld, %u).",

From 8ba5c1e6fb0c56fbb100e8c5ad00b66f2cb071cb Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 18 Apr 2023 15:54:27 +0900
Subject: [PATCH 02/49] Add a new message type and its description

---
 include/core/federated/net_common.h | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/include/core/federated/net_common.h b/include/core/federated/net_common.h
index 8ace6d9b0..d98f7abbf 100644
--- a/include/core/federated/net_common.h
+++ b/include/core/federated/net_common.h
@@ -683,6 +683,20 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define MSG_TYPE_NEIGHBOR_STRUCTURE 24
 #define MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE 9
 
+/** 
+ * Byte identifying a next downstream event tag (NDET) message sent from a downstream
+ * federate via the RTI in centralized coordination. 
+ * The next two bytes are the source federate ID.
+ * The next eight bytes will be the timestamp. 
+ * The next four bytes will be the microstep. 
+ * This message from the RTI tells the federate the tag of the earliest event on the  
+ * source federate's event queue. In other words, this federate only has to send LTC 
+ * and NULL messages greater than or equal to this tag. If this federate has no its 
+ * upstream federates, this federate also can skip sending NET messages that are 
+ * greater than or equal to this tag. 
+ */
+#define MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG 25
+
 /////////////////////////////////////////////
 //// Rejection codes
 

From 81288447844888d73a670058b4932d8b9df8f3cd Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 19 Apr 2023 13:34:03 +0900
Subject: [PATCH 03/49] Add skeletons of NDET

---
 core/federated/RTI/rti_lib.c |  6 ++++++
 core/federated/federate.c    | 12 ++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 46a293844..43ce262cc 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -495,6 +495,10 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
     free(visited);
 }
 
+void send_upstream_next_downstream_event_tag(federate_t* fed, tag_t next_event_tag) {
+    // Fixme: Fill this function.
+}
+
 void handle_port_absent_message(federate_t* sending_federate, unsigned char* buffer) {
     size_t message_size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int64_t) + sizeof(uint32_t);
 
@@ -762,6 +766,8 @@ void handle_next_event_tag(federate_t* fed) {
         fed->id,
         intended_tag
     );
+    // FIXME: uncomment below function after implementing it.
+    //send_upstream_next_downstream_event_tag(fed, intended_tag);
     pthread_mutex_unlock(&_RTI.rti_mutex);
 }
 
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 2c1d08345..87d4894e7 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2394,6 +2394,15 @@ void handle_stop_request_message() {
     lf_mutex_unlock(&mutex);
 }
 
+/**
+ * Handle a MSG_TYPE_NEXT_DOWNSTREAM_MESSAGE from the RTI
+ * 
+ * 
+*/
+void handle_next_downstream_event_tag() {
+    // FIXME: Fill this function.
+}
+
 /**
  * Close sockets used to communicate with other federates, if they are open,
  * and send a MSG_TYPE_RESIGN message to the RTI. This implements the function
@@ -2596,6 +2605,9 @@ void* listen_to_rti_TCP(void* args) {
             case MSG_TYPE_PORT_ABSENT:
                 handle_port_absent_message(_fed.socket_TCP_RTI, -1);
                 break;
+            case MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG:
+                handle_next_downstream_event_tag();
+                break;
             case MSG_TYPE_CLOCK_SYNC_T1:
             case MSG_TYPE_CLOCK_SYNC_T4:
                 lf_print_error("Federate %d received unexpected clock sync message from RTI on TCP socket.",

From ae2bfb3b451fae09c16a621f623d81d4b664db96 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 17 May 2023 21:11:30 +0900
Subject: [PATCH 04/49] Add some comments and start filling functions

---
 core/federated/RTI/rti_lib.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 43ce262cc..3286c80bc 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -497,6 +497,16 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
 
 void send_upstream_next_downstream_event_tag(federate_t* fed, tag_t next_event_tag) {
     // Fixme: Fill this function.
+    // The RTI receives next_event_tag from the federated fed. 
+    // It has to send NDET messages to the upstream federates of fed
+    // if the LTC message from an upstream federate is ealrier than the next_event_tag.
+
+    for (int i = 0; i < fed->num_upstream; i++) {
+        if (lf_tag_compare(_RTI.federates[i].completed, next_event_tag) < 0) {
+            // completed or granted?
+            // send next downstream event tag to the _RTI.federates[i]
+        }
+    }
 }
 
 void handle_port_absent_message(federate_t* sending_federate, unsigned char* buffer) {

From b06d638f6bd1ec023857a35e65a61505603d5702 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 21 May 2023 15:00:25 +0900
Subject: [PATCH 05/49] Add encoding and sending part of NDET

---
 core/federated/RTI/rti_lib.c        | 7 +++++--
 core/federated/RTI/rti_lib.h        | 9 +++++++++
 include/core/federated/net_common.h | 8 +++++++-
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 3286c80bc..17e15cf5e 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -496,15 +496,18 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
 }
 
 void send_upstream_next_downstream_event_tag(federate_t* fed, tag_t next_event_tag) {
-    // Fixme: Fill this function.
+    // TODO: Fill this function.
     // The RTI receives next_event_tag from the federated fed. 
     // It has to send NDET messages to the upstream federates of fed
     // if the LTC message from an upstream federate is ealrier than the next_event_tag.
+    unsigned char next_downstream_event_tag_buffer[MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG_LENGTH];
+    ENCODE_NEXT_DOWNSTREAM_EVENT_TAG(next_downstream_event_tag_buffer, next_event_tag.time, next_event_tag.microstep);
 
     for (int i = 0; i < fed->num_upstream; i++) {
         if (lf_tag_compare(_RTI.federates[i].completed, next_event_tag) < 0) {
-            // completed or granted?
             // send next downstream event tag to the _RTI.federates[i]
+            write_to_socket_errexit(_RTI.federates[i].socket, MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG_LENGTH, next_downstream_event_tag_buffer,
+                    "RTI failed to send MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG message to federate %d.", _RTI.federates[i].id);            
         }
     }
 }
diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h
index 410d6dbb1..8b8389173 100644
--- a/core/federated/RTI/rti_lib.h
+++ b/core/federated/RTI/rti_lib.h
@@ -350,6 +350,15 @@ void send_downstream_advance_grants_if_safe(federate_t* fed, bool visited[]);
  */
 void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag);
 
+
+/**
+ * @brief Send the next downstream event tag.
+ * 
+ * @param fed The downstream federate.
+ * @param next_event_tag The next event tag from the downstream federate.
+*/
+void send_upstream_next_downstream_event_tag(federate_t* fed, tag_t next_event_tag)
+
 /**
  * Handle a port absent message being received rom a federate via the RIT.
  *
diff --git a/include/core/federated/net_common.h b/include/core/federated/net_common.h
index d98f7abbf..288accd6d 100644
--- a/include/core/federated/net_common.h
+++ b/include/core/federated/net_common.h
@@ -686,7 +686,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 /** 
  * Byte identifying a next downstream event tag (NDET) message sent from a downstream
  * federate via the RTI in centralized coordination. 
- * The next two bytes are the source federate ID.
  * The next eight bytes will be the timestamp. 
  * The next four bytes will be the microstep. 
  * This message from the RTI tells the federate the tag of the earliest event on the  
@@ -696,6 +695,13 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * greater than or equal to this tag. 
  */
 #define MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG 25
+#define MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG_LENGTH (1 + sizeof(instant_t) + sizeof(microstep_t))
+#define ENCODE_NEXT_DOWNSTREAM_EVENT_TAG(buffer, time, microstep) do { \
+    buffer[0] = MSG_TYPE_STOP_REQUEST; \
+    encode_int64(time, &(buffer[1])); \
+    assert(microstep >= 0); \
+    encode_int32((int32_t)microstep, &(buffer[1 + sizeof(instant_t)])); \
+} while(0)
 
 /////////////////////////////////////////////
 //// Rejection codes

From 871e9d5e8e9c2c261561270880905febe25c7ae2 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Thu, 31 Aug 2023 15:16:48 +0900
Subject: [PATCH 06/49] Send NDT and rename NDET to NDT

---
 core/federated/RTI/rti_lib.c        | 23 +++++++++++++----------
 core/federated/RTI/rti_lib.h        |  4 ++--
 core/federated/federate.c           |  2 +-
 include/core/federated/net_common.h |  9 +--------
 4 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 6ccd4ed50..bd6a3d2ec 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -282,19 +282,22 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
     update_enclave_next_event_tag_locked(&(fed->enclave), next_event_tag);
 }
 
-void send_upstream_next_downstream_event_tag(federate_t* fed, tag_t next_event_tag) {
+void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
     // TODO: Fill this function.
     // The RTI receives next_event_tag from the federated fed. 
     // It has to send NDET messages to the upstream federates of fed
     // if the LTC message from an upstream federate is ealrier than the next_event_tag.
-    unsigned char next_downstream_event_tag_buffer[MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG_LENGTH];
-    ENCODE_NEXT_DOWNSTREAM_EVENT_TAG(next_downstream_event_tag_buffer, next_event_tag.time, next_event_tag.microstep);
-
-    for (int i = 0; i < fed->num_upstream; i++) {
-        if (lf_tag_compare(_RTI.federates[i].completed, next_event_tag) < 0) {
-            // send next downstream event tag to the _RTI.federates[i]
-            write_to_socket_errexit(_RTI.federates[i].socket, MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG_LENGTH, next_downstream_event_tag_buffer,
-                    "RTI failed to send MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG message to federate %d.", _RTI.federates[i].id);            
+    size_t message_length = 1 + sizeof(int64_t) + sizeof(uint32_t);
+    unsigned char buffer[message_length];
+    buffer[0] = MSG_TYPE_NEXT_DOWNSTREAM_TAG;
+    encode_int64(next_event_tag.time, &(buffer[1]));
+    encode_int32((int32_t)next_event_tag.microstep, &(buffer[1 + sizeof(int64_t)]));
+
+    for (int i = 0; i < fed->enclave.num_upstream; i++) {
+        if (lf_tag_compare(_f_rti->enclaves[i]->enclave.completed, next_event_tag) < 0) {
+            // send next downstream tag to upstream federates that do not complete the next_event_tag
+            write_to_socket_errexit(_f_rti->enclaves[i]->socket, message_length, buffer,
+                    "RTI failed to send MSG_TYPE_NEXT_DOWNSTREAM_TAG message to federate %d.", _f_rti->enclaves[i]->socket);
         }
     }
 }
@@ -553,7 +556,7 @@ void handle_next_event_tag(federate_t* fed) {
         intended_tag
     );
     // FIXME: uncomment below function after implementing it.
-    //send_upstream_next_downstream_event_tag(fed, intended_tag);
+    //send_upstream_next_downstream_tag(fed, intended_tag);
     lf_mutex_unlock(&rti_mutex);
 }
 
diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h
index 2d62c9f40..80ac69881 100644
--- a/core/federated/RTI/rti_lib.h
+++ b/core/federated/RTI/rti_lib.h
@@ -233,12 +233,12 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
 
 
 /**
- * @brief Send the next downstream event tag.
+ * @brief Send the next downstream tag.
  * 
  * @param fed The downstream federate.
  * @param next_event_tag The next event tag from the downstream federate.
 */
-void send_upstream_next_downstream_event_tag(federate_t* fed, tag_t next_event_tag)
+void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag);
 
 /**
  * Handle a port absent message being received rom a federate via the RIT.
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 083ac9e41..e44ae477d 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2536,7 +2536,7 @@ void* listen_to_rti_TCP(void* args) {
             case MSG_TYPE_PORT_ABSENT:
                 handle_port_absent_message(_fed.socket_TCP_RTI, -1);
                 break;
-            case MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG:
+            case MSG_TYPE_NEXT_DOWNSTREAM_TAG:
                 handle_next_downstream_event_tag();
                 break;
             case MSG_TYPE_CLOCK_SYNC_T1:
diff --git a/include/core/federated/net_common.h b/include/core/federated/net_common.h
index 837bcfd93..f9884fd0d 100644
--- a/include/core/federated/net_common.h
+++ b/include/core/federated/net_common.h
@@ -696,14 +696,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * upstream federates, this federate also can skip sending NET messages that are 
  * greater than or equal to this tag. 
  */
-#define MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG 25
-#define MSG_TYPE_NEXT_DOWNSTREAM_EVENT_TAG_LENGTH (1 + sizeof(instant_t) + sizeof(microstep_t))
-#define ENCODE_NEXT_DOWNSTREAM_EVENT_TAG(buffer, time, microstep) do { \
-    buffer[0] = MSG_TYPE_STOP_REQUEST; \
-    encode_int64(time, &(buffer[1])); \
-    assert(microstep >= 0); \
-    encode_int32((int32_t)microstep, &(buffer[1 + sizeof(instant_t)])); \
-} while(0)
+#define MSG_TYPE_NEXT_DOWNSTREAM_TAG 25
 
 /////////////////////////////////////////////
 //// Rejection codes

From 13ebb41c1846e3d782a17a71aa32f8f7a4858db3 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Fri, 1 Sep 2023 10:46:05 +0900
Subject: [PATCH 07/49] Send NDT if the RTI needs info from upstreams

---
 core/federated/RTI/rti_lib.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index bd6a3d2ec..409a12534 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -293,6 +293,7 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
     encode_int64(next_event_tag.time, &(buffer[1]));
     encode_int32((int32_t)next_event_tag.microstep, &(buffer[1 + sizeof(int64_t)]));
 
+    // FIXME: Send NDT to transitive upstreams either
     for (int i = 0; i < fed->enclave.num_upstream; i++) {
         if (lf_tag_compare(_f_rti->enclaves[i]->enclave.completed, next_event_tag) < 0) {
             // send next downstream tag to upstream federates that do not complete the next_event_tag
@@ -555,8 +556,10 @@ void handle_next_event_tag(federate_t* fed) {
         fed->enclave.id,
         intended_tag
     );
-    // FIXME: uncomment below function after implementing it.
-    //send_upstream_next_downstream_tag(fed, intended_tag);
+    // If fed cannot get the grant of the intended tag, send NDTs to its upstream federates.
+    if (lf_tag_compare(fed->enclave.last_granted, intended_tag) < 0) {
+        send_upstream_next_downstream_tag(fed, intended_tag);
+    }
     lf_mutex_unlock(&rti_mutex);
 }
 

From 5005afdf674781c3fcb0c3719023c17a72b20a22 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sat, 2 Sep 2023 16:42:03 +0900
Subject: [PATCH 08/49] Start implementing handle_next_downstream_tag

---
 core/federated/RTI/rti_lib.c |  7 +++++--
 core/federated/federate.c    | 12 ++++++++++--
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 409a12534..fdaeaf9b0 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -295,9 +295,12 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
 
     // FIXME: Send NDT to transitive upstreams either
     for (int i = 0; i < fed->enclave.num_upstream; i++) {
-        if (lf_tag_compare(_f_rti->enclaves[i]->enclave.completed, next_event_tag) < 0) {
+        int upstream_id = fed->enclave.upstream[i];
+        if (lf_tag_compare(_f_rti->enclaves[upstream_id]->enclave.completed, next_event_tag) < 0) {
             // send next downstream tag to upstream federates that do not complete the next_event_tag
-            write_to_socket_errexit(_f_rti->enclaves[i]->socket, message_length, buffer,
+            LF_PRINT_LOG("RTI sending next downstream event message to federate %u.",
+                upstream_id);
+            write_to_socket_errexit(_f_rti->enclaves[upstream_id]->socket, message_length, buffer,
                     "RTI failed to send MSG_TYPE_NEXT_DOWNSTREAM_TAG message to federate %d.", _f_rti->enclaves[i]->socket);
         }
     }
diff --git a/core/federated/federate.c b/core/federated/federate.c
index e44ae477d..6f4d36f29 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2315,8 +2315,16 @@ void handle_stop_request_message() {
  * 
  * 
 */
-void handle_next_downstream_event_tag() {
+void handle_next_downstream_tag() {
     // FIXME: Fill this function.
+    size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t);
+    unsigned char buffer[bytes_to_read];
+    read_from_socket_errexit(_fed.socket_TCP_RTI, bytes_to_read, buffer,
+            "Failed to read next downstream tag from RTI.");
+    tag_t NDT = extract_tag(buffer);
+
+    LF_PRINT_LOG("Received from RTI a MSG_TYPE_NEXT_DOWNSTREAM_TAG message with elapsed tag " PRINTF_TAG ".",
+            NDT.time - start_time, NDT.microstep);
 }
 
 /**
@@ -2537,7 +2545,7 @@ void* listen_to_rti_TCP(void* args) {
                 handle_port_absent_message(_fed.socket_TCP_RTI, -1);
                 break;
             case MSG_TYPE_NEXT_DOWNSTREAM_TAG:
-                handle_next_downstream_event_tag();
+                handle_next_downstream_tag();
                 break;
             case MSG_TYPE_CLOCK_SYNC_T1:
             case MSG_TYPE_CLOCK_SYNC_T4:

From a58cf3a1637adf305005c786459d5e7910f55398 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Thu, 7 Sep 2023 13:28:20 +0900
Subject: [PATCH 09/49] Start add ndt queue to store NDTs

---
 core/environment.c           | 4 ++++
 core/federated/RTI/rti_lib.c | 2 +-
 include/core/environment.h   | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/core/environment.c b/core/environment.c
index 4216800dc..f04b7e942 100644
--- a/core/environment.c
+++ b/core/environment.c
@@ -106,6 +106,10 @@ static void environment_init_modes(environment_t* env, int num_modes, int num_st
  * @brief Initialize the federation-specific parts of the environment struct.
  */
 static void environment_init_federated(environment_t* env, int num_is_present_fields) {
+#ifdef FEDERATED_CENTRALIZED
+    // TODO: init ndt_queue with proper functions.
+    // env->ndt_queue = pqueue_init(10, tag_in_reverse_order, )
+#endif
 #ifdef FEDERATED_DECENTRALIZED
     env->_lf_intended_tag_fields = (tag_t**) calloc(num_is_present_fields, sizeof(tag_t*));
     lf_assert(env->_lf_intended_tag_fields != NULL, "Out of memory");
diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index fdaeaf9b0..68308d70c 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -301,7 +301,7 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
             LF_PRINT_LOG("RTI sending next downstream event message to federate %u.",
                 upstream_id);
             write_to_socket_errexit(_f_rti->enclaves[upstream_id]->socket, message_length, buffer,
-                    "RTI failed to send MSG_TYPE_NEXT_DOWNSTREAM_TAG message to federate %d.", _f_rti->enclaves[i]->socket);
+                    "RTI failed to send MSG_TYPE_NEXT_DOWNSTREAM_TAG message to federate %d.", upstream_id);
         }
     }
 }
diff --git a/include/core/environment.h b/include/core/environment.h
index ed93b72d2..6fb98ee48 100644
--- a/include/core/environment.h
+++ b/include/core/environment.h
@@ -38,6 +38,7 @@
 
 #include "lf_types.h"
 #include "platform.h"
+#include "pqueue.h"
 
 // Forward declarations so that a pointers can appear in the environment struct.
 typedef struct lf_scheduler_t lf_scheduler_t;
@@ -104,6 +105,9 @@ typedef struct environment_t {
     tag_t** _lf_intended_tag_fields;
     int _lf_intended_tag_fields_size;
 #endif // FEDERATED
+#ifdef FEDERATED_CENTRALIZED
+    pqueue_t* ndt_queue;
+#endif // FEDERATED_CENTRALIZED
 } environment_t;
 
 #ifdef MODAL_REACTORS

From 6c08c13e3a03a79c558e516aff0d481b4eb895b9 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Thu, 7 Sep 2023 13:56:23 +0900
Subject: [PATCH 10/49] Add command line argument v to check the version number

---
 core/federated/RTI/rti_lib.c | 3 +++
 core/federated/RTI/rti_lib.h | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 68308d70c..9365f2bb8 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -1820,6 +1820,9 @@ int process_args(int argc, const char* argv[]) {
             _f_rti->authentication_enabled = true;
         } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) {
             _f_rti->tracing_enabled = true;
+        } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--version") == 0) {
+            lf_print("%s", version_info);
+            return;
         } else if (strcmp(argv[i], " ") == 0) {
             // Tolerate spaces
             continue;
diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h
index 80ac69881..6df506bc3 100644
--- a/core/federated/RTI/rti_lib.h
+++ b/core/federated/RTI/rti_lib.h
@@ -34,6 +34,8 @@
 /////////////////////////////////////////////
 //// Data structures
 
+static char version_info[] = "0.5.0"; // Simply use Lingua Franca version name now.
+
 typedef enum socket_type_t {
     TCP,
     UDP

From 19969eb84820cbdf45fed6fd265b2dd59fcbd243 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <78055940+byeong-gil@users.noreply.github.com>
Date: Thu, 7 Sep 2023 20:35:02 +0900
Subject: [PATCH 11/49] Minor fix

---
 core/federated/RTI/rti_lib.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 9365f2bb8..a0c49a25c 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -1822,7 +1822,7 @@ int process_args(int argc, const char* argv[]) {
             _f_rti->tracing_enabled = true;
         } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--version") == 0) {
             lf_print("%s", version_info);
-            return;
+            return 0;
         } else if (strcmp(argv[i], " ") == 0) {
             // Tolerate spaces
             continue;
@@ -1862,4 +1862,4 @@ void initialize_RTI(){
     _f_rti->authentication_enabled = false,
     _f_rti->tracing_enabled = false;
     _f_rti->stop_in_progress = false;
-}
\ No newline at end of file
+}

From 79eef2a65512a439bb04cb1e5ebf1ed01aabac08 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 10 Sep 2023 11:59:30 +0900
Subject: [PATCH 12/49] Initialized ndt queue in the same manner with the event
 queue

For reusing the pqueue.c, save ndt_queue stores dummy events that has
the tag of NDTs. Nonetheless, current pqueue.c can only use time as the
priority. So new pqueue should be implemented.
---
 core/environment.c         |  6 ++++--
 core/federated/federate.c  | 10 ++++++++++
 core/reactor_common.c      |  6 ++++++
 include/core/environment.h |  2 +-
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/core/environment.c b/core/environment.c
index f04b7e942..9c569633b 100644
--- a/core/environment.c
+++ b/core/environment.c
@@ -107,8 +107,10 @@ static void environment_init_modes(environment_t* env, int num_modes, int num_st
  */
 static void environment_init_federated(environment_t* env, int num_is_present_fields) {
 #ifdef FEDERATED_CENTRALIZED
-    // TODO: init ndt_queue with proper functions.
-    // env->ndt_queue = pqueue_init(10, tag_in_reverse_order, )
+    // FIXME: Create a queue saving tags instead of events. For now, ndt_q stores 
+    // dummy events.
+    env->ndt_q = pqueue_init(INITIAL_EVENT_QUEUE_SIZE, in_reverse_order, get_event_time,
+        get_event_position, set_event_position, event_matches, print_event);
 #endif
 #ifdef FEDERATED_DECENTRALIZED
     env->_lf_intended_tag_fields = (tag_t**) calloc(num_is_present_fields, sizeof(tag_t*));
diff --git a/core/federated/federate.c b/core/federated/federate.c
index bd7e7142f..da013fb5d 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2321,6 +2321,16 @@ void handle_next_downstream_tag() {
 
     LF_PRINT_LOG("Received from RTI a MSG_TYPE_NEXT_DOWNSTREAM_TAG message with elapsed tag " PRINTF_TAG ".",
             NDT.time - start_time, NDT.microstep);
+    environment_t* env;
+    _lf_get_environments(&env);
+
+    if (lf_tag_compare(env->current_tag, NDT) < 0) {
+        // The current tag is less than NDT. Push NDT to ndt_q.
+        event_t* dummy = _lf_create_dummy_events(env, NULL, NDT.time, NULL, NDT.microstep);
+        pqueue_insert(env->ndt_q, dummy);
+    } else {
+        // The current tag is greater than or equal to NDT. Send LTC, NET, and ABS messages.
+    }
 }
 
 /**
diff --git a/core/reactor_common.c b/core/reactor_common.c
index b7a3c9ac7..6f7d33d4d 100644
--- a/core/reactor_common.c
+++ b/core/reactor_common.c
@@ -292,6 +292,12 @@ void _lf_start_time_step(environment_t *env) {
         }
     }
 
+#ifdef FEDERATED_CENTRALIZED
+    while (lf_tag_compare(((event_t*) pqueue_peek(env->ndt_q))->time, env->current_tag.time) < 0) {
+        // Remove elements of ndt_q with tag less than the current tag.
+        pqueue_remove(env->ndt_q, pqueue_peek(env->ndt_q));
+    }
+#endif
 #ifdef FEDERATED_DECENTRALIZED
     for (int i = 0; i < env->is_present_fields_size; i++) {
         // FIXME: For now, an intended tag of (NEVER, 0)
diff --git a/include/core/environment.h b/include/core/environment.h
index 6fb98ee48..62476f540 100644
--- a/include/core/environment.h
+++ b/include/core/environment.h
@@ -106,7 +106,7 @@ typedef struct environment_t {
     int _lf_intended_tag_fields_size;
 #endif // FEDERATED
 #ifdef FEDERATED_CENTRALIZED
-    pqueue_t* ndt_queue;
+    pqueue_t* ndt_q;
 #endif // FEDERATED_CENTRALIZED
 } environment_t;
 

From 2ebc563b660ca6ca21f96a723cbb011c58b0ebae Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 12 Sep 2023 21:01:49 +0900
Subject: [PATCH 13/49] Add supports for ndt_queue

---
 core/utils/ndt_pqueue_support.h | 81 +++++++++++++++++++++++++++++++++
 core/utils/pqueue.c             | 58 +++++++++++++++++++++++
 2 files changed, 139 insertions(+)
 create mode 100644 core/utils/ndt_pqueue_support.h

diff --git a/core/utils/ndt_pqueue_support.h b/core/utils/ndt_pqueue_support.h
new file mode 100644
index 000000000..53e3e37da
--- /dev/null
+++ b/core/utils/ndt_pqueue_support.h
@@ -0,0 +1,81 @@
+/*************
+Copyright (c) 2022, The University of California at Berkeley.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+***************/
+
+/**
+ * @file pqueue_support.h
+ * @author Edward A. Lee
+ * @author Marten Lohstroh
+ * @brief Header-only support functions for pqueue.
+ */
+
+#ifndef NDT_PQUEUE_SUPPORT_H
+#define NDT_PQUEUE_SUPPORT_H
+
+
+#include "../reactor.h"
+#include "tag.h"
+
+// ********** NDT Priority Queue Support Start
+
+typedef struct {
+    tag_t tag;
+    size_t pos;
+} ndt_node;
+
+/**
+ * Return whether the tags contained by the first and second argument 
+ * are given in reverse order.
+*/
+static int tag_in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
+
+/**
+ * Return whether or not the tags contained by given pointers are identical.
+ */
+static int tag_matches(void* next, void* curr);
+
+/**
+ * Report a priority equal to the pointer to an ndt_node.
+ * Used for sorting pointers to ndt_node in the NDT queue.
+ */
+static pqueue_pri_t get_ndt_priority(void *a);
+
+/**
+ * Return the given ndt_node's position in the ndt_queue.
+ */
+static size_t get_ndtq_position(void *a);
+
+/**
+ * Return the given ndt_node's position in the ndt_queue.
+ */
+static void set_ndtq_position(void *a, size_t pos);
+
+/**
+ * Print some information about the given ndt_node.
+ * 
+ * DEBUG function only.
+ */
+static void print_tag(void *reaction);
+
+// ********** NDT Priority Queue Support End
+#endif
diff --git a/core/utils/pqueue.c b/core/utils/pqueue.c
index aa1f1bd15..2e1fc79c0 100644
--- a/core/utils/pqueue.c
+++ b/core/utils/pqueue.c
@@ -39,6 +39,10 @@
 
 #include "platform.h"
 #include "pqueue.h"
+#include "ndt_pqueue_support.h"
+#ifdef FEDERATED_CENTRALIZED
+#include "ndt_pqueue_support.h"
+#endif
 #include "util.h"
 #include "lf_types.h"
 
@@ -473,3 +477,57 @@ void print_event(void *event) {
     LF_PRINT_DEBUG("time: " PRINTF_TIME ", trigger: %p, token: %p",
             e->time, e->trigger, e->token);
 }
+
+
+// ********** NDT Priority Queue Support Start
+#ifdef FEDERATED_CENTRALIZED
+/**
+ * Return whether the tags contained by the first and second argument 
+ * are given in reverse order.
+*/
+int tag_in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that) {
+    return lf_tag_compare(((ndt_node*) thiz)->tag, ((ndt_node*) that)->tag);
+}
+
+/**
+ * Return whether or not the tags contained by given pointers are identical.
+ */
+static int tag_matches(void* next, void* curr) {
+    return lf_tag_compare(((ndt_node*) next)->tag, ((ndt_node*) curr)->tag);
+}
+
+/**
+ * Report a priority equal to the pointer to an ndt_node.
+ * Used for sorting pointers to ndt_node in the NDT queue.
+ */
+static pqueue_pri_t get_ndt_node(void *a) {
+    // Note that NDT queue stores pointers to ndt_nodes and it is its 
+    // pqueue_pri_t as well. Return itself.
+    return (pqueue_pri_t) a;
+}
+
+/**
+ * Return the given ndt_node's position in the ndt_queue.
+ */
+static size_t get_ndtq_position(void *a) {
+    return ((ndt_node*) a)->pos;
+}
+
+/**
+ * Set the given ndt_node's position in the ndt_queue.
+ */
+static void set_ndtq_position(void *a, size_t pos) {
+    ((ndt_node*) a)->pos = pos;
+}
+
+/**
+ * Print some information about the given ndt_node.
+ * 
+ * DEBUG function only.
+ */
+static void print_tag(void *node) {
+    tag_t tag = ((ndt_node*) node)->tag;
+    LF_PRINT_DEBUG("Elapsed logical time:" PRINTF_TIME ", microstep: %d, position: %ld", 
+    tag.time, tag.microstep, ((ndt_node*) node)->pos);
+}
+#endif
\ No newline at end of file

From 5939d5abc751aac964938f00876ab518c2da97e6 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 3 Oct 2023 12:26:26 +0900
Subject: [PATCH 14/49] Add the NDT option in the RTI

---
 core/federated/RTI/rti_lib.c | 9 +++++++--
 core/federated/RTI/rti_lib.h | 5 +++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 5ecf135f5..8577d4d19 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -560,8 +560,10 @@ void handle_next_event_tag(federate_t* fed) {
         intended_tag
     );
     // If fed cannot get the grant of the intended tag, send NDTs to its upstream federates.
-    if (lf_tag_compare(fed->enclave.last_granted, intended_tag) < 0) {
-        send_upstream_next_downstream_tag(fed, intended_tag);
+    if (_f_rti->ndt_enabled) {
+        if (lf_tag_compare(fed->enclave.last_granted, intended_tag) < 0) {
+            send_upstream_next_downstream_tag(fed, intended_tag);
+        }
     }
     lf_mutex_unlock(&rti_mutex);
 }
@@ -1789,6 +1791,8 @@ int process_args(int argc, const char* argv[]) {
         } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--version") == 0) {
             lf_print("%s", version_info);
             return 0;
+        } else if (strcmp(argv[i], "--ndt" == 0)) {
+            _f_rti->ndt_enabled = true;
         } else if (strcmp(argv[i], " ") == 0) {
             // Tolerate spaces
             continue;
@@ -1827,5 +1831,6 @@ void initialize_RTI(){
     _f_rti->clock_sync_exchanges_per_interval = 10,
     _f_rti->authentication_enabled = false,
     _f_rti->tracing_enabled = false;
+    _f_rti->ndt_enabled = true; // FIXME: Initialize this with true to test the feature
     _f_rti->stop_in_progress = false;
 }
diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h
index 6df506bc3..cd107e23b 100644
--- a/core/federated/RTI/rti_lib.h
+++ b/core/federated/RTI/rti_lib.h
@@ -180,6 +180,11 @@ typedef struct federation_rti_t {
      */
     bool authentication_enabled;
 
+    /**
+     * Boolean indicating that NDT message is enabled
+    */
+    bool ndt_enabled;
+
     /**
      * Boolean indicating that a stop request is already in progress.
      */

From 53a6f38724c8042086c5cfff2df920c64b6766f4 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 3 Oct 2023 13:08:17 +0900
Subject: [PATCH 15/49] Initialzie ndt_q properly

---
 core/environment.c              |  8 ++++----
 core/federated/federate.c       |  6 ++++--
 core/reactor_common.c           |  6 +++---
 core/utils/ndt_pqueue_support.h |  5 +++++
 core/utils/pqueue.c             | 18 ++++++++++--------
 include/core/environment.h      |  4 ++--
 include/core/utils/pqueue.h     | 28 ++++++++++++++++------------
 7 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/core/environment.c b/core/environment.c
index 9c569633b..affde4d76 100644
--- a/core/environment.c
+++ b/core/environment.c
@@ -106,12 +106,12 @@ static void environment_init_modes(environment_t* env, int num_modes, int num_st
  * @brief Initialize the federation-specific parts of the environment struct.
  */
 static void environment_init_federated(environment_t* env, int num_is_present_fields) {
-#ifdef FEDERATED_CENTRALIZED
+// #ifdef FEDERATED_CENTRALIZED
     // FIXME: Create a queue saving tags instead of events. For now, ndt_q stores 
     // dummy events.
-    env->ndt_q = pqueue_init(INITIAL_EVENT_QUEUE_SIZE, in_reverse_order, get_event_time,
-        get_event_position, set_event_position, event_matches, print_event);
-#endif
+    env->ndt_q = pqueue_init(INITIAL_EVENT_QUEUE_SIZE, tag_in_reverse_order, get_ndt_priority,
+        get_ndtq_position, set_ndtq_position, ndt_node_matches, print_event);
+// #endif
 #ifdef FEDERATED_DECENTRALIZED
     env->_lf_intended_tag_fields = (tag_t**) calloc(num_is_present_fields, sizeof(tag_t*));
     lf_assert(env->_lf_intended_tag_fields != NULL, "Out of memory");
diff --git a/core/federated/federate.c b/core/federated/federate.c
index e8efd2996..7b2174ceb 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -67,6 +67,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <openssl/rand.h> // For secure random number generation.
 #include <openssl/hmac.h> // For HMAC-based authentication of federates.
 #endif
+#include <pqueue.h>
 
 // Global variables defined in tag.c:
 extern instant_t _lf_last_reported_unadjusted_physical_time_ns;
@@ -2321,8 +2322,9 @@ void handle_next_downstream_tag() {
 
     if (lf_tag_compare(env->current_tag, NDT) < 0) {
         // The current tag is less than NDT. Push NDT to ndt_q.
-        event_t* dummy = _lf_create_dummy_events(env, NULL, NDT.time, NULL, NDT.microstep);
-        pqueue_insert(env->ndt_q, dummy);
+        ndt_node* node = (ndt_node*) malloc(sizeof(ndt_node));
+        node->tag = env->current_tag;
+        pqueue_insert(env->ndt_q, node);
     } else {
         // The current tag is greater than or equal to NDT. Send LTC, NET, and ABS messages.
     }
diff --git a/core/reactor_common.c b/core/reactor_common.c
index 6f7d33d4d..31e710d80 100644
--- a/core/reactor_common.c
+++ b/core/reactor_common.c
@@ -292,12 +292,12 @@ void _lf_start_time_step(environment_t *env) {
         }
     }
 
-#ifdef FEDERATED_CENTRALIZED
-    while (lf_tag_compare(((event_t*) pqueue_peek(env->ndt_q))->time, env->current_tag.time) < 0) {
+// #ifdef FEDERATED_CENTRALIZED
+    while (lf_tag_compare(((ndt_node*) pqueue_peek(env->ndt_q))->tag, env->current_tag) < 0) {
         // Remove elements of ndt_q with tag less than the current tag.
         pqueue_remove(env->ndt_q, pqueue_peek(env->ndt_q));
     }
-#endif
+// #endif
 #ifdef FEDERATED_DECENTRALIZED
     for (int i = 0; i < env->is_present_fields_size; i++) {
         // FIXME: For now, an intended tag of (NEVER, 0)
diff --git a/core/utils/ndt_pqueue_support.h b/core/utils/ndt_pqueue_support.h
index 53e3e37da..aa00c71c7 100644
--- a/core/utils/ndt_pqueue_support.h
+++ b/core/utils/ndt_pqueue_support.h
@@ -70,6 +70,11 @@ static size_t get_ndtq_position(void *a);
  */
 static void set_ndtq_position(void *a, size_t pos);
 
+/**
+ * 
+*/
+static int ndt_node_matches(void* next, void* curr);
+
 /**
  * Print some information about the given ndt_node.
  * 
diff --git a/core/utils/pqueue.c b/core/utils/pqueue.c
index 2e1fc79c0..545d2f7fc 100644
--- a/core/utils/pqueue.c
+++ b/core/utils/pqueue.c
@@ -39,10 +39,6 @@
 
 #include "platform.h"
 #include "pqueue.h"
-#include "ndt_pqueue_support.h"
-#ifdef FEDERATED_CENTRALIZED
-#include "ndt_pqueue_support.h"
-#endif
 #include "util.h"
 #include "lf_types.h"
 
@@ -480,7 +476,6 @@ void print_event(void *event) {
 
 
 // ********** NDT Priority Queue Support Start
-#ifdef FEDERATED_CENTRALIZED
 /**
  * Return whether the tags contained by the first and second argument 
  * are given in reverse order.
@@ -500,7 +495,7 @@ static int tag_matches(void* next, void* curr) {
  * Report a priority equal to the pointer to an ndt_node.
  * Used for sorting pointers to ndt_node in the NDT queue.
  */
-static pqueue_pri_t get_ndt_node(void *a) {
+static pqueue_pri_t get_ndt_priority(void *a) {
     // Note that NDT queue stores pointers to ndt_nodes and it is its 
     // pqueue_pri_t as well. Return itself.
     return (pqueue_pri_t) a;
@@ -520,6 +515,14 @@ static void set_ndtq_position(void *a, size_t pos) {
     ((ndt_node*) a)->pos = pos;
 }
 
+/**
+ * 
+*/
+static int ndt_node_matches(void* next, void* curr) {
+    return ((((ndt_node*) next)->pos == ((ndt_node*) curr)->pos) &&
+    ((lf_tag_compare(((ndt_node*) next)->tag, ((ndt_node*) curr)->tag)) == 0));
+}
+
 /**
  * Print some information about the given ndt_node.
  * 
@@ -529,5 +532,4 @@ static void print_tag(void *node) {
     tag_t tag = ((ndt_node*) node)->tag;
     LF_PRINT_DEBUG("Elapsed logical time:" PRINTF_TIME ", microstep: %d, position: %ld", 
     tag.time, tag.microstep, ((ndt_node*) node)->pos);
-}
-#endif
\ No newline at end of file
+}
\ No newline at end of file
diff --git a/include/core/environment.h b/include/core/environment.h
index 62476f540..396e85756 100644
--- a/include/core/environment.h
+++ b/include/core/environment.h
@@ -105,9 +105,9 @@ typedef struct environment_t {
     tag_t** _lf_intended_tag_fields;
     int _lf_intended_tag_fields_size;
 #endif // FEDERATED
-#ifdef FEDERATED_CENTRALIZED
+// #ifdef FEDERATED_CENTRALIZED
     pqueue_t* ndt_q;
-#endif // FEDERATED_CENTRALIZED
+// #endif // FEDERATED_CENTRALIZED
 } environment_t;
 
 #ifdef MODAL_REACTORS
diff --git a/include/core/utils/pqueue.h b/include/core/utils/pqueue.h
index 5c3e7fe2b..d1cda7a7b 100644
--- a/include/core/utils/pqueue.h
+++ b/include/core/utils/pqueue.h
@@ -42,6 +42,10 @@
 #define PQUEUE_H
 
 #include <stddef.h>
+#include <pqueue_support.h>
+// #ifdef FEDERATED_CENTRALIZED
+#include "ndt_pqueue_support.h"
+// #endif
 
 /** priority data type */
 typedef unsigned long long pqueue_pri_t;
@@ -213,18 +217,18 @@ pqueue_dump(pqueue_t *q,
 int pqueue_is_valid(pqueue_t *q);
 
 // ********** Priority Queue Support Start
-int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
-int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that);
-int event_matches(void* next, void* curr);
-int reaction_matches(void* next, void* curr);
-pqueue_pri_t get_event_time(void *a);
-pqueue_pri_t get_reaction_index(void *a);
-size_t get_event_position(void *a);
-size_t get_reaction_position(void *a);
-void set_event_position(void *a, size_t pos);
-void set_reaction_position(void *a, size_t pos);
-void print_reaction(void *reaction);
-void print_event(void *event);
+// int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
+// int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that);
+// int event_matches(void* next, void* curr);
+// int reaction_matches(void* next, void* curr);
+// pqueue_pri_t get_event_time(void *a);
+// pqueue_pri_t get_reaction_index(void *a);
+// size_t get_event_position(void *a);
+// size_t get_reaction_position(void *a);
+// void set_event_position(void *a, size_t pos);
+// void set_reaction_position(void *a, size_t pos);
+// void print_reaction(void *reaction);
+// void print_event(void *event);
 
 #endif /* PQUEUE_H */
 /** @} */

From d8e179ba00cc009c865f2ab8f7ca7af329557f88 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 3 Oct 2023 14:44:09 +0900
Subject: [PATCH 16/49] Copy the ndt_q support code into pqueue.h

---
 include/core/utils/pqueue.h | 42 +++++++++++++++++++++++--------------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/include/core/utils/pqueue.h b/include/core/utils/pqueue.h
index d1cda7a7b..efa59a681 100644
--- a/include/core/utils/pqueue.h
+++ b/include/core/utils/pqueue.h
@@ -42,10 +42,7 @@
 #define PQUEUE_H
 
 #include <stddef.h>
-#include <pqueue_support.h>
-// #ifdef FEDERATED_CENTRALIZED
-#include "ndt_pqueue_support.h"
-// #endif
+#include "../tag.h"
 
 /** priority data type */
 typedef unsigned long long pqueue_pri_t;
@@ -217,18 +214,31 @@ pqueue_dump(pqueue_t *q,
 int pqueue_is_valid(pqueue_t *q);
 
 // ********** Priority Queue Support Start
-// int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
-// int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that);
-// int event_matches(void* next, void* curr);
-// int reaction_matches(void* next, void* curr);
-// pqueue_pri_t get_event_time(void *a);
-// pqueue_pri_t get_reaction_index(void *a);
-// size_t get_event_position(void *a);
-// size_t get_reaction_position(void *a);
-// void set_event_position(void *a, size_t pos);
-// void set_reaction_position(void *a, size_t pos);
-// void print_reaction(void *reaction);
-// void print_event(void *event);
+int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
+int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that);
+int event_matches(void* next, void* curr);
+int reaction_matches(void* next, void* curr);
+pqueue_pri_t get_event_time(void *a);
+pqueue_pri_t get_reaction_index(void *a);
+size_t get_event_position(void *a);
+size_t get_reaction_position(void *a);
+void set_event_position(void *a, size_t pos);
+void set_reaction_position(void *a, size_t pos);
+void print_reaction(void *reaction);
+void print_event(void *event);
+
+// ********** NDT Priority Queue Support Start
+typedef struct {
+    tag_t tag;
+    size_t pos;
+} ndt_node;
+static int tag_in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
+static int tag_matches(void* next, void* curr);
+static pqueue_pri_t get_ndt_priority(void *a);
+static size_t get_ndtq_position(void *a);
+static void set_ndtq_position(void *a, size_t pos);
+static int ndt_node_matches(void* next, void* curr);
+static void print_tag(void *reaction);
 
 #endif /* PQUEUE_H */
 /** @} */

From 95e0e7928e3b5c6852e7f1ad57dc2c4923af4050 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 10 Oct 2023 17:44:56 +0900
Subject: [PATCH 17/49] Fixing the reference issues

---
 core/environment.c              |  2 +-
 core/federated/federate.c       |  3 ++-
 core/utils/ndt_pqueue_support.h |  2 +-
 core/utils/pqueue.c             | 12 ++++++------
 include/core/utils/pqueue.h     | 26 ++++++++++++++------------
 5 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/core/environment.c b/core/environment.c
index affde4d76..6e6b36652 100644
--- a/core/environment.c
+++ b/core/environment.c
@@ -109,7 +109,7 @@ static void environment_init_federated(environment_t* env, int num_is_present_fi
 // #ifdef FEDERATED_CENTRALIZED
     // FIXME: Create a queue saving tags instead of events. For now, ndt_q stores 
     // dummy events.
-    env->ndt_q = pqueue_init(INITIAL_EVENT_QUEUE_SIZE, tag_in_reverse_order, get_ndt_priority,
+    env->ndt_q = pqueue_init(INITIAL_EVENT_QUEUE_SIZE, tag_in_reverse_order, get_ndtq_priority,
         get_ndtq_position, set_ndtq_position, ndt_node_matches, print_event);
 // #endif
 #ifdef FEDERATED_DECENTRALIZED
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 7b2174ceb..0d2851daf 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2308,7 +2308,8 @@ void handle_stop_request_message() {
  * 
 */
 void handle_next_downstream_tag() {
-    // FIXME: Fill this function.
+    // FIXME: maybe the mutex lock is needed. Is it possible to remove an element 
+    // simultaneously with this function?
     size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t);
     unsigned char buffer[bytes_to_read];
     read_from_socket_errexit(_fed.socket_TCP_RTI, bytes_to_read, buffer,
diff --git a/core/utils/ndt_pqueue_support.h b/core/utils/ndt_pqueue_support.h
index aa00c71c7..26b622c8d 100644
--- a/core/utils/ndt_pqueue_support.h
+++ b/core/utils/ndt_pqueue_support.h
@@ -58,7 +58,7 @@ static int tag_matches(void* next, void* curr);
  * Report a priority equal to the pointer to an ndt_node.
  * Used for sorting pointers to ndt_node in the NDT queue.
  */
-static pqueue_pri_t get_ndt_priority(void *a);
+static pqueue_pri_t get_ndtq_priority(void *a);
 
 /**
  * Return the given ndt_node's position in the ndt_queue.
diff --git a/core/utils/pqueue.c b/core/utils/pqueue.c
index 545d2f7fc..86fed89c8 100644
--- a/core/utils/pqueue.c
+++ b/core/utils/pqueue.c
@@ -487,7 +487,7 @@ int tag_in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that) {
 /**
  * Return whether or not the tags contained by given pointers are identical.
  */
-static int tag_matches(void* next, void* curr) {
+int tag_matches(void* next, void* curr) {
     return lf_tag_compare(((ndt_node*) next)->tag, ((ndt_node*) curr)->tag);
 }
 
@@ -495,7 +495,7 @@ static int tag_matches(void* next, void* curr) {
  * Report a priority equal to the pointer to an ndt_node.
  * Used for sorting pointers to ndt_node in the NDT queue.
  */
-static pqueue_pri_t get_ndt_priority(void *a) {
+pqueue_pri_t get_ndtq_priority(void *a) {
     // Note that NDT queue stores pointers to ndt_nodes and it is its 
     // pqueue_pri_t as well. Return itself.
     return (pqueue_pri_t) a;
@@ -504,21 +504,21 @@ static pqueue_pri_t get_ndt_priority(void *a) {
 /**
  * Return the given ndt_node's position in the ndt_queue.
  */
-static size_t get_ndtq_position(void *a) {
+size_t get_ndtq_position(void *a) {
     return ((ndt_node*) a)->pos;
 }
 
 /**
  * Set the given ndt_node's position in the ndt_queue.
  */
-static void set_ndtq_position(void *a, size_t pos) {
+void set_ndtq_position(void *a, size_t pos) {
     ((ndt_node*) a)->pos = pos;
 }
 
 /**
  * 
 */
-static int ndt_node_matches(void* next, void* curr) {
+int ndt_node_matches(void* next, void* curr) {
     return ((((ndt_node*) next)->pos == ((ndt_node*) curr)->pos) &&
     ((lf_tag_compare(((ndt_node*) next)->tag, ((ndt_node*) curr)->tag)) == 0));
 }
@@ -528,7 +528,7 @@ static int ndt_node_matches(void* next, void* curr) {
  * 
  * DEBUG function only.
  */
-static void print_tag(void *node) {
+void print_tag(void *node) {
     tag_t tag = ((ndt_node*) node)->tag;
     LF_PRINT_DEBUG("Elapsed logical time:" PRINTF_TIME ", microstep: %d, position: %ld", 
     tag.time, tag.microstep, ((ndt_node*) node)->pos);
diff --git a/include/core/utils/pqueue.h b/include/core/utils/pqueue.h
index efa59a681..35ff9a31f 100644
--- a/include/core/utils/pqueue.h
+++ b/include/core/utils/pqueue.h
@@ -42,7 +42,7 @@
 #define PQUEUE_H
 
 #include <stddef.h>
-#include "../tag.h"
+#include "tag.h"
 
 /** priority data type */
 typedef unsigned long long pqueue_pri_t;
@@ -74,6 +74,11 @@ typedef struct pqueue_t
     pqueue_print_entry_f prt;   /**< callback to print elements */
     void **d;                   /**< The actual queue in binary heap form */
 } pqueue_t;
+typedef struct ndt_node 
+{
+    tag_t tag;
+    size_t pos;
+} ndt_node;
 
 /**
  * initialize the queue
@@ -228,17 +233,14 @@ void print_reaction(void *reaction);
 void print_event(void *event);
 
 // ********** NDT Priority Queue Support Start
-typedef struct {
-    tag_t tag;
-    size_t pos;
-} ndt_node;
-static int tag_in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
-static int tag_matches(void* next, void* curr);
-static pqueue_pri_t get_ndt_priority(void *a);
-static size_t get_ndtq_position(void *a);
-static void set_ndtq_position(void *a, size_t pos);
-static int ndt_node_matches(void* next, void* curr);
-static void print_tag(void *reaction);
+
+int tag_in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that);
+int tag_matches(void* next, void* curr);
+pqueue_pri_t get_ndtq_priority(void *a);
+size_t get_ndtq_position(void *a);
+void set_ndtq_position(void *a, size_t pos);
+int ndt_node_matches(void* next, void* curr);
+void print_tag(void *reaction);
 
 #endif /* PQUEUE_H */
 /** @} */

From 8b0ccc39c0c3c605a4a0829f6460eb7f6e7a8a43 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <78055940+byeong-gil@users.noreply.github.com>
Date: Tue, 10 Oct 2023 21:32:43 +0900
Subject: [PATCH 18/49] Update lingua-franca-ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index fe4328b6c..1f7391f92 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-c-remove-deprecated-schedulers
+master

From b4c4aff14c48dac56964c2ff724137095bf0353a Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 10 Oct 2023 22:22:46 +0900
Subject: [PATCH 19/49] Check emptiness of ndt_q before processing it

---
 core/reactor_common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/reactor_common.c b/core/reactor_common.c
index 31e710d80..d6cf5b032 100644
--- a/core/reactor_common.c
+++ b/core/reactor_common.c
@@ -293,7 +293,7 @@ void _lf_start_time_step(environment_t *env) {
     }
 
 // #ifdef FEDERATED_CENTRALIZED
-    while (lf_tag_compare(((ndt_node*) pqueue_peek(env->ndt_q))->tag, env->current_tag) < 0) {
+    while (pqueue_peek(env->ndt_q) != NULL && lf_tag_compare(((ndt_node*) pqueue_peek(env->ndt_q))->tag, env->current_tag) < 0) {
         // Remove elements of ndt_q with tag less than the current tag.
         pqueue_remove(env->ndt_q, pqueue_peek(env->ndt_q));
     }

From afe8cc92d0e53305b2dacf2433bf6b0669c75a8c Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sat, 14 Oct 2023 16:29:06 +0900
Subject: [PATCH 20/49] Add debuging messages

---
 core/federated/RTI/rti_lib.c | 10 +++++++---
 core/federated/federate.c    |  2 +-
 core/reactor_common.c        |  7 ++++++-
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 8577d4d19..7b63dfddf 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -298,8 +298,8 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
         int upstream_id = fed->enclave.upstream[i];
         if (lf_tag_compare(_f_rti->enclaves[upstream_id]->enclave.completed, next_event_tag) < 0) {
             // send next downstream tag to upstream federates that do not complete the next_event_tag
-            LF_PRINT_LOG("RTI sending next downstream event message to federate %u.",
-                upstream_id);
+            LF_PRINT_LOG("RTI sending the next downstream event message (NDT) " PRINTF_TAG "to federate %u.",
+                next_event_tag.time - start_time, next_event_tag.microstep, upstream_id);
             write_to_socket_errexit(_f_rti->enclaves[upstream_id]->socket, message_length, buffer,
                     "RTI failed to send MSG_TYPE_NEXT_DOWNSTREAM_TAG message to federate %d.", upstream_id);
         }
@@ -1656,6 +1656,8 @@ void usage(int argc, const char* argv[]) {
     lf_print("          clock sync attempt (default is 10). Applies to 'init' and 'on'.");
     lf_print("  -a, --auth Turn on HMAC authentication options.");
     lf_print("  -t, --tracing Turn on tracing.");
+    lf_print("  -v, --version The minimum required version of Lingua Franca.");
+    lf_print("  --ndt Turn on the ndt_downstream_tag message option.");
 
     lf_print("Command given:");
     for (int i = 0; i < argc; i++) {
@@ -1791,10 +1793,12 @@ int process_args(int argc, const char* argv[]) {
         } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--version") == 0) {
             lf_print("%s", version_info);
             return 0;
-        } else if (strcmp(argv[i], "--ndt" == 0)) {
+        } else if (strcmp(argv[i], "--ndt") == 0) {
+            lf_print("ndt is enabled");
             _f_rti->ndt_enabled = true;
         } else if (strcmp(argv[i], " ") == 0) {
             // Tolerate spaces
+            lf_print("space");
             continue;
         }  else {
            lf_print_error("Unrecognized command-line argument: %s", argv[i]);
diff --git a/core/federated/federate.c b/core/federated/federate.c
index beb452143..760102e81 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2337,7 +2337,7 @@ void handle_next_downstream_tag() {
     if (lf_tag_compare(env->current_tag, NDT) < 0) {
         // The current tag is less than NDT. Push NDT to ndt_q.
         ndt_node* node = (ndt_node*) malloc(sizeof(ndt_node));
-        node->tag = env->current_tag;
+        node->tag = NDT;
         pqueue_insert(env->ndt_q, node);
     } else {
         // The current tag is greater than or equal to NDT. Send LTC, NET, and ABS messages.
diff --git a/core/reactor_common.c b/core/reactor_common.c
index d6cf5b032..12821b0a1 100644
--- a/core/reactor_common.c
+++ b/core/reactor_common.c
@@ -293,8 +293,13 @@ void _lf_start_time_step(environment_t *env) {
     }
 
 // #ifdef FEDERATED_CENTRALIZED
-    while (pqueue_peek(env->ndt_q) != NULL && lf_tag_compare(((ndt_node*) pqueue_peek(env->ndt_q))->tag, env->current_tag) < 0) {
+    while (pqueue_peek(env->ndt_q) != NULL 
+    && lf_tag_compare(((ndt_node*) pqueue_peek(env->ndt_q))->tag, env->current_tag) <= 0) {
         // Remove elements of ndt_q with tag less than the current tag.
+        tag_t tag_to_remove = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
+        LF_PRINT_DEBUG("Remove the tag " PRINTF_TAG " from the ndt_q is before the current tag " PRINTF_TAG ". Remove it.",
+        tag_to_remove.time - start_time, tag_to_remove.microstep,
+        env->current_tag.time - start_time, env->current_tag.microstep);
         pqueue_remove(env->ndt_q, pqueue_peek(env->ndt_q));
     }
 // #endif

From f2e4329e1f0a8bcdcb05baa6fb534fafebdacf9d Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sat, 14 Oct 2023 17:03:47 +0900
Subject: [PATCH 21/49] Eliminate unnecessary port absent messages

However, an upstream federate cannot send an ABS message when it
receives NDT earlier than the current tag cause NDT messages do not
contain the fed ID
---
 core/federated/federate.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 760102e81..8f0fdb85c 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1392,10 +1392,6 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
                                   unsigned short fed_ID) {
     assert(env != GLOBAL_ENVIRONMENT);
 
-    // Construct the message
-    size_t message_length = 1 + sizeof(port_ID) + sizeof(fed_ID) + sizeof(instant_t) + sizeof(microstep_t);
-    unsigned char buffer[message_length];
-
     // Apply the additional delay to the current tag and use that as the intended
     // tag of the outgoing message. Note that if there is delay on the connection,
     // then we cannot promise no message with tag = current_tag + delay because a
@@ -1403,6 +1399,18 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
     // message with a tag strictly less than current_tag + delay.
     tag_t current_message_intended_tag = lf_delay_strict(env->current_tag,
                                                     additional_delay);
+    
+    if (pqueue_peek(env->ndt_q) != NULL) {
+        tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
+        if (lf_tag_compare(current_message_intended_tag, ndt_q_barrier) < 0) {
+            // No events exist in any downstream federates
+            return;
+        }
+    }
+
+    // Construct the message
+    size_t message_length = 1 + sizeof(port_ID) + sizeof(fed_ID) + sizeof(instant_t) + sizeof(microstep_t);
+    unsigned char buffer[message_length];
 
     LF_PRINT_LOG("Sending port "
             "absent for tag " PRINTF_TAG " for port %d to federate %d.",
@@ -2341,6 +2349,7 @@ void handle_next_downstream_tag() {
         pqueue_insert(env->ndt_q, node);
     } else {
         // The current tag is greater than or equal to NDT. Send LTC, NET, and ABS messages.
+        // FIXME: How do we know which federate invoke this NDT? This info is needed for sending ABS messages.
     }
 }
 

From 8a5d11a9a66dc61c5c5d3711d7c8c3b5ddb26731 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 15 Oct 2023 15:16:42 +0900
Subject: [PATCH 22/49] Remove unnecessary NET messages

---
 core/federated/federate.c | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 8f0fdb85c..e68a8c5d2 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1404,6 +1404,10 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
         tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
         if (lf_tag_compare(current_message_intended_tag, ndt_q_barrier) < 0) {
             // No events exist in any downstream federates
+            LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is smaller than the tag barrier " PRINTF_TAG "."
+            "Don't have to send the port absent message.",
+            current_message_intended_tag.time - start_time, current_message_intended_tag.microstep,
+            ndt_q_barrier.time - start_time, ndt_q_barrier.microstep);
             return;
         }
     }
@@ -2761,10 +2765,27 @@ tag_t _lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply
             // This if statement does not fall through but rather returns.
             // NET is not bounded by physical time or has no downstream federates.
             // Normal case.
-            _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag, wait_for_reply);
-            _fed.last_sent_NET = tag;
-            LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.",
-                    tag.time - start_time, tag.microstep);
+
+            // If there is no downstream events that require the NET of the current tag,
+            // do not send the NET.
+            bool need_to_send_NET = true;
+            if (pqueue_peek(env->ndt_q) != NULL) {
+                tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
+                if (lf_tag_compare(tag, ndt_q_barrier) < 0) {
+                    // No events exist in any downstream federates
+                    LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is smaller than the tag barrier " PRINTF_TAG "."
+                    "Don't have to send the next event tag.",
+                    tag.time - start_time, tag.microstep,
+                    ndt_q_barrier.time - start_time, ndt_q_barrier.microstep);
+                    need_to_send_NET = false;
+                }
+            }
+            if (need_to_send_NET) {
+                _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag, wait_for_reply);
+                _fed.last_sent_NET = tag;
+                LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.",
+                        tag.time - start_time, tag.microstep);
+            }
 
             if (!wait_for_reply) {
                 LF_PRINT_LOG("Not waiting for reply to NET.");

From 8b63d24119429c9b6d2e769d3b297c3ecbd72624 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 15 Oct 2023 16:16:59 +0900
Subject: [PATCH 23/49] Remove unnecessary LTC messages

There is an error that an upstream federate does not send NET, LTC
messages even when there is an actual message. Need to fix.
---
 core/federated/federate.c | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index e68a8c5d2..fa7b06c68 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1922,11 +1922,28 @@ void _lf_logical_tag_complete(tag_t tag_to_send) {
     if (compare_with_last_tag >= 0) {
         return;
     }
-    LF_PRINT_LOG("Sending Logical Time Complete (LTC) " PRINTF_TAG " to the RTI.",
+
+    environment_t *env;
+    _lf_get_environments(&env);
+    bool need_to_send_LTC = true;
+    if (pqueue_peek(env->ndt_q) != NULL) {
+        tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
+        if (lf_tag_compare(tag_to_send, ndt_q_barrier) < 0) {
+            // No events exist in any downstream federates
+            LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is smaller than the tag barrier " PRINTF_TAG "."
+            "Don't have to send the logical tag complete.",
+            tag_to_send.time - start_time, tag_to_send.microstep,
+            ndt_q_barrier.time - start_time, ndt_q_barrier.microstep);
+            need_to_send_LTC = false;
+        }
+    }
+    if (need_to_send_LTC) {
+        LF_PRINT_LOG("Sending Logical Tag Complete (LTC) " PRINTF_TAG " to the RTI.",
             tag_to_send.time - start_time,
             tag_to_send.microstep);
-    _lf_send_tag(MSG_TYPE_LOGICAL_TAG_COMPLETE, tag_to_send, true);
-    _fed.last_sent_LTC = tag_to_send;
+        _lf_send_tag(MSG_TYPE_LOGICAL_TAG_COMPLETE, tag_to_send, true);
+        _fed.last_sent_LTC = tag_to_send;
+    }
 }
 
 bool update_max_level(tag_t tag, bool is_provisional) {
@@ -2133,6 +2150,7 @@ void handle_provisional_tag_advance_grant() {
         // TAG. In either case, we know that at the PTAG tag, all outputs
         // have either been sent or are absent, so we can send an LTC.
         // Send an LTC to indicate absent outputs.
+        LF_PRINT_DEBUG("Inside handle_provisional_tag_advance_grant");
         _lf_logical_tag_complete(PTAG);
         // Nothing more to do.
         lf_mutex_unlock(&env->mutex);

From f9e07777e3e03249172a46f63325b1adccd5cdcb Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 15 Oct 2023 17:16:27 +0900
Subject: [PATCH 24/49] Add tracepoints for NDT

---
 core/federated/RTI/rti_lib.c        | 3 +++
 core/federated/federate.c           | 3 +++
 include/core/trace.h                | 4 ++++
 util/tracing/visualization/fedsd.py | 3 +++
 4 files changed, 13 insertions(+)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 7b63dfddf..efc1639e4 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -300,6 +300,9 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
             // send next downstream tag to upstream federates that do not complete the next_event_tag
             LF_PRINT_LOG("RTI sending the next downstream event message (NDT) " PRINTF_TAG "to federate %u.",
                 next_event_tag.time - start_time, next_event_tag.microstep, upstream_id);
+            if (_f_rti->tracing_enabled) {
+                tracepoint_rti_to_federate(_f_rti->trace, send_NDT, upstream_id, &next_event_tag);
+            }
             write_to_socket_errexit(_f_rti->enclaves[upstream_id]->socket, message_length, buffer,
                     "RTI failed to send MSG_TYPE_NEXT_DOWNSTREAM_TAG message to federate %d.", upstream_id);
         }
diff --git a/core/federated/federate.c b/core/federated/federate.c
index fa7b06c68..156ad0c9d 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2361,6 +2361,9 @@ void handle_next_downstream_tag() {
 
     LF_PRINT_LOG("Received from RTI a MSG_TYPE_NEXT_DOWNSTREAM_TAG message with elapsed tag " PRINTF_TAG ".",
             NDT.time - start_time, NDT.microstep);
+    // Trace the event when tracing is enabled
+    tracepoint_federate_from_rti(_fed.trace, receive_NDT, _lf_my_fed_id, &NDT);
+    
     environment_t* env;
     _lf_get_environments(&env);
 
diff --git a/include/core/trace.h b/include/core/trace.h
index 72e2d466a..659427752 100644
--- a/include/core/trace.h
+++ b/include/core/trace.h
@@ -98,6 +98,7 @@ typedef enum
     send_P2P_MSG,
     send_ADR_AD,
     send_ADR_QR,
+    send_NDT,
     // Receiving messages
     receive_ACK,
     receive_TIMESTAMP,
@@ -119,6 +120,7 @@ typedef enum
     receive_P2P_MSG,
     receive_ADR_AD,
     receive_ADR_QR,
+    receive_NDT,
     receive_UNIDENTIFIED,
     NUM_EVENT_TYPES
 } trace_event_t;
@@ -161,6 +163,7 @@ static const char *trace_event_names[] = {
     "Sending P2P_MSG",
     "Sending ADR_AD",
     "Sending ADR_QR",
+    "Sending NDT",
     // Receiving messages
     "Receiving ACK",
     "Receiving TIMESTAMP",
@@ -182,6 +185,7 @@ static const char *trace_event_names[] = {
     "Receiving P2P_MSG",
     "Receiving ADR_AD",
     "Receiving ADR_QR",
+    "Receiving NDT",
     "Receiving UNIDENTIFIED",
 };
 
diff --git a/util/tracing/visualization/fedsd.py b/util/tracing/visualization/fedsd.py
index e91249995..f0e13032a 100644
--- a/util/tracing/visualization/fedsd.py
+++ b/util/tracing/visualization/fedsd.py
@@ -26,6 +26,7 @@
     .NET { stroke: #118ab2; fill: #118ab2} \
     .PTAG { stroke: #06d6a0; fill: #06d6a0} \
     .TAG { stroke: #08a578; fill: #08a578} \
+    .NDT { stroke: purple; fill: purple} \
     .TIMESTAMP { stroke: grey; fill: grey } \
     .FED_ID {stroke: #80DD99; fill: #80DD99 } \
     .ADV {stroke-linecap="round" ; stroke: "red" ; fill: "red"} \
@@ -60,6 +61,7 @@
     "Sending P2P_MSG": "P2P_MSG",
     "Sending ADR_AD": "ADR_AD",
     "Sending ADR_QR": "ADR_QR",
+    "Sending NDT": "NDT",
     "Receiving ACK": "ACK",
     "Receiving TIMESTAMP": "TIMESTAMP",
     "Receiving NET": "NET",
@@ -80,6 +82,7 @@
     "Receiving P2P_MSG": "P2P_MSG",
     "Receiving ADR_AD": "ADR_AD",
     "Receiving ADR_QR": "ADR_QR",
+    "Receiving NDT": "NDT",
     "Receiving UNIDENTIFIED": "UNIDENTIFIED",
     "Scheduler advancing time ends": "AdvLT"
 }

From 693db2e07572b11ce9097c8dc31b7217e23e5145 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 15 Oct 2023 17:56:59 +0900
Subject: [PATCH 25/49] Send NET messages when downstream federates need it

---
 core/federated/federate.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 156ad0c9d..152102cac 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2363,18 +2363,20 @@ void handle_next_downstream_tag() {
             NDT.time - start_time, NDT.microstep);
     // Trace the event when tracing is enabled
     tracepoint_federate_from_rti(_fed.trace, receive_NDT, _lf_my_fed_id, &NDT);
-    
+
     environment_t* env;
     _lf_get_environments(&env);
 
-    if (lf_tag_compare(env->current_tag, NDT) < 0) {
-        // The current tag is less than NDT. Push NDT to ndt_q.
+    if (lf_tag_compare(env->current_tag, NDT) <= 0) {
+        // The current tag is less than or equal to the NDT. Push NDT to ndt_q.
         ndt_node* node = (ndt_node*) malloc(sizeof(ndt_node));
         node->tag = NDT;
         pqueue_insert(env->ndt_q, node);
-    } else {
-        // The current tag is greater than or equal to NDT. Send LTC, NET, and ABS messages.
-        // FIXME: How do we know which federate invoke this NDT? This info is needed for sending ABS messages.
+    }
+    if (lf_tag_compare(env->current_tag, NDT) >= 0) {
+        // The current tag is greater than or equal to NDT. Send the appropriate NET message.
+        tag_t next_event_tag = get_next_event_tag(env);
+        send_next_event_tag(env, next_event_tag, false);
     }
 }
 

From 29728e763718e71c5f545d1985d6f6b50dbdc99b Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 17 Oct 2023 10:23:28 +0900
Subject: [PATCH 26/49] When a sender sends a tagged message, insert the tag to
 ndt_q to send the LTC message

---
 core/federated/federate.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 152102cac..bdf111e04 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -421,6 +421,12 @@ int send_timed_message(environment_t* env,
         lf_mutex_unlock(&outbound_socket_mutex);
         return 0;
     }
+
+    // Insert the ndt_node at the tag to send LTC to the RTI.
+    ndt_node* node = (ndt_node*) malloc(sizeof(ndt_node));
+    node->tag = current_message_intended_tag;
+    pqueue_insert(env->ndt_q, node);
+
     // Trace the event when tracing is enabled
     if (message_type == MSG_TYPE_TAGGED_MESSAGE) {
         tracepoint_federate_to_rti(_fed.trace, send_TAGGED_MSG, _lf_my_fed_id, &current_message_intended_tag);

From 1a774035a43b1e2850c58d627b49acc2d719c350 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Fri, 20 Oct 2023 15:36:43 +0900
Subject: [PATCH 27/49] Minor fix

---
 core/reactor_common.c | 2 +-
 core/utils/pqueue.c   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/reactor_common.c b/core/reactor_common.c
index 12821b0a1..8c380b71a 100644
--- a/core/reactor_common.c
+++ b/core/reactor_common.c
@@ -294,7 +294,7 @@ void _lf_start_time_step(environment_t *env) {
 
 // #ifdef FEDERATED_CENTRALIZED
     while (pqueue_peek(env->ndt_q) != NULL 
-    && lf_tag_compare(((ndt_node*) pqueue_peek(env->ndt_q))->tag, env->current_tag) <= 0) {
+    && lf_tag_compare(((ndt_node*) pqueue_peek(env->ndt_q))->tag, env->current_tag) < 0) {
         // Remove elements of ndt_q with tag less than the current tag.
         tag_t tag_to_remove = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
         LF_PRINT_DEBUG("Remove the tag " PRINTF_TAG " from the ndt_q is before the current tag " PRINTF_TAG ". Remove it.",
diff --git a/core/utils/pqueue.c b/core/utils/pqueue.c
index 86fed89c8..cee4e4fd5 100644
--- a/core/utils/pqueue.c
+++ b/core/utils/pqueue.c
@@ -481,7 +481,7 @@ void print_event(void *event) {
  * are given in reverse order.
 */
 int tag_in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that) {
-    return lf_tag_compare(((ndt_node*) thiz)->tag, ((ndt_node*) that)->tag);
+    return lf_tag_compare(((ndt_node*) thiz)->tag, ((ndt_node*) that)->tag) == 1;
 }
 
 /**

From b97fd914b41a1319532bb240d053f348ec952a67 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 24 Oct 2023 09:20:34 +0900
Subject: [PATCH 28/49] Disable NDT when there is a cycle between federates

---
 core/federated/RTI/rti_lib.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index efc1639e4..6df0624cb 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -283,9 +283,8 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
 }
 
 void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
-    // TODO: Fill this function.
     // The RTI receives next_event_tag from the federated fed. 
-    // It has to send NDET messages to the upstream federates of fed
+    // It has to send NDT messages to the upstream federates of fed
     // if the LTC message from an upstream federate is ealrier than the next_event_tag.
     size_t message_length = 1 + sizeof(int64_t) + sizeof(uint32_t);
     unsigned char buffer[message_length];
@@ -296,8 +295,21 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
     // FIXME: Send NDT to transitive upstreams either
     for (int i = 0; i < fed->enclave.num_upstream; i++) {
         int upstream_id = fed->enclave.upstream[i];
-        if (lf_tag_compare(_f_rti->enclaves[upstream_id]->enclave.completed, next_event_tag) < 0) {
-            // send next downstream tag to upstream federates that do not complete the next_event_tag
+        federate_t* upstream_federate = _f_rti->enclaves[upstream_id];
+        bool has_cycle = false;
+        for (int j = 0; j < upstream_federate->enclave.num_upstream; j++) {
+            if (upstream_federate->enclave.upstream[j] == fed->enclave.id) {
+                has_cycle = true;
+            }
+        }
+        if (has_cycle) {
+            // fed and upstream_fed consist a cycle. Do not use NDT.
+            continue;
+        }
+
+        if (lf_tag_compare(upstream_federate->enclave.completed, next_event_tag) < 0 &&
+        lf_tag_compare(upstream_federate->enclave.next_event, next_event_tag) <= 0) {
+            // send next downstream tag to upstream federates that do not complete at next_event_tag
             LF_PRINT_LOG("RTI sending the next downstream event message (NDT) " PRINTF_TAG "to federate %u.",
                 next_event_tag.time - start_time, next_event_tag.microstep, upstream_id);
             if (_f_rti->tracing_enabled) {
@@ -1796,9 +1808,6 @@ int process_args(int argc, const char* argv[]) {
         } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--version") == 0) {
             lf_print("%s", version_info);
             return 0;
-        } else if (strcmp(argv[i], "--ndt") == 0) {
-            lf_print("ndt is enabled");
-            _f_rti->ndt_enabled = true;
         } else if (strcmp(argv[i], " ") == 0) {
             // Tolerate spaces
             lf_print("space");

From 5ca3de03d5ca61b00269c2481da75ff6467f4471 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 24 Oct 2023 21:59:09 +0900
Subject: [PATCH 29/49] Remove the wrong cycle detection code and add FIXME to
 solve it in the future

---
 core/federated/RTI/rti_lib.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 6df0624cb..57fb8c1f7 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -292,16 +292,15 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
     encode_int64(next_event_tag.time, &(buffer[1]));
     encode_int32((int32_t)next_event_tag.microstep, &(buffer[1 + sizeof(int64_t)]));
 
-    // FIXME: Send NDT to transitive upstreams either
+    // FIXME: Send NDT to transitive upstreams either. 
+    // Also, the RTI has to check the sparsity of a federate and determine whether
+    // it sends NDT to it or not.
     for (int i = 0; i < fed->enclave.num_upstream; i++) {
         int upstream_id = fed->enclave.upstream[i];
         federate_t* upstream_federate = _f_rti->enclaves[upstream_id];
         bool has_cycle = false;
-        for (int j = 0; j < upstream_federate->enclave.num_upstream; j++) {
-            if (upstream_federate->enclave.upstream[j] == fed->enclave.id) {
-                has_cycle = true;
-            }
-        }
+        // FIXME: Add a util function in enclave.c to detect whether a federate is
+        // in a cycle. It needs to be a recursive algorithm.
         if (has_cycle) {
             // fed and upstream_fed consist a cycle. Do not use NDT.
             continue;

From e67988a9bc8b0058ed73eb892b4ecf92583c13c8 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Tue, 24 Oct 2023 22:03:10 +0900
Subject: [PATCH 30/49] Temporarily disable the use of NDT for port absent
 messages

---
 core/federated/federate.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index bdf111e04..ef45e80c6 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1406,17 +1406,19 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
     tag_t current_message_intended_tag = lf_delay_strict(env->current_tag,
                                                     additional_delay);
     
-    if (pqueue_peek(env->ndt_q) != NULL) {
-        tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
-        if (lf_tag_compare(current_message_intended_tag, ndt_q_barrier) < 0) {
-            // No events exist in any downstream federates
-            LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is smaller than the tag barrier " PRINTF_TAG "."
-            "Don't have to send the port absent message.",
-            current_message_intended_tag.time - start_time, current_message_intended_tag.microstep,
-            ndt_q_barrier.time - start_time, ndt_q_barrier.microstep);
-            return;
-        }
-    }
+    // FIXME: Currently, a federate cannot send skipped port absent messages
+    // when it receives NDT lately. So every port absent messages are sent tentatively.
+    // if (pqueue_peek(env->ndt_q) != NULL) {
+    //     tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
+    //     if (lf_tag_compare(current_message_intended_tag, ndt_q_barrier) < 0) {
+    //         // No events exist in any downstream federates
+    //         LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is smaller than the tag barrier " PRINTF_TAG "."
+    //         "Don't have to send the port absent message.",
+    //         current_message_intended_tag.time - start_time, current_message_intended_tag.microstep,
+    //         ndt_q_barrier.time - start_time, ndt_q_barrier.microstep);
+    //         return;
+    //     }
+    // }
 
     // Construct the message
     size_t message_length = 1 + sizeof(port_ID) + sizeof(fed_ID) + sizeof(instant_t) + sizeof(microstep_t);

From 2667c1c22c240f4fd49b360cbef3cc07ff8a5603 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 1 Nov 2023 16:19:07 +0900
Subject: [PATCH 31/49] Minor fix

---
 core/federated/federate.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index ef45e80c6..8a9b5fe6a 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2381,8 +2381,8 @@ void handle_next_downstream_tag() {
         node->tag = NDT;
         pqueue_insert(env->ndt_q, node);
     }
-    if (lf_tag_compare(env->current_tag, NDT) >= 0) {
-        // The current tag is greater than or equal to NDT. Send the appropriate NET message.
+    if (lf_tag_compare(env->current_tag, NDT) > 0) {
+        // The current tag is greater than the NDT. Send the appropriate NET message.
         tag_t next_event_tag = get_next_event_tag(env);
         send_next_event_tag(env, next_event_tag, false);
     }

From 057ac08d8bc29fa9d99b96684a35b7268bbf9b0c Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 1 Nov 2023 16:27:39 +0900
Subject: [PATCH 32/49] Instead of sending NET as the response of NDT, insert
 the current tag to the ndt_q

---
 core/federated/federate.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 8a9b5fe6a..351b5d5b5 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2382,9 +2382,12 @@ void handle_next_downstream_tag() {
         pqueue_insert(env->ndt_q, node);
     }
     if (lf_tag_compare(env->current_tag, NDT) > 0) {
-        // The current tag is greater than the NDT. Send the appropriate NET message.
-        tag_t next_event_tag = get_next_event_tag(env);
-        send_next_event_tag(env, next_event_tag, false);
+        // The current tag is greater than the NDT. Send the LTC with the NDT and
+        // push the current tag to ndt_q so that this federate notify the appropriate NET message.
+        _lf_send_tag(MSG_TYPE_LOGICAL_TAG_COMPLETE, NDT, true);
+        ndt_node* node = (ndt_node*) malloc(sizeof(ndt_node));
+        node->tag = env->current_tag;
+        pqueue_insert(env->ndt_q, node);
     }
 }
 

From d4f4c95db07dbd559806ad9089d205e4c2822eae Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 5 Nov 2023 14:14:58 +0900
Subject: [PATCH 33/49] Detect a cycle and disable the NDT

---
 core/federated/RTI/enclave.c | 19 +++++++++++++++++++
 core/federated/RTI/enclave.h |  2 ++
 core/federated/RTI/rti_lib.c | 15 ++++++++-------
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c
index 9af2d60ac..06425f401 100644
--- a/core/federated/RTI/enclave.c
+++ b/core/federated/RTI/enclave.c
@@ -306,3 +306,22 @@ tag_t transitive_next_event(enclave_t* e, tag_t candidate, bool visited[]) {
     }
     return result;
 }
+
+bool has_cycle(enclave_t *e, int target_id, bool visited[]) {
+    if (visited[e->id] || e->state == NOT_CONNECTED) {
+        if (e->id == target_id) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    visited[e->id] = true;
+
+    for (int i = 0; i < e->num_upstream; i++) {
+        if (has_cycle(_e_rti->enclaves[e->upstream[i]], target_id, visited)) {
+            return true;
+        }
+    }
+    return false;
+}
diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h
index 51e2cc0ee..32062a349 100644
--- a/core/federated/RTI/enclave.h
+++ b/core/federated/RTI/enclave.h
@@ -260,4 +260,6 @@ void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag);
  */
 tag_t transitive_next_event(enclave_t *e, tag_t candidate, bool visited[]);
 
+bool has_cycle(enclave_t *e, int target_id, bool visited[]);
+
 #endif // ENCLAVE_H
\ No newline at end of file
diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 57fb8c1f7..af68ab3da 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -293,18 +293,19 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
     encode_int32((int32_t)next_event_tag.microstep, &(buffer[1 + sizeof(int64_t)]));
 
     // FIXME: Send NDT to transitive upstreams either. 
+    bool *visited = (bool *)calloc(_f_rti->number_of_enclaves, sizeof(bool)); // Initializes to 0.
+    if (has_cycle(&fed->enclave, fed->enclave.id, visited)) {
+        // This fed is a part of a cycle. Don't apply the NDT optimization.
+        LF_PRINT_DEBUG("There is a cycle including this federate.");
+        return;
+    }
+    LF_PRINT_DEBUG("There is no cycle including this federate.");
+
     // Also, the RTI has to check the sparsity of a federate and determine whether
     // it sends NDT to it or not.
     for (int i = 0; i < fed->enclave.num_upstream; i++) {
         int upstream_id = fed->enclave.upstream[i];
         federate_t* upstream_federate = _f_rti->enclaves[upstream_id];
-        bool has_cycle = false;
-        // FIXME: Add a util function in enclave.c to detect whether a federate is
-        // in a cycle. It needs to be a recursive algorithm.
-        if (has_cycle) {
-            // fed and upstream_fed consist a cycle. Do not use NDT.
-            continue;
-        }
 
         if (lf_tag_compare(upstream_federate->enclave.completed, next_event_tag) < 0 &&
         lf_tag_compare(upstream_federate->enclave.next_event, next_event_tag) <= 0) {

From f590af6267579be61c364e4a649566f28b81953a Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 5 Nov 2023 15:19:59 +0900
Subject: [PATCH 34/49] Don't insert NDT to ndt_q when send tagged messages if
 it haven't received any NDTs

---
 core/federated/federate.c | 15 +++++++++------
 core/reactor_common.c     |  2 +-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 351b5d5b5..36f6b7d30 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -423,9 +423,12 @@ int send_timed_message(environment_t* env,
     }
 
     // Insert the ndt_node at the tag to send LTC to the RTI.
-    ndt_node* node = (ndt_node*) malloc(sizeof(ndt_node));
-    node->tag = current_message_intended_tag;
-    pqueue_insert(env->ndt_q, node);
+    if (pqueue_size(env->ndt_q) != 0) {
+        LF_PRINT_DEBUG("Insert NDT at the intended to send LTC and NET quickly.");
+        ndt_node* node = (ndt_node*) malloc(sizeof(ndt_node));
+        node->tag = current_message_intended_tag;
+        pqueue_insert(env->ndt_q, node);
+    }
 
     // Trace the event when tracing is enabled
     if (message_type == MSG_TYPE_TAGGED_MESSAGE) {
@@ -1408,7 +1411,7 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
     
     // FIXME: Currently, a federate cannot send skipped port absent messages
     // when it receives NDT lately. So every port absent messages are sent tentatively.
-    // if (pqueue_peek(env->ndt_q) != NULL) {
+    // if (pqueue_size(env->ndt_q) != 0 ) {
     //     tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
     //     if (lf_tag_compare(current_message_intended_tag, ndt_q_barrier) < 0) {
     //         // No events exist in any downstream federates
@@ -1934,7 +1937,7 @@ void _lf_logical_tag_complete(tag_t tag_to_send) {
     environment_t *env;
     _lf_get_environments(&env);
     bool need_to_send_LTC = true;
-    if (pqueue_peek(env->ndt_q) != NULL) {
+    if (pqueue_size(env->ndt_q) != 0 ) {
         tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
         if (lf_tag_compare(tag_to_send, ndt_q_barrier) < 0) {
             // No events exist in any downstream federates
@@ -2803,7 +2806,7 @@ tag_t _lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply
             // If there is no downstream events that require the NET of the current tag,
             // do not send the NET.
             bool need_to_send_NET = true;
-            if (pqueue_peek(env->ndt_q) != NULL) {
+            if (pqueue_size(env->ndt_q) != 0 ) {
                 tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
                 if (lf_tag_compare(tag, ndt_q_barrier) < 0) {
                     // No events exist in any downstream federates
diff --git a/core/reactor_common.c b/core/reactor_common.c
index 8c380b71a..b2deafd9e 100644
--- a/core/reactor_common.c
+++ b/core/reactor_common.c
@@ -293,7 +293,7 @@ void _lf_start_time_step(environment_t *env) {
     }
 
 // #ifdef FEDERATED_CENTRALIZED
-    while (pqueue_peek(env->ndt_q) != NULL 
+    while (pqueue_size(env->ndt_q) != 0 
     && lf_tag_compare(((ndt_node*) pqueue_peek(env->ndt_q))->tag, env->current_tag) < 0) {
         // Remove elements of ndt_q with tag less than the current tag.
         tag_t tag_to_remove = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;

From 8b7670518c7c89be79c33b13b09fda3e1835054a Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Mon, 6 Nov 2023 15:34:19 +0900
Subject: [PATCH 35/49] Check whether each enclave is in a cylce when the RTI
 first constructs the connection

---
 core/federated/RTI/enclave.c |  5 +++--
 core/federated/RTI/enclave.h | 16 +++++++++++++++-
 core/federated/RTI/rti_lib.c | 33 +++++++++++++++++++++++++--------
 core/federated/RTI/rti_lib.h |  4 ++++
 core/federated/federate.c    |  2 ++
 5 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c
index 06425f401..fecbe8d02 100644
--- a/core/federated/RTI/enclave.c
+++ b/core/federated/RTI/enclave.c
@@ -31,6 +31,7 @@ void initialize_enclave(enclave_t* e, uint16_t id) {
     e->downstream = NULL;
     e->num_downstream = 0;
     e->mode = REALTIME;
+    e->is_in_cycle = false;
 
     // Initialize the next event condition variable.
     lf_cond_init(&e->next_event_condition, &rti_mutex);
@@ -307,7 +308,7 @@ tag_t transitive_next_event(enclave_t* e, tag_t candidate, bool visited[]) {
     return result;
 }
 
-bool has_cycle(enclave_t *e, int target_id, bool visited[]) {
+bool check_cycle(enclave_t *e, int target_id, bool visited[]) {
     if (visited[e->id] || e->state == NOT_CONNECTED) {
         if (e->id == target_id) {
             return true;
@@ -319,7 +320,7 @@ bool has_cycle(enclave_t *e, int target_id, bool visited[]) {
     visited[e->id] = true;
 
     for (int i = 0; i < e->num_upstream; i++) {
-        if (has_cycle(_e_rti->enclaves[e->upstream[i]], target_id, visited)) {
+        if (check_cycle(_e_rti->enclaves[e->upstream[i]], target_id, visited)) {
             return true;
         }
     }
diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h
index 32062a349..37360c6a7 100644
--- a/core/federated/RTI/enclave.h
+++ b/core/federated/RTI/enclave.h
@@ -61,6 +61,7 @@ typedef struct enclave_t {
     execution_mode_t mode;  // FAST or REALTIME.
     lf_cond_t next_event_condition; // Condition variable used by enclaves to notify an enclave
                                     // that it's call to next_event_tag() should unblock.
+    bool is_in_cycle;
 } enclave_t;
 
 /**
@@ -260,6 +261,19 @@ void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag);
  */
 tag_t transitive_next_event(enclave_t *e, tag_t candidate, bool visited[]);
 
-bool has_cycle(enclave_t *e, int target_id, bool visited[]);
+/**
+ * Check whether the target enclave is a part of a cycle or not.
+ * This function follows the edge of reaction graph in the depth
+ * first manner. It can conclude that the enclave is in a cylce
+ * if it meets the target enclave again.
+ *
+ * @param e The current enclave.
+ * @param target_id The id of the target enclave.
+ * @param visited An array of booleans indicating which federates
+ *  have been visited (for the first invocation, this should be
+ *  an array of falses of size _RTI.number_of_federates).
+ * @return Whether the target enclave is in a cycle.
+*/
+bool check_cycle(enclave_t *e, int target_id, bool visited[]);
 
 #endif // ENCLAVE_H
\ No newline at end of file
diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index af68ab3da..1206a99a5 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -282,6 +282,23 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
     update_enclave_next_event_tag_locked(&(fed->enclave), next_event_tag);
 }
 
+/**
+ * Update the cycle information of every federate.
+*/
+void update_cycle_information() {
+    bool *visited = (bool *)calloc(_f_rti->number_of_enclaves, sizeof(bool));
+    for (int i = 0; i < _f_rti->number_of_enclaves; i++) {
+        enclave_t* target_enclave = &_f_rti->enclaves[i]->enclave;
+        if (check_cycle(target_enclave, i, visited)) {
+            target_enclave->is_in_cycle = true;
+            LF_PRINT_DEBUG("There is a cycle including federate %d.", i);
+        } else {
+            target_enclave->is_in_cycle = false;
+            LF_PRINT_DEBUG("There is no cycle including federate %d.", i);
+        }
+    }
+}
+
 void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
     // The RTI receives next_event_tag from the federated fed. 
     // It has to send NDT messages to the upstream federates of fed
@@ -293,13 +310,6 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
     encode_int32((int32_t)next_event_tag.microstep, &(buffer[1 + sizeof(int64_t)]));
 
     // FIXME: Send NDT to transitive upstreams either. 
-    bool *visited = (bool *)calloc(_f_rti->number_of_enclaves, sizeof(bool)); // Initializes to 0.
-    if (has_cycle(&fed->enclave, fed->enclave.id, visited)) {
-        // This fed is a part of a cycle. Don't apply the NDT optimization.
-        LF_PRINT_DEBUG("There is a cycle including this federate.");
-        return;
-    }
-    LF_PRINT_DEBUG("There is no cycle including this federate.");
 
     // Also, the RTI has to check the sparsity of a federate and determine whether
     // it sends NDT to it or not.
@@ -307,9 +317,14 @@ void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
         int upstream_id = fed->enclave.upstream[i];
         federate_t* upstream_federate = _f_rti->enclaves[upstream_id];
 
+        if (upstream_federate->enclave.is_in_cycle) {
+            LF_PRINT_DEBUG("There is a cycle including federate %d. Do not send the NDT", i);
+            continue;
+        }
+
         if (lf_tag_compare(upstream_federate->enclave.completed, next_event_tag) < 0 &&
         lf_tag_compare(upstream_federate->enclave.next_event, next_event_tag) <= 0) {
-            // send next downstream tag to upstream federates that do not complete at next_event_tag
+            // Send next downstream tag to upstream federates that do not complete at next_event_tag
             LF_PRINT_LOG("RTI sending the next downstream event message (NDT) " PRINTF_TAG "to federate %u.",
                 next_event_tag.time - start_time, next_event_tag.microstep, upstream_id);
             if (_f_rti->tracing_enabled) {
@@ -1528,9 +1543,11 @@ void connect_to_federates(int socket_descriptor) {
             i--;
         }
     }
+    update_cycle_information();
     // All federates have connected.
     LF_PRINT_DEBUG("All federates have connected to RTI.");
 
+
     if (_f_rti->clock_sync_global_status >= clock_sync_on) {
         // Create the thread that performs periodic PTP clock synchronization sessions
         // over the UDP channel, but only if the UDP channel is open and at least one
diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h
index cd107e23b..757bce710 100644
--- a/core/federated/RTI/rti_lib.h
+++ b/core/federated/RTI/rti_lib.h
@@ -238,6 +238,10 @@ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_ty
  */
 void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag);
 
+/**
+ * Update the cycle information of every federate.
+*/
+void update_cycle_information();
 
 /**
  * @brief Send the next downstream tag.
diff --git a/core/federated/federate.c b/core/federated/federate.c
index bb8602a8c..7cfa6adf8 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -424,6 +424,8 @@ int send_timed_message(environment_t* env,
 
     // Insert the ndt_node at the tag to send LTC to the RTI.
     if (pqueue_size(env->ndt_q) != 0) {
+        // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
+        // is not enough.
         LF_PRINT_DEBUG("Insert NDT at the intended to send LTC and NET quickly.");
         ndt_node* node = (ndt_node*) malloc(sizeof(ndt_node));
         node->tag = current_message_intended_tag;

From 915c4d61997b92ef04c491435202054621e2d53b Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Mon, 6 Nov 2023 16:11:58 +0900
Subject: [PATCH 36/49] Clean the array visited before reusing it

---
 core/federated/RTI/rti_lib.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 1206a99a5..0fefc9137 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -296,7 +296,11 @@ void update_cycle_information() {
             target_enclave->is_in_cycle = false;
             LF_PRINT_DEBUG("There is no cycle including federate %d.", i);
         }
+        for (int j = 0; j < _f_rti->number_of_enclaves; j++) {
+            visited[j] = false;
+        }
     }
+    free(visited);
 }
 
 void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {

From ce6ba0f320eefdb7f4fcbbe023bc12ecfa6c9bd8 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Mon, 6 Nov 2023 16:18:50 +0900
Subject: [PATCH 37/49] Try to remove port absent messages again

---
 core/federated/federate.c | 36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 7cfa6adf8..db0c73a0b 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1412,19 +1412,19 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
     tag_t current_message_intended_tag = lf_delay_strict(env->current_tag,
                                                     additional_delay);
     
-    // FIXME: Currently, a federate cannot send skipped port absent messages
-    // when it receives NDT lately. So every port absent messages are sent tentatively.
-    // if (pqueue_size(env->ndt_q) != 0 ) {
-    //     tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
-    //     if (lf_tag_compare(current_message_intended_tag, ndt_q_barrier) < 0) {
-    //         // No events exist in any downstream federates
-    //         LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is smaller than the tag barrier " PRINTF_TAG "."
-    //         "Don't have to send the port absent message.",
-    //         current_message_intended_tag.time - start_time, current_message_intended_tag.microstep,
-    //         ndt_q_barrier.time - start_time, ndt_q_barrier.microstep);
-    //         return;
-    //     }
-    // }
+    if (pqueue_size(env->ndt_q) != 0 ) {
+        // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
+        // is not enough.
+        tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
+        if (lf_tag_compare(current_message_intended_tag, ndt_q_barrier) < 0) {
+            // No events exist in any downstream federates
+            LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is less than the earliest NDT " PRINTF_TAG "."
+            "Skip sending the port absent message.",
+            current_message_intended_tag.time - start_time, current_message_intended_tag.microstep,
+            ndt_q_barrier.time - start_time, ndt_q_barrier.microstep);
+            return;
+        }
+    }
 
     // Construct the message
     size_t message_length = 1 + sizeof(port_ID) + sizeof(fed_ID) + sizeof(instant_t) + sizeof(microstep_t);
@@ -1944,8 +1944,8 @@ void _lf_logical_tag_complete(tag_t tag_to_send) {
         tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
         if (lf_tag_compare(tag_to_send, ndt_q_barrier) < 0) {
             // No events exist in any downstream federates
-            LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is smaller than the tag barrier " PRINTF_TAG "."
-            "Don't have to send the logical tag complete.",
+            LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is less than the earliest NDT " PRINTF_TAG "."
+            "Skip sending the logical tag complete.",
             tag_to_send.time - start_time, tag_to_send.microstep,
             ndt_q_barrier.time - start_time, ndt_q_barrier.microstep);
             need_to_send_LTC = false;
@@ -2832,11 +2832,13 @@ tag_t _lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply
             // do not send the NET.
             bool need_to_send_NET = true;
             if (pqueue_size(env->ndt_q) != 0 ) {
+                // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
+                // is not enough.
                 tag_t ndt_q_barrier = ((ndt_node*) pqueue_peek(env->ndt_q))->tag;
                 if (lf_tag_compare(tag, ndt_q_barrier) < 0) {
                     // No events exist in any downstream federates
-                    LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is smaller than the tag barrier " PRINTF_TAG "."
-                    "Don't have to send the next event tag.",
+                    LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is less than the earliest NDT " PRINTF_TAG "."
+                    "Skip sending the next event tag.",
                     tag.time - start_time, tag.microstep,
                     ndt_q_barrier.time - start_time, ndt_q_barrier.microstep);
                     need_to_send_NET = false;

From 3faa18840cce38d89d8d833ba8ce03ba097b6606 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <78055940+byeong-gil@users.noreply.github.com>
Date: Sun, 12 Nov 2023 16:37:46 +0900
Subject: [PATCH 38/49] Fix the function `tag_matches` based on @Edwardalee's
 comment

---
 core/utils/pqueue.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/utils/pqueue.c b/core/utils/pqueue.c
index cee4e4fd5..e2029b985 100644
--- a/core/utils/pqueue.c
+++ b/core/utils/pqueue.c
@@ -488,7 +488,7 @@ int tag_in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that) {
  * Return whether or not the tags contained by given pointers are identical.
  */
 int tag_matches(void* next, void* curr) {
-    return lf_tag_compare(((ndt_node*) next)->tag, ((ndt_node*) curr)->tag);
+    return lf_tag_compare(((ndt_node*) next)->tag, ((ndt_node*) curr)->tag) == 0;
 }
 
 /**
@@ -532,4 +532,4 @@ void print_tag(void *node) {
     tag_t tag = ((ndt_node*) node)->tag;
     LF_PRINT_DEBUG("Elapsed logical time:" PRINTF_TIME ", microstep: %d, position: %ld", 
     tag.time, tag.microstep, ((ndt_node*) node)->pos);
-}
\ No newline at end of file
+}

From ffa5efb4f506f0d857ad4a47944ea9944ec8012a Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Mon, 13 Nov 2023 22:13:50 +0900
Subject: [PATCH 39/49] Add functions for not sending NDTs to an upstream
 federate of a downstream federate with a physical action

---
 core/federated/RTI/enclave.c | 21 +++++++++++++++++++++
 core/federated/RTI/enclave.h |  4 ++++
 core/federated/RTI/rti_lib.c | 23 +++++++++++++++++++++++
 core/federated/RTI/rti_lib.h |  5 +++++
 core/federated/federate.c    |  2 ++
 5 files changed, 55 insertions(+)

diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c
index fecbe8d02..1374d92eb 100644
--- a/core/federated/RTI/enclave.c
+++ b/core/federated/RTI/enclave.c
@@ -32,6 +32,8 @@ void initialize_enclave(enclave_t* e, uint16_t id) {
     e->num_downstream = 0;
     e->mode = REALTIME;
     e->is_in_cycle = false;
+    e->has_physical_action = false;
+    e->enable_ndt = false;
 
     // Initialize the next event condition variable.
     lf_cond_init(&e->next_event_condition, &rti_mutex);
@@ -326,3 +328,22 @@ bool check_cycle(enclave_t *e, int target_id, bool visited[]) {
     }
     return false;
 }
+
+bool check_physical_action_of_transitive_downstreams(enclave_t *e, bool visited[]) {
+    if (visited[e->id] || e->state == NOT_CONNECTED) {
+        return false;
+    }
+
+    visited[e->id] = true;
+
+    for (int i = 0; i < e->num_downstream; i++) {
+        if (check_physical_action_of_transitive_downstreams(_e_rti->enclaves[e->downstream[i]], visited)) {
+            return true;
+        }
+    }
+    if (e->has_physical_action) {
+        return true;
+    } else {
+        return false;
+    }
+}
diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h
index 37360c6a7..46bcd50d6 100644
--- a/core/federated/RTI/enclave.h
+++ b/core/federated/RTI/enclave.h
@@ -62,6 +62,8 @@ typedef struct enclave_t {
     lf_cond_t next_event_condition; // Condition variable used by enclaves to notify an enclave
                                     // that it's call to next_event_tag() should unblock.
     bool is_in_cycle;
+    bool has_physical_action;
+    bool enable_ndt;
 } enclave_t;
 
 /**
@@ -276,4 +278,6 @@ tag_t transitive_next_event(enclave_t *e, tag_t candidate, bool visited[]);
 */
 bool check_cycle(enclave_t *e, int target_id, bool visited[]);
 
+bool check_physical_action_of_transitive_downstreams(enclave_t *e, bool visited[]);
+
 #endif // ENCLAVE_H
\ No newline at end of file
diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c
index 0fefc9137..ef5572403 100644
--- a/core/federated/RTI/rti_lib.c
+++ b/core/federated/RTI/rti_lib.c
@@ -303,6 +303,28 @@ void update_cycle_information() {
     free(visited);
 }
 
+/**
+ * Determine whether or not to use the NDT messages.
+*/
+void determine_the_ndt_condition() {
+    bool *visited = (bool *)calloc(_f_rti->number_of_enclaves, sizeof(bool));
+    for (int i = 0; i < _f_rti->number_of_enclaves; i++) {
+        enclave_t* target_enclave = &_f_rti->enclaves[i]->enclave;
+        if (target_enclave->is_in_cycle || check_physical_action_of_transitive_downstreams(target_enclave, visited)) {
+            target_enclave->enable_ndt = false;
+            LF_PRINT_DEBUG("There is a cycle including federate %d or a transitive downstream federate of"
+            " the federate %d has a physical action.", i, i);
+        } else {
+            target_enclave->enable_ndt = true;
+            LF_PRINT_DEBUG("Enable NDT messages for federate %d.", i);
+        }
+        for (int j = 0; j < _f_rti->number_of_enclaves; j++) {
+            visited[j] = false;
+        }
+    }
+    free(visited);
+}
+
 void send_upstream_next_downstream_tag(federate_t* fed, tag_t next_event_tag) {
     // The RTI receives next_event_tag from the federated fed. 
     // It has to send NDT messages to the upstream federates of fed
@@ -1548,6 +1570,7 @@ void connect_to_federates(int socket_descriptor) {
         }
     }
     update_cycle_information();
+    determine_the_ndt_condition();
     // All federates have connected.
     LF_PRINT_DEBUG("All federates have connected to RTI.");
 
diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h
index 757bce710..9e520f74f 100644
--- a/core/federated/RTI/rti_lib.h
+++ b/core/federated/RTI/rti_lib.h
@@ -243,6 +243,11 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even
 */
 void update_cycle_information();
 
+/**
+ * Determine whether or not to use the NDT messages.
+*/
+void determine_the_ndt_condition();
+
 /**
  * @brief Send the next downstream tag.
  * 
diff --git a/core/federated/federate.c b/core/federated/federate.c
index db0c73a0b..3ec02dcaf 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1412,6 +1412,8 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
     tag_t current_message_intended_tag = lf_delay_strict(env->current_tag,
                                                     additional_delay);
     
+    // FIXME: If port absent messages are not used when there is no zero-delay cycle,
+    // This part is not needed as we don't apply the NDT optimization for cycles.
     if (pqueue_size(env->ndt_q) != 0 ) {
         // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
         // is not enough.

From 41074e5be37b3aac1964feef0bbecfd294c9fe41 Mon Sep 17 00:00:00 2001
From: Byeonggil Jun <78055940+byeong-gil@users.noreply.github.com>
Date: Tue, 14 Nov 2023 15:05:21 +0900
Subject: [PATCH 40/49] Update lingua-franca-ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 7e9223177..1f7391f92 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-enclaves4
+master

From 356cdbae79fbcf995bb60e5d5fb85d128e9c40a6 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 15 Nov 2023 15:58:55 +0900
Subject: [PATCH 41/49] Do not insert the duplicate tag into the ndt_q

---
 core/federated/federate.c | 18 ++++++++++++------
 core/reactor_common.c     |  2 +-
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index f3b9ce20f..0d5175625 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -425,7 +425,7 @@ int send_timed_message(environment_t* env,
     // Insert the intended tag into the ndt_q to send LTC to the RTI quickly.
     if (pqueue_tag_size(env->ndt_q) != 0) {
         // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
-        // is not enough.
+        // is not enough to know whether this federate using the NDT optimization or not.
         LF_PRINT_DEBUG("Insert NDT at the intended to send LTC and NET quickly.");
         pqueue_tag_insert_tag(env->ndt_q, current_message_intended_tag);
     }
@@ -1414,7 +1414,7 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
     // This part is not needed as we don't apply the NDT optimization for cycles.
     if (pqueue_tag_size(env->ndt_q) != 0 ) {
         // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
-        // is not enough.
+        // is not enough to know whether this federate using the NDT optimization or not.
         tag_t earliest_ndt = pqueue_tag_peek(env->ndt_q)->tag;
         if (lf_tag_compare(current_message_intended_tag, earliest_ndt) < 0) {
             // No events exist in any downstream federates
@@ -1941,6 +1941,8 @@ void _lf_logical_tag_complete(tag_t tag_to_send) {
     _lf_get_environments(&env);
     bool need_to_send_LTC = true;
     if (pqueue_tag_size(env->ndt_q) != 0 ) {
+        // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
+        // is not enough to know whether this federate using the NDT optimization or not.
         tag_t earliest_ndt = pqueue_tag_peek(env->ndt_q)->tag;
         if (lf_tag_compare(tag_to_send, earliest_ndt) < 0) {
             // No events exist in any downstream federates
@@ -2403,15 +2405,19 @@ void handle_next_downstream_tag() {
     environment_t* env;
     _lf_get_environments(&env);
 
-    if (lf_tag_compare(env->current_tag, NDT) <= 0) {
-        // The current tag is less than or equal to the NDT. Push NDT to ndt_q.
+    if (lf_tag_compare(env->current_tag, NDT) <= 0
+    && pqueue_tag_find_equal_same_tag(env->ndt_q, NDT) == NULL) {
+        // The current tag is less than or equal to the NDT and the tag doesn't exists
+        // in the ndt_q. Insert NDT to ndt_q.
         pqueue_tag_insert_tag(env->ndt_q, NDT);
     }
     if (lf_tag_compare(env->current_tag, NDT) > 0) {
         // The current tag is greater than the NDT. Send the LTC with the NDT and
         // push the current tag to ndt_q so that this federate notify the appropriate NET message.
         _lf_send_tag(MSG_TYPE_LOGICAL_TAG_COMPLETE, NDT, true);
-        pqueue_tag_insert_tag(env->ndt_q, env->current_tag);
+        if (pqueue_tag_find_equal_same_tag(env->ndt_q, NDT) == NULL) {
+            pqueue_tag_insert_tag(env->ndt_q, env->current_tag);
+        }
     }
 }
 
@@ -2829,7 +2835,7 @@ tag_t _lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply
             bool need_to_send_NET = true;
             if (pqueue_tag_size(env->ndt_q) != 0 ) {
                 // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
-                // is not enough.
+                // is not enough to know whether this federate using the NDT optimization or not.
                 tag_t earliest_ndt = pqueue_tag_peek(env->ndt_q)->tag;
                 if (lf_tag_compare(tag, earliest_ndt) < 0) {
                     // No events exist in any downstream federates
diff --git a/core/reactor_common.c b/core/reactor_common.c
index cc315fea5..0f45b6103 100644
--- a/core/reactor_common.c
+++ b/core/reactor_common.c
@@ -298,7 +298,7 @@ void _lf_start_time_step(environment_t *env) {
     && lf_tag_compare(pqueue_tag_peek(env->ndt_q)->tag, env->current_tag) < 0) {
         // Remove elements of ndt_q with tag less than the current tag.
         tag_t tag_to_remove = pqueue_tag_pop_tag(env->ndt_q);
-        LF_PRINT_DEBUG("Remove the tag " PRINTF_TAG " from the ndt_q is before the current tag " PRINTF_TAG ". Remove it.",
+        LF_PRINT_DEBUG("Remove the tag " PRINTF_TAG " from the ndt_q is less than the current tag " PRINTF_TAG ". Remove it.",
         tag_to_remove.time - start_time, tag_to_remove.microstep,
         env->current_tag.time - start_time, env->current_tag.microstep);
     }

From 720ec567a2a4fbf9e4b760889310bd132b05fe1d Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Wed, 15 Nov 2023 21:27:30 +0900
Subject: [PATCH 42/49] Fix Typo

---
 core/federated/federate.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index 0d5175625..a614459f8 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -2842,7 +2842,7 @@ tag_t _lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply
                     LF_PRINT_DEBUG("The intended tag " PRINTF_TAG " is less than the earliest NDT " PRINTF_TAG "."
                     "Skip sending the next event tag.",
                     tag.time - start_time, tag.microstep,
-                    ndt_q_barrier.time - start_time, earliest_ndt.microstep);
+                    earliest_ndt.time - start_time, earliest_ndt.microstep);
                     need_to_send_NET = false;
                 }
             }

From 6dbfd2058c9afe2564402f1e62257cf545c50692 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 19 Nov 2023 15:45:13 +0900
Subject: [PATCH 43/49] Use the function `pqueue_tag_insert_if_no_match` when
 inserting a tag into the ndt queue

---
 core/federated/federate.c | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/core/federated/federate.c b/core/federated/federate.c
index a614459f8..419976f42 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -427,7 +427,7 @@ int send_timed_message(environment_t* env,
         // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
         // is not enough to know whether this federate using the NDT optimization or not.
         LF_PRINT_DEBUG("Insert NDT at the intended to send LTC and NET quickly.");
-        pqueue_tag_insert_tag(env->ndt_q, current_message_intended_tag);
+        pqueue_tag_insert_if_no_match(env->ndt_q, current_message_intended_tag);
     }
 
     // Trace the event when tracing is enabled
@@ -2405,19 +2405,16 @@ void handle_next_downstream_tag() {
     environment_t* env;
     _lf_get_environments(&env);
 
-    if (lf_tag_compare(env->current_tag, NDT) <= 0
-    && pqueue_tag_find_equal_same_tag(env->ndt_q, NDT) == NULL) {
+    if (lf_tag_compare(env->current_tag, NDT) <= 0) {
         // The current tag is less than or equal to the NDT and the tag doesn't exists
         // in the ndt_q. Insert NDT to ndt_q.
-        pqueue_tag_insert_tag(env->ndt_q, NDT);
+        pqueue_tag_insert_if_no_match(env->ndt_q, NDT);
     }
     if (lf_tag_compare(env->current_tag, NDT) > 0) {
         // The current tag is greater than the NDT. Send the LTC with the NDT and
         // push the current tag to ndt_q so that this federate notify the appropriate NET message.
         _lf_send_tag(MSG_TYPE_LOGICAL_TAG_COMPLETE, NDT, true);
-        if (pqueue_tag_find_equal_same_tag(env->ndt_q, NDT) == NULL) {
-            pqueue_tag_insert_tag(env->ndt_q, env->current_tag);
-        }
+        pqueue_tag_insert_if_no_match(env->ndt_q, env->current_tag);
     }
 }
 

From 20151f2724b4f60fb0f017ad7e9ab747b4a76853 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 19 Nov 2023 15:54:33 +0900
Subject: [PATCH 44/49] Free an ndt queue at the end of the program

---
 core/environment.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/environment.c b/core/environment.c
index 1141db66f..c0715e5e1 100644
--- a/core/environment.c
+++ b/core/environment.c
@@ -118,7 +118,7 @@ static void environment_init_federated(environment_t* env, int num_is_present_fi
 #endif
 }
 
-void environment_init_tags( environment_t *env, instant_t start_time, interval_t duration) {
+void environment_init_tags(environment_t *env, instant_t start_time, interval_t duration) {
     env->current_tag = (tag_t){.time = start_time, .microstep = 0u};
     
     tag_t stop_tag = FOREVER_TAG_INITIALIZER;
@@ -170,6 +170,7 @@ void environment_free(environment_t* env) {
     pqueue_free(env->event_q);
     pqueue_free(env->recycle_q);
     pqueue_free(env->next_q);
+    pqueue_tag_free(env->ndt_q);
 
     environment_free_threaded(env);
     environment_free_single_threaded(env);

From 7a951d8e58bdd5576221c23b95b9ab87696b9701 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Mon, 20 Nov 2023 16:41:17 +0900
Subject: [PATCH 45/49] Inform the RTI whether a federate has a physical action

---
 core/federated/RTI/rti_remote.c     | 23 +++++++++++++++++++++++
 core/federated/federate.c           |  8 ++++++++
 include/core/federated/net_common.h |  9 ++++++++-
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index e11886c10..92a5e4a43 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1391,6 +1391,28 @@ int receive_connection_information(int socket_id, uint16_t fed_id) {
     }
 }
 
+int receive_physical_action_information(int socket_id, uint16_t fed_id) {
+    LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_PHYSICAL_ACTION from federate %d.", fed_id);
+    unsigned char response[1 + sizeof(uint16_t)];
+    read_from_socket_errexit(socket_id, 1 + sizeof(uint16_t) , response,
+            "RTI failed to read MSG_TYPE_PHYSICAL_ACTION message from federate %d.", fed_id);
+    if (response[0] != MSG_TYPE_PHYSICAL_ACTION) {
+        lf_print_error("RTI was expecting a MSG_TYPE_PHYSICAL_ACTION message from federate %d. Got %u instead. "
+                "Rejecting federate.", fed_id, response[0]);
+        send_reject(socket_id, UNEXPECTED_MESSAGE);
+        return 0;
+    } else {
+        federate_info_t *fed = GET_FED_INFO(fed_id);
+        uint16_t federate_has_physical_action = extract_uint16(&(response[1]));
+
+        LF_PRINT_DEBUG("RTI got MSG_TYPE_PHYSICAL_ACTION %u from federate %d.", federate_has_physical_action, fed_id);
+
+        fed->enclave.has_physical_action = (bool) federate_has_physical_action;
+    }
+
+    return 1;
+}
+
 int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) {
     // Read the MSG_TYPE_UDP_PORT message from the federate regardless of the status of
     // clock synchronization. This message will tell the RTI whether the federate
@@ -1564,6 +1586,7 @@ void connect_to_federates(int socket_descriptor) {
         int32_t fed_id = receive_and_check_fed_id_message(socket_id, (struct sockaddr_in*)&client_fd);
         if (fed_id >= 0
                 && receive_connection_information(socket_id, (uint16_t)fed_id)
+                && receive_physical_action_information(socket_id, (uint16_t)fed_id)
                 && receive_udp_message_and_set_up_clock_sync(socket_id, (uint16_t)fed_id)) {
 
             // Create a thread to communicate with the federate.
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 419976f42..77ef59005 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -1153,6 +1153,14 @@ void connect_to_rti(const char* hostname, int port) {
                 // @see MSG_TYPE_NEIGHBOR_STRUCTURE in net_common.h
                 send_neighbor_structure_to_RTI(_fed.socket_TCP_RTI);
 
+                // Send whether this federate has a physical action to the RTI.
+                uint16_t has_physical_action = _fed.min_delay_from_physical_action_to_federate_output != NEVER;
+                unsigned char physical_action_message_buffer[1 + sizeof(uint16_t)];
+                physical_action_message_buffer[0] = MSG_TYPE_PHYSICAL_ACTION;
+                encode_uint16(has_physical_action, &(physical_action_message_buffer[1]));
+                write_to_socket_errexit(_fed.socket_TCP_RTI, 1 + sizeof(uint16_t), physical_action_message_buffer,
+                    "Failed to send physical action info to RTI.");
+
                 uint16_t udp_port = setup_clock_synchronization_with_rti();
 
                 // Write the returned port number to the RTI
diff --git a/include/core/federated/net_common.h b/include/core/federated/net_common.h
index 43dbddef9..cd9ae218d 100644
--- a/include/core/federated/net_common.h
+++ b/include/core/federated/net_common.h
@@ -698,7 +698,14 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #define MSG_TYPE_NEXT_DOWNSTREAM_TAG 25
 
-#define MSG_TYPE_PHYSICAL_ACTION
+/**
+ * A message that informs the RTI about its presence of a physical action. The
+ * information is used for deciding whether the RTI sends NDT messages to the source
+ * federate's upstreams.
+ * The next four byte will be 1 if the source federate has a physical action
+ * and 0 otherwise.
+*/
+#define MSG_TYPE_PHYSICAL_ACTION 26
 
 /////////////////////////////////////////////
 //// Rejection codes

From 83e0009839fc2c01f3896c24ec0ca8dd2de50fc7 Mon Sep 17 00:00:00 2001
From: Byeonggil Jun <78055940+byeong-gil@users.noreply.github.com>
Date: Mon, 20 Nov 2023 21:28:58 +0900
Subject: [PATCH 46/49] Update lingua-franca-ref.txt

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 1f7391f92..4b247b944 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-master
+rti-ndt

From e337d38f2de29792032a6d782cee3b2e4e66d3ed Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Sun, 26 Nov 2023 15:04:14 +0900
Subject: [PATCH 47/49] Listen for and handle set up messages more flexibly

---
 core/federated/RTI/rti_remote.c | 303 ++++++++++++++++++--------------
 core/federated/RTI/rti_remote.h |  34 +++-
 2 files changed, 200 insertions(+), 137 deletions(-)

diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 92a5e4a43..1961deac1 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1322,161 +1322,205 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie
     return (int32_t)fed_id;
 }
 
-int receive_connection_information(int socket_id, uint16_t fed_id) {
-    LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_NEIGHBOR_STRUCTURE from federate %d.", fed_id);
-    unsigned char connection_info_header[MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE];
+int receive_set_up_information(int socket_id, uint16_t fed_id) {
+    LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_NEIGHBOR_STRUCTURE, MSG_TYPE_PHYSICAL_ACTION, and "
+    "MSG_TYPE_UDP_PORT from federate %d.", fed_id);
+
+    bool waiting_connection_information = true;
+    bool waiting_physical_action_information = true;
+    bool waiting_udp_message_and_set_up_clock_sync = true;
+
+    // Buffer for incoming messages.
+    // This does not constrain the message size because messages
+    // are forwarded piece by piece.
+    unsigned char buffer[FED_COM_BUFFER_SIZE];
+
+    // Listen for messages from the federate.
+    while (waiting_connection_information
+    || waiting_physical_action_information
+    || waiting_udp_message_and_set_up_clock_sync) {
+        // Read no more than one byte to get the message type.
+        ssize_t bytes_read = read_from_socket(socket_id, 1, buffer);
+        if (bytes_read < 1) {
+            // Socket is closed
+            lf_print_warning("RTI: Socket to federate %d is closed.", fed_id);
+            return 0;
+        }
+        LF_PRINT_DEBUG("RTI: Received message type %u from federate %d.", buffer[0], fed_id);
+        switch(buffer[0]) {
+            case MSG_TYPE_NEIGHBOR_STRUCTURE:
+                if (handle_connection_information(socket_id, (uint16_t)fed_id)) {
+                    waiting_connection_information = false;
+                } else {
+                    LF_PRINT_DEBUG("RTI failed to handle MSG_TYPE_NEIGHBOR_STRUCTURE message header from federate %d.", fed_id);
+                    return 0;
+                }
+                break;
+            case MSG_TYPE_PHYSICAL_ACTION:
+                if (handle_physical_action_information(socket_id, (uint16_t)fed_id)) {
+                    waiting_physical_action_information = false;
+                } else {
+                    LF_PRINT_DEBUG("RTI failed to handle MSG_TYPE_NEIGHBOR_STRUCTURE message header from federate %d.", fed_id);
+                    return 0;
+                }
+                break;
+            case MSG_TYPE_UDP_PORT:
+                if (handle_udp_message_and_set_up_clock_sync(socket_id, (uint16_t)fed_id)) {
+                    waiting_udp_message_and_set_up_clock_sync = false;
+                } else {
+                    LF_PRINT_DEBUG("RTI failed to handle MSG_TYPE_NEIGHBOR_STRUCTURE message header from federate %d.", fed_id);
+                    return 0;
+                }
+                break;
+            default:
+                lf_print_error("RTI was expecting a MSG_TYPE_NEIGHBOR_STRUCTURE, MSG_TYPE_PHYSICAL_ACTION, or MSG_TYPE_UDP_PORT "
+                "message from federate %d. Got %u instead. Rejecting federate.", fed_id, buffer[0]);
+                send_reject(socket_id, UNEXPECTED_MESSAGE);
+                return 0;
+        }
+    }
+    return 1;
+}
+
+int handle_connection_information(int socket_id, uint16_t fed_id) {
+    LF_PRINT_DEBUG("RTI received MSG_TYPE_NEIGHBOR_STRUCTURE from federate %d.", fed_id);
+    unsigned char connection_info_header[MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE - 1];
     read_from_socket_errexit(
         socket_id,
-        MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE,
+        MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE - 1,
         connection_info_header,
         "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message header from federate %d.",
         fed_id
     );
 
-    if (connection_info_header[0] != MSG_TYPE_NEIGHBOR_STRUCTURE) {
-        lf_print_error("RTI was expecting a MSG_TYPE_UDP_PORT message from federate %d. Got %u instead. "
-                "Rejecting federate.", fed_id, connection_info_header[0]);
-        send_reject(socket_id, UNEXPECTED_MESSAGE);
-        return 0;
-    } else {
-        federate_info_t* fed = GET_FED_INFO(fed_id);
-        // Read the number of upstream and downstream connections
-        fed->enclave.num_upstream = extract_int32(&(connection_info_header[1]));
-        fed->enclave.num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)]));
-        LF_PRINT_DEBUG(
-            "RTI got %d upstreams and %d downstreams from federate %d.",
-            fed->enclave.num_upstream,
-            fed->enclave.num_downstream,
-            fed_id);
-
-        // Allocate memory for the upstream and downstream pointers
-        fed->enclave.upstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_upstream);
-        fed->enclave.downstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_downstream);
-
-        // Allocate memory for the upstream delay pointers
-        fed->enclave.upstream_delay =
-            (interval_t*)malloc(
-                sizeof(interval_t) * fed->enclave.num_upstream
-            );
-
-        size_t connections_info_body_size = ((sizeof(uint16_t) + sizeof(int64_t)) *
-            fed->enclave.num_upstream) + (sizeof(uint16_t) * fed->enclave.num_downstream);
-        unsigned char* connections_info_body = (unsigned char*)malloc(connections_info_body_size);
-        read_from_socket_errexit(
-            socket_id,
-            connections_info_body_size,
-            connections_info_body,
-            "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message body from federate %d.",
-            fed_id
+    federate_info_t* fed = GET_FED_INFO(fed_id);
+    // Read the number of upstream and downstream connections
+    fed->enclave.num_upstream = extract_int32(&(connection_info_header[0]));
+    fed->enclave.num_downstream = extract_int32(&(connection_info_header[sizeof(int32_t)]));
+    LF_PRINT_DEBUG(
+        "RTI got %d upstreams and %d downstreams from federate %d.",
+        fed->enclave.num_upstream,
+        fed->enclave.num_downstream,
+        fed_id);
+
+    // Allocate memory for the upstream and downstream pointers
+    fed->enclave.upstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_upstream);
+    fed->enclave.downstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_downstream);
+
+    // Allocate memory for the upstream delay pointers
+    fed->enclave.upstream_delay =
+        (interval_t*)malloc(
+            sizeof(interval_t) * fed->enclave.num_upstream
         );
 
-        // Keep track of where we are in the buffer
-        size_t message_head = 0;
-        // First, read the info about upstream federates
-        for (int i=0; i<fed->enclave.num_upstream; i++) {
-            fed->enclave.upstream[i] = extract_uint16(&(connections_info_body[message_head]));
-            message_head += sizeof(uint16_t);
-            fed->enclave.upstream_delay[i] = extract_int64(&(connections_info_body[message_head]));
-            message_head += sizeof(int64_t);
-        }
+    size_t connections_info_body_size = ((sizeof(uint16_t) + sizeof(int64_t)) *
+        fed->enclave.num_upstream) + (sizeof(uint16_t) * fed->enclave.num_downstream);
+    unsigned char* connections_info_body = (unsigned char*)malloc(connections_info_body_size);
+    read_from_socket_errexit(
+        socket_id,
+        connections_info_body_size,
+        connections_info_body,
+        "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message body from federate %d.",
+        fed_id
+    );
 
-        // Next, read the info about downstream federates
-        for (int i=0; i<fed->enclave.num_downstream; i++) {
-            fed->enclave.downstream[i] = extract_uint16(&(connections_info_body[message_head]));
-            message_head += sizeof(uint16_t);
-        }
+    // Keep track of where we are in the buffer
+    size_t message_head = 0;
+    // First, read the info about upstream federates
+    for (int i=0; i<fed->enclave.num_upstream; i++) {
+        fed->enclave.upstream[i] = extract_uint16(&(connections_info_body[message_head]));
+        LF_PRINT_DEBUG("fed->enclave.upstream[i] = %d", fed->enclave.upstream[i]); // Remove
+        message_head += sizeof(uint16_t);
+        fed->enclave.upstream_delay[i] = extract_int64(&(connections_info_body[message_head]));
+        LF_PRINT_DEBUG("fed->enclave.upstream_delay[i] = %ld", fed->enclave.upstream_delay[i]); // Remove
+        message_head += sizeof(int64_t);
+    }
 
-        free(connections_info_body);
-        return 1;
+    // Next, read the info about downstream federates
+    for (int i=0; i<fed->enclave.num_downstream; i++) {
+        fed->enclave.downstream[i] = extract_uint16(&(connections_info_body[message_head]));
+        LF_PRINT_DEBUG("fed->enclave.downstream[i] = %d", fed->enclave.downstream[i]); // Remove
+        message_head += sizeof(uint16_t);
     }
+
+    free(connections_info_body);
+    return 1;
 }
 
-int receive_physical_action_information(int socket_id, uint16_t fed_id) {
-    LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_PHYSICAL_ACTION from federate %d.", fed_id);
-    unsigned char response[1 + sizeof(uint16_t)];
-    read_from_socket_errexit(socket_id, 1 + sizeof(uint16_t) , response,
+int handle_physical_action_information(int socket_id, uint16_t fed_id) {
+    LF_PRINT_DEBUG("RTI received MSG_TYPE_PHYSICAL_ACTION from federate %d.", fed_id);
+    unsigned char response[sizeof(uint16_t)];
+    read_from_socket_errexit(socket_id, sizeof(uint16_t) , response,
             "RTI failed to read MSG_TYPE_PHYSICAL_ACTION message from federate %d.", fed_id);
-    if (response[0] != MSG_TYPE_PHYSICAL_ACTION) {
-        lf_print_error("RTI was expecting a MSG_TYPE_PHYSICAL_ACTION message from federate %d. Got %u instead. "
-                "Rejecting federate.", fed_id, response[0]);
-        send_reject(socket_id, UNEXPECTED_MESSAGE);
-        return 0;
-    } else {
-        federate_info_t *fed = GET_FED_INFO(fed_id);
-        uint16_t federate_has_physical_action = extract_uint16(&(response[1]));
 
-        LF_PRINT_DEBUG("RTI got MSG_TYPE_PHYSICAL_ACTION %u from federate %d.", federate_has_physical_action, fed_id);
+    federate_info_t *fed = GET_FED_INFO(fed_id);
+    uint16_t federate_has_physical_action = extract_uint16(&(response[0]));
 
-        fed->enclave.has_physical_action = (bool) federate_has_physical_action;
-    }
+    LF_PRINT_DEBUG("RTI got MSG_TYPE_PHYSICAL_ACTION %u from federate %d.", federate_has_physical_action, fed_id);
+
+    fed->enclave.has_physical_action = (bool) federate_has_physical_action;
 
     return 1;
 }
 
-int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) {
+int handle_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) {
     // Read the MSG_TYPE_UDP_PORT message from the federate regardless of the status of
     // clock synchronization. This message will tell the RTI whether the federate
     // is doing clock synchronization, and if it is, what port to use for UDP.
-    LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_UDP_PORT from federate %d.", fed_id);
-    unsigned char response[1 + sizeof(uint16_t)];
-    read_from_socket_errexit(socket_id, 1 + sizeof(uint16_t) , response,
+    LF_PRINT_DEBUG("RTI received MSG_TYPE_UDP_PORT from federate %d.", fed_id);
+    unsigned char response[sizeof(uint16_t)];
+    read_from_socket_errexit(socket_id, sizeof(uint16_t) , response,
             "RTI failed to read MSG_TYPE_UDP_PORT message from federate %d.", fed_id);
-    if (response[0] != MSG_TYPE_UDP_PORT) {
-        lf_print_error("RTI was expecting a MSG_TYPE_UDP_PORT message from federate %d. Got %u instead. "
-                "Rejecting federate.", fed_id, response[0]);
-        send_reject(socket_id, UNEXPECTED_MESSAGE);
-        return 0;
-    } else {
-        federate_info_t *fed = GET_FED_INFO(fed_id);
-        if (rti_remote->clock_sync_global_status >= clock_sync_init) {// If no initial clock sync, no need perform initial clock sync.
-            uint16_t federate_UDP_port_number = extract_uint16(&(response[1]));
-
-            LF_PRINT_DEBUG("RTI got MSG_TYPE_UDP_PORT %u from federate %d.", federate_UDP_port_number, fed_id);
-
-            // A port number of UINT16_MAX means initial clock sync should not be performed.
-            if (federate_UDP_port_number != UINT16_MAX) {
-                // Perform the initialization clock synchronization with the federate.
-                // Send the required number of messages for clock synchronization
-                for (int i=0; i < rti_remote->clock_sync_exchanges_per_interval; i++) {
-                    // Send the RTI's current physical time T1 to the federate.
-                    send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed, TCP);
-
-                    // Listen for reply message, which should be T3.
-                    size_t message_size = 1 + sizeof(int32_t);
-                    unsigned char buffer[message_size];
-                    read_from_socket_errexit(socket_id, message_size, buffer,
-                            "Socket to federate %d unexpectedly closed.", fed_id);
-                    if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) {
-                        int32_t fed_id = extract_int32(&(buffer[1]));
-                        assert(fed_id > -1);
-                        assert(fed_id < 65536);
-                        LF_PRINT_DEBUG("RTI received T3 clock sync message from federate %d.", fed_id);
-                        handle_physical_clock_sync_message(fed, TCP);
-                    } else {
-                        lf_print_error("Unexpected message %u from federate %d.", buffer[0], fed_id);
-                        send_reject(socket_id, UNEXPECTED_MESSAGE);
-                        return 0;
-                    }
+
+    federate_info_t *fed = GET_FED_INFO(fed_id);
+    if (rti_remote->clock_sync_global_status >= clock_sync_init) {// If no initial clock sync, no need perform initial clock sync.
+        uint16_t federate_UDP_port_number = extract_uint16(&(response[0]));
+
+        LF_PRINT_DEBUG("RTI got MSG_TYPE_UDP_PORT %u from federate %d.", federate_UDP_port_number, fed_id);
+
+        // A port number of UINT16_MAX means initial clock sync should not be performed.
+        if (federate_UDP_port_number != UINT16_MAX) {
+            // Perform the initialization clock synchronization with the federate.
+            // Send the required number of messages for clock synchronization
+            for (int i=0; i < rti_remote->clock_sync_exchanges_per_interval; i++) {
+                // Send the RTI's current physical time T1 to the federate.
+                send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed, TCP);
+
+                // Listen for reply message, which should be T3.
+                size_t message_size = 1 + sizeof(int32_t);
+                unsigned char buffer[message_size];
+                read_from_socket_errexit(socket_id, message_size, buffer,
+                        "Socket to federate %d unexpectedly closed.", fed_id);
+                if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) {
+                    int32_t fed_id = extract_int32(&(buffer[1]));
+                    assert(fed_id > -1);
+                    assert(fed_id < 65536);
+                    LF_PRINT_DEBUG("RTI received T3 clock sync message from federate %d.", fed_id);
+                    handle_physical_clock_sync_message(fed, TCP);
+                } else {
+                    lf_print_error("Unexpected message %u from federate %d.", buffer[0], fed_id);
+                    send_reject(socket_id, UNEXPECTED_MESSAGE);
+                    return 0;
                 }
-                LF_PRINT_DEBUG("RTI finished initial clock synchronization with federate %d.", fed_id);
-            }
-            if (rti_remote->clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port.
-                    if (federate_UDP_port_number > 0) {
-                        // Initialize the UDP_addr field of the federate struct
-                        fed->UDP_addr.sin_family = AF_INET;
-                        fed->UDP_addr.sin_port = htons(federate_UDP_port_number);
-                        fed->UDP_addr.sin_addr = fed->server_ip_addr;
-                    }
-            } else {
-                    // Disable clock sync after initial round.
-                    fed->clock_synchronization_enabled = false;
             }
-        } else { // No clock synchronization at all.
-            // Clock synchronization is universally disabled via the clock-sync command-line parameter
-            // (-c off was passed to the RTI).
-            // Note that the federates are still going to send a MSG_TYPE_UDP_PORT message but with a payload (port) of -1.
-            fed->clock_synchronization_enabled = false;
+            LF_PRINT_DEBUG("RTI finished initial clock synchronization with federate %d.", fed_id);
+        }
+        if (rti_remote->clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port.
+                if (federate_UDP_port_number > 0) {
+                    // Initialize the UDP_addr field of the federate struct
+                    fed->UDP_addr.sin_family = AF_INET;
+                    fed->UDP_addr.sin_port = htons(federate_UDP_port_number);
+                    fed->UDP_addr.sin_addr = fed->server_ip_addr;
+                }
+        } else {
+                // Disable clock sync after initial round.
+                fed->clock_synchronization_enabled = false;
         }
+    } else { // No clock synchronization at all.
+        // Clock synchronization is universally disabled via the clock-sync command-line parameter
+        // (-c off was passed to the RTI).
+        // Note that the federates are still going to send a MSG_TYPE_UDP_PORT message but with a payload (port) of -1.
+        fed->clock_synchronization_enabled = false;
     }
     return 1;
 }
@@ -1584,10 +1628,7 @@ void connect_to_federates(int socket_descriptor) {
         
         // The first message from the federate should contain its ID and the federation ID.
         int32_t fed_id = receive_and_check_fed_id_message(socket_id, (struct sockaddr_in*)&client_fd);
-        if (fed_id >= 0
-                && receive_connection_information(socket_id, (uint16_t)fed_id)
-                && receive_physical_action_information(socket_id, (uint16_t)fed_id)
-                && receive_udp_message_and_set_up_clock_sync(socket_id, (uint16_t)fed_id)) {
+        if (fed_id >= 0 && receive_set_up_information(socket_id, (uint16_t)fed_id)) {
 
             // Create a thread to communicate with the federate.
             // This has to be done after clock synchronization is finished
diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index e800da4dd..55753fde2 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -435,14 +435,36 @@ void send_reject(int socket_id, unsigned char error_code);
 int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* client_fd);
 
 /**
- * Listen for a MSG_TYPE_NEIGHBOR_STRUCTURE message, and upon receiving it, fill
- * out the relevant information in the federate's struct.
+ * Listen for MSG_TYPE_NEIGHBOR_STRUCTURE, MSG_TYPE_PHYSICAL_ACTION, and
+ * MSG_TYPE_UDP_PORT messages, and upon receiving them, handle a payload
+ * of each message.
+ * @param socket_id The socket on which to listen.
+ * @param fed_id The federate ID.
+ * @return 1 for success, 0 for failure.
+*/
+int receive_set_up_information(int socket_id, uint16_t fed_id);
+
+/**
+ * Handle a MSG_TYPE_NEIGHBOR_STRUCTURE message. Fill out 
+ * the relevant information in the federate's struct.
+ * @param socket_id The socket on which to listen.
+ * @param fed_id The federate ID.
+ * @return 1 for success, 0 for failure.
  */
-int receive_connection_information(int socket_id, uint16_t fed_id);
+int handle_connection_information(int socket_id, uint16_t fed_id);
+
+/**
+ * Handle a MSG_TYPE_PHYSICAL_ACTION message. Update the information of 
+ * presence of a physical action.
+ * @param socket_id The socket on which to listen.
+ * @param fed_id The federate ID.
+ * @return 1 for success, 0 for failure.
+*/
+int handle_physical_action_information(int socket_id, uint16_t fed_id);
 
 /**
- * Listen for a MSG_TYPE_UDP_PORT message, and upon receiving it, set up
- * clock synchronization and perform the initial clock synchronization.
+ * Handle a MSG_TYPE_UDP_PORT message, set up clock synchronization and
+ * perform the initial clock synchronization.
  * Initial clock synchronization is performed only if the MSG_TYPE_UDP_PORT message
  * payload is not UINT16_MAX. If it is also not 0, then this function sets
  * up to perform runtime clock synchronization using the UDP port number
@@ -452,7 +474,7 @@ int receive_connection_information(int socket_id, uint16_t fed_id);
  * @param fed_id The federate ID.
  * @return 1 for success, 0 for failure.
  */
-int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id);
+int handle_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id);
 
 #ifdef __RTI_AUTH__
 /**

From db9db08365670a0872d7685f345acfa9d7245ae1 Mon Sep 17 00:00:00 2001
From: Byeong-gil Jun <junbg@hanyang.ac.kr>
Date: Mon, 27 Nov 2023 14:55:43 +0900
Subject: [PATCH 48/49] Make the NDT things optional in the RTI and federates

---
 core/environment.c              |  6 ++++--
 core/federated/RTI/main.c       | 11 +++++++++--
 core/federated/RTI/rti_remote.c |  8 ++++++++
 core/federated/RTI/rti_remote.h |  7 ++++++-
 core/federated/federate.c       | 16 ++++++++++++++++
 core/reactor_common.c           |  4 ++--
 include/core/environment.h      |  4 ++--
 7 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/core/environment.c b/core/environment.c
index c0715e5e1..a9e10e9f4 100644
--- a/core/environment.c
+++ b/core/environment.c
@@ -106,11 +106,11 @@ static void environment_init_modes(environment_t* env, int num_modes, int num_st
  * @brief Initialize the federation-specific parts of the environment struct.
  */
 static void environment_init_federated(environment_t* env, int num_is_present_fields) {
-// #ifdef FEDERATED_CENTRALIZED
+#ifdef FEDERATED_NDT_ENABLED
     // FIXME: Create a queue saving tags instead of events. For now, ndt_q stores 
     // dummy events.
     env->ndt_q = pqueue_tag_init(10);
-// #endif
+#endif // FEDERATED_NDT_ENABLED
 #ifdef FEDERATED_DECENTRALIZED
     env->_lf_intended_tag_fields = (tag_t**) calloc(num_is_present_fields, sizeof(tag_t*));
     LF_ASSERT(env->_lf_intended_tag_fields, "Out of memory");
@@ -170,7 +170,9 @@ void environment_free(environment_t* env) {
     pqueue_free(env->event_q);
     pqueue_free(env->recycle_q);
     pqueue_free(env->next_q);
+    #ifdef FEDERATED_NDT_ENABLED
     pqueue_tag_free(env->ndt_q);
+    #endif // FEDERATED_NDT_ENABLED
 
     environment_free_threaded(env);
     environment_free_single_threaded(env);
diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c
index 941590285..bfe68ec37 100644
--- a/core/federated/RTI/main.c
+++ b/core/federated/RTI/main.c
@@ -99,6 +99,8 @@ void usage(int argc, const char* argv[]) {
     lf_print("          clock sync attempt (default is 10). Applies to 'init' and 'on'.\n");
     lf_print("  -a, --auth Turn on HMAC authentication options.\n");
     lf_print("  -t, --tracing Turn on tracing.\n");
+    lf_print("  -v, --version The minimum required version of Lingua Franca.");
+    lf_print("      --ndt Turn on ndt optimization.\n");
 
     lf_print("Command given:");
     for (int i = 0; i < argc; i++) {
@@ -170,8 +172,11 @@ int process_clock_sync_args(int argc, const char* argv[]) {
 }
 
 int process_args(int argc, const char* argv[]) {
-    for (int i = 1; i < argc; i++) {
-        if (strcmp(argv[i], "-i") == 0 || strcmp(argv[i], "--id") == 0) {
+    for (int i = 1; i < argc; i++) { 
+        if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--version") == 0) {
+            lf_print("%s", version_info);
+            return 0;
+        } else if (strcmp(argv[i], "-i") == 0 || strcmp(argv[i], "--id") == 0) {
             if (argc < i + 2) {
                 lf_print_error("--id needs a string argument.");
                 usage(argc, argv);
@@ -232,6 +237,8 @@ int process_args(int argc, const char* argv[]) {
             rti.authentication_enabled = true;
         } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) {
             rti.base.tracing_enabled = true;
+        } else if (strcmp(argv[i], "--ndt" == 0)) {
+            rti.ndt_enabled = true;
         } else if (strcmp(argv[i], " ") == 0) {
             // Tolerate spaces
             continue;
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 1961deac1..8e0352ba6 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -335,6 +335,9 @@ void determine_the_ndt_condition() {
 }
 
 void send_upstream_next_downstream_tag(federate_info_t* fed, tag_t next_event_tag) {
+    if (!rti_remote->ndt_enabled) {
+        return;
+    }
     // The RTI receives next_event_tag from the federated fed. 
     // It has to send NDT messages to the upstream federates of fed
     // if the LTC message from an upstream federate is ealrier than the next_event_tag.
@@ -1330,6 +1333,10 @@ int receive_set_up_information(int socket_id, uint16_t fed_id) {
     bool waiting_physical_action_information = true;
     bool waiting_udp_message_and_set_up_clock_sync = true;
 
+    if (!rti_remote->ndt_enabled) {
+        waiting_physical_action_information = false;
+    }
+
     // Buffer for incoming messages.
     // This does not constrain the message size because messages
     // are forwarded piece by piece.
@@ -1794,6 +1801,7 @@ void initialize_RTI(rti_remote_t *rti){
     rti_remote->clock_sync_period_ns = MSEC(10);
     rti_remote->clock_sync_exchanges_per_interval = 10;
     rti_remote->authentication_enabled = false;
+    rti_remote->ndt_enabled = false;
     rti_remote->base.tracing_enabled = false;
     rti_remote->stop_in_progress = false;
 }
diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h
index 55753fde2..bfe4370bf 100644
--- a/core/federated/RTI/rti_remote.h
+++ b/core/federated/RTI/rti_remote.h
@@ -35,7 +35,7 @@
 /////////////////////////////////////////////
 //// Data structures
 
-static char version_info[] = "0.5.0"; // Simply use Lingua Franca version name now.
+static char version_info[] = "0.5.0"; // Simply use Lingua Franca version name for now.
 
 typedef enum socket_type_t {
     TCP,
@@ -163,6 +163,11 @@ typedef struct rti_remote_t {
      */
     bool authentication_enabled;
 
+    /**
+     * Boolean indicating that NDT message is enabled.
+    */
+    bool ndt_enabled;
+
     /**
      * Boolean indicating that a stop request is already in progress.
      */
diff --git a/core/federated/federate.c b/core/federated/federate.c
index 77ef59005..60b9588a9 100644
--- a/core/federated/federate.c
+++ b/core/federated/federate.c
@@ -67,7 +67,9 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <openssl/rand.h> // For secure random number generation.
 #include <openssl/hmac.h> // For HMAC-based authentication of federates.
 #endif
+#ifdef FEDERATED_NDT_ENABLED
 #include "pqueue_tag.h"
+#endif // FEDERATED_NDT_ENABLED
 
 // Global variables defined in tag.c:
 extern instant_t _lf_last_reported_unadjusted_physical_time_ns;
@@ -423,12 +425,14 @@ int send_timed_message(environment_t* env,
     }
 
     // Insert the intended tag into the ndt_q to send LTC to the RTI quickly.
+#ifdef FEDERATED_NDT_ENABLED
     if (pqueue_tag_size(env->ndt_q) != 0) {
         // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
         // is not enough to know whether this federate using the NDT optimization or not.
         LF_PRINT_DEBUG("Insert NDT at the intended to send LTC and NET quickly.");
         pqueue_tag_insert_if_no_match(env->ndt_q, current_message_intended_tag);
     }
+#endif // FEDERATED_NDT_ENABLED
 
     // Trace the event when tracing is enabled
     if (message_type == MSG_TYPE_TAGGED_MESSAGE) {
@@ -1153,6 +1157,7 @@ void connect_to_rti(const char* hostname, int port) {
                 // @see MSG_TYPE_NEIGHBOR_STRUCTURE in net_common.h
                 send_neighbor_structure_to_RTI(_fed.socket_TCP_RTI);
 
+#ifdef FEDERATED_NDT_ENABLED
                 // Send whether this federate has a physical action to the RTI.
                 uint16_t has_physical_action = _fed.min_delay_from_physical_action_to_federate_output != NEVER;
                 unsigned char physical_action_message_buffer[1 + sizeof(uint16_t)];
@@ -1160,6 +1165,7 @@ void connect_to_rti(const char* hostname, int port) {
                 encode_uint16(has_physical_action, &(physical_action_message_buffer[1]));
                 write_to_socket_errexit(_fed.socket_TCP_RTI, 1 + sizeof(uint16_t), physical_action_message_buffer,
                     "Failed to send physical action info to RTI.");
+#endif // FEDERATED_NDT_ENABLED
 
                 uint16_t udp_port = setup_clock_synchronization_with_rti();
 
@@ -1420,6 +1426,7 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
     
     // FIXME: If port absent messages are not used when there is no zero-delay cycle,
     // This part is not needed as we don't apply the NDT optimization for cycles.
+#ifdef FEDERATED_NDT_ENABLED
     if (pqueue_tag_size(env->ndt_q) != 0 ) {
         // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
         // is not enough to know whether this federate using the NDT optimization or not.
@@ -1433,6 +1440,7 @@ void send_port_absent_to_federate(environment_t* env, interval_t additional_dela
             return;
         }
     }
+#endif // FEDERATED_NDT_ENABLED
 
     // Construct the message
     size_t message_length = 1 + sizeof(port_ID) + sizeof(fed_ID) + sizeof(instant_t) + sizeof(microstep_t);
@@ -1948,6 +1956,7 @@ void _lf_logical_tag_complete(tag_t tag_to_send) {
     environment_t *env;
     _lf_get_environments(&env);
     bool need_to_send_LTC = true;
+#ifdef FEDERATED_NDT_ENABLED
     if (pqueue_tag_size(env->ndt_q) != 0 ) {
         // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
         // is not enough to know whether this federate using the NDT optimization or not.
@@ -1961,6 +1970,7 @@ void _lf_logical_tag_complete(tag_t tag_to_send) {
             need_to_send_LTC = false;
         }
     }
+#endif // FEDERATED_NDT_ENABLED
     if (need_to_send_LTC) {
         LF_PRINT_LOG("Sending Logical Tag Complete (LTC) " PRINTF_TAG " to the RTI.",
             tag_to_send.time - start_time,
@@ -2391,6 +2401,7 @@ void handle_stop_request_message() {
     lf_mutex_unlock(&outbound_socket_mutex);
 }
 
+#ifdef FEDERATED_NDT_ENABLED
 /**
  * Handle a MSG_TYPE_NEXT_DOWNSTREAM_MESSAGE from the RTI
  * 
@@ -2425,6 +2436,7 @@ void handle_next_downstream_tag() {
         pqueue_tag_insert_if_no_match(env->ndt_q, env->current_tag);
     }
 }
+#endif // FEDERATED_NDT_ENABLED
 
 /**
  * Close sockets used to communicate with other federates, if they are open,
@@ -2643,9 +2655,11 @@ void* listen_to_rti_TCP(void* args) {
             case MSG_TYPE_PORT_ABSENT:
                 handle_port_absent_message(_fed.socket_TCP_RTI, -1);
                 break;
+#ifdef FEDERATED_NDT_ENABLED
             case MSG_TYPE_NEXT_DOWNSTREAM_TAG:
                 handle_next_downstream_tag();
                 break;
+#endif // FEDERATED_NDT_ENABLED
             case MSG_TYPE_CLOCK_SYNC_T1:
             case MSG_TYPE_CLOCK_SYNC_T4:
                 lf_print_error("Federate %d received unexpected clock sync message from RTI on TCP socket.",
@@ -2838,6 +2852,7 @@ tag_t _lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply
             // If there is no downstream events that require the NET of the current tag,
             // do not send the NET.
             bool need_to_send_NET = true;
+#ifdef FEDERATED_NDT_ENABLED
             if (pqueue_tag_size(env->ndt_q) != 0 ) {
                 // FIXME: If the RTI changes the use of NDTs dynamically, merely checking the size
                 // is not enough to know whether this federate using the NDT optimization or not.
@@ -2851,6 +2866,7 @@ tag_t _lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply
                     need_to_send_NET = false;
                 }
             }
+#endif // FEDERATED_NDT_ENABLED
             if (need_to_send_NET) {
                 _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag, wait_for_reply);
                 _fed.last_sent_NET = tag;
diff --git a/core/reactor_common.c b/core/reactor_common.c
index 0f45b6103..111cab93a 100644
--- a/core/reactor_common.c
+++ b/core/reactor_common.c
@@ -293,7 +293,7 @@ void _lf_start_time_step(environment_t *env) {
         }
     }
 
-// #ifdef FEDERATED_CENTRALIZED
+#ifdef FEDERATED_NDT_ENABLED
     while (pqueue_tag_size(env->ndt_q) != 0 
     && lf_tag_compare(pqueue_tag_peek(env->ndt_q)->tag, env->current_tag) < 0) {
         // Remove elements of ndt_q with tag less than the current tag.
@@ -302,7 +302,7 @@ void _lf_start_time_step(environment_t *env) {
         tag_to_remove.time - start_time, tag_to_remove.microstep,
         env->current_tag.time - start_time, env->current_tag.microstep);
     }
-// #endif
+#endif
 #ifdef FEDERATED_DECENTRALIZED
     for (int i = 0; i < env->is_present_fields_size; i++) {
         // FIXME: For now, an intended tag of (NEVER, 0)
diff --git a/include/core/environment.h b/include/core/environment.h
index 9684cb057..717c1b4ff 100644
--- a/include/core/environment.h
+++ b/include/core/environment.h
@@ -107,9 +107,9 @@ typedef struct environment_t {
     tag_t** _lf_intended_tag_fields;
     int _lf_intended_tag_fields_size;
 #endif // FEDERATED
-// #ifdef FEDERATED_CENTRALIZED
+#ifdef FEDERATED_NDT_ENABLED
     pqueue_tag_t* ndt_q;
-// #endif // FEDERATED_CENTRALIZED
+#endif // FEDERATED_NDT_ENABLED
 #ifdef LF_ENCLAVES // TODO: Consider dropping #ifdef
     enclave_info_t *enclave_info;
 #endif

From 7a02876808628cecfbed5f32906e01df5dfc7a6b Mon Sep 17 00:00:00 2001
From: Byeonggil Jun <byeonggil@asu.edu>
Date: Tue, 5 Dec 2023 08:54:35 +0900
Subject: [PATCH 49/49] Remove the duplicate cycle detection function

---
 core/federated/RTI/main.c       |  6 ++--
 core/federated/RTI/rti_common.c | 53 ++++++++++-----------------------
 core/federated/RTI/rti_remote.c |  1 -
 3 files changed, 19 insertions(+), 41 deletions(-)

diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c
index c1fa0c0ae..6833ec9e9 100644
--- a/core/federated/RTI/main.c
+++ b/core/federated/RTI/main.c
@@ -172,7 +172,7 @@ int process_clock_sync_args(int argc, const char* argv[]) {
 }
 
 int process_args(int argc, const char* argv[]) {
-    for (int i = 1; i < argc; i++) { 
+    for (int i = 1; i < argc; i++) {
         if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--version") == 0) {
             lf_print("%s", version_info);
             return 0;
@@ -237,7 +237,7 @@ int process_args(int argc, const char* argv[]) {
             rti.authentication_enabled = true;
         } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) {
             rti.base.tracing_enabled = true;
-        } else if (strcmp(argv[i], "--ndt" == 0)) {
+        } else if (strcmp(argv[i], "--ndt") == 0) {
             rti.ndt_enabled = true;
         } else if (strcmp(argv[i], " ") == 0) {
             // Tolerate spaces
@@ -246,7 +246,7 @@ int process_args(int argc, const char* argv[]) {
            lf_print_error("Unrecognized command-line argument: %s", argv[i]);
            usage(argc, argv);
            return 0;
-       }
+        }
     }
     if (rti.base.number_of_scheduling_nodes == 0) {
         lf_print_error("--number_of_federates needs a valid positive integer argument.");
diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c
index 088a0b49f..b70a86b6e 100644
--- a/core/federated/RTI/rti_common.c
+++ b/core/federated/RTI/rti_common.c
@@ -300,44 +300,23 @@ static void _update_min_delays_upstream(scheduling_node_t* end, scheduling_node_
     }
 }
 
-// bool check_cycle(scheduling_node_t* e, int target_id, bool visited[]) {
-//     if (visited[e->id] || e->state == NOT_CONNECTED) {
-//         if (e->id == target_id) {
-//             return true;
-//         } else {
-//             return false;
-//         }
-//     }
-
-//     visited[e->id] = true;
-
-//     for (int i = 0; i < e->num_upstream; i++) {
-//         if (check_cycle(rti_common->scheduling_nodes[e->upstream[i]], target_id, visited)) {
-//             return true;
-//         }
-//     }
-//     return false;
-// }
-
-// FIXME: Implement this function again by adopting the function update_min_delays_upstream
 bool check_physical_action_of_transitive_downstreams(scheduling_node_t* e, bool visited[]) {
-    return false;
-    // if (visited[e->id] || e->state == NOT_CONNECTED) {
-    //     return false;
-    // }
-
-    // visited[e->id] = true;
-
-    // for (int i = 0; i < e->num_downstream; i++) {
-    //     if (check_physical_action_of_transitive_downstreams(rti_common->scheduling_nodes[e->downstream[i]], visited)) {
-    //         return true;
-    //     }
-    // }
-    // if (e->has_physical_action) {
-    //     return true;
-    // } else {
-    //     return false;
-    // }
+    if (visited[e->id] || e->state == NOT_CONNECTED) {
+        return false;
+    }
+
+    visited[e->id] = true;
+
+    for (int i = 0; i < e->num_downstream; i++) {
+        if (check_physical_action_of_transitive_downstreams(rti_common->scheduling_nodes[e->downstream[i]], visited)) {
+            return true;
+        }
+    }
+    if (e->has_physical_action) {
+        return true;
+    } else {
+        return false;
+    }
 }
 
 void update_min_delays_upstream(scheduling_node_t* node) {
diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c
index 8c23dc559..5194e4254 100644
--- a/core/federated/RTI/rti_remote.c
+++ b/core/federated/RTI/rti_remote.c
@@ -1644,7 +1644,6 @@ void connect_to_federates(int socket_descriptor) {
             i--;
         }
     }
-    update_cycle_information();
     determine_the_ndt_condition();
     // All federates have connected.
     LF_PRINT_DEBUG("All federates have connected to RTI.");