From 98a569dbace3a6c7ae7735603bf68fb6627ddcac Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Sun, 28 Jan 2024 14:35:14 -0700 Subject: [PATCH 01/43] Prepare for adding a new message type, Downstream Next Event Tag (DNET) --- include/core/federated/network/net_common.h | 11 +++++++++++ include/core/trace.h | 4 ++++ util/tracing/visualization/fedsd.py | 3 +++ 3 files changed, 18 insertions(+) diff --git a/include/core/federated/network/net_common.h b/include/core/federated/network/net_common.h index 9ea720fd7..4ee5ab68f 100644 --- a/include/core/federated/network/net_common.h +++ b/include/core/federated/network/net_common.h @@ -665,6 +665,17 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define MSG_TYPE_FAILED 25 +/** + * Byte identifying a downstream next event tag (DNET) message sent + * from the RTI in centralized coordination. + * The next eight bytes will be the timestamp. + * The next four bytes will be the microstep. + * This message from the RTI tells the federate the tag of the earliest event on the + * downstream federate's event queue. In other words, this federate doesn't have to send + * LTC and NET messages earlier than this tag. + */ +#define MSG_TYPE_DOWNSTREAM_NEXT_EVENT_TAG 26 + ///////////////////////////////////////////// //// Rejection codes diff --git a/include/core/trace.h b/include/core/trace.h index d88abc291..6b7b95e49 100644 --- a/include/core/trace.h +++ b/include/core/trace.h @@ -99,6 +99,7 @@ typedef enum send_P2P_MSG, send_ADR_AD, send_ADR_QR, + send_DNET, // Receiving messages receive_ACK, receive_FAILED, @@ -121,6 +122,7 @@ typedef enum receive_P2P_MSG, receive_ADR_AD, receive_ADR_QR, + receive_DNET, receive_UNIDENTIFIED, NUM_EVENT_TYPES } trace_event_t; @@ -164,6 +166,7 @@ static const char *trace_event_names[] = { "Sending P2P_MSG", "Sending ADR_AD", "Sending ADR_QR", + "Sending DNET", // Receiving messages "Receiving ACK", "Receiving FAILED", @@ -186,6 +189,7 @@ static const char *trace_event_names[] = { "Receiving P2P_MSG", "Receiving ADR_AD", "Receiving ADR_QR", + "Receiving DNET", "Receiving UNIDENTIFIED", }; diff --git a/util/tracing/visualization/fedsd.py b/util/tracing/visualization/fedsd.py index b35e96dd7..2f4b4d389 100644 --- a/util/tracing/visualization/fedsd.py +++ b/util/tracing/visualization/fedsd.py @@ -26,6 +26,7 @@ .NET { stroke: #118ab2; fill: #118ab2} \ .PTAG { stroke: #06d6a0; fill: #06d6a0} \ .TAG { stroke: #08a578; fill: #08a578} \ + .DNET { stroke: purple; fill: purple} \ .TIMESTAMP { stroke: grey; fill: grey } \ .FED_ID {stroke: #80DD99; fill: #80DD99 } \ .ADV {stroke-linecap="round" ; stroke: "red" ; fill: "red"} \ @@ -61,6 +62,7 @@ "Sending P2P_MSG": "P2P_MSG", "Sending ADR_AD": "ADR_AD", "Sending ADR_QR": "ADR_QR", + "Sending DNET": "DNET", "Receiving ACK": "ACK", "Receiving FAILED": "FAILED", "Receiving TIMESTAMP": "TIMESTAMP", @@ -82,6 +84,7 @@ "Receiving P2P_MSG": "P2P_MSG", "Receiving ADR_AD": "ADR_AD", "Receiving ADR_QR": "ADR_QR", + "Receiving DNET": "DNET", "Receiving UNIDENTIFIED": "UNIDENTIFIED", "Scheduler advancing time ends": "AdvLT" } From af3d083d0c2efda6daa22df15f87719c677996e6 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Mon, 5 Feb 2024 11:17:39 -0700 Subject: [PATCH 02/43] Rename upstream and downstream to immediate_upstreams and immediate_downstreams --- core/federated/RTI/rti_common.c | 48 +++++++++++++++--------------- core/federated/RTI/rti_common.h | 34 ++++++++++----------- core/federated/RTI/rti_local.c | 8 ++--- core/federated/RTI/rti_remote.c | 52 ++++++++++++++++----------------- 4 files changed, 71 insertions(+), 71 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 33049db50..d6dee9100 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -48,11 +48,11 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) { e->last_provisionally_granted = NEVER_TAG; e->next_event = NEVER_TAG; e->state = NOT_CONNECTED; - e->upstream = NULL; - e->upstream_delay = NULL; - e->num_upstream = 0; - e->downstream = NULL; - e->num_downstream = 0; + e->immediate_upstreams = NULL; + e->immediate_upstream_delays = NULL; + e->num_immediate_upstreams = 0; + e->immediate_downstreams = NULL; + e->num_immediate_downstreams = 0; e->mode = REALTIME; invalidate_min_delays_upstream(e); } @@ -68,8 +68,8 @@ void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) { enclave->id, enclave->completed.time - start_time, enclave->completed.microstep); // Check downstream scheduling_nodes to see whether they should now be granted a TAG. - for (int i = 0; i < enclave->num_downstream; i++) { - scheduling_node_t *downstream = rti_common->scheduling_nodes[enclave->downstream[i]]; + for (int i = 0; i < enclave->num_immediate_downstreams; i++) { + scheduling_node_t *downstream = rti_common->scheduling_nodes[enclave->immediate_downstreams[i]]; // Notify downstream enclave if appropriate. notify_advance_grant_if_safe(downstream); bool *visited = (bool *)calloc(rti_common->number_of_scheduling_nodes, sizeof(bool)); // Initializes to 0. @@ -125,8 +125,8 @@ tag_t eimt_strict(scheduling_node_t* e) { // This will be the smallest upstream NET plus the least delay. // This could be NEVER_TAG if the RTI has not seen a NET from some upstream node. tag_t t_d = FOREVER_TAG; - for (int i = 0; i < e->num_upstream; i++) { - scheduling_node_t* upstream = rti_common->scheduling_nodes[e->upstream[i]]; + for (int i = 0; i < e->num_immediate_upstreams; i++) { + scheduling_node_t* upstream = rti_common->scheduling_nodes[e->immediate_upstreams[i]]; // Skip this node if it is part of a zero-delay cycle. if (is_in_zero_delay_cycle(upstream)) continue; // If we haven't heard from the upstream node, then assume it can send an event at the start time. @@ -141,7 +141,7 @@ tag_t eimt_strict(scheduling_node_t* e) { if (lf_tag_compare(upstream->next_event, earliest) < 0) { earliest = upstream->next_event; } - tag_t earliest_tag_from_upstream = lf_delay_tag(earliest, e->upstream_delay[i]); + tag_t earliest_tag_from_upstream = lf_delay_tag(earliest, e->immediate_upstream_delays[i]); LF_PRINT_DEBUG("RTI: Strict EIMT of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".", e->id, upstream->id, @@ -159,8 +159,8 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) { // Find the earliest LTC of upstream scheduling_nodes (M). tag_t min_upstream_completed = FOREVER_TAG; - for (int j = 0; j < e->num_upstream; j++) { - scheduling_node_t *upstream = rti_common->scheduling_nodes[e->upstream[j]]; + for (int j = 0; j < e->num_immediate_upstreams; j++) { + scheduling_node_t *upstream = rti_common->scheduling_nodes[e->immediate_upstreams[j]]; // Ignore this enclave/federate if it is not connected. if (upstream->state == NOT_CONNECTED) continue; @@ -168,7 +168,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) { // Adjust by the "after" delay. // Note that "no delay" is encoded as NEVER, // whereas one microstep delay is encoded as 0LL. - tag_t candidate = lf_delay_strict(upstream->completed, e->upstream_delay[j]); + tag_t candidate = lf_delay_strict(upstream->completed, e->immediate_upstream_delays[j]); if (lf_tag_compare(candidate, min_upstream_completed) < 0) { min_upstream_completed = candidate; @@ -245,8 +245,8 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) { void notify_downstream_advance_grant_if_safe(scheduling_node_t* e, bool visited[]) { visited[e->id] = true; - for (int i = 0; i < e->num_downstream; i++) { - scheduling_node_t* downstream = rti_common->scheduling_nodes[e->downstream[i]]; + for (int i = 0; i < e->num_immediate_downstreams; i++) { + scheduling_node_t* downstream = rti_common->scheduling_nodes[e->immediate_downstreams[i]]; if (visited[downstream->id]) continue; notify_advance_grant_if_safe(downstream); notify_downstream_advance_grant_if_safe(downstream, visited); @@ -266,7 +266,7 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne // Check to see whether we can reply now with a tag advance grant. // If the enclave has no upstream scheduling_nodes, then it does not wait for // nor expect a reply. It just proceeds to advance time. - if (e->num_upstream > 0) { + if (e->num_immediate_upstreams > 0) { notify_advance_grant_if_safe(e); } else { // Even though there was no grant, mark the tag as if there was. @@ -315,26 +315,26 @@ static void _update_min_delays_upstream( // NOTE: It would be better to iterate through these sorted by minimum delay, // but for most programs, the gain might be negligible since there are relatively few // upstream nodes. - for (int i = 0; i < intermediate->num_upstream; i++) { + for (int i = 0; i < intermediate->num_immediate_upstreams; i++) { // Add connection delay to path delay so far. - tag_t path_delay = lf_delay_tag(delay_from_intermediate_so_far, intermediate->upstream_delay[i]); + tag_t path_delay = lf_delay_tag(delay_from_intermediate_so_far, intermediate->immediate_upstream_delays[i]); // If the path delay is less than the so-far recorded path delay from upstream, update upstream. - if (lf_tag_compare(path_delay, path_delays[intermediate->upstream[i]]) < 0) { - if (path_delays[intermediate->upstream[i]].time == FOREVER) { + if (lf_tag_compare(path_delay, path_delays[intermediate->immediate_upstreams[i]]) < 0) { + if (path_delays[intermediate->immediate_upstreams[i]].time == FOREVER) { // Found a finite path. *count = *count + 1; } - path_delays[intermediate->upstream[i]] = path_delay; + path_delays[intermediate->immediate_upstreams[i]] = path_delay; // Since the path delay to upstream has changed, recursively update those upstream of it. // Do not do this, however, if the upstream node is the end node because this means we have // completed a cycle. - if (end->id != intermediate->upstream[i]) { - _update_min_delays_upstream(end, rti_common->scheduling_nodes[intermediate->upstream[i]], path_delays, count); + if (end->id != intermediate->immediate_upstreams[i]) { + _update_min_delays_upstream(end, rti_common->scheduling_nodes[intermediate->immediate_upstreams[i]], path_delays, count); } else { // Found a cycle. end->flags = end->flags | IS_IN_CYCLE; // Is it a zero-delay cycle? - if (lf_tag_compare(path_delay, ZERO_TAG) == 0 && intermediate->upstream_delay[i] < 0) { + if (lf_tag_compare(path_delay, ZERO_TAG) == 0 && intermediate->immediate_upstream_delays[i] < 0) { end->flags = end->flags | IS_IN_ZERO_DELAY_CYCLE; } else { // Clear the flag. diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 770918d5b..8a0458f75 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -50,23 +50,23 @@ typedef struct minimum_delay_t { * any scheduling constraints. */ typedef struct scheduling_node_t { - uint16_t id; // ID of this scheduling node. - tag_t completed; // The largest logical tag completed by the scheduling node - // (or NEVER if no LTC has been received). - tag_t last_granted; // The maximum TAG that has been granted so far (or NEVER if none granted) - tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) - tag_t next_event; // Most recent NET received from the scheduling node (or NEVER if none received). - scheduling_node_state_t state; // State of the scheduling node. - int* upstream; // Array of upstream scheduling node ids. - interval_t* upstream_delay; // Minimum delay on connections from upstream scheduling nodes. - // Here, NEVER encodes no delay. 0LL is a microstep delay. - int num_upstream; // Size of the array of upstream scheduling nodes and delays. - int* downstream; // Array of downstream scheduling node ids. - int num_downstream; // Size of the array of downstream scheduling nodes. - execution_mode_t mode; // FAST or REALTIME. - minimum_delay_t* min_delays; // Array of minimum delays from upstream nodes, not including this node. - size_t num_min_delays; // Size of min_delays array. - int flags; // Or of IS_IN_ZERO_DELAY_CYCLE, IS_IN_CYCLE + uint16_t id; // ID of this scheduling node. + tag_t completed; // The largest logical tag completed by the scheduling node + // (or NEVER if no LTC has been received). + tag_t last_granted; // The maximum TAG that has been granted so far (or NEVER if none granted) + tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) + tag_t next_event; // Most recent NET received from the scheduling node (or NEVER if none received). + scheduling_node_state_t state; // State of the scheduling node. + int* immediate_upstreams; // Array of immediate upstream scheduling node ids. + interval_t* immediate_upstream_delays; // Minimum delay on connections from immdediate upstream scheduling nodes. + // Here, NEVER encodes no delay. 0LL is a microstep delay. + int num_immediate_upstreams; // Size of the array of immediate upstream scheduling nodes and delays. + int* immediate_downstreams; // Array of immediate downstream scheduling node ids. + int num_immediate_downstreams; // Size of the array of immediate downstream scheduling nodes. + execution_mode_t mode; // FAST or REALTIME. + minimum_delay_t* min_delays; // Array of minimum delays from upstream nodes, not including this node. + size_t num_min_delays; // Size of min_delays array. + int flags; // Or of IS_IN_ZERO_DELAY_CYCLE, IS_IN_CYCLE } scheduling_node_t; /** diff --git a/core/federated/RTI/rti_local.c b/core/federated/RTI/rti_local.c index c75605426..04952f0e9 100644 --- a/core/federated/RTI/rti_local.c +++ b/core/federated/RTI/rti_local.c @@ -53,9 +53,9 @@ void initialize_local_rti(environment_t *envs, int num_envs) { rti_local->base.scheduling_nodes[i] = (scheduling_node_t *) enclave_info; // Encode the connection topology into the enclave_info object. - enclave_info->base.num_downstream = _lf_get_downstream_of(i, &enclave_info->base.downstream); - enclave_info->base.num_upstream = _lf_get_upstream_of(i, &enclave_info->base.upstream); - _lf_get_upstream_delay_of(i, &enclave_info->base.upstream_delay); + enclave_info->base.num_immediate_downstreams = _lf_get_downstream_of(i, &enclave_info->base.immediate_downstreams); + enclave_info->base.num_immediate_upstreams = _lf_get_upstream_of(i, &enclave_info->base.immediate_upstreams); + _lf_get_upstream_delay_of(i, &enclave_info->base.immediate_upstream_delays); enclave_info->base.state = GRANTED; } @@ -111,7 +111,7 @@ tag_t rti_next_event_tag_locked(enclave_info_t* e, tag_t next_event_tag) { } // If this enclave has no upstream, then we give a TAG till forever straight away. - if (e->base.num_upstream == 0) { + if (e->base.num_immediate_upstreams == 0) { LF_PRINT_LOG("RTI: enclave %u has no upstream. Giving it a to the NET", e->base.id); e->base.last_granted = next_event_tag; } diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index 373a109df..e05212742 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -281,8 +281,8 @@ void notify_provisional_tag_advance_grant(scheduling_node_t *e, tag_t tag) { // Note that this is transitive. // NOTE: This is not needed for enclaves because zero-delay loops are prohibited. // It's only needed for federates, which is why this is implemented here. - for (int j = 0; j < e->num_upstream; j++) { - scheduling_node_t *upstream = rti_remote->base.scheduling_nodes[e->upstream[j]]; + for (int j = 0; j < e->num_immediate_upstreams; j++) { + scheduling_node_t *upstream = rti_remote->base.scheduling_nodes[e->immediate_upstreams[j]]; // Ignore this federate if it has resigned. if (upstream->state == NOT_CONNECTED) @@ -1395,33 +1395,33 @@ static int receive_connection_information(int *socket_id, uint16_t fed_id) { } else { federate_info_t *fed = GET_FED_INFO(fed_id); // Read the number of upstream and downstream connections - fed->enclave.num_upstream = extract_int32(&(connection_info_header[1])); - fed->enclave.num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); + fed->enclave.num_immediate_upstreams = extract_int32(&(connection_info_header[1])); + fed->enclave.num_immediate_downstreams = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); LF_PRINT_DEBUG( "RTI got %d upstreams and %d downstreams from federate %d.", - fed->enclave.num_upstream, - fed->enclave.num_downstream, + fed->enclave.num_immediate_upstreams, + fed->enclave.num_immediate_downstreams, fed_id); // Allocate memory for the upstream and downstream pointers - if (fed->enclave.num_upstream > 0) { - fed->enclave.upstream = (int *)malloc(sizeof(uint16_t) * fed->enclave.num_upstream); + if (fed->enclave.num_immediate_upstreams > 0) { + fed->enclave.immediate_upstreams = (int *)malloc(sizeof(uint16_t) * fed->enclave.num_immediate_upstreams); // Allocate memory for the upstream delay pointers - fed->enclave.upstream_delay = (interval_t *)malloc( - sizeof(interval_t) * fed->enclave.num_upstream); + fed->enclave.immediate_upstream_delays = (interval_t *)malloc( + sizeof(interval_t) * fed->enclave.num_immediate_upstreams); } else { - fed->enclave.upstream = (int *)NULL; - fed->enclave.upstream_delay = (interval_t *)NULL; + fed->enclave.immediate_upstreams = (int *)NULL; + fed->enclave.immediate_upstream_delays = (interval_t *)NULL; } - if (fed->enclave.num_downstream > 0) { - fed->enclave.downstream = (int *)malloc(sizeof(uint16_t) * fed->enclave.num_downstream); + if (fed->enclave.num_immediate_downstreams > 0) { + fed->enclave.immediate_downstreams = (int *)malloc(sizeof(uint16_t) * fed->enclave.num_immediate_downstreams); } else { - fed->enclave.downstream = (int *)NULL; + fed->enclave.immediate_downstreams = (int *)NULL; } size_t connections_info_body_size = ( - (sizeof(uint16_t) + sizeof(int64_t)) * fed->enclave.num_upstream) - + (sizeof(uint16_t) * fed->enclave.num_downstream); + (sizeof(uint16_t) + sizeof(int64_t)) * fed->enclave.num_immediate_upstreams) + + (sizeof(uint16_t) * fed->enclave.num_immediate_downstreams); unsigned char *connections_info_body = NULL; if (connections_info_body_size > 0) { connections_info_body = (unsigned char *)malloc(connections_info_body_size); @@ -1435,16 +1435,16 @@ static int receive_connection_information(int *socket_id, uint16_t fed_id) { // Keep track of where we are in the buffer size_t message_head = 0; // First, read the info about upstream federates - for (int i = 0; i < fed->enclave.num_upstream; i++) { - fed->enclave.upstream[i] = extract_uint16(&(connections_info_body[message_head])); + for (int i = 0; i < fed->enclave.num_immediate_upstreams; i++) { + fed->enclave.immediate_upstreams[i] = extract_uint16(&(connections_info_body[message_head])); message_head += sizeof(uint16_t); - fed->enclave.upstream_delay[i] = extract_int64(&(connections_info_body[message_head])); + fed->enclave.immediate_upstream_delays[i] = extract_int64(&(connections_info_body[message_head])); message_head += sizeof(int64_t); } // Next, read the info about downstream federates - for (int i = 0; i < fed->enclave.num_downstream; i++) { - fed->enclave.downstream[i] = extract_uint16(&(connections_info_body[message_head])); + for (int i = 0; i < fed->enclave.num_immediate_downstreams; i++) { + fed->enclave.immediate_downstreams[i] = extract_uint16(&(connections_info_body[message_head])); message_head += sizeof(uint16_t); } @@ -1826,10 +1826,10 @@ void free_scheduling_nodes(scheduling_node_t **scheduling_nodes, uint16_t number for (uint16_t i = 0; i < number_of_scheduling_nodes; i++) { // FIXME: Gives error freeing memory not allocated!!!! scheduling_node_t *node = scheduling_nodes[i]; - if (node->upstream != NULL) - free(node->upstream); - if (node->downstream != NULL) - free(node->downstream); + if (node->immediate_upstreams != NULL) + free(node->immediate_upstreams); + if (node->immediate_downstreams != NULL) + free(node->immediate_downstreams); } free(scheduling_nodes); } From 2373e9f56bdae327d4e02b55c0580b670931500d Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Mon, 5 Feb 2024 13:20:05 -0700 Subject: [PATCH 03/43] Use a matrix instead of multiple arrays to store minimum delays of every possible connection This is a preparation step for the DNET message calculation. This matrix enables each federate to search min_delays from upstreams as well as to downstreams. Use a matrix instead of multiple arrays to store minimum delays of every possible connection This is a preparation step for the DNET message calculation. This matrix enables each federate to search min_delays from upstreams as well as to downstreams. --- core/federated/RTI/main.c | 10 ++++++++-- core/federated/RTI/rti_common.c | 31 ++++++++++++++++++------------- core/federated/RTI/rti_common.h | 23 +++++++++++++++-------- core/federated/RTI/rti_remote.c | 8 ++++---- 4 files changed, 45 insertions(+), 27 deletions(-) diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c index 700304aea..6beb6a8d3 100644 --- a/core/federated/RTI/main.c +++ b/core/federated/RTI/main.c @@ -320,8 +320,14 @@ int main(int argc, const char* argv[]) { assert(rti.base.number_of_scheduling_nodes < UINT16_MAX); // Allocate memory for the federates - rti.base.scheduling_nodes = (scheduling_node_t**)calloc(rti.base.number_of_scheduling_nodes, sizeof(scheduling_node_t*)); - for (uint16_t i = 0; i < rti.base.number_of_scheduling_nodes; i++) { + int n = rti.base.number_of_scheduling_nodes; + rti.base.scheduling_nodes = (scheduling_node_t**)calloc(n, sizeof(scheduling_node_t*)); + // Allocate memory for min_delays. + rti.base.min_delays = (tag_t*)calloc((n*n), sizeof(tag_t)); + for (uint16_t i = 0; i < n; i++) { + for (uint16_t j = 0; j < n; j++) { + rti.base.min_delays[i+j*n] = FOREVER_TAG; + } federate_info_t *fed_info = (federate_info_t *) calloc(1, sizeof(federate_info_t)); initialize_federate(fed_info, i); rti.base.scheduling_nodes[i] = (scheduling_node_t *) fed_info; diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index d6dee9100..712364324 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -35,9 +35,12 @@ void initialize_rti_common(rti_common_t * _rti_common) { #define IS_IN_CYCLE 2 void invalidate_min_delays_upstream(scheduling_node_t* node) { - if(node->min_delays != NULL) free(node->min_delays); - node->min_delays = NULL; - node->num_min_delays = 0; + if(node->all_upstreams != NULL) free(node->all_upstreams); + if(node->all_downstreams != NULL) free(node->all_downstreams); + node->all_upstreams = NULL; + node->num_all_upstreams = 0; + node->all_downstreams = NULL; + node->num_all_downstreams = 0; node->flags = 0; // All flags cleared because they get set lazily. } @@ -91,9 +94,11 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) { // federates, which will be the smallest upstream NET plus the least delay. // This could be NEVER_TAG if the RTI has not seen a NET from some upstream node. tag_t t_d = FOREVER_TAG; - for (int i = 0; i < e->num_min_delays; i++) { - // Node e->min_delays[i].id is upstream of e with min delay e->min_delays[i].min_delay. - scheduling_node_t* upstream = rti_common->scheduling_nodes[e->min_delays[i].id]; + int n = rti_common->number_of_scheduling_nodes; + for (int i = 0; i < e->num_all_upstreams; i++) { + // Node e->all_upstreams[i] is upstream of e with + // min delay rti_common->min_delays[e->all_upstreams[i]*n + e->id] + scheduling_node_t* upstream = rti_common->scheduling_nodes[e->all_upstreams[i]]; // If we haven't heard from the upstream node, then assume it can send an event at the start time. if (lf_tag_compare(upstream->next_event, NEVER_TAG) == 0) { tag_t start_tag = {.time = start_time, .microstep = 0}; @@ -104,7 +109,7 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) { // by (0,1). If the time part of the delay is greater than 0, then we want to ignore // the microstep in upstream->next_event because that microstep will have been lost. // Otherwise, we want preserve it and add to it. This is handled by lf_tag_add(). - tag_t earliest_tag_from_upstream = lf_tag_add(upstream->next_event, e->min_delays[i].min_delay); + tag_t earliest_tag_from_upstream = lf_tag_add(upstream->next_event, rti_common->min_delays[e->all_upstreams[i]*n + e->id]); /* Following debug message is too verbose for normal use: LF_PRINT_DEBUG("RTI: Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".", @@ -347,7 +352,7 @@ static void _update_min_delays_upstream( void update_min_delays_upstream(scheduling_node_t* node) { // Check whether cached result is valid. - if (node->min_delays == NULL) { + if (node->all_upstreams == NULL) { // This is not Dijkstra's algorithm, but rather one optimized for sparse upstream nodes. // There must be a name for this algorithm. @@ -362,9 +367,9 @@ void update_min_delays_upstream(scheduling_node_t* node) { } _update_min_delays_upstream(node, NULL, path_delays, &count); - // Put the results onto the node's struct. - node->num_min_delays = count; - node->min_delays = (minimum_delay_t*)calloc(count, sizeof(minimum_delay_t)); + // Put the results onto the matrix. + node->num_all_upstreams = count; + node->all_upstreams = (uint16_t*)calloc(count, sizeof(uint16_t)); LF_PRINT_DEBUG("++++ Node %hu is in ZDC: %d", node->id, is_in_zero_delay_cycle(node)); int k = 0; for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) { @@ -373,8 +378,8 @@ void update_min_delays_upstream(scheduling_node_t* node) { if (k >= count) { lf_print_error_and_exit("Internal error! Count of upstream nodes %zu for node %d is wrong!", count, i); } - minimum_delay_t min_delay = {.id = i, .min_delay = path_delays[i]}; - node->min_delays[k++] = min_delay; + rti_common->min_delays[node->id + i*rti_common->number_of_scheduling_nodes] = path_delays[i]; + node->all_upstreams[k++] = i; // N^2 debug statement could be a problem with large benchmarks. // LF_PRINT_DEBUG("++++ Node %hu is upstream with delay" PRINTF_TAG "\n", i, path_delays[i].time, path_delays[i].microstep); } diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 8a0458f75..37aa2d490 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -57,15 +57,17 @@ typedef struct scheduling_node_t { tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) tag_t next_event; // Most recent NET received from the scheduling node (or NEVER if none received). scheduling_node_state_t state; // State of the scheduling node. - int* immediate_upstreams; // Array of immediate upstream scheduling node ids. + uint16_t* immediate_upstreams; // Array of immediate upstream scheduling node ids. interval_t* immediate_upstream_delays; // Minimum delay on connections from immdediate upstream scheduling nodes. // Here, NEVER encodes no delay. 0LL is a microstep delay. - int num_immediate_upstreams; // Size of the array of immediate upstream scheduling nodes and delays. - int* immediate_downstreams; // Array of immediate downstream scheduling node ids. - int num_immediate_downstreams; // Size of the array of immediate downstream scheduling nodes. + uint16_t num_immediate_upstreams; // Size of the array of immediate upstream scheduling nodes and delays. + uint16_t* immediate_downstreams; // Array of immediate downstream scheduling node ids. + uint16_t num_immediate_downstreams; // Size of the array of immediate downstream scheduling nodes. execution_mode_t mode; // FAST or REALTIME. - minimum_delay_t* min_delays; // Array of minimum delays from upstream nodes, not including this node. - size_t num_min_delays; // Size of min_delays array. + uint16_t* all_upstreams; // Array of all upstream scheduling node ids. + uint16_t num_all_upstreams; // Size of the array of all upstream scheduling nodes and delays. + uint16_t* all_downstreams; // Array of all downstream scheduling node ids. + uint16_t num_all_downstreams; // Size of the array of all downstream scheduling nodes. int flags; // Or of IS_IN_ZERO_DELAY_CYCLE, IS_IN_CYCLE } scheduling_node_t; @@ -79,7 +81,12 @@ typedef struct rti_common_t { scheduling_node_t **scheduling_nodes; // Number of scheduling nodes - int32_t number_of_scheduling_nodes; + uint16_t number_of_scheduling_nodes; + + // Matrix of minimum delays between each nodes + // Rows represent upstreams and Columns represent downstreams. + // FOREVER_TAG means there is no path, ZERO_TAG means there is no delay. + tag_t* min_delays; // RTI's decided stop tag for the scheduling nodes tag_t max_stop_tag; @@ -264,7 +271,7 @@ bool is_in_cycle(scheduling_node_t* node); /** * For the given scheduling node (enclave or federate), if necessary, update the `min_delays`, - * `num_min_delays`, and the fields that indicate cycles. These fields will be + * `num_all_upstreams`, and the fields that indicate cycles. These fields will be * updated only if they have not been previously updated or if invalidate_min_delays_upstream * has been called since they were last updated. * @param node The node. diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index e05212742..8c7ba68a1 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -1405,18 +1405,18 @@ static int receive_connection_information(int *socket_id, uint16_t fed_id) { // Allocate memory for the upstream and downstream pointers if (fed->enclave.num_immediate_upstreams > 0) { - fed->enclave.immediate_upstreams = (int *)malloc(sizeof(uint16_t) * fed->enclave.num_immediate_upstreams); + fed->enclave.immediate_upstreams = (uint16_t *)malloc(sizeof(uint16_t) * fed->enclave.num_immediate_upstreams); // Allocate memory for the upstream delay pointers fed->enclave.immediate_upstream_delays = (interval_t *)malloc( sizeof(interval_t) * fed->enclave.num_immediate_upstreams); } else { - fed->enclave.immediate_upstreams = (int *)NULL; + fed->enclave.immediate_upstreams = (uint16_t *)NULL; fed->enclave.immediate_upstream_delays = (interval_t *)NULL; } if (fed->enclave.num_immediate_downstreams > 0) { - fed->enclave.immediate_downstreams = (int *)malloc(sizeof(uint16_t) * fed->enclave.num_immediate_downstreams); + fed->enclave.immediate_downstreams = (uint16_t *)malloc(sizeof(uint16_t) * fed->enclave.num_immediate_downstreams); } else { - fed->enclave.immediate_downstreams = (int *)NULL; + fed->enclave.immediate_downstreams = (uint16_t *)NULL; } size_t connections_info_body_size = ( From 6b37e3695d7cd3dbd8959f45a3657cd58cc2700e Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 7 Feb 2024 14:18:13 -0700 Subject: [PATCH 04/43] Save IDs of all downstreams for faster access to the min_dleays matrix --- core/federated/RTI/rti_common.c | 49 ++++++++++++++++++++++++++++++++- core/federated/RTI/rti_common.h | 10 ++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index f2c084285..aa5ba0bad 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -35,7 +35,12 @@ void initialize_rti_common(rti_common_t * _rti_common) { #define IS_IN_CYCLE 2 void invalidate_min_delays_upstream(scheduling_node_t* node) { - if(node->all_upstreams != NULL) free(node->all_upstreams); + if(node->all_upstreams != NULL) { + free(node->all_upstreams); + for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) { + rti_common->min_delays[i*rti_common->number_of_scheduling_nodes + node->id] = FOREVER_TAG; + } + } if(node->all_downstreams != NULL) free(node->all_downstreams); node->all_upstreams = NULL; node->num_all_upstreams = 0; @@ -89,6 +94,7 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) { // and then find the minimum of the node's recorded NET plus the minimum path delay. // Update the shortest paths, if necessary. update_min_delays_upstream(e); + update_all_downstreams(e); // Next, find the tag of the earliest possible incoming message from upstream enclaves or // federates, which will be the smallest upstream NET plus the least delay. @@ -387,6 +393,47 @@ void update_min_delays_upstream(scheduling_node_t* node) { } } +void update_all_downstreams(scheduling_node_t* node) { + if (node->all_downstreams == NULL) { + bool visited[rti_common->number_of_scheduling_nodes]; + for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) { + visited[i] = false; + } + + int queue[rti_common->number_of_scheduling_nodes]; + int front = 0, rear = 0; + + visited[node->id] = true; + queue[rear++] = node->id; + + int count = 0; + while (front != rear) { + int current_id = queue[front++]; + scheduling_node_t* current_node = rti_common->scheduling_nodes[current_id]; + for (int i = 0; i < current_node->num_immediate_downstreams; i++) { + int downstream_id = current_node->immediate_downstreams[i]; + if (visited[downstream_id] == false) { + visited[downstream_id] = true; + queue[rear++] = downstream_id; + count++; + } + } + } + + int k = 0; + node->all_downstreams = (uint16_t*)calloc(count, sizeof(uint16_t)); + node->num_all_downstreams = count; + for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) { + if (node->num_all_downstreams <= k) { + // Error + } + if (visited[i] == true && i != node->id) { + node->all_downstreams[k++] = i; + } + } + } +} + bool is_in_zero_delay_cycle(scheduling_node_t* node) { update_min_delays_upstream(node); return node->flags & IS_IN_ZERO_DELAY_CYCLE; diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 37aa2d490..358d379b9 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -271,13 +271,21 @@ bool is_in_cycle(scheduling_node_t* node); /** * For the given scheduling node (enclave or federate), if necessary, update the `min_delays`, - * `num_all_upstreams`, and the fields that indicate cycles. These fields will be + * `all_upstremas`, `num_all_upstreams`, and the fields that indicate cycles. These fields will be * updated only if they have not been previously updated or if invalidate_min_delays_upstream * has been called since they were last updated. * @param node The node. */ void update_min_delays_upstream(scheduling_node_t* node); +/** + * For the given scheduling node (enclave or federate), if necessary, update the `all_downstreams`, + * `num_all_downstreams`. These fields will be updated only if they have not been previously updated + * or if invalidate_min_delays_upstream has been called since they were last updated. + * @param node The node. +*/ +void update_all_downstreams(scheduling_node_t* node); + /** * For the given scheduling node (enclave or federate), invalidate the `min_delays`, * `num_min_delays`, and the fields that indicate cycles. From eea1edf4c5424b357095773962d165c12f2e498c Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 7 Feb 2024 16:04:19 -0700 Subject: [PATCH 05/43] Add a function for subtracting tags --- core/tag.c | 29 +++++++++++++++++++++++++++++ include/core/tag.h | 5 +++++ 2 files changed, 34 insertions(+) diff --git a/core/tag.c b/core/tag.c index 955f3b5ae..bf0d4dcca 100644 --- a/core/tag.c +++ b/core/tag.c @@ -128,6 +128,35 @@ tag_t lf_tag_add(tag_t a, tag_t b) { return result; } +tag_t lf_tag_subtract(tag_t a, tag_t b) { + // FIXME: Overflow is not handled. + // (a.t, a.m) - (b.t - b.m) + // 1) a.t = NEVER + // b.t can be NEVER, FOREVER, normal and the result will be 0, NEVER, NEVER, respectively. + // 2) a.t = FOREVER + // b.t can be NEVER, FOREVER, normal and the result will be FOREVER, 0, FOREVER. + // 3) a.t is not NEVER neither FOREVER + // a) a.t < b.t + // the result will be NEVER + // b) a.t >= b.t + // i) a.m <= b.m + // the result should be (a.t - b.t, 0) + // ii) a.m > b.m + // if b is ZERO, the result is a itself else, the result is (a.t - b.t + 1, 0) + if (a.time == NEVER && b.time != NEVER) return NEVER_TAG; + if (a.time == FOREVER && b.time != FOREVER) return FOREVER_TAG; + if (b.time == 0 && b.microstep == 0) return a; + tag_t result = {.time = a.time - b.time, .microstep = a.microstep - b.microstep}; + if (result.time < 0) { + return NEVER_TAG; + } else { + if (result.microstep > 0) { + result.time++; + } + result.microstep = 0; + } +} + int lf_tag_compare(tag_t tag1, tag_t tag2) { if (tag1.time < tag2.time) { return -1; diff --git a/include/core/tag.h b/include/core/tag.h index e38ea7de5..8163222e1 100644 --- a/include/core/tag.h +++ b/include/core/tag.h @@ -96,6 +96,11 @@ tag_t lf_tag(void* env); */ tag_t lf_tag_add(tag_t a, tag_t b); +/** + * Subtract the tag a from the tag b. +*/ +tag_t lf_tag_subtract(tag_t a, tag_t b); + /** * Compare two tags. Return -1 if the first is less than * the second, 0 if they are equal, and +1 if the first is From b4900f37f3861d1d466bbfc75a7450752d4b176e Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 7 Feb 2024 17:04:38 -0700 Subject: [PATCH 06/43] Calculate and send DNET messages --- core/federated/RTI/rti_common.c | 45 +++++++++++++++++++++++++------ core/federated/RTI/rti_common.h | 14 ++++++++++ core/federated/RTI/rti_remote.c | 17 ++++++++++++ core/federated/federate.c | 24 +++++++++++++++++ include/core/federated/federate.h | 5 ++++ 5 files changed, 97 insertions(+), 8 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index aa5ba0bad..8a3cc660e 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -55,6 +55,7 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) { e->last_granted = NEVER_TAG; e->last_provisionally_granted = NEVER_TAG; e->next_event = NEVER_TAG; + e->last_DNET = NEVER_TAG; e->state = NOT_CONNECTED; e->immediate_upstreams = NULL; e->immediate_upstream_delays = NULL; @@ -286,9 +287,17 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne // Check downstream scheduling_nodes to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which downstream scheduling_nodes have been visited. + // FIXME: As we have all_downstreams field now, we don't need the function notify_downstream_davnace_grnat_if_safe. + update_all_downstreams(e); bool *visited = (bool *)calloc(rti_common->number_of_scheduling_nodes, sizeof(bool)); // Initializes to 0. notify_downstream_advance_grant_if_safe(e, visited); free(visited); + + // Send DNET to the node's upstreams if needed + for (int i = 0; i < e->num_all_upstreams; i++) { + int target_upstream_id = e->all_upstreams[i]; + send_downstream_next_event_tag_if_needed(rti_common->scheduling_nodes[target_upstream_id], next_event_tag); + } } void notify_advance_grant_if_safe(scheduling_node_t* e) { @@ -400,18 +409,18 @@ void update_all_downstreams(scheduling_node_t* node) { visited[i] = false; } - int queue[rti_common->number_of_scheduling_nodes]; + uint16_t queue[rti_common->number_of_scheduling_nodes]; int front = 0, rear = 0; visited[node->id] = true; queue[rear++] = node->id; - int count = 0; + size_t count = 0; while (front != rear) { int current_id = queue[front++]; scheduling_node_t* current_node = rti_common->scheduling_nodes[current_id]; - for (int i = 0; i < current_node->num_immediate_downstreams; i++) { - int downstream_id = current_node->immediate_downstreams[i]; + for (uint16_t i = 0; i < current_node->num_immediate_downstreams; i++) { + uint16_t downstream_id = current_node->immediate_downstreams[i]; if (visited[downstream_id] == false) { visited[downstream_id] = true; queue[rear++] = downstream_id; @@ -423,17 +432,37 @@ void update_all_downstreams(scheduling_node_t* node) { int k = 0; node->all_downstreams = (uint16_t*)calloc(count, sizeof(uint16_t)); node->num_all_downstreams = count; - for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) { - if (node->num_all_downstreams <= k) { - // Error - } + for (uint16_t i = 0; i < rti_common->number_of_scheduling_nodes; i++) { if (visited[i] == true && i != node->id) { + if (k >= count) { + lf_print_error_and_exit("Internal error! Count of downstream nodes %zu for node %d is wrong!", count, i); + } node->all_downstreams[k++] = i; } } } } +void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new_NET) { + tag_t DNET = FOREVER_TAG; + if (lf_tag_compare(node->last_DNET, new_NET) >= 0) { + DNET = new_NET; + } else { + for (int i = 0; i < node->num_all_downstreams; i++) { + uint16_t target_downstream_id = node->all_downstreams[i]; + scheduling_node_t* target_dowstream = rti_common->scheduling_nodes[target_downstream_id]; + int index = node->id * rti_common->number_of_scheduling_nodes + target_downstream_id; + tag_t DNET_candidate = lf_tag_subtract(target_dowstream->next_event, rti_common->min_delays[index]); + if (lf_tag_compare(DNET, DNET_candidate) > 0) { + DNET = DNET_candidate; + } + } + } + if (lf_tag_compare(node->last_DNET, DNET) != 0) { + send_downstream_next_event_tag(node, DNET); + } +} + bool is_in_zero_delay_cycle(scheduling_node_t* node) { update_min_delays_upstream(node); return node->flags & IS_IN_ZERO_DELAY_CYCLE; diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 358d379b9..18e6653c3 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -56,6 +56,7 @@ typedef struct scheduling_node_t { tag_t last_granted; // The maximum TAG that has been granted so far (or NEVER if none granted) tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) tag_t next_event; // Most recent NET received from the scheduling node (or NEVER if none received). + tag_t last_DNET; // Most recent DNET. scheduling_node_state_t state; // State of the scheduling node. uint16_t* immediate_upstreams; // Array of immediate upstream scheduling node ids. interval_t* immediate_upstream_delays; // Minimum delay on connections from immdediate upstream scheduling nodes. @@ -286,6 +287,19 @@ void update_min_delays_upstream(scheduling_node_t* node); */ void update_all_downstreams(scheduling_node_t* node); +/** + * FIXME: Add this function to rti_local either. + * @param e The target node. + * @param tag The downstream next event tag for e. +*/ +void send_downstream_next_event_tag(scheduling_node_t *e, tag_t tag); + +/** + * @param node + * @param new_NET +*/ +void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new_NET); + /** * For the given scheduling node (enclave or federate), invalidate the `min_delays`, * `num_min_delays`, and the fields that indicate cycles. diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index d8854f7d0..e76ce86ba 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -302,6 +302,23 @@ void notify_provisional_tag_advance_grant(scheduling_node_t *e, tag_t tag) { } } +void send_downstream_next_event_tag(scheduling_node_t *e, tag_t tag) { + size_t message_length = 1 + sizeof(int64_t) + sizeof(uint32_t); + unsigned char buffer[message_length]; + buffer[0] = MSG_TYPE_DOWNSTREAM_NEXT_EVENT_TAG; + encode_int64(tag.time, &(buffer[1])); + encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); + + if (write_to_socket(((federate_info_t *)e)->socket, message_length, buffer)) { + lf_print_error("RTI failed to send downstream next event tag to federate %d.", e->id); + e->state = NOT_CONNECTED; + } else { + e->last_DNET = tag; + LF_PRINT_LOG("RTI sent to federate %d the Downstream Next Event Tag (DNET) " PRINTF_TAG ".", + e->id, tag.time - start_time, tag.microstep); + } +} + void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag) { federate_info_t *fed = GET_FED_INFO(federate_id); tag_t min_in_transit_tag = pqueue_tag_peek_tag(fed->in_transit_message_tags); diff --git a/core/federated/federate.c b/core/federated/federate.c index afcb5fa98..2e8abf5a2 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -88,6 +88,7 @@ federate_instance_t _fed = { .is_last_TAG_provisional = false, .has_upstream = false, .has_downstream = false, + .last_DNET = (tag_t) {.time = NEVER, .microstep = 0u}, .received_stop_request_from_rti = false, .last_sent_LTC = (tag_t) {.time = NEVER, .microstep = 0u}, .last_sent_NET = (tag_t) {.time = NEVER, .microstep = 0u}, @@ -1464,6 +1465,26 @@ static void handle_stop_request_message() { tag_to_stop.time, tag_to_stop.microstep); } +/** + * Handle a downstream next event tag (DNET) message from the RTI. + * FIXME: Use this tag to eliminate unncessary LTC or NET messages. +*/ +static void handle_downstream_next_event_tag() { + size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); + unsigned char buffer[bytes_to_read]; + read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL, + "Failed to read downstream next event tag from RTI."); + tag_t DNET = extract_tag(buffer); + + // Trace the event when tracing is enabled + tracepoint_federate_from_rti(_fed.trace, receive_DNET, _lf_my_fed_id, &DNET); + + LF_PRINT_LOG("Received Downstream Next Event Tag (DNET): " PRINTF_TAG ".", + _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); + + _fed.last_DNET = DNET; +} + /** * Send a resign signal to the RTI. */ @@ -1583,6 +1604,9 @@ static void* listen_to_rti_TCP(void* args) { lf_print_error_and_exit("Failed to complete the reading of an absent message from the RTI."); } break; + case MSG_TYPE_DOWNSTREAM_NEXT_EVENT_TAG: + handle_downstream_next_event_tag(); + break; case MSG_TYPE_FAILED: handle_rti_failed_message(); break; diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h index e035d94c0..adc3f50ae 100644 --- a/include/core/federated/federate.h +++ b/include/core/federated/federate.h @@ -142,6 +142,11 @@ typedef struct federate_instance_t { */ bool has_downstream; + /** + * + */ + tag_t last_DNET; + /** * Used to prevent the federate from sending a REQUEST_STOP * message if it has already received a stop request from the RTI. From 42fe08f4310a01667b653318d4842e1e20b95fbc Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Thu, 8 Feb 2024 13:43:06 -0700 Subject: [PATCH 07/43] Remove unnecessary LTC and NET messages by DNET --- core/federated/RTI/rti_common.c | 8 ++++++++ core/federated/RTI/rti_remote.c | 5 ++++- core/federated/federate.c | 19 +++++++++++++------ core/tag.c | 1 + 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 8a3cc660e..0716e7be5 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -444,6 +444,9 @@ void update_all_downstreams(scheduling_node_t* node) { } void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new_NET) { + if (is_in_zero_delay_cycle(node)) { + return; + } tag_t DNET = FOREVER_TAG; if (lf_tag_compare(node->last_DNET, new_NET) >= 0) { DNET = new_NET; @@ -451,8 +454,13 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new for (int i = 0; i < node->num_all_downstreams; i++) { uint16_t target_downstream_id = node->all_downstreams[i]; scheduling_node_t* target_dowstream = rti_common->scheduling_nodes[target_downstream_id]; + update_min_delays_upstream(target_dowstream); int index = node->id * rti_common->number_of_scheduling_nodes + target_downstream_id; tag_t DNET_candidate = lf_tag_subtract(target_dowstream->next_event, rti_common->min_delays[index]); + + if (DNET_candidate.time < start_time) { + DNET_candidate = NEVER_TAG; + } if (lf_tag_compare(DNET, DNET_candidate) > 0) { DNET = DNET_candidate; } diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index e76ce86ba..a73391535 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -307,8 +307,11 @@ void send_downstream_next_event_tag(scheduling_node_t *e, tag_t tag) { unsigned char buffer[message_length]; buffer[0] = MSG_TYPE_DOWNSTREAM_NEXT_EVENT_TAG; encode_int64(tag.time, &(buffer[1])); - encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); + encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); + if (rti_remote->base.tracing_enabled) { + tracepoint_rti_to_federate(rti_remote->base.trace, send_DNET, e->id, &tag); + } if (write_to_socket(((federate_info_t *)e)->socket, message_length, buffer)) { lf_print_error("RTI failed to send downstream next event tag to federate %d.", e->id); e->state = NOT_CONNECTED; diff --git a/core/federated/federate.c b/core/federated/federate.c index 2e8abf5a2..1d20bc04c 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2302,8 +2302,9 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { } void lf_latest_tag_complete(tag_t tag_to_send) { - int compare_with_last_tag = lf_tag_compare(_fed.last_sent_LTC, tag_to_send); - if (compare_with_last_tag >= 0) { + int compare_with_last_LTC = lf_tag_compare(_fed.last_sent_LTC, tag_to_send); + int compare_with_last_DNET = lf_tag_compare(_fed.last_DNET, tag_to_send); + if (compare_with_last_LTC >= 0 || compare_with_last_DNET > 0) { return; } LF_PRINT_LOG("Sending Latest Tag Complete (LTC) " PRINTF_TAG " to the RTI.", @@ -2463,10 +2464,12 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) // This if statement does not fall through but rather returns. // NET is not bounded by physical time or has no downstream federates. // Normal case. - send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag); - _fed.last_sent_NET = tag; - LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.", - tag.time - start_time, tag.microstep); + if (lf_tag_compare(_fed.last_DNET, tag) <= 0) { + send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag); + _fed.last_sent_NET = tag; + LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.", + tag.time - start_time, tag.microstep); + } if (!wait_for_reply) { LF_PRINT_LOG("Not waiting for reply to NET."); @@ -2728,6 +2731,10 @@ int lf_send_tagged_message(environment_t* env, tracepoint_federate_to_rti(_fed.trace, send_TAGGED_MSG, _lf_my_fed_id, ¤t_message_intended_tag); } + if (lf_tag_compare(_fed.last_DNET, current_message_intended_tag) > 0) { + _fed.last_DNET = current_message_intended_tag; + } + int result = write_to_socket_close_on_error(socket, header_length, header_buffer); if (result == 0) { // Header sent successfully. Send the body. diff --git a/core/tag.c b/core/tag.c index bf0d4dcca..b8c61310a 100644 --- a/core/tag.c +++ b/core/tag.c @@ -155,6 +155,7 @@ tag_t lf_tag_subtract(tag_t a, tag_t b) { } result.microstep = 0; } + return result; } int lf_tag_compare(tag_t tag1, tag_t tag2) { From d3e462ca48f0e09b41a2def48489a2d5f3151c49 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Fri, 9 Feb 2024 20:51:24 -0700 Subject: [PATCH 08/43] Sends last skipped DNETs when sending T_MSGs --- core/federated/RTI/rti_common.c | 4 +++- core/federated/federate.c | 14 +++++++++++++- core/tag.c | 10 ++-------- include/core/federated/federate.h | 5 +++++ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 0716e7be5..fa45b0300 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -466,7 +466,9 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new } } } - if (lf_tag_compare(node->last_DNET, DNET) != 0) { + if (lf_tag_compare(node->last_DNET, DNET) != 0 + && lf_tag_compare(node->completed, DNET) < 0 + && lf_tag_compare(node->next_event, DNET) <= 0) { send_downstream_next_event_tag(node, DNET); } } diff --git a/core/federated/federate.c b/core/federated/federate.c index 1d20bc04c..8cd845894 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -88,6 +88,7 @@ federate_instance_t _fed = { .is_last_TAG_provisional = false, .has_upstream = false, .has_downstream = false, + .last_skipped_LTC = (tag_t) {.time = NEVER, .microstep = 0u}, .last_DNET = (tag_t) {.time = NEVER, .microstep = 0u}, .received_stop_request_from_rti = false, .last_sent_LTC = (tag_t) {.time = NEVER, .microstep = 0u}, @@ -2304,14 +2305,21 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { void lf_latest_tag_complete(tag_t tag_to_send) { int compare_with_last_LTC = lf_tag_compare(_fed.last_sent_LTC, tag_to_send); int compare_with_last_DNET = lf_tag_compare(_fed.last_DNET, tag_to_send); - if (compare_with_last_LTC >= 0 || compare_with_last_DNET > 0) { + if (compare_with_last_LTC >= 0) { return; } + if (compare_with_last_DNET > 0) { + LF_PRINT_LOG("Skipping Latest Tag Complete (LTC) " PRINTF_TAG " .", + tag_to_send.time - start_time, + tag_to_send.microstep); + _fed.last_skipped_LTC = tag_to_send; + } LF_PRINT_LOG("Sending Latest Tag Complete (LTC) " PRINTF_TAG " to the RTI.", tag_to_send.time - start_time, tag_to_send.microstep); send_tag(MSG_TYPE_LATEST_TAG_COMPLETE, tag_to_send); _fed.last_sent_LTC = tag_to_send; + _fed.last_skipped_LTC = NEVER_TAG; } parse_rti_code_t lf_parse_rti_addr(const char* rti_addr) { @@ -2733,6 +2741,10 @@ int lf_send_tagged_message(environment_t* env, if (lf_tag_compare(_fed.last_DNET, current_message_intended_tag) > 0) { _fed.last_DNET = current_message_intended_tag; + if (lf_tag_compare(_fed.last_skipped_LTC, NEVER_TAG) != 0) { + send_tag(MSG_TYPE_LATEST_TAG_COMPLETE, _fed.last_skipped_LTC); + _fed.last_skipped_LTC = NEVER_TAG; + } } int result = write_to_socket_close_on_error(socket, header_length, header_buffer); diff --git a/core/tag.c b/core/tag.c index b8c61310a..a16124356 100644 --- a/core/tag.c +++ b/core/tag.c @@ -137,12 +137,9 @@ tag_t lf_tag_subtract(tag_t a, tag_t b) { // b.t can be NEVER, FOREVER, normal and the result will be FOREVER, 0, FOREVER. // 3) a.t is not NEVER neither FOREVER // a) a.t < b.t - // the result will be NEVER + // The result will be NEVER // b) a.t >= b.t - // i) a.m <= b.m - // the result should be (a.t - b.t, 0) - // ii) a.m > b.m - // if b is ZERO, the result is a itself else, the result is (a.t - b.t + 1, 0) + // If b is ZERO, the result is a itself. Otherwise, the result is (a.t - b.t, 0) if (a.time == NEVER && b.time != NEVER) return NEVER_TAG; if (a.time == FOREVER && b.time != FOREVER) return FOREVER_TAG; if (b.time == 0 && b.microstep == 0) return a; @@ -150,9 +147,6 @@ tag_t lf_tag_subtract(tag_t a, tag_t b) { if (result.time < 0) { return NEVER_TAG; } else { - if (result.microstep > 0) { - result.time++; - } result.microstep = 0; } return result; diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h index adc3f50ae..f9eec6112 100644 --- a/include/core/federated/federate.h +++ b/include/core/federated/federate.h @@ -142,6 +142,11 @@ typedef struct federate_instance_t { */ bool has_downstream; + /** + * + */ + tag_t last_skipped_LTC; + /** * */ From 9967d9a1e84857b40dbdadfcd569f2371cb5b4f1 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 13 Feb 2024 14:12:36 -0700 Subject: [PATCH 09/43] Do not send DNET to upstreams of ZDC --- core/federated/RTI/rti_common.c | 14 ++++++++++---- core/federated/federate.c | 13 ++++++++----- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index f51bf33d0..81bf90a2e 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -460,17 +460,23 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new for (int i = 0; i < node->num_all_downstreams; i++) { uint16_t target_downstream_id = node->all_downstreams[i]; scheduling_node_t* target_dowstream = rti_common->scheduling_nodes[target_downstream_id]; - update_min_delays_upstream(target_dowstream); + + if (is_in_zero_delay_cycle(target_dowstream)) { + // This node is an upstream of ZDC. Do not send DNET to this node. + return; + } + int index = node->id * rti_common->number_of_scheduling_nodes + target_downstream_id; tag_t DNET_candidate = lf_tag_subtract(target_dowstream->next_event, rti_common->min_delays[index]); - if (DNET_candidate.time < start_time) { - DNET_candidate = NEVER_TAG; - } if (lf_tag_compare(DNET, DNET_candidate) > 0) { DNET = DNET_candidate; } } + if (DNET.time < start_time) { + // DNET is NEVER and no need to be sent. + return; + } } if (lf_tag_compare(node->last_DNET, DNET) != 0 && lf_tag_compare(node->completed, DNET) < 0 diff --git a/core/federated/federate.c b/core/federated/federate.c index 1d9b79ba7..e066e8f34 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -1480,9 +1480,15 @@ static void handle_downstream_next_event_tag() { tracepoint_federate_from_rti(_fed.trace, receive_DNET, _lf_my_fed_id, &DNET); LF_PRINT_LOG("Received Downstream Next Event Tag (DNET): " PRINTF_TAG ".", - _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); + DNET.time - start_time, DNET.microstep); _fed.last_DNET = DNET; + + if (lf_tag_compare(_fed.last_skipped_LTC, NEVER_TAG) != 0 + && lf_tag_compare(_fed.last_skipped_LTC, _fed.last_DNET) >= 0) { + send_tag(MSG_TYPE_LATEST_TAG_COMPLETE, _fed.last_skipped_LTC); + _fed.last_skipped_LTC = NEVER_TAG; + } } /** @@ -2312,6 +2318,7 @@ void lf_latest_tag_complete(tag_t tag_to_send) { tag_to_send.time - start_time, tag_to_send.microstep); _fed.last_skipped_LTC = tag_to_send; + return; } LF_PRINT_LOG("Sending Latest Tag Complete (LTC) " PRINTF_TAG " to the RTI.", tag_to_send.time - start_time, @@ -2734,10 +2741,6 @@ int lf_send_tagged_message(environment_t* env, if (lf_tag_compare(_fed.last_DNET, current_message_intended_tag) > 0) { _fed.last_DNET = current_message_intended_tag; - if (lf_tag_compare(_fed.last_skipped_LTC, NEVER_TAG) != 0) { - send_tag(MSG_TYPE_LATEST_TAG_COMPLETE, _fed.last_skipped_LTC); - _fed.last_skipped_LTC = NEVER_TAG; - } } int result = write_to_socket_close_on_error(socket, header_length, header_buffer); From 400d138b0ad06e96ce95aa09548618913b4a1808 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 13 Feb 2024 14:40:45 -0700 Subject: [PATCH 10/43] Disable the RTI unit tests until it reflects the changes of the RTI --- test/Tests.cmake | 91 ++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/test/Tests.cmake b/test/Tests.cmake index fef6dc0aa..3c9144895 100644 --- a/test/Tests.cmake +++ b/test/Tests.cmake @@ -38,53 +38,54 @@ foreach(FILE ${TEST_FILES}) endforeach(FILE ${TEST_FILES}) # Add the test for the RTI. -if (NOT DEFINED LF_SINGLE_THREADED) - # Check which system we are running on to select the correct platform support - # file and assign the file's path to LF_PLATFORM_FILE - if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") - set(LF_PLATFORM_FILE ${CoreLibPath}/platform/lf_linux_support.c) - elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") - set(LF_PLATFORM_FILE ${CoreLibPath}/platform/lf_macos_support.c) - else() - message(FATAL_ERROR "Your platform is not supported! RTI supports Linux and MacOS.") - endif() +# FIXME: Comment out the tests for the RTI until the tests are aligned to the new RTI. +# if (NOT DEFINED LF_SINGLE_THREADED) +# # Check which system we are running on to select the correct platform support +# # file and assign the file's path to LF_PLATFORM_FILE +# if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") +# set(LF_PLATFORM_FILE ${CoreLibPath}/platform/lf_linux_support.c) +# elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") +# set(LF_PLATFORM_FILE ${CoreLibPath}/platform/lf_macos_support.c) +# else() +# message(FATAL_ERROR "Your platform is not supported! RTI supports Linux and MacOS.") +# endif() - set(IncludeDir include/core) +# set(IncludeDir include/core) - set(RTI_DIR ${CoreLibPath}/federated/RTI) - add_executable( - rti_common_test - ${TEST_DIR}/RTI/rti_common_test.c - ${RTI_DIR}/rti_common.c - ${RTI_DIR}/rti_remote.c - ${CoreLibPath}/trace.c - ${LF_PLATFORM_FILE} - ${CoreLibPath}/platform/lf_atomic_gcc_clang.c - ${CoreLibPath}/platform/lf_unix_clock_support.c - ${CoreLibPath}/utils/util.c - ${CoreLibPath}/tag.c - ${CoreLibPath}/clock.c - ${CoreLibPath}/federated/network/net_util.c - ${CoreLibPath}/utils/pqueue_base.c - ${CoreLibPath}/utils/pqueue_tag.c - ${CoreLibPath}/utils/pqueue.c - ) - add_test(NAME rti_common_test COMMAND rti_common_test) - target_include_directories(rti_common_test PUBLIC ${RTI_DIR}) - target_include_directories(rti_common_test PUBLIC ${IncludeDir}) - target_include_directories(rti_common_test PUBLIC ${IncludeDir}/federated) - target_include_directories(rti_common_test PUBLIC ${IncludeDir}/modal_models) - target_include_directories(rti_common_test PUBLIC ${IncludeDir}/platform) - target_include_directories(rti_common_test PUBLIC ${IncludeDir}/utils) - # Set the STANDALONE_RTI flag to include the rti_remote and rti_common. - target_compile_definitions(rti_common_test PUBLIC STANDALONE_RTI=1) +# set(RTI_DIR ${CoreLibPath}/federated/RTI) +# add_executable( +# rti_common_test +# ${TEST_DIR}/RTI/rti_common_test.c +# ${RTI_DIR}/rti_common.c +# ${RTI_DIR}/rti_remote.c +# ${CoreLibPath}/trace.c +# ${LF_PLATFORM_FILE} +# ${CoreLibPath}/platform/lf_atomic_gcc_clang.c +# ${CoreLibPath}/platform/lf_unix_clock_support.c +# ${CoreLibPath}/utils/util.c +# ${CoreLibPath}/tag.c +# ${CoreLibPath}/clock.c +# ${CoreLibPath}/federated/network/net_util.c +# ${CoreLibPath}/utils/pqueue_base.c +# ${CoreLibPath}/utils/pqueue_tag.c +# ${CoreLibPath}/utils/pqueue.c +# ) +# add_test(NAME rti_common_test COMMAND rti_common_test) +# target_include_directories(rti_common_test PUBLIC ${RTI_DIR}) +# target_include_directories(rti_common_test PUBLIC ${IncludeDir}) +# target_include_directories(rti_common_test PUBLIC ${IncludeDir}/federated) +# target_include_directories(rti_common_test PUBLIC ${IncludeDir}/modal_models) +# target_include_directories(rti_common_test PUBLIC ${IncludeDir}/platform) +# target_include_directories(rti_common_test PUBLIC ${IncludeDir}/utils) +# # Set the STANDALONE_RTI flag to include the rti_remote and rti_common. +# target_compile_definitions(rti_common_test PUBLIC STANDALONE_RTI=1) - # Set FEDERATED to get federated compilation support - target_compile_definitions(rti_common_test PUBLIC FEDERATED=1) +# # Set FEDERATED to get federated compilation support +# target_compile_definitions(rti_common_test PUBLIC FEDERATED=1) - target_compile_definitions(rti_common_test PUBLIC PLATFORM_${CMAKE_SYSTEM_NAME}) +# target_compile_definitions(rti_common_test PUBLIC PLATFORM_${CMAKE_SYSTEM_NAME}) - # Find threads and link to it - find_package(Threads REQUIRED) - target_link_libraries(rti_common_test Threads::Threads) -endif() +# # Find threads and link to it +# find_package(Threads REQUIRED) +# target_link_libraries(rti_common_test Threads::Threads) +# endif() From 1719951042a12598f677c0b7ced8d7a70f0aee7c Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 13 Feb 2024 16:14:25 -0700 Subject: [PATCH 11/43] Send NET regardless of DNET if it needs TAG from the RTI --- core/federated/RTI/rti_common.c | 8 ++++++-- core/federated/federate.c | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 81bf90a2e..dda4ce15a 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -296,6 +296,10 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne // Send DNET to the node's upstreams if needed for (int i = 0; i < e->num_all_upstreams; i++) { int target_upstream_id = e->all_upstreams[i]; + if (target_upstream_id == e->id) { + // FIXME: This shouldn't be entered, but currently, it's entered. + continue; + } send_downstream_next_event_tag_if_needed(rti_common->scheduling_nodes[target_upstream_id], next_event_tag); } } @@ -474,8 +478,8 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new } } if (DNET.time < start_time) { - // DNET is NEVER and no need to be sent. - return; + // DNET is NEVER. + DNET = NEVER_TAG; } } if (lf_tag_compare(node->last_DNET, DNET) != 0 diff --git a/core/federated/federate.c b/core/federated/federate.c index e066e8f34..8ac180f21 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2478,11 +2478,15 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) // This if statement does not fall through but rather returns. // NET is not bounded by physical time or has no downstream federates. // Normal case. - if (lf_tag_compare(_fed.last_DNET, tag) <= 0) { + if (lf_tag_compare(_fed.last_DNET, tag) <= 0 + || lf_tag_compare(_fed.last_TAG, tag) < 0) { send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag); _fed.last_sent_NET = tag; LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.", tag.time - start_time, tag.microstep); + } else { + LF_PRINT_LOG("Skip next event tag (NET) " PRINTF_TAG " to RTI.", + tag.time - start_time, tag.microstep); } if (!wait_for_reply) { From a6e2b5d7816721fcb70c722ab38c7b6ae94b1747 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 14 Feb 2024 15:03:39 -0700 Subject: [PATCH 12/43] Using ID of the federate which sends NET to calculate DNET --- core/federated/RTI/rti_common.c | 22 ++++++++++++++++------ core/federated/RTI/rti_common.h | 2 +- include/core/tag.h | 1 + 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index dda4ce15a..86e69c940 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -300,7 +300,7 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne // FIXME: This shouldn't be entered, but currently, it's entered. continue; } - send_downstream_next_event_tag_if_needed(rti_common->scheduling_nodes[target_upstream_id], next_event_tag); + send_downstream_next_event_tag_if_needed(rti_common->scheduling_nodes[target_upstream_id], e->id); } } @@ -453,13 +453,23 @@ void update_all_downstreams(scheduling_node_t* node) { } } -void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new_NET) { +// It should be static because it's used only in this file. Remove from the header file. +void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id) { if (is_in_zero_delay_cycle(node)) { return; } + tag_t DNET = FOREVER_TAG; - if (lf_tag_compare(node->last_DNET, new_NET) >= 0) { - DNET = new_NET; + scheduling_node_t* new_NET_source_federate = rti_common->scheduling_nodes[new_NET_source_federate_id]; + if (is_in_zero_delay_cycle(new_NET_source_federate)) { + return; + } + + int index = node->id * rti_common->number_of_scheduling_nodes + new_NET_source_federate_id; + tag_t DNET_candidate = lf_tag_subtract(new_NET_source_federate->next_event, rti_common->min_delays[index]); + + if (lf_tag_compare(node->last_DNET, DNET_candidate) >= 0) { + DNET = DNET_candidate; } else { for (int i = 0; i < node->num_all_downstreams; i++) { uint16_t target_downstream_id = node->all_downstreams[i]; @@ -470,8 +480,8 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new return; } - int index = node->id * rti_common->number_of_scheduling_nodes + target_downstream_id; - tag_t DNET_candidate = lf_tag_subtract(target_dowstream->next_event, rti_common->min_delays[index]); + index = node->id * rti_common->number_of_scheduling_nodes + target_downstream_id; + DNET_candidate = lf_tag_subtract(target_dowstream->next_event, rti_common->min_delays[index]); if (lf_tag_compare(DNET, DNET_candidate) > 0) { DNET = DNET_candidate; diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 18e6653c3..423f152ad 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -298,7 +298,7 @@ void send_downstream_next_event_tag(scheduling_node_t *e, tag_t tag); * @param node * @param new_NET */ -void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, tag_t new_NET); +void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id); /** * For the given scheduling node (enclave or federate), invalidate the `min_delays`, diff --git a/include/core/tag.h b/include/core/tag.h index ae95a0a09..65920361b 100644 --- a/include/core/tag.h +++ b/include/core/tag.h @@ -98,6 +98,7 @@ tag_t lf_tag_add(tag_t a, tag_t b); /** * Subtract the tag a from the tag b. + * FIXME: This function if for DNET, move to rti_common.c. */ tag_t lf_tag_subtract(tag_t a, tag_t b); From 86649093ef9b128202143bcb7d57dc78ac2f29e0 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Sun, 18 Feb 2024 19:58:03 -0700 Subject: [PATCH 13/43] Move DNET calculation function from tag.h to rti_common.h --- core/federated/RTI/rti_common.c | 36 +++++++++++++++++++++++++++++++-- core/federated/RTI/rti_common.h | 8 ++++++++ core/tag.c | 24 ---------------------- include/core/tag.h | 6 ------ 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 86e69c940..48a91605c 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -453,6 +453,38 @@ void update_all_downstreams(scheduling_node_t* node) { } } +tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay) { + // FIXME: Overflow is not handled. + // (a.t, a.m) - (b.t - b.m) + // The least b is (0, 0) which indicates NEVER delay. + // 1) If a.t = NEVER, return NEVER. + // 2) IF a.t = FOREVER, return FOREVER. + // 3) a.t is not NEVER neither FOREVER + // a) b.t = 0 + // i) b. + // b) a.t >= b.t + // If b is ZERO, the result is a itself. Otherwise, the result is (a.t - b.t, 0) + if (received_tag.time == NEVER || received_tag.time > minimum_delay.time) return NEVER_TAG; + if (received_tag.time == FOREVER) return FOREVER_TAG; + tag_t result = {.time = received_tag.time - minimum_delay.time, .microstep = received_tag.microstep - minimum_delay.microstep}; + if (minimum_delay.time == 0) { + if (received_tag.microstep >= minimum_delay.microstep) { + // + } else { + result.time = received_tag.time -1; + result.microstep = UINT_MAX; + } + } else { + if (received_tag.microstep >= minimum_delay.microstep) { + result.microstep = UINT_MAX; + } else { + result.time = result.time -1; + result.microstep = UINT_MAX; + } + } + return result; +} + // It should be static because it's used only in this file. Remove from the header file. void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id) { if (is_in_zero_delay_cycle(node)) { @@ -466,7 +498,7 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t } int index = node->id * rti_common->number_of_scheduling_nodes + new_NET_source_federate_id; - tag_t DNET_candidate = lf_tag_subtract(new_NET_source_federate->next_event, rti_common->min_delays[index]); + tag_t DNET_candidate = get_DNET_candidate(new_NET_source_federate->next_event, rti_common->min_delays[index]); if (lf_tag_compare(node->last_DNET, DNET_candidate) >= 0) { DNET = DNET_candidate; @@ -481,7 +513,7 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t } index = node->id * rti_common->number_of_scheduling_nodes + target_downstream_id; - DNET_candidate = lf_tag_subtract(target_dowstream->next_event, rti_common->min_delays[index]); + DNET_candidate = get_DNET_candidate(target_dowstream->next_event, rti_common->min_delays[index]); if (lf_tag_compare(DNET, DNET_candidate) > 0) { DNET = DNET_candidate; diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 423f152ad..810b3cfd8 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -287,6 +287,14 @@ void update_min_delays_upstream(scheduling_node_t* node); */ void update_all_downstreams(scheduling_node_t* node); +/** + * Subtract the tag a from the tag b. + * minimum_delay cannot be NEVER. + * @param received_tag + * @param minimum_delay +*/ +tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay); + /** * FIXME: Add this function to rti_local either. * @param e The target node. diff --git a/core/tag.c b/core/tag.c index 537a711ed..1b7b9feda 100644 --- a/core/tag.c +++ b/core/tag.c @@ -62,30 +62,6 @@ tag_t lf_tag_add(tag_t a, tag_t b) { return result; } -tag_t lf_tag_subtract(tag_t a, tag_t b) { - // FIXME: Overflow is not handled. - // (a.t, a.m) - (b.t - b.m) - // 1) a.t = NEVER - // b.t can be NEVER, FOREVER, normal and the result will be 0, NEVER, NEVER, respectively. - // 2) a.t = FOREVER - // b.t can be NEVER, FOREVER, normal and the result will be FOREVER, 0, FOREVER. - // 3) a.t is not NEVER neither FOREVER - // a) a.t < b.t - // The result will be NEVER - // b) a.t >= b.t - // If b is ZERO, the result is a itself. Otherwise, the result is (a.t - b.t, 0) - if (a.time == NEVER && b.time != NEVER) return NEVER_TAG; - if (a.time == FOREVER && b.time != FOREVER) return FOREVER_TAG; - if (b.time == 0 && b.microstep == 0) return a; - tag_t result = {.time = a.time - b.time, .microstep = a.microstep - b.microstep}; - if (result.time < 0) { - return NEVER_TAG; - } else { - result.microstep = 0; - } - return result; -} - int lf_tag_compare(tag_t tag1, tag_t tag2) { if (tag1.time < tag2.time) { return -1; diff --git a/include/core/tag.h b/include/core/tag.h index 65920361b..55d83c35d 100644 --- a/include/core/tag.h +++ b/include/core/tag.h @@ -96,12 +96,6 @@ tag_t lf_tag(void* env); */ tag_t lf_tag_add(tag_t a, tag_t b); -/** - * Subtract the tag a from the tag b. - * FIXME: This function if for DNET, move to rti_common.c. -*/ -tag_t lf_tag_subtract(tag_t a, tag_t b); - /** * Compare two tags. Return -1 if the first is less than * the second, 0 if they are equal, and +1 if the first is From ccc9b8bd4f69cdfbbd64ca31afe980d0e4bef0dd Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Mon, 19 Feb 2024 16:09:39 -0700 Subject: [PATCH 14/43] Fix the calculation of DNET candidates --- core/federated/RTI/rti_common.c | 48 ++++++++++++++++----------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 48a91605c..519719645 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -454,31 +454,28 @@ void update_all_downstreams(scheduling_node_t* node) { } tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay) { - // FIXME: Overflow is not handled. - // (a.t, a.m) - (b.t - b.m) - // The least b is (0, 0) which indicates NEVER delay. - // 1) If a.t = NEVER, return NEVER. - // 2) IF a.t = FOREVER, return FOREVER. - // 3) a.t is not NEVER neither FOREVER - // a) b.t = 0 - // i) b. - // b) a.t >= b.t - // If b is ZERO, the result is a itself. Otherwise, the result is (a.t - b.t, 0) - if (received_tag.time == NEVER || received_tag.time > minimum_delay.time) return NEVER_TAG; + // FIXME: FOREVER - FOREVER is not handled. + // (A.t, A.m) - (B.t - B.m) + // The least B is (0, 0) which indicates NEVER delay. + // 1) If A.t = NEVER, return NEVER. + // 2) If A.t = FOREVER, return FOREVER. + // 3) A.t is not NEVER neither FOREVER + // a) If A < B (A.t < B.t or A.t == B.t and A.m < B.m) return NEVER + // b) A >= B + // i) If A.m < B.m return (A.t - B.t - 1, UINT_MAX) + // ii) If A.m >= B.m + // If B.t is 0 return (A.t, A.m - B.m) + // Else, return (A.t - B.t, UINT_MAX) + if (received_tag.time == NEVER || lf_tag_compare(received_tag, minimum_delay) < 0) return NEVER_TAG; if (received_tag.time == FOREVER) return FOREVER_TAG; tag_t result = {.time = received_tag.time - minimum_delay.time, .microstep = received_tag.microstep - minimum_delay.microstep}; - if (minimum_delay.time == 0) { - if (received_tag.microstep >= minimum_delay.microstep) { - // - } else { - result.time = received_tag.time -1; - result.microstep = UINT_MAX; - } + if (received_tag.microstep < minimum_delay.microstep) { + result.time -= 1; + result.microstep = UINT_MAX; } else { - if (received_tag.microstep >= minimum_delay.microstep) { - result.microstep = UINT_MAX; + if (minimum_delay.time == 0) { + // } else { - result.time = result.time -1; result.microstep = UINT_MAX; } } @@ -519,11 +516,12 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t DNET = DNET_candidate; } } - if (DNET.time < start_time) { - // DNET is NEVER. - DNET = NEVER_TAG; - } } + if (DNET.time < start_time) { + // DNET is NEVER. + DNET = NEVER_TAG; + } + if (lf_tag_compare(node->last_DNET, DNET) != 0 && lf_tag_compare(node->completed, DNET) < 0 && lf_tag_compare(node->next_event, DNET) <= 0) { From e9ae4645be6335452d712c91c429b88ddc691b89 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 21 Feb 2024 14:02:33 -0700 Subject: [PATCH 15/43] Re-enable the RTI unit tests --- test/RTI/rti_common_test.c | 158 ++++++++++++++++++++++--------------- test/Tests.cmake | 91 +++++++++++---------- 2 files changed, 138 insertions(+), 111 deletions(-) diff --git a/test/RTI/rti_common_test.c b/test/RTI/rti_common_test.c index 8d0eb9371..c34088cf2 100644 --- a/test/RTI/rti_common_test.c +++ b/test/RTI/rti_common_test.c @@ -17,14 +17,14 @@ static rti_common_t test_rti; * @param node The node to be freed */ void delete_scheduling_node(scheduling_node_t* node) { - if (node->upstream != NULL) { - free(node->upstream); + if (node->immediate_upstreams != NULL) { + free(node->immediate_upstreams); } - if (node->upstream_delay != NULL) { - free(node->upstream_delay); + if (node->immediate_upstream_delays != NULL) { + free(node->immediate_upstream_delays); } - if (node->downstream != NULL) { - free(node->downstream); + if (node->immediate_downstreams != NULL) { + free(node->immediate_downstreams); } invalidate_min_delays_upstream(node); } @@ -34,37 +34,37 @@ void delete_scheduling_node(scheduling_node_t* node) { * Before calling this function, reset_common_RTI should be called to * reset every scheduling nodes. * @param id The ID of the scheduling node. - * @param num_upstream The number of upstreams of the scheduling node. - * @param num_downstream The number of downstreams of the scheduling node. - * @param upstream The array of IDs from upstream nodes. - * @param upstream_delay The array of delays from upstream nodes. - * @param downstream The array of IDs from downstream nodes. + * @param num_immediate_upstreams The number of upstreams of the scheduling node. + * @param num_immediate_downstreams The number of downstreams of the scheduling node. + * @param immediate_upstreams The array of IDs from immediate upstream nodes. + * @param immediate_upstream_delays The array of delays from immediate upstream nodes. + * @param immediate_downstreams The array of IDs from immediate downstream nodes. */ void set_scheduling_node( int id, - int num_upstream, - int num_downstream, - int* upstream, - interval_t* upstream_delay, - int* downstream) { - // Save the number of upstream and downstream nodes. - test_rti.scheduling_nodes[id]->num_upstream = num_upstream; - test_rti.scheduling_nodes[id]->num_downstream = num_downstream; - - // If there is any upstream nodes, store IDs and delays from the upstream nodes into the structure. - if (test_rti.scheduling_nodes[id]->num_upstream > 0) { - test_rti.scheduling_nodes[id]->upstream = (int*) calloc(test_rti.scheduling_nodes[id]->num_upstream, sizeof(int)); - test_rti.scheduling_nodes[id]->upstream_delay = (interval_t*) calloc(test_rti.scheduling_nodes[id]->num_upstream, sizeof(interval_t)); - for (int i = 0; i < test_rti.scheduling_nodes[id]->num_upstream; i++) { - test_rti.scheduling_nodes[id]->upstream[i] = upstream[i]; - test_rti.scheduling_nodes[id]->upstream_delay[i] = upstream_delay[i]; + int num_immediate_upstreams, + int num_immediate_downstreams, + int* immediate_upstreams, + interval_t* immediate_upstream_delays, + int* immediate_downstreams) { + // Save the number of immediate upstream and immediate downstream nodes. + test_rti.scheduling_nodes[id]->num_immediate_upstreams = num_immediate_upstreams; + test_rti.scheduling_nodes[id]->num_immediate_downstreams = num_immediate_downstreams; + + // If there is any immediate upstream nodes, store IDs and delays from the upstream nodes into the structure. + if (test_rti.scheduling_nodes[id]->num_immediate_upstreams > 0) { + test_rti.scheduling_nodes[id]->immediate_upstreams = (uint16_t*) calloc(test_rti.scheduling_nodes[id]->num_immediate_upstreams, sizeof(uint16_t)); + test_rti.scheduling_nodes[id]->immediate_upstream_delays = (interval_t*) calloc(test_rti.scheduling_nodes[id]->num_immediate_upstreams, sizeof(interval_t)); + for (int i = 0; i < test_rti.scheduling_nodes[id]->num_immediate_upstreams; i++) { + test_rti.scheduling_nodes[id]->immediate_upstreams[i] = immediate_upstreams[i]; + test_rti.scheduling_nodes[id]->immediate_upstream_delays[i] = immediate_upstream_delays[i]; } } - // If there is any downstream nodes, store IDs of the downstream nodes into the structure. - if (test_rti.scheduling_nodes[id]->num_downstream > 0) { - test_rti.scheduling_nodes[id]->downstream = (int*) calloc(test_rti.scheduling_nodes[id]->num_downstream, sizeof(int)); - for (int i = 0; i < test_rti.scheduling_nodes[id]->num_downstream; i++) { - test_rti.scheduling_nodes[id]->downstream[i] = downstream[i]; + // If there is any immediate downstream nodes, store IDs of the downstream nodes into the structure. + if (test_rti.scheduling_nodes[id]->num_immediate_downstreams > 0) { + test_rti.scheduling_nodes[id]->immediate_downstreams = (uint16_t*) calloc(test_rti.scheduling_nodes[id]->num_immediate_downstreams, sizeof(uint16_t)); + for (int i = 0; i < test_rti.scheduling_nodes[id]->num_immediate_downstreams; i++) { + test_rti.scheduling_nodes[id]->immediate_downstreams[i] = immediate_downstreams[i]; } } } @@ -98,6 +98,7 @@ void set_common_RTI(uint16_t num_nodes) { // Allocate memory for the scheduling nodes test_rti.scheduling_nodes = (scheduling_node_t**)calloc(test_rti.number_of_scheduling_nodes, sizeof(scheduling_node_t*)); + test_rti.min_delays = (tag_t*)calloc((num_nodes*num_nodes), sizeof(tag_t)); for (uint16_t i = 0; i < test_rti.number_of_scheduling_nodes; i++) { scheduling_node_t *scheduling_node = (scheduling_node_t *) malloc(sizeof(scheduling_node_t)); initialize_scheduling_node(scheduling_node, i); @@ -128,10 +129,10 @@ void valid_cache() { set_state_of_nodes(GRANTED); // If min_delays is not null (the cached data is valid), nothing should be changed. - test_rti.scheduling_nodes[1]->num_min_delays = 1; - test_rti.scheduling_nodes[1]->min_delays = (minimum_delay_t*)calloc(1, sizeof(minimum_delay_t)); + test_rti.scheduling_nodes[1]->num_all_upstreams = 1; + test_rti.scheduling_nodes[1]->all_upstreams = (uint16_t*)calloc(1, sizeof(uint16_t)); update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_min_delays == 1); + assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 1); } void not_connected() { @@ -144,13 +145,14 @@ void not_connected() { set_state_of_nodes(NOT_CONNECTED); - // If the nodes are not connected, num_min_delays should not be changed. + // If the nodes are not connected, num_all_upstreams should not be changed. update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_min_delays == 0); + assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 0); } static void two_nodes_no_delay() { set_common_RTI(2); + uint16_t n = test_rti.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --> node[1] @@ -159,18 +161,21 @@ static void two_nodes_no_delay() { set_state_of_nodes(GRANTED); + // Test update_min_delays_upstream update_min_delays_upstream(test_rti.scheduling_nodes[0]); - assert(test_rti.scheduling_nodes[1]->num_min_delays == 0); // node[0] has no upstream nodes. + assert(test_rti.scheduling_nodes[0]->num_all_upstreams == 0); // node[0] has no upstream nodes. update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_min_delays == 1); // node[1] has one upstream nodes. - assert(test_rti.scheduling_nodes[1]->min_delays[0].id == 0); // node[1]'s upstream node is node[0]. + assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 1); // node[1] has one upstream nodes. + assert(test_rti.scheduling_nodes[1]->all_upstreams[0] == 0); // node[1]'s upstream node is node[0]. // The min_delay between them is node[0] and node[1] which means no delay. - assert(lf_tag_compare(test_rti.scheduling_nodes[1]->min_delays[0].min_delay, ZERO_TAG) == 0); + // assert(lf_tag_compare(test_rti.scheduling_nodes[1]->min_delays[0].min_delay, ZERO_TAG) == 0); + assert(lf_tag_compare(test_rti.min_delays[0*n + 1], ZERO_TAG) == 0); } static void two_nodes_zero_delay() { set_common_RTI(2); + uint16_t n = test_rti.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --/0/--> node[1] @@ -179,18 +184,27 @@ static void two_nodes_zero_delay() { set_state_of_nodes(GRANTED); + // Test update_min_delays_upstream update_min_delays_upstream(test_rti.scheduling_nodes[0]); - assert(test_rti.scheduling_nodes[1]->num_min_delays == 0); // node[0] has no upstream nodes. + assert(test_rti.scheduling_nodes[0]->num_all_upstreams == 0); // node[0] has no upstream nodes. update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_min_delays == 1); // node[1] has one upstream nodes. - assert(test_rti.scheduling_nodes[1]->min_delays[0].id == 0); // node[1]'s upstream node is node[0]. + assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 1); // node[1] has one upstream nodes. + assert(test_rti.scheduling_nodes[1]->all_upstreams[0] == 0); // node[1]'s upstream node is node[0]. // The min_delay between node[0] and node[1] is (0, 1) which means zero delay. - assert(lf_tag_compare(test_rti.scheduling_nodes[1]->min_delays[0].min_delay, (tag_t) {.time = 0, .microstep = 1}) == 0); + assert(lf_tag_compare(test_rti.min_delays[0*n + 1], (tag_t) {.time = 0, .microstep = 1}) == 0); + + // Test update_all_downstreams + update_all_downstreams(test_rti.scheduling_nodes[0]); + assert(test_rti.scheduling_nodes[0]->num_all_downstreams == 1); // node[0] has one downstream nodes. + + update_all_downstreams(test_rti.scheduling_nodes[1]); + assert(test_rti.scheduling_nodes[1]->num_all_downstreams == 0); // node[1] has no downstream nodes. } static void two_nodes_normal_delay() { set_common_RTI(2); + uint16_t n = test_rti.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --/1 nsec/--> node[1] @@ -200,17 +214,18 @@ static void two_nodes_normal_delay() { set_state_of_nodes(GRANTED); update_min_delays_upstream(test_rti.scheduling_nodes[0]); - assert(test_rti.scheduling_nodes[1]->num_min_delays == 0); // node[0] has no upstream nodes. + assert(test_rti.scheduling_nodes[0]->num_all_upstreams == 0); // node[0] has no upstream nodes. update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_min_delays == 1); // node[1] has one upstream nodes. - assert(test_rti.scheduling_nodes[1]->min_delays[0].id == 0); // node[1]'s upstream node is node[0]. + assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 1); // node[1] has one upstream nodes. + assert(test_rti.scheduling_nodes[1]->all_upstreams[0] == 0); // node[1]'s upstream node is node[0]. // The min_delay between node[0] and node[1] is (1 nsec, 0). - assert(lf_tag_compare(test_rti.scheduling_nodes[1]->min_delays[0].min_delay, (tag_t) {.time = NSEC(1), .microstep = 0}) == 0); + assert(lf_tag_compare(test_rti.min_delays[0*n + 1], (tag_t) {.time = NSEC(1), .microstep = 0}) == 0); } static void multiple_nodes() { set_common_RTI(4); + uint16_t n = test_rti.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --/1 nsec/--> node[1] --/0/--> node[2] --/2 nsec/--> node[3] @@ -221,32 +236,45 @@ static void multiple_nodes() { set_state_of_nodes(GRANTED); + // Test update_min_delays_upstream update_min_delays_upstream(test_rti.scheduling_nodes[2]); - assert(test_rti.scheduling_nodes[2]->num_min_delays == 2); // node[2] has two upstream nodes. - assert(test_rti.scheduling_nodes[2]->min_delays[1].id == 1); // node[2]'s first upstream node is node[1]. - // The min_delay between node[1] and node[2] is (0, 1), which denotes zero delay. - assert(lf_tag_compare(test_rti.scheduling_nodes[2]->min_delays[1].min_delay, (tag_t) {0, 1}) == 0); - assert(test_rti.scheduling_nodes[2]->min_delays[0].id == 0); // node[2]'s second upstream node is node[0]. + assert(test_rti.scheduling_nodes[2]->num_all_upstreams == 2); // node[2] has two upstream nodes. + assert(test_rti.scheduling_nodes[2]->all_upstreams[0] == 0); // node[0] is an upstream node of node[2]. // The min_delay between node[0] and node[2] is (1 nsec, 1) = 1 nsec + zero delay. - assert(lf_tag_compare(test_rti.scheduling_nodes[2]->min_delays[0].min_delay, (tag_t) {NSEC(1), 1}) == 0); + assert(lf_tag_compare(test_rti.min_delays[0*n + 2], (tag_t) {NSEC(1), 1}) == 0); + assert(test_rti.scheduling_nodes[2]->all_upstreams[1] == 1); // node[1] is an upstream node of node[2]. + // The min_delay between node[1] and node[2] is (0, 1), which denotes zero delay. + assert(lf_tag_compare(test_rti.min_delays[1*n + 2], (tag_t) {0, 1}) == 0); update_min_delays_upstream(test_rti.scheduling_nodes[3]); - assert(test_rti.scheduling_nodes[3]->num_min_delays == 3); // node[3] has three upstream nodes. - assert(test_rti.scheduling_nodes[3]->min_delays[2].id == 2); // node[3]'s first upstream node is node [2]. - // The min_delay between node[2] and node[3] is (2 nsec, 0). - assert(lf_tag_compare(test_rti.scheduling_nodes[3]->min_delays[2].min_delay, (tag_t) {NSEC(2), 0}) == 0); - assert(test_rti.scheduling_nodes[3]->min_delays[1].id == 1); // node[3]'s second upstream node is node [1]. - // The min_delay between node[1] and node[3] is (2 nsec, 0) = zero_delay + 2 nsec. - assert(lf_tag_compare(test_rti.scheduling_nodes[3]->min_delays[1].min_delay, (tag_t) {NSEC(2), 0}) == 0); - assert(test_rti.scheduling_nodes[3]->min_delays[0].id == 0); // node[3]'s third upstream node is node [0]. + assert(test_rti.scheduling_nodes[3]->num_all_upstreams == 3); // node[3] has three upstream nodes. + assert(test_rti.scheduling_nodes[3]->all_upstreams[0] == 0); // node[0] is an upstream node of node[3]. // The min_delay between node[0] and node[3] is (3 nsec, 0) = 1 nsec + zero_delay + 2 nsec. - assert(lf_tag_compare(test_rti.scheduling_nodes[3]->min_delays[0].min_delay, (tag_t) {NSEC(3), 0}) == 0); + assert(lf_tag_compare(test_rti.min_delays[0*n + 3], (tag_t) {NSEC(3), 0}) == 0); + assert(test_rti.scheduling_nodes[3]->all_upstreams[1] == 1); // node[1] is an upstream node of node[3]. + // The min_delay between node[1] and node[3] is (2 nsec, 0) = zero_delay + 2 nsec. + assert(lf_tag_compare(test_rti.min_delays[1*n + 3], (tag_t) {NSEC(2), 0}) == 0); + assert(test_rti.scheduling_nodes[3]->all_upstreams[2] == 2); // node[2] is an upstream node of node[3]. + // The min_delay between node[2] and node[3] is (2 nsec, 0). + assert(lf_tag_compare(test_rti.min_delays[2*n + 3], (tag_t) {NSEC(2), 0}) == 0); + + // Test update_all_downstreams + update_all_downstreams(test_rti.scheduling_nodes[0]); + assert(test_rti.scheduling_nodes[0]->num_all_downstreams == 3); // node[0] has three downstream nodes. + assert(test_rti.scheduling_nodes[0]->all_downstreams[0] == 1); // node[1] is a downstream node of node[3]. + assert(test_rti.scheduling_nodes[0]->all_downstreams[1] == 2); // node[2] is a downstream node of node[3]. + assert(test_rti.scheduling_nodes[0]->all_downstreams[2] == 3); // node[3] is a downstream node of node[3]. + + update_all_downstreams(test_rti.scheduling_nodes[1]); + assert(test_rti.scheduling_nodes[1]->num_all_downstreams == 2); // node[1] has two downstream nodes. + assert(test_rti.scheduling_nodes[1]->all_downstreams[0] == 2); // node[2] is a downstream node of node[3]. + assert(test_rti.scheduling_nodes[1]->all_downstreams[1] == 3); // node[3] is a downstream node of node[3]. } int main(int argc, char **argv) { initialize_rti_common(&test_rti); - // Tests for the function update_min_delays_upstream() + // Tests for the function update_min_delays_upstream() and update_all_downstreams() valid_cache(); not_connected(); two_nodes_no_delay(); diff --git a/test/Tests.cmake b/test/Tests.cmake index 3c9144895..fef6dc0aa 100644 --- a/test/Tests.cmake +++ b/test/Tests.cmake @@ -38,54 +38,53 @@ foreach(FILE ${TEST_FILES}) endforeach(FILE ${TEST_FILES}) # Add the test for the RTI. -# FIXME: Comment out the tests for the RTI until the tests are aligned to the new RTI. -# if (NOT DEFINED LF_SINGLE_THREADED) -# # Check which system we are running on to select the correct platform support -# # file and assign the file's path to LF_PLATFORM_FILE -# if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") -# set(LF_PLATFORM_FILE ${CoreLibPath}/platform/lf_linux_support.c) -# elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") -# set(LF_PLATFORM_FILE ${CoreLibPath}/platform/lf_macos_support.c) -# else() -# message(FATAL_ERROR "Your platform is not supported! RTI supports Linux and MacOS.") -# endif() +if (NOT DEFINED LF_SINGLE_THREADED) + # Check which system we are running on to select the correct platform support + # file and assign the file's path to LF_PLATFORM_FILE + if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + set(LF_PLATFORM_FILE ${CoreLibPath}/platform/lf_linux_support.c) + elseif(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + set(LF_PLATFORM_FILE ${CoreLibPath}/platform/lf_macos_support.c) + else() + message(FATAL_ERROR "Your platform is not supported! RTI supports Linux and MacOS.") + endif() -# set(IncludeDir include/core) + set(IncludeDir include/core) -# set(RTI_DIR ${CoreLibPath}/federated/RTI) -# add_executable( -# rti_common_test -# ${TEST_DIR}/RTI/rti_common_test.c -# ${RTI_DIR}/rti_common.c -# ${RTI_DIR}/rti_remote.c -# ${CoreLibPath}/trace.c -# ${LF_PLATFORM_FILE} -# ${CoreLibPath}/platform/lf_atomic_gcc_clang.c -# ${CoreLibPath}/platform/lf_unix_clock_support.c -# ${CoreLibPath}/utils/util.c -# ${CoreLibPath}/tag.c -# ${CoreLibPath}/clock.c -# ${CoreLibPath}/federated/network/net_util.c -# ${CoreLibPath}/utils/pqueue_base.c -# ${CoreLibPath}/utils/pqueue_tag.c -# ${CoreLibPath}/utils/pqueue.c -# ) -# add_test(NAME rti_common_test COMMAND rti_common_test) -# target_include_directories(rti_common_test PUBLIC ${RTI_DIR}) -# target_include_directories(rti_common_test PUBLIC ${IncludeDir}) -# target_include_directories(rti_common_test PUBLIC ${IncludeDir}/federated) -# target_include_directories(rti_common_test PUBLIC ${IncludeDir}/modal_models) -# target_include_directories(rti_common_test PUBLIC ${IncludeDir}/platform) -# target_include_directories(rti_common_test PUBLIC ${IncludeDir}/utils) -# # Set the STANDALONE_RTI flag to include the rti_remote and rti_common. -# target_compile_definitions(rti_common_test PUBLIC STANDALONE_RTI=1) + set(RTI_DIR ${CoreLibPath}/federated/RTI) + add_executable( + rti_common_test + ${TEST_DIR}/RTI/rti_common_test.c + ${RTI_DIR}/rti_common.c + ${RTI_DIR}/rti_remote.c + ${CoreLibPath}/trace.c + ${LF_PLATFORM_FILE} + ${CoreLibPath}/platform/lf_atomic_gcc_clang.c + ${CoreLibPath}/platform/lf_unix_clock_support.c + ${CoreLibPath}/utils/util.c + ${CoreLibPath}/tag.c + ${CoreLibPath}/clock.c + ${CoreLibPath}/federated/network/net_util.c + ${CoreLibPath}/utils/pqueue_base.c + ${CoreLibPath}/utils/pqueue_tag.c + ${CoreLibPath}/utils/pqueue.c + ) + add_test(NAME rti_common_test COMMAND rti_common_test) + target_include_directories(rti_common_test PUBLIC ${RTI_DIR}) + target_include_directories(rti_common_test PUBLIC ${IncludeDir}) + target_include_directories(rti_common_test PUBLIC ${IncludeDir}/federated) + target_include_directories(rti_common_test PUBLIC ${IncludeDir}/modal_models) + target_include_directories(rti_common_test PUBLIC ${IncludeDir}/platform) + target_include_directories(rti_common_test PUBLIC ${IncludeDir}/utils) + # Set the STANDALONE_RTI flag to include the rti_remote and rti_common. + target_compile_definitions(rti_common_test PUBLIC STANDALONE_RTI=1) -# # Set FEDERATED to get federated compilation support -# target_compile_definitions(rti_common_test PUBLIC FEDERATED=1) + # Set FEDERATED to get federated compilation support + target_compile_definitions(rti_common_test PUBLIC FEDERATED=1) -# target_compile_definitions(rti_common_test PUBLIC PLATFORM_${CMAKE_SYSTEM_NAME}) + target_compile_definitions(rti_common_test PUBLIC PLATFORM_${CMAKE_SYSTEM_NAME}) -# # Find threads and link to it -# find_package(Threads REQUIRED) -# target_link_libraries(rti_common_test Threads::Threads) -# endif() + # Find threads and link to it + find_package(Threads REQUIRED) + target_link_libraries(rti_common_test Threads::Threads) +endif() From 4ebd09e6c6563e31a55b21589b7c8b9b1e26c292 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 23 Apr 2024 10:07:26 -0700 Subject: [PATCH 16/43] Fix a trace point for DNET --- core/federated/RTI/rti_remote.c | 9 +++++++++ util/tracing/trace_util.c | 2 ++ 2 files changed, 11 insertions(+) diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index 9fbb440b7..8c9fd69e1 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -273,6 +273,15 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag) { } void send_downstream_next_event_tag(scheduling_node_t* e, tag_t tag) { + if (e->state == NOT_CONNECTED) { + return; + } + // Need to make sure that the destination federate's thread has already + // sent the starting MSG_TYPE_TIMESTAMP message. + while (e->state == PENDING) { + // Need to wait here. + lf_cond_wait(&sent_start_time); + } size_t message_length = 1 + sizeof(int64_t) + sizeof(uint32_t); unsigned char buffer[message_length]; buffer[0] = MSG_TYPE_DOWNSTREAM_NEXT_EVENT_TAG; diff --git a/util/tracing/trace_util.c b/util/tracing/trace_util.c index ffe0c6b8f..9c1a77592 100644 --- a/util/tracing/trace_util.c +++ b/util/tracing/trace_util.c @@ -96,6 +96,7 @@ const char* trace_event_names[] = { "Sending P2P_MSG", "Sending ADR_AD", "Sending ADR_QR", + "Sending DNET", // Receiving messages "Receiving ACK", "Receiving FAILED", @@ -118,6 +119,7 @@ const char* trace_event_names[] = { "Receiving P2P_MSG", "Receiving ADR_AD", "Receiving ADR_QR", + "Receiving DNET", "Receiving UNIDENTIFIED", }; From 9a9d12089ba2597bc4fe118aa76c4f1a40f42f5b Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 23 Apr 2024 13:58:09 -0700 Subject: [PATCH 17/43] Skip sending LTCs if a network input reaction has been scheduled at the current tag --- core/environment.c | 2 ++ core/federated/RTI/rti_common.c | 3 +-- core/federated/federate.c | 21 ++++----------------- core/reactor_common.c | 2 ++ core/threaded/reactor_threaded.c | 5 +++++ include/core/environment.h | 1 + include/core/federated/federate.h | 5 ----- 7 files changed, 15 insertions(+), 24 deletions(-) diff --git a/core/environment.c b/core/environment.c index 4523c4721..2fd3072d7 100644 --- a/core/environment.c +++ b/core/environment.c @@ -147,6 +147,8 @@ static void environment_init_federated(environment_t* env, int num_is_present_fi env->_lf_intended_tag_fields = NULL; env->_lf_intended_tag_fields_size = 0; } +#elif FEDERATED_CENTRALIZED + env->need_to_send_LTC = false; #else (void)env; (void)num_is_present_fields; diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index dfd37ac36..0480e9dff 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -516,8 +516,7 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t DNET = NEVER_TAG; } - if (lf_tag_compare(node->last_DNET, DNET) != 0 && lf_tag_compare(node->completed, DNET) < 0 && - lf_tag_compare(node->next_event, DNET) <= 0) { + if (lf_tag_compare(node->last_DNET, DNET) != 0 && lf_tag_compare(node->next_event, DNET) <= 0) { send_downstream_next_event_tag(node, DNET); } } diff --git a/core/federated/federate.c b/core/federated/federate.c index a96f98598..985f14de2 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -87,7 +87,6 @@ federate_instance_t _fed = {.socket_TCP_RTI = -1, .is_last_TAG_provisional = false, .has_upstream = false, .has_downstream = false, - .last_skipped_LTC = {.time = NEVER, .microstep = 0u}, .last_DNET = {.time = NEVER, .microstep = 0u}, .received_stop_request_from_rti = false, .last_sent_LTC = {.time = NEVER, .microstep = 0u}, @@ -1451,12 +1450,6 @@ static void handle_downstream_next_event_tag() { LF_PRINT_LOG("Received Downstream Next Event Tag (DNET): " PRINTF_TAG ".", DNET.time - start_time, DNET.microstep); _fed.last_DNET = DNET; - - if (lf_tag_compare(_fed.last_skipped_LTC, NEVER_TAG) != 0 && - lf_tag_compare(_fed.last_skipped_LTC, _fed.last_DNET) >= 0) { - send_tag(MSG_TYPE_LATEST_TAG_COMPLETE, _fed.last_skipped_LTC); - _fed.last_skipped_LTC = NEVER_TAG; - } } /** @@ -2230,22 +2223,16 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { } void lf_latest_tag_complete(tag_t tag_to_send) { + environment_t* env; + _lf_get_environments(&env); int compare_with_last_LTC = lf_tag_compare(_fed.last_sent_LTC, tag_to_send); - int compare_with_last_DNET = lf_tag_compare(_fed.last_DNET, tag_to_send); - if (compare_with_last_LTC >= 0) { - return; - } - if (compare_with_last_DNET > 0) { - LF_PRINT_LOG("Skipping Latest Tag Complete (LTC) " PRINTF_TAG " .", tag_to_send.time - start_time, - tag_to_send.microstep); - _fed.last_skipped_LTC = tag_to_send; + if (compare_with_last_LTC >= 0 || !env->need_to_send_LTC) { return; } LF_PRINT_LOG("Sending Latest Tag Complete (LTC) " PRINTF_TAG " to the RTI.", tag_to_send.time - start_time, tag_to_send.microstep); send_tag(MSG_TYPE_LATEST_TAG_COMPLETE, tag_to_send); _fed.last_sent_LTC = tag_to_send; - _fed.last_skipped_LTC = NEVER_TAG; } parse_rti_code_t lf_parse_rti_addr(const char* rti_addr) { @@ -2391,7 +2378,7 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) // This if statement does not fall through but rather returns. // NET is not bounded by physical time or has no downstream federates. // Normal case. - if (lf_tag_compare(_fed.last_DNET, tag) <= 0 || lf_tag_compare(_fed.last_TAG, tag) < 0) { + if (lf_tag_compare(_fed.last_DNET, tag) < 0 || (_fed.has_upstream && lf_tag_compare(_fed.last_TAG, tag) < 0)) { send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag); _fed.last_sent_NET = tag; LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.", tag.time - start_time, tag.microstep); diff --git a/core/reactor_common.c b/core/reactor_common.c index 33e5582f5..54de1a906 100644 --- a/core/reactor_common.c +++ b/core/reactor_common.c @@ -210,6 +210,8 @@ void _lf_start_time_step(environment_t* env) { } #endif // FEDERATED_DECENTRALIZED + env->need_to_send_LTC = false; + // Reset absent fields on network ports because // their status is unknown lf_reset_status_fields_on_input_port_triggers(); diff --git a/core/threaded/reactor_threaded.c b/core/threaded/reactor_threaded.c index 57f888fc2..54c74b19f 100644 --- a/core/threaded/reactor_threaded.c +++ b/core/threaded/reactor_threaded.c @@ -886,6 +886,11 @@ void _lf_worker_do_work(environment_t* env, int worker_number) { worker_number, current_reaction_to_execute->name, LF_LEVEL(current_reaction_to_execute->index), current_reaction_to_execute->is_an_input_reaction, current_reaction_to_execute->chain_id, current_reaction_to_execute->deadline); +#ifdef FEDERATED + if (current_reaction_to_execute->is_an_input_reaction) { + env->need_to_send_LTC = true; + } +#endif // FEDERATED bool violation = _lf_worker_handle_violations(env, worker_number, current_reaction_to_execute); diff --git a/include/core/environment.h b/include/core/environment.h index a776dee95..2e6b99e1f 100644 --- a/include/core/environment.h +++ b/include/core/environment.h @@ -105,6 +105,7 @@ typedef struct environment_t { lf_cond_t global_tag_barrier_requestors_reached_zero; #endif // LF_SINGLE_THREADED #if defined(FEDERATED) + bool need_to_send_LTC; tag_t** _lf_intended_tag_fields; int _lf_intended_tag_fields_size; #endif // FEDERATED diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h index fd7b23a3d..339fa1e69 100644 --- a/include/core/federated/federate.h +++ b/include/core/federated/federate.h @@ -142,11 +142,6 @@ typedef struct federate_instance_t { */ bool has_downstream; - /** - * - */ - tag_t last_skipped_LTC; - /** * */ From 9b742afd6b5caee9f90a3ad1bda596a76168d065 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Sat, 27 Apr 2024 15:31:09 -0700 Subject: [PATCH 18/43] Maintain a variable for storing the last skipped NET signal --- core/federated/federate.c | 15 +++++++++++++++ include/core/federated/federate.h | 15 ++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/core/federated/federate.c b/core/federated/federate.c index 985f14de2..589b2c2b4 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -91,6 +91,7 @@ federate_instance_t _fed = {.socket_TCP_RTI = -1, .received_stop_request_from_rti = false, .last_sent_LTC = {.time = NEVER, .microstep = 0u}, .last_sent_NET = {.time = NEVER, .microstep = 0u}, + .last_skipped_NET = {.time = NEVER, .microstep = 0u}, .min_delay_from_physical_action_to_federate_output = NEVER}; federation_metadata_t federation_metadata = { @@ -1449,6 +1450,17 @@ static void handle_downstream_next_event_tag() { LF_PRINT_LOG("Received Downstream Next Event Tag (DNET): " PRINTF_TAG ".", DNET.time - start_time, DNET.microstep); + environment_t* env; + _lf_get_environments(&env); + if (lf_tag_compare(DNET, _fed.last_skipped_NET) < 0) { + LF_PRINT_LOG("The incoming DNET " PRINTF_TAG " is earlier than the last skipped NET " PRINTF_TAG + ". Send the skipped NET", + DNET.time - start_time, DNET.microstep, _fed.last_skipped_NET.time, _fed.last_skipped_NET.microstep); + send_tag(MSG_TYPE_NEXT_EVENT_TAG, _fed.last_skipped_NET); + _fed.last_sent_NET = _fed.last_skipped_NET; + _fed.last_skipped_NET = NEVER_TAG; + } + _fed.last_DNET = DNET; } @@ -2381,8 +2393,10 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) if (lf_tag_compare(_fed.last_DNET, tag) < 0 || (_fed.has_upstream && lf_tag_compare(_fed.last_TAG, tag) < 0)) { send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag); _fed.last_sent_NET = tag; + _fed.last_skipped_NET = NEVER_TAG; LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.", tag.time - start_time, tag.microstep); } else { + _fed.last_skipped_NET = tag; LF_PRINT_LOG("Skip next event tag (NET) " PRINTF_TAG " to RTI.", tag.time - start_time, tag.microstep); } @@ -2420,6 +2434,7 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) if (lf_tag_compare(next_tag, tag) != 0) { send_tag(MSG_TYPE_NEXT_EVENT_TAG, next_tag); _fed.last_sent_NET = next_tag; + _fed.last_skipped_NET = NEVER_TAG; LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI from loop.", next_tag.time - lf_time_start(), next_tag.microstep); } diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h index 339fa1e69..16fe8f3e9 100644 --- a/include/core/federated/federate.h +++ b/include/core/federated/federate.h @@ -142,11 +142,6 @@ typedef struct federate_instance_t { */ bool has_downstream; - /** - * - */ - tag_t last_DNET; - /** * Used to prevent the federate from sending a REQUEST_STOP * message if it has already received a stop request from the RTI. @@ -173,6 +168,16 @@ typedef struct federate_instance_t { */ tag_t last_sent_NET; + /** + * A record of the most recently skipped NET (next event tag) message. + */ + tag_t last_skipped_NET; + + /** + * + */ + tag_t last_DNET; + /** * For use in federates with centralized coordination, the minimum * time delay between a physical action within this federate and an From a7601aec7668f19f9b518b8d59de19724bbea92b Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 8 May 2024 15:51:55 -0700 Subject: [PATCH 19/43] Compute TAG values using EIMT --- core/federated/RTI/rti_common.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 0480e9dff..61ef2cb3e 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -18,6 +18,20 @@ static rti_common_t* rti_common = NULL; // Global variables defined in tag.c: extern instant_t start_time; +static tag_t latest_earlier_tag(tag_t tag) { + if (tag.time == NEVER || tag.time == FOREVER) { + return tag; + } else if (tag.time == 0 && tag.microstep == 0) { + return NEVER_TAG; + } else if (tag.microstep == 0) { + tag.time -= 1; + tag.microstep = UINT_MAX; + } else { + tag.microstep -= 1; + } + return tag; +} + void initialize_rti_common(rti_common_t* _rti_common) { rti_common = _rti_common; rti_common->max_stop_tag = NEVER_TAG; @@ -230,7 +244,8 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) { "(adjusted by after delay). Granting tag advance (TAG) for " PRINTF_TAG, e->id, t_d.time - lf_time_start(), t_d.microstep, e->next_event.time - lf_time_start(), e->next_event.microstep); - result.tag = e->next_event; + // result.tag = e->next_event; + result.tag = latest_earlier_tag(t_d); } else if ( // Scenario (2) above lf_tag_compare(t_d, e->next_event) == 0 // EIMT equal to NET && is_in_zero_delay_cycle(e) // The node is part of a ZDC From 99aa14f3d929cc720ed321620a90a758be77ee1a Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 18 Jun 2024 14:47:37 +0900 Subject: [PATCH 20/43] Revert changes relating to LTC signals --- core/environment.c | 2 -- core/federated/federate.c | 4 +--- core/reactor_common.c | 2 -- core/threaded/reactor_threaded.c | 5 ----- include/core/environment.h | 1 - 5 files changed, 1 insertion(+), 13 deletions(-) diff --git a/core/environment.c b/core/environment.c index e167e36b3..d2d56a593 100644 --- a/core/environment.c +++ b/core/environment.c @@ -126,8 +126,6 @@ static void environment_init_federated(environment_t* env, int num_is_present_fi env->_lf_intended_tag_fields = NULL; env->_lf_intended_tag_fields_size = 0; } -#elif FEDERATED_CENTRALIZED - env->need_to_send_LTC = false; #else (void)env; (void)num_is_present_fields; diff --git a/core/federated/federate.c b/core/federated/federate.c index e4cd7e313..80a8e4be2 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2235,10 +2235,8 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { } void lf_latest_tag_complete(tag_t tag_to_send) { - environment_t* env; - _lf_get_environments(&env); int compare_with_last_LTC = lf_tag_compare(_fed.last_sent_LTC, tag_to_send); - if (compare_with_last_LTC >= 0 || !env->need_to_send_LTC) { + if (compare_with_last_LTC >= 0) { return; } LF_PRINT_LOG("Sending Latest Tag Complete (LTC) " PRINTF_TAG " to the RTI.", tag_to_send.time - start_time, diff --git a/core/reactor_common.c b/core/reactor_common.c index 406117a82..26a489bec 100644 --- a/core/reactor_common.c +++ b/core/reactor_common.c @@ -210,8 +210,6 @@ void _lf_start_time_step(environment_t* env) { } #endif // FEDERATED_DECENTRALIZED - env->need_to_send_LTC = false; - // Reset absent fields on network ports because // their status is unknown lf_reset_status_fields_on_input_port_triggers(); diff --git a/core/threaded/reactor_threaded.c b/core/threaded/reactor_threaded.c index 68676210a..392f49c31 100644 --- a/core/threaded/reactor_threaded.c +++ b/core/threaded/reactor_threaded.c @@ -879,11 +879,6 @@ void _lf_worker_do_work(environment_t* env, int worker_number) { "level: %lld, is input reaction: %d, and deadline " PRINTF_TIME ".", worker_number, current_reaction_to_execute->name, LF_LEVEL(current_reaction_to_execute->index), current_reaction_to_execute->is_an_input_reaction, current_reaction_to_execute->deadline); -#ifdef FEDERATED - if (current_reaction_to_execute->is_an_input_reaction) { - env->need_to_send_LTC = true; - } -#endif // FEDERATED bool violation = _lf_worker_handle_violations(env, worker_number, current_reaction_to_execute); diff --git a/include/core/environment.h b/include/core/environment.h index b738ecd1e..038f97e4e 100644 --- a/include/core/environment.h +++ b/include/core/environment.h @@ -85,7 +85,6 @@ typedef struct environment_t { lf_cond_t global_tag_barrier_requestors_reached_zero; #endif // LF_SINGLE_THREADED #if defined(FEDERATED) - bool need_to_send_LTC; tag_t** _lf_intended_tag_fields; int _lf_intended_tag_fields_size; #endif // FEDERATED From 6275943d06878b6e00ef3bf6c91eee1250a8dbb0 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 18 Jun 2024 14:49:57 +0900 Subject: [PATCH 21/43] Minor fix --- trace/api/types/trace_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace/api/types/trace_types.h b/trace/api/types/trace_types.h index 56c54f8b6..b7cc7be0c 100644 --- a/trace/api/types/trace_types.h +++ b/trace/api/types/trace_types.h @@ -72,7 +72,7 @@ typedef enum { receive_P2P_MSG, receive_ADR_AD, receive_ADR_QR, - received_DNET, + receive_DNET, receive_UNIDENTIFIED, NUM_EVENT_TYPES } trace_event_t; From 013aa0c52197a8383a873b3d62afa18f202c1ef6 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 18 Jun 2024 15:24:15 +0900 Subject: [PATCH 22/43] Resolve FIXMEs --- core/federated/RTI/main.c | 2 +- core/federated/RTI/rti_common.c | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c index 9c58d3cff..06613c87e 100644 --- a/core/federated/RTI/main.c +++ b/core/federated/RTI/main.c @@ -318,7 +318,7 @@ int main(int argc, const char* argv[]) { // Allocate memory for the federates int n = rti.base.number_of_scheduling_nodes; rti.base.scheduling_nodes = (scheduling_node_t**)calloc(n, sizeof(scheduling_node_t*)); - // Allocate memory for min_delays. + // Allocate memory for the array of min_delays. rti.base.min_delays = (tag_t*)calloc((n * n), sizeof(tag_t)); for (uint16_t i = 0; i < n; i++) { for (uint16_t j = 0; j < n; j++) { diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 61ef2cb3e..1c42941ec 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -292,19 +292,16 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne e->last_granted = next_event_tag; } // Check downstream scheduling_nodes to see whether they should now be granted a TAG. - // To handle cycles, need to create a boolean array to keep - // track of which downstream scheduling_nodes have been visited. - // FIXME: As we have all_downstreams field now, we don't need the function notify_downstream_davnace_grnat_if_safe. update_all_downstreams(e); - bool* visited = (bool*)calloc(rti_common->number_of_scheduling_nodes, sizeof(bool)); // Initializes to 0. - notify_downstream_advance_grant_if_safe(e, visited); - free(visited); + for (int i = 0; i < rti_common->num_all_downstreams; i++) { + scheduling_node_t* downstream = rti_common->scheduling_nodes[e->num_all_downstreams[i]]; + notify_advance_grant_if_safe(downstream); + } - // Send DNET to the node's upstreams if needed + // Send DNET to the node e's upstream federates if needed for (int i = 0; i < e->num_all_upstreams; i++) { int target_upstream_id = e->all_upstreams[i]; if (target_upstream_id == e->id) { - // FIXME: This shouldn't be entered, but currently, it's entered. continue; } send_downstream_next_event_tag_if_needed(rti_common->scheduling_nodes[target_upstream_id], e->id); @@ -460,18 +457,18 @@ void update_all_downstreams(scheduling_node_t* node) { } tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay) { - // FIXME: FOREVER - FOREVER is not handled. // (A.t, A.m) - (B.t - B.m) - // The least B is (0, 0) which indicates NEVER delay. + // B cannot be NEVER_TAG as (0, 0) denotes no delay. + // Also, we assume B is not FOREVER_TAG because FOREVER_TAG delay means that there is no connection. // 1) If A.t = NEVER, return NEVER. // 2) If A.t = FOREVER, return FOREVER. // 3) A.t is not NEVER neither FOREVER // a) If A < B (A.t < B.t or A.t == B.t and A.m < B.m) return NEVER // b) A >= B - // i) If A.m < B.m return (A.t - B.t - 1, UINT_MAX) - // ii) If A.m >= B.m - // If B.t is 0 return (A.t, A.m - B.m) - // Else, return (A.t - B.t, UINT_MAX) + // i) If A.t >= B.t = 0 and A.m >= B.m return (A.t - B.t, A.m - B.m) + // ii) If A.t >= B.t > 0 and A.m >= B.m return (A.t - B.t, UINT_MAX) + // iii) If A.t >= B.t > 0 and A.m < B.m return (A.t - B.t - 1, UINT_MAX) + if (received_tag.time == NEVER || lf_tag_compare(received_tag, minimum_delay) < 0) return NEVER_TAG; if (received_tag.time == FOREVER) From b99ace2a4ebbf7aa4417de9d700b412d20e98cfc Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 19 Jun 2024 15:33:59 +0900 Subject: [PATCH 23/43] Minor Fix --- core/federated/RTI/rti_common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 1c42941ec..1c853fc2b 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -293,8 +293,8 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne } // Check downstream scheduling_nodes to see whether they should now be granted a TAG. update_all_downstreams(e); - for (int i = 0; i < rti_common->num_all_downstreams; i++) { - scheduling_node_t* downstream = rti_common->scheduling_nodes[e->num_all_downstreams[i]]; + for (int i = 0; i < e->num_all_downstreams; i++) { + scheduling_node_t* downstream = rti_common->scheduling_nodes[e->all_downstreams[i]]; notify_advance_grant_if_safe(downstream); } From 925985b170046b4d24ec294efdf051c690395f3e Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 13 Aug 2024 11:36:23 -0700 Subject: [PATCH 24/43] Add an option for turning on and off DNET signal --- core/federated/RTI/main.c | 3 +++ core/federated/RTI/rti_common.c | 20 +++++++++++--------- core/federated/RTI/rti_common.h | 8 +++++--- core/federated/RTI/rti_local.c | 4 ++++ core/federated/RTI/rti_remote.c | 3 ++- core/federated/RTI/rti_remote.h | 1 + lingua-franca-ref.txt | 2 +- 7 files changed, 27 insertions(+), 14 deletions(-) diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c index 06613c87e..c24415a04 100644 --- a/core/federated/RTI/main.c +++ b/core/federated/RTI/main.c @@ -136,6 +136,7 @@ void usage(int argc, const char* argv[]) { lf_print(" clock sync attempt (default is 10). Applies to 'init' and 'on'.\n"); lf_print(" -a, --auth Turn on HMAC authentication options.\n"); lf_print(" -t, --tracing Turn on tracing.\n"); + lf_print(" -d, --dnet Turn on DNET signals for reducing network messages.\n"); lf_print("Command given:"); for (int i = 0; i < argc; i++) { @@ -263,6 +264,8 @@ int process_args(int argc, const char* argv[]) { rti.authentication_enabled = true; } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) { rti.base.tracing_enabled = true; + } else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--dnet") == 0) { + rti.base.dnet_enabled = true; } else if (strcmp(argv[i], " ") == 0) { // Tolerate spaces continue; diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 636103903..5e02400e5 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -298,13 +298,15 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne notify_advance_grant_if_safe(downstream); } - // Send DNET to the node e's upstream federates if needed - for (int i = 0; i < e->num_all_upstreams; i++) { - int target_upstream_id = e->all_upstreams[i]; - if (target_upstream_id == e->id) { - continue; + if (rti_common->dnet_enabled) { + // Send DNET to the node e's upstream federates if needed + for (int i = 0; i < e->num_all_upstreams; i++) { + int target_upstream_id = e->all_upstreams[i]; + if (target_upstream_id == e->id) { + continue; + } + downstream_next_event_tag_if_needed(rti_common->scheduling_nodes[target_upstream_id], e->id); } - send_downstream_next_event_tag_if_needed(rti_common->scheduling_nodes[target_upstream_id], e->id); } } @@ -488,8 +490,8 @@ tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay) { return result; } -// It should be static because it's used only in this file. Remove from the header file. -void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id) { +// FIXME: It should be static because it's used only in this file. Remove from the header file. +void downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id) { if (is_in_zero_delay_cycle(node)) { return; } @@ -529,7 +531,7 @@ void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t } if (lf_tag_compare(node->last_DNET, DNET) != 0 && lf_tag_compare(node->next_event, DNET) <= 0) { - send_downstream_next_event_tag(node, DNET); + notify_downstream_next_event_tag(node, DNET); } } diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 56f4f8709..7f9d3ea4c 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -95,6 +95,9 @@ typedef struct rti_common_t { // Boolean indicating that tracing is enabled. bool tracing_enabled; + // Boolean indicating that DNET is enabled. + bool dnet_enabled; + // The RTI mutex for making thread-safe access to the shared state. lf_mutex_t* mutex; } rti_common_t; @@ -291,17 +294,16 @@ void update_all_downstreams(scheduling_node_t* node); tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay); /** - * FIXME: Add this function to rti_local either. * @param e The target node. * @param tag The downstream next event tag for e. */ -void send_downstream_next_event_tag(scheduling_node_t* e, tag_t tag); +void notify_downstream_next_event_tag(scheduling_node_t* e, tag_t tag); /** * @param node * @param new_NET */ -void send_downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id); +void downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id); /** * For the given scheduling node (enclave or federate), invalidate the `min_delays`, diff --git a/core/federated/RTI/rti_local.c b/core/federated/RTI/rti_local.c index 416c81557..f1a91e1cf 100644 --- a/core/federated/RTI/rti_local.c +++ b/core/federated/RTI/rti_local.c @@ -191,6 +191,10 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag) { LF_PRINT_LOG("RTI: enclave %u callback with PTAG " PRINTF_TAG " ", e->id, tag.time - lf_time_start(), tag.microstep); } +void notify_downstream_next_event_tag(scheduling_node_t* e, tag_t tag) { + // Nothing to do here. +} + void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number_of_scheduling_nodes) { // Nothing to do here. } diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index a67214e01..c1b886bac 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -272,7 +272,7 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag) { } } -void send_downstream_next_event_tag(scheduling_node_t* e, tag_t tag) { +void notify_downstream_next_event_tag(scheduling_node_t* e, tag_t tag) { if (e->state == NOT_CONNECTED) { return; } @@ -1765,6 +1765,7 @@ void initialize_RTI(rti_remote_t* rti) { rti_remote->clock_sync_exchanges_per_interval = 10; rti_remote->authentication_enabled = false; rti_remote->base.tracing_enabled = false; + rti_remote->base.dnet_enabled = false; rti_remote->stop_in_progress = false; } diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h index 8d3012b95..f810d428c 100644 --- a/core/federated/RTI/rti_remote.h +++ b/core/federated/RTI/rti_remote.h @@ -154,6 +154,7 @@ typedef struct rti_remote_t { * Boolean indicating that authentication is enabled. */ bool authentication_enabled; + /** * Boolean indicating that a stop request is already in progress. */ diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt index 1f7391f92..8ba7da0a2 100644 --- a/lingua-franca-ref.txt +++ b/lingua-franca-ref.txt @@ -1 +1 @@ -master +rti-DNET From fc970bbd1b0a47615821943ef348c9e02c83167c Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 14 Aug 2024 10:41:18 -0700 Subject: [PATCH 25/43] Send a necessary NET even the federate can advance its tag --- core/federated/federate.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/federated/federate.c b/core/federated/federate.c index c873bd0b4..e9cc9df66 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2406,6 +2406,14 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) if (lf_tag_compare(_fed.last_TAG, tag) >= 0) { LF_PRINT_DEBUG("Granted tag " PRINTF_TAG " because TAG or PTAG has been received.", _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); + + // In case a downstream federate needs the NET of this tag, send NET. + if (lf_tag_compare(_fed.last_DNET, tag) < 0 && lf_tag_compare(_fed.last_DNET, _fed.last_sent_NET) >= 0) { + send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag); + _fed.last_sent_NET = tag; + _fed.last_skipped_NET = NEVER_TAG; + LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.", tag.time - start_time, tag.microstep); + } return _fed.last_TAG; } From a2ab5bc04dfbac189434d0d39e2afc3ee5a00644 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 14 Aug 2024 16:19:06 -0700 Subject: [PATCH 26/43] Add some comments and remove a FIXME --- core/federated/RTI/rti_common.c | 4 ++-- core/federated/RTI/rti_common.h | 9 ++++++--- core/federated/federate.c | 7 ++++--- include/core/federated/federate.h | 6 +++--- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 5e02400e5..f259d6af8 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -490,7 +490,6 @@ tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay) { return result; } -// FIXME: It should be static because it's used only in this file. Remove from the header file. void downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id) { if (is_in_zero_delay_cycle(node)) { return; @@ -526,7 +525,8 @@ void downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_N } } if (DNET.time < start_time) { - // DNET is NEVER. + // DNET with the time smaller than the start time acts as the same as DNET of the NEVER tag. + // Thus, set DNET as NEVER_TAG to prevent sending unnecessary DNETs. DNET = NEVER_TAG; } diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 7f9d3ea4c..ab8494c8a 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -286,7 +286,8 @@ void update_min_delays_upstream(scheduling_node_t* node); void update_all_downstreams(scheduling_node_t* node); /** - * Subtract the tag a from the tag b. + * Find the tag g that is the latest tag that satisfies lf_tag_add(g, minimum_delay) < received_tag. + * This function behaves like the tag subtraction, received_tag - minimum_delay. * minimum_delay cannot be NEVER. * @param received_tag * @param minimum_delay @@ -300,8 +301,10 @@ tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay); void notify_downstream_next_event_tag(scheduling_node_t* e, tag_t tag); /** - * @param node - * @param new_NET + * @param node The target node that may receive a new DNET. + * @param new_NET_source_federate_id The ID of the federate that sends a new NET. If this federate's new NET does not + * change the DNET value, we can exit this function immediately. If it does, we have to look up the target federate's + * downstream federates to compute the exact new DNET value. */ void downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id); diff --git a/core/federated/federate.c b/core/federated/federate.c index e9cc9df66..517caf1bb 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -1468,7 +1468,6 @@ static void handle_stop_request_message() { /** * Handle a downstream next event tag (DNET) message from the RTI. - * FIXME: Use this tag to eliminate unncessary LTC or NET messages. */ static void handle_downstream_next_event_tag() { size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); @@ -2412,7 +2411,9 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag); _fed.last_sent_NET = tag; _fed.last_skipped_NET = NEVER_TAG; - LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.", tag.time - start_time, tag.microstep); + LF_PRINT_LOG("Sent the last skipped next event tag (NET) " PRINTF_TAG + " to RTI based on the last DNET " PRINTF_TAG ".", + _fed.last_DNET - start_time, _fed.last_DNET.microstep, tag.time - start_time, tag.microstep); } return _fed.last_TAG; } @@ -2442,7 +2443,7 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) LF_PRINT_LOG("Sent next event tag (NET) " PRINTF_TAG " to RTI.", tag.time - start_time, tag.microstep); } else { _fed.last_skipped_NET = tag; - LF_PRINT_LOG("Skip next event tag (NET) " PRINTF_TAG " to RTI.", tag.time - start_time, tag.microstep); + LF_PRINT_LOG("Skip sending next event tag (NET) " PRINTF_TAG " to RTI.", tag.time - start_time, tag.microstep); } if (!wait_for_reply) { diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h index a4df3cb2f..bfbd24314 100644 --- a/include/core/federated/federate.h +++ b/include/core/federated/federate.h @@ -164,17 +164,17 @@ typedef struct federate_instance_t { tag_t last_sent_LTC; /** - * A record of the most recently sent NET (next event tag) message. + * A record of the most recently sent NET (next event tag) signal. */ tag_t last_sent_NET; /** - * A record of the most recently skipped NET (next event tag) message. + * A record of the most recently skipped NET (next event tag) signal. */ tag_t last_skipped_NET; /** - * + * A record of the most recent DNET (downstream next event tag) signal. */ tag_t last_DNET; From 5d52945c6f540eb3e065082ab7e893f84912440f Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Thu, 15 Aug 2024 10:32:10 -0700 Subject: [PATCH 27/43] Minor refactoring --- core/federated/RTI/rti_common.c | 46 +++++++++++++++++---------------- core/federated/RTI/rti_common.h | 8 +++--- core/federated/federate.c | 2 +- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index f259d6af8..e4781a038 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -305,7 +305,11 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne if (target_upstream_id == e->id) { continue; } - downstream_next_event_tag_if_needed(rti_common->scheduling_nodes[target_upstream_id], e->id); + scheduling_node_t* node = rti_common->scheduling_nodes[target_upstream_id]; + tag_t dnet = downstream_next_event_tag(node, e->id); + if (lf_tag_compare(node->last_DNET, dnet) != 0 && lf_tag_compare(node->next_event, dnet) <= 0) { + notify_downstream_next_event_tag(node, dnet); + } } } } @@ -458,7 +462,7 @@ void update_all_downstreams(scheduling_node_t* node) { } } -tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay) { +tag_t get_dnet_candidate(tag_t received_tag, tag_t minimum_delay) { // (A.t, A.m) - (B.t - B.m) // B cannot be NEVER_TAG as (0, 0) denotes no delay. // Also, we assume B is not FOREVER_TAG because FOREVER_TAG delay means that there is no connection. @@ -490,22 +494,22 @@ tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay) { return result; } -void downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id) { +tag_t downstream_next_event_tag(scheduling_node_t* node, uint16_t node_sending_new_net_id) { if (is_in_zero_delay_cycle(node)) { - return; + return NEVER_TAG; } - tag_t DNET = FOREVER_TAG; - scheduling_node_t* new_NET_source_federate = rti_common->scheduling_nodes[new_NET_source_federate_id]; - if (is_in_zero_delay_cycle(new_NET_source_federate)) { - return; + tag_t result = FOREVER_TAG; + scheduling_node_t* node_sending_new_net = rti_common->scheduling_nodes[node_sending_new_net_id]; + if (is_in_zero_delay_cycle(node_sending_new_net)) { + return NEVER_TAG; } - int index = node->id * rti_common->number_of_scheduling_nodes + new_NET_source_federate_id; - tag_t DNET_candidate = get_DNET_candidate(new_NET_source_federate->next_event, rti_common->min_delays[index]); + int index = node->id * rti_common->number_of_scheduling_nodes + node_sending_new_net_id; + tag_t candidate = get_dnet_candidate(node_sending_new_net->next_event, rti_common->min_delays[index]); - if (lf_tag_compare(node->last_DNET, DNET_candidate) >= 0) { - DNET = DNET_candidate; + if (lf_tag_compare(node->last_DNET, candidate) >= 0) { + result = candidate; } else { for (int i = 0; i < node->num_all_downstreams; i++) { uint16_t target_downstream_id = node->all_downstreams[i]; @@ -513,26 +517,24 @@ void downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_N if (is_in_zero_delay_cycle(target_dowstream)) { // This node is an upstream of ZDC. Do not send DNET to this node. - return; + return NEVER_TAG; } index = node->id * rti_common->number_of_scheduling_nodes + target_downstream_id; - DNET_candidate = get_DNET_candidate(target_dowstream->next_event, rti_common->min_delays[index]); + candidate = get_dnet_candidate(target_dowstream->next_event, rti_common->min_delays[index]); - if (lf_tag_compare(DNET, DNET_candidate) > 0) { - DNET = DNET_candidate; + if (lf_tag_compare(result, candidate) > 0) { + result = candidate; } } } - if (DNET.time < start_time) { + if (result.time < start_time) { // DNET with the time smaller than the start time acts as the same as DNET of the NEVER tag. - // Thus, set DNET as NEVER_TAG to prevent sending unnecessary DNETs. - DNET = NEVER_TAG; + // Thus, set the result as NEVER_TAG to prevent sending unnecessary DNETs. + result = NEVER_TAG; } - if (lf_tag_compare(node->last_DNET, DNET) != 0 && lf_tag_compare(node->next_event, DNET) <= 0) { - notify_downstream_next_event_tag(node, DNET); - } + return result; } bool is_in_zero_delay_cycle(scheduling_node_t* node) { diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index ab8494c8a..2ac967acb 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -292,7 +292,7 @@ void update_all_downstreams(scheduling_node_t* node); * @param received_tag * @param minimum_delay */ -tag_t get_DNET_candidate(tag_t received_tag, tag_t minimum_delay); +tag_t get_dnet_candidate(tag_t received_tag, tag_t minimum_delay); /** * @param e The target node. @@ -302,11 +302,11 @@ void notify_downstream_next_event_tag(scheduling_node_t* e, tag_t tag); /** * @param node The target node that may receive a new DNET. - * @param new_NET_source_federate_id The ID of the federate that sends a new NET. If this federate's new NET does not - * change the DNET value, we can exit this function immediately. If it does, we have to look up the target federate's + * @param node_sending_new_net_id The ID of the node that sends a new NET. If this node's new NET does not + * change the DNET value, we can exit this function immediately. If it does, we have to look up the target node's * downstream federates to compute the exact new DNET value. */ -void downstream_next_event_tag_if_needed(scheduling_node_t* node, uint16_t new_NET_source_federate_id); +tag_t downstream_next_event_tag(scheduling_node_t* node, uint16_t node_sending_new_net_id); /** * For the given scheduling node (enclave or federate), invalidate the `min_delays`, diff --git a/core/federated/federate.c b/core/federated/federate.c index 517caf1bb..3d3ea2c71 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2413,7 +2413,7 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) _fed.last_skipped_NET = NEVER_TAG; LF_PRINT_LOG("Sent the last skipped next event tag (NET) " PRINTF_TAG " to RTI based on the last DNET " PRINTF_TAG ".", - _fed.last_DNET - start_time, _fed.last_DNET.microstep, tag.time - start_time, tag.microstep); + _fed.last_DNET.time - start_time, _fed.last_DNET.microstep, tag.time - start_time, tag.microstep); } return _fed.last_TAG; } From 5e09b528e39252f8df402366c8d6d5cb9aad3f0d Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Thu, 15 Aug 2024 11:14:41 -0700 Subject: [PATCH 28/43] Update comments. --- core/federated/RTI/rti_common.c | 12 ++--- core/federated/RTI/rti_common.h | 57 +++++++++++++-------- include/core/federated/network/net_common.h | 7 +-- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index e4781a038..19ec326bf 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -462,7 +462,7 @@ void update_all_downstreams(scheduling_node_t* node) { } } -tag_t get_dnet_candidate(tag_t received_tag, tag_t minimum_delay) { +tag_t get_dnet_candidate(tag_t next_event_tag, tag_t minimum_delay) { // (A.t, A.m) - (B.t - B.m) // B cannot be NEVER_TAG as (0, 0) denotes no delay. // Also, we assume B is not FOREVER_TAG because FOREVER_TAG delay means that there is no connection. @@ -475,13 +475,13 @@ tag_t get_dnet_candidate(tag_t received_tag, tag_t minimum_delay) { // ii) If A.t >= B.t > 0 and A.m >= B.m return (A.t - B.t, UINT_MAX) // iii) If A.t >= B.t > 0 and A.m < B.m return (A.t - B.t - 1, UINT_MAX) - if (received_tag.time == NEVER || lf_tag_compare(received_tag, minimum_delay) < 0) + if (next_event_tag.time == NEVER || lf_tag_compare(next_event_tag, minimum_delay) < 0) return NEVER_TAG; - if (received_tag.time == FOREVER) + if (next_event_tag.time == FOREVER) return FOREVER_TAG; - tag_t result = {.time = received_tag.time - minimum_delay.time, - .microstep = received_tag.microstep - minimum_delay.microstep}; - if (received_tag.microstep < minimum_delay.microstep) { + tag_t result = {.time = next_event_tag.time - minimum_delay.time, + .microstep = next_event_tag.microstep - minimum_delay.microstep}; + if (next_event_tag.microstep < minimum_delay.microstep) { result.time -= 1; result.microstep = UINT_MAX; } else { diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 2ac967acb..5582d527a 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -256,18 +256,6 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e); */ tag_t eimt_strict(scheduling_node_t* e); -/** - * Return true if the node is in a zero-delay cycle. - * @param node The node. - */ -bool is_in_zero_delay_cycle(scheduling_node_t* node); - -/** - * Return true if the node is in a cycle (possibly a zero-delay cycle). - * @param node The node. - */ -bool is_in_cycle(scheduling_node_t* node); - /** * For the given scheduling node (enclave or federate), if necessary, update the `min_delays`, * `all_upstremas`, `num_all_upstreams`, and the fields that indicate cycles. These fields will be @@ -286,27 +274,52 @@ void update_min_delays_upstream(scheduling_node_t* node); void update_all_downstreams(scheduling_node_t* node); /** - * Find the tag g that is the latest tag that satisfies lf_tag_add(g, minimum_delay) < received_tag. - * This function behaves like the tag subtraction, received_tag - minimum_delay. + * Find the tag g that is the latest tag that satisfies lf_tag_add(g, minimum_delay) < next_event_tag. + * This function behaves like the tag subtraction, next_event_tag - minimum_delay. * minimum_delay cannot be NEVER. - * @param received_tag - * @param minimum_delay + * + * This function is called in function downstream_next_event_tag. + * @param next_event_tag The next event tag of a downstream node. + * @param minimum_delay The minimum delay between the target upstream node and the downstream node. + */ +tag_t get_dnet_candidate(tag_t next_event_tag, tag_t minimum_delay); + +/** + * @brief Determine whether the specified scheduling node is needed to receive a downstream next event tag (DNET), + * and, if so, return the details. + * + * This function is called upon receiving a NET from one of the specified node's downstream nodes. + * + * This function calculates the minimum tag M over + * all downstream scheduling nodes of the most recent NET from that node minus the "after delay" (see function + * get_dnet_candidate). If M is earlier than the startup tag, then set the result as the NEVER_TAG. + * + * @param node The target node that may receive a new DNET. + * @param node_sending_new_net_id The ID of the node that sends a new NET. If this node's new NET does not + * change the DNET value, we can exit this function immediately. If it does, we have to look up the target node's + * downstream federates to compute the exact new DNET value. + * @return If needed, return the tag value. Otherwise, return the NEVER_TAG. */ -tag_t get_dnet_candidate(tag_t received_tag, tag_t minimum_delay); +tag_t downstream_next_event_tag(scheduling_node_t* node, uint16_t node_sending_new_net_id); /** + * Notify a downstream next event tag (DNET) signal to the specified scheduling node. * @param e The target node. * @param tag The downstream next event tag for e. */ void notify_downstream_next_event_tag(scheduling_node_t* e, tag_t tag); /** - * @param node The target node that may receive a new DNET. - * @param node_sending_new_net_id The ID of the node that sends a new NET. If this node's new NET does not - * change the DNET value, we can exit this function immediately. If it does, we have to look up the target node's - * downstream federates to compute the exact new DNET value. + * Return true if the node is in a zero-delay cycle. + * @param node The node. */ -tag_t downstream_next_event_tag(scheduling_node_t* node, uint16_t node_sending_new_net_id); +bool is_in_zero_delay_cycle(scheduling_node_t* node); + +/** + * Return true if the node is in a cycle (possibly a zero-delay cycle). + * @param node The node. + */ +bool is_in_cycle(scheduling_node_t* node); /** * For the given scheduling node (enclave or federate), invalidate the `min_delays`, diff --git a/include/core/federated/network/net_common.h b/include/core/federated/network/net_common.h index 129bc7741..999685e48 100644 --- a/include/core/federated/network/net_common.h +++ b/include/core/federated/network/net_common.h @@ -675,9 +675,10 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * from the RTI in centralized coordination. * The next eight bytes will be the timestamp. * The next four bytes will be the microstep. - * This message from the RTI tells the federate the tag of the earliest event on the - * downstream federate's event queue. In other words, this federate doesn't have to send - * LTC and NET messages earlier than this tag. + * This signal from the RTI tells the destination federate the latest tag that + * the federate can safely skip sending a next event tag (NET) signal. + * In other words, the federate doesn't have to send NET signals with tags + * earlier than or equal to this tag unless it cannot advance to its next event tag. */ #define MSG_TYPE_DOWNSTREAM_NEXT_EVENT_TAG 26 From 530b0cd2783c8dc8c9aab563add96f3dbc38eec6 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun <78055940+byeonggiljun@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:38:01 -0700 Subject: [PATCH 29/43] Apply suggestions from code review Co-authored-by: Edward A. Lee --- core/federated/RTI/rti_common.c | 6 ++++++ core/federated/RTI/rti_common.h | 15 ++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 19ec326bf..2b673cd6a 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -18,6 +18,12 @@ static rti_common_t* rti_common = NULL; // Global variables defined in tag.c: extern instant_t start_time; +/** + * @brief Return the greatest tag earlier than the given tag. + * + * If the given tag is `FOREVER_TAG` or `NEVER_TAG`, however, just return the given tag. + * @param tag The tag. + */ static tag_t latest_earlier_tag(tag_t tag) { if (tag.time == NEVER || tag.time == FOREVER) { return tag; diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 5582d527a..2ca9a9a9d 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -66,7 +66,7 @@ typedef struct scheduling_node_t { uint16_t num_all_upstreams; // Size of the array of all upstream scheduling nodes and delays. uint16_t* all_downstreams; // Array of all downstream scheduling node ids. uint16_t num_all_downstreams; // Size of the array of all downstream scheduling nodes. - int flags; // Or of IS_IN_ZERO_DELAY_CYCLE, IS_IN_CYCLE + int flags; // One of IS_IN_ZERO_DELAY_CYCLE, IS_IN_CYCLE } scheduling_node_t; /** @@ -81,9 +81,10 @@ typedef struct rti_common_t { // Number of scheduling nodes uint16_t number_of_scheduling_nodes; - // Matrix of minimum delays between each nodes - // Rows represent upstreams and Columns represent downstreams. - // FOREVER_TAG means there is no path, ZERO_TAG means there is no delay. + // Matrix of minimum delays between pairs of nodes. + // Rows represent upstream nodes and Columns represent downstream nodes. + // FOREVER_TAG means there is no path, and ZERO_TAG means there is no delay. + // This could be NULL if the matrix is not being used, so accesses should test for NULL first. tag_t* min_delays; // RTI's decided stop tag for the scheduling nodes @@ -258,7 +259,7 @@ tag_t eimt_strict(scheduling_node_t* e); /** * For the given scheduling node (enclave or federate), if necessary, update the `min_delays`, - * `all_upstremas`, `num_all_upstreams`, and the fields that indicate cycles. These fields will be + * `all_upstreams`, `num_all_upstreams`, and the fields that indicate cycles. These fields will be * updated only if they have not been previously updated or if invalidate_min_delays_upstream * has been called since they were last updated. * @param node The node. @@ -266,8 +267,8 @@ tag_t eimt_strict(scheduling_node_t* e); void update_min_delays_upstream(scheduling_node_t* node); /** - * For the given scheduling node (enclave or federate), if necessary, update the `all_downstreams`, - * `num_all_downstreams`. These fields will be updated only if they have not been previously updated + * For the given scheduling node (enclave or federate), if necessary, update the `all_downstreams` and + * `num_all_downstreams` fields. These fields will be updated only if they have not been previously updated * or if invalidate_min_delays_upstream has been called since they were last updated. * @param node The node. */ From 2857ab87ff374b9a818eedae119327299a9e45fa Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Fri, 23 Aug 2024 16:01:05 -0700 Subject: [PATCH 30/43] Move function `lf_tag_latest_earlier` to tag.c --- core/federated/RTI/rti_common.c | 22 +--------------------- core/tag.c | 20 ++++++++++++++++++++ tag/api/tag.h | 8 ++++++++ 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 2b673cd6a..a571507ab 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -18,26 +18,6 @@ static rti_common_t* rti_common = NULL; // Global variables defined in tag.c: extern instant_t start_time; -/** - * @brief Return the greatest tag earlier than the given tag. - * - * If the given tag is `FOREVER_TAG` or `NEVER_TAG`, however, just return the given tag. - * @param tag The tag. - */ -static tag_t latest_earlier_tag(tag_t tag) { - if (tag.time == NEVER || tag.time == FOREVER) { - return tag; - } else if (tag.time == 0 && tag.microstep == 0) { - return NEVER_TAG; - } else if (tag.microstep == 0) { - tag.time -= 1; - tag.microstep = UINT_MAX; - } else { - tag.microstep -= 1; - } - return tag; -} - void initialize_rti_common(rti_common_t* _rti_common) { rti_common = _rti_common; rti_common->max_stop_tag = NEVER_TAG; @@ -251,7 +231,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) { e->id, t_d.time - lf_time_start(), t_d.microstep, e->next_event.time - lf_time_start(), e->next_event.microstep); // result.tag = e->next_event; - result.tag = latest_earlier_tag(t_d); + result.tag = lf_tag_latest_earlier(t_d); } else if ( // Scenario (2) above lf_tag_compare(t_d, e->next_event) == 0 // EIMT equal to NET && is_in_zero_delay_cycle(e) // The node is part of a ZDC diff --git a/core/tag.c b/core/tag.c index c5f8f88c6..d20daf6ae 100644 --- a/core/tag.c +++ b/core/tag.c @@ -124,6 +124,26 @@ tag_t lf_delay_strict(tag_t tag, interval_t interval) { return result; } +/** + * @brief Return the greatest tag earlier than the given tag. + * + * If the given tag is `FOREVER_TAG` or `NEVER_TAG`, however, just return the given tag. + * @param tag The tag. + */ +tag_t lf_tag_latest_earlier(tag_t tag) { + if (lf_tag_compare(tag, NEVER_TAG) || lf_tag_compare(tag, FOREVER_TAG)) { + return tag; + } else if (tag.time == 0 && tag.microstep == 0) { + return NEVER_TAG; + } else if (tag.microstep == 0) { + tag.time -= 1; + tag.microstep = UINT_MAX; + } else { + tag.microstep -= 1; + } + return tag; +} + instant_t lf_time_logical(void* env) { assert(env != GLOBAL_ENVIRONMENT); return ((environment_t*)env)->current_tag.time; diff --git a/tag/api/tag.h b/tag/api/tag.h index c40e490f8..f4f77fd56 100644 --- a/tag/api/tag.h +++ b/tag/api/tag.h @@ -162,6 +162,14 @@ tag_t lf_delay_tag(tag_t tag, interval_t interval); */ tag_t lf_delay_strict(tag_t tag, interval_t interval); +/** + * @brief Return the greatest tag earlier than the given tag. + * + * If the given tag is `FOREVER_TAG` or `NEVER_TAG`, however, just return the given tag. + * @param tag The tag. + */ +tag_t lf_tag_latest_earlier(tag_t tag); + /** * Return the current logical time in nanoseconds. * On many platforms, this is the number of nanoseconds From 4507043ca782ed8e19901fc58fd5cf5ea06422d9 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Fri, 23 Aug 2024 18:00:44 -0700 Subject: [PATCH 31/43] Invalidate every node's delay information in `invalidate_min_delays` --- core/federated/RTI/rti_common.c | 38 +++++++++++++++-------- core/federated/RTI/rti_common.h | 13 +++----- core/federated/RTI/rti_remote.c | 8 ++--- core/federated/RTI/test/rti_common_test.c | 2 +- 4 files changed, 34 insertions(+), 27 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index a571507ab..1c7038d04 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -33,20 +33,28 @@ void initialize_rti_common(rti_common_t* _rti_common) { #define IS_IN_ZERO_DELAY_CYCLE 1 #define IS_IN_CYCLE 2 -void invalidate_min_delays_upstream(scheduling_node_t* node) { - if (node->all_upstreams != NULL) { - free(node->all_upstreams); - for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) { - rti_common->min_delays[i * rti_common->number_of_scheduling_nodes + node->id] = FOREVER_TAG; +void invalidate_min_delays() { + if (rti_common->min_delays != NULL) { + uint16_t n = rti_common->number_of_scheduling_nodes; + for (uint16_t i = 0; i < n; i++) { + for (uint16_t j = 0; j < n; j++) { + rti_common->min_delays[i + j * n] = FOREVER_TAG; + } + scheduling_node_t* node = rti_common->scheduling_nodes[i]; + if (node->all_upstreams != NULL) + free(node->all_upstreams); + + if (node->all_downstreams != NULL) + free(node->all_downstreams); + + node->all_upstreams = NULL; + node->num_all_upstreams = 0; + node->all_downstreams = NULL; + node->num_all_downstreams = 0; + node->flags = 0; // All flags cleared because they get set lazily. } + free(rti_common->min_delays); } - if (node->all_downstreams != NULL) - free(node->all_downstreams); - node->all_upstreams = NULL; - node->num_all_upstreams = 0; - node->all_downstreams = NULL; - node->num_all_downstreams = 0; - node->flags = 0; // All flags cleared because they get set lazily. } void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) { @@ -63,7 +71,11 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) { e->immediate_downstreams = NULL; e->num_immediate_downstreams = 0; e->mode = REALTIME; - invalidate_min_delays_upstream(e); + e->all_upstreams = NULL; + e->num_all_upstreams = 0; + e->all_downstreams = NULL; + e->num_all_upstreams = 0; + e->flags = 0; } void _logical_tag_complete(scheduling_node_t* enclave, tag_t completed) { diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 2ca9a9a9d..9db3f541b 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -260,7 +260,7 @@ tag_t eimt_strict(scheduling_node_t* e); /** * For the given scheduling node (enclave or federate), if necessary, update the `min_delays`, * `all_upstreams`, `num_all_upstreams`, and the fields that indicate cycles. These fields will be - * updated only if they have not been previously updated or if invalidate_min_delays_upstream + * updated only if they have not been previously updated or if invalidate_min_delays * has been called since they were last updated. * @param node The node. */ @@ -269,7 +269,7 @@ void update_min_delays_upstream(scheduling_node_t* node); /** * For the given scheduling node (enclave or federate), if necessary, update the `all_downstreams` and * `num_all_downstreams` fields. These fields will be updated only if they have not been previously updated - * or if invalidate_min_delays_upstream has been called since they were last updated. + * or if invalidate_min_delays has been called since they were last updated. * @param node The node. */ void update_all_downstreams(scheduling_node_t* node); @@ -323,13 +323,10 @@ bool is_in_zero_delay_cycle(scheduling_node_t* node); bool is_in_cycle(scheduling_node_t* node); /** - * For the given scheduling node (enclave or federate), invalidate the `min_delays`, - * `num_min_delays`, and the fields that indicate cycles. - * This should be called whenever the structure of the connections upstream of the - * given node have changed. - * @param node The node. + * Invalidate the `min_delays`, `num_min_delays`, and the fields that indicate cycles + * of all nodes. This should be called whenever the structure of the connections have changed. */ -void invalidate_min_delays_upstream(scheduling_node_t* node); +void invalidate_min_delays(); /** * Free dynamically allocated memory on the scheduling nodes and the scheduling node array itself. diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index c1b886bac..81b3645a3 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -1776,22 +1776,20 @@ void clock_sync_add_offset(instant_t* t) { (void)t; } void clock_sync_subtract_offset(instant_t* t) { (void)t; } void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number_of_scheduling_nodes) { + invalidate_min_delays(); for (uint16_t i = 0; i < number_of_scheduling_nodes; i++) { scheduling_node_t* node = scheduling_nodes[i]; if (node->immediate_upstreams != NULL) { free(node->immediate_upstreams); free(node->immediate_upstream_delays); - free(node->all_upstreams); + // free(node->all_upstreams); } if (node->immediate_downstreams != NULL) { free(node->immediate_downstreams); - free(node->all_downstreams); + // free(node->all_downstreams); } free(node); } - if (rti_remote->base.min_delays != NULL) { - free(rti_remote->base.min_delays); - } free(scheduling_nodes); } diff --git a/core/federated/RTI/test/rti_common_test.c b/core/federated/RTI/test/rti_common_test.c index a077018e5..d7354d2aa 100644 --- a/core/federated/RTI/test/rti_common_test.c +++ b/core/federated/RTI/test/rti_common_test.c @@ -26,7 +26,6 @@ void delete_scheduling_node(scheduling_node_t* node) { if (node->immediate_downstreams != NULL) { free(node->immediate_downstreams); } - invalidate_min_delays_upstream(node); } /** @@ -72,6 +71,7 @@ void set_scheduling_node(int id, int num_immediate_upstreams, int num_immediate_ * This includes freeing every scheduling node and the array of nodes. */ void reset_common_RTI() { + invalidate_min_delays(); // For every scheduling nodes, delete them and free themselves, too. for (uint16_t i = 0; i < test_rti.number_of_scheduling_nodes; i++) { delete_scheduling_node(test_rti.scheduling_nodes[i]); From 896e15293bd21958a124d7f73b02c455340f81d7 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Sat, 24 Aug 2024 21:59:41 -0700 Subject: [PATCH 32/43] Apply suggestions from @edwardalee and minor fix --- core/tag.c | 10 +--------- include/core/federated/network/net_common.h | 8 ++++---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/core/tag.c b/core/tag.c index d20daf6ae..cc8d17545 100644 --- a/core/tag.c +++ b/core/tag.c @@ -124,17 +124,9 @@ tag_t lf_delay_strict(tag_t tag, interval_t interval) { return result; } -/** - * @brief Return the greatest tag earlier than the given tag. - * - * If the given tag is `FOREVER_TAG` or `NEVER_TAG`, however, just return the given tag. - * @param tag The tag. - */ tag_t lf_tag_latest_earlier(tag_t tag) { - if (lf_tag_compare(tag, NEVER_TAG) || lf_tag_compare(tag, FOREVER_TAG)) { + if (lf_tag_compare(tag, NEVER_TAG) == 0 || lf_tag_compare(tag, FOREVER_TAG) == 0) { return tag; - } else if (tag.time == 0 && tag.microstep == 0) { - return NEVER_TAG; } else if (tag.microstep == 0) { tag.time -= 1; tag.microstep = UINT_MAX; diff --git a/include/core/federated/network/net_common.h b/include/core/federated/network/net_common.h index 999685e48..47d8bade5 100644 --- a/include/core/federated/network/net_common.h +++ b/include/core/federated/network/net_common.h @@ -675,10 +675,10 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * from the RTI in centralized coordination. * The next eight bytes will be the timestamp. * The next four bytes will be the microstep. - * This signal from the RTI tells the destination federate the latest tag that - * the federate can safely skip sending a next event tag (NET) signal. - * In other words, the federate doesn't have to send NET signals with tags - * earlier than or equal to this tag unless it cannot advance to its next event tag. + * This signal from the RTI tells the destination federate that downstream + * federates do not need for it to send any next event tag (NET) signal + * with a tag _g_ less than the specified tag. Thus, it should only send + * those signals if needs permission from the RTI to advance to _g_. */ #define MSG_TYPE_DOWNSTREAM_NEXT_EVENT_TAG 26 From 0fab3b6392b4a6cf87516b120fa9ec6d565c2562 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Sat, 24 Aug 2024 23:50:42 -0700 Subject: [PATCH 33/43] Run Clang format --- include/core/federated/network/net_common.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/core/federated/network/net_common.h b/include/core/federated/network/net_common.h index 47d8bade5..a658ef7c2 100644 --- a/include/core/federated/network/net_common.h +++ b/include/core/federated/network/net_common.h @@ -675,8 +675,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * from the RTI in centralized coordination. * The next eight bytes will be the timestamp. * The next four bytes will be the microstep. - * This signal from the RTI tells the destination federate that downstream - * federates do not need for it to send any next event tag (NET) signal + * This signal from the RTI tells the destination federate that downstream + * federates do not need for it to send any next event tag (NET) signal * with a tag _g_ less than the specified tag. Thus, it should only send * those signals if needs permission from the RTI to advance to _g_. */ From a757105d20c8faa0313f4278e35e27d7c701892f Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Fri, 18 Oct 2024 16:43:03 -0700 Subject: [PATCH 34/43] Remove the fields all_upstream and all_downstream --- core/federated/RTI/main.c | 5 - core/federated/RTI/rti_common.c | 262 ++++++++----------- core/federated/RTI/rti_common.h | 11 +- core/federated/RTI/test/rti_common_test.c | 292 ++++++++++++++-------- 4 files changed, 297 insertions(+), 273 deletions(-) diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c index c24415a04..4853491cd 100644 --- a/core/federated/RTI/main.c +++ b/core/federated/RTI/main.c @@ -321,12 +321,7 @@ int main(int argc, const char* argv[]) { // Allocate memory for the federates int n = rti.base.number_of_scheduling_nodes; rti.base.scheduling_nodes = (scheduling_node_t**)calloc(n, sizeof(scheduling_node_t*)); - // Allocate memory for the array of min_delays. - rti.base.min_delays = (tag_t*)calloc((n * n), sizeof(tag_t)); for (uint16_t i = 0; i < n; i++) { - for (uint16_t j = 0; j < n; j++) { - rti.base.min_delays[i + j * n] = FOREVER_TAG; - } federate_info_t* fed_info = (federate_info_t*)calloc(1, sizeof(federate_info_t)); initialize_federate(fed_info, i); rti.base.scheduling_nodes[i] = (scheduling_node_t*)fed_info; diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 1c7038d04..1837a30f6 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -20,6 +20,7 @@ extern instant_t start_time; void initialize_rti_common(rti_common_t* _rti_common) { rti_common = _rti_common; + rti_common->min_delays = NULL; rti_common->max_stop_tag = NEVER_TAG; rti_common->number_of_scheduling_nodes = 0; rti_common->num_scheduling_nodes_handling_stop = 0; @@ -37,20 +38,7 @@ void invalidate_min_delays() { if (rti_common->min_delays != NULL) { uint16_t n = rti_common->number_of_scheduling_nodes; for (uint16_t i = 0; i < n; i++) { - for (uint16_t j = 0; j < n; j++) { - rti_common->min_delays[i + j * n] = FOREVER_TAG; - } scheduling_node_t* node = rti_common->scheduling_nodes[i]; - if (node->all_upstreams != NULL) - free(node->all_upstreams); - - if (node->all_downstreams != NULL) - free(node->all_downstreams); - - node->all_upstreams = NULL; - node->num_all_upstreams = 0; - node->all_downstreams = NULL; - node->num_all_downstreams = 0; node->flags = 0; // All flags cleared because they get set lazily. } free(rti_common->min_delays); @@ -71,10 +59,6 @@ void initialize_scheduling_node(scheduling_node_t* e, uint16_t id) { e->immediate_downstreams = NULL; e->num_immediate_downstreams = 0; e->mode = REALTIME; - e->all_upstreams = NULL; - e->num_all_upstreams = 0; - e->all_downstreams = NULL; - e->num_all_upstreams = 0; e->flags = 0; } @@ -106,39 +90,39 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) { // First, we need to find the shortest path (minimum delay) path to each upstream node // and then find the minimum of the node's recorded NET plus the minimum path delay. // Update the shortest paths, if necessary. - update_min_delays_upstream(e); - update_all_downstreams(e); + update_min_delays(); // Next, find the tag of the earliest possible incoming message from upstream enclaves or // federates, which will be the smallest upstream NET plus the least delay. // This could be NEVER_TAG if the RTI has not seen a NET from some upstream node. tag_t t_d = FOREVER_TAG; int n = rti_common->number_of_scheduling_nodes; - for (int i = 0; i < e->num_all_upstreams; i++) { - // Node e->all_upstreams[i] is upstream of e with - // min delay rti_common->min_delays[e->all_upstreams[i]*n + e->id] - scheduling_node_t* upstream = rti_common->scheduling_nodes[e->all_upstreams[i]]; - // If we haven't heard from the upstream node, then assume it can send an event at the start time. - if (lf_tag_compare(upstream->next_event, NEVER_TAG) == 0) { - tag_t start_tag = {.time = start_time, .microstep = 0}; - upstream->next_event = start_tag; - } - // The min_delay here is a tag_t, not an interval_t because it may account for more than - // one connection. No delay at all is represented by (0,0). A delay of 0 is represented - // by (0,1). If the time part of the delay is greater than 0, then we want to ignore - // the microstep in upstream->next_event because that microstep will have been lost. - // Otherwise, we want preserve it and add to it. This is handled by lf_tag_add(). - tag_t earliest_tag_from_upstream = - lf_tag_add(upstream->next_event, rti_common->min_delays[e->all_upstreams[i] * n + e->id]); - - /* Following debug message is too verbose for normal use: - LF_PRINT_DEBUG("RTI: Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".", - e->id, - upstream->id, - earliest_tag_from_upstream.time - start_time, earliest_tag_from_upstream.microstep); - */ - if (lf_tag_compare(earliest_tag_from_upstream, t_d) < 0) { - t_d = earliest_tag_from_upstream; + for (int i = 0; i < n; i++) { + if (lf_tag_compare(rti_common->min_delays[i * n + e->id], FOREVER_TAG) != 0) { + // Node i is upstream of e with min delay rti_common->min_delays[i * n + e->id] + scheduling_node_t* upstream = rti_common->scheduling_nodes[i]; + // If we haven't heard from the upstream node, then assume it can send an event at the start time. + if (lf_tag_compare(upstream->next_event, NEVER_TAG) == 0) { + tag_t start_tag = {.time = start_time, .microstep = 0}; + upstream->next_event = start_tag; + } + // The min_delay here is a tag_t, not an interval_t because it may account for more than + // one connection. No delay at all is represented by (0,0). A delay of 0 is represented + // by (0,1). If the time part of the delay is greater than 0, then we want to ignore + // the microstep in upstream->next_event because that microstep will have been lost. + // Otherwise, we want preserve it and add to it. This is handled by lf_tag_add(). + tag_t earliest_tag_from_upstream = + lf_tag_add(upstream->next_event, rti_common->min_delays[i * n + e->id]); + + /* Following debug message is too verbose for normal use: + LF_PRINT_DEBUG("RTI: Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".", + e->id, + upstream->id, + earliest_tag_from_upstream.time - start_time, earliest_tag_from_upstream.microstep); + */ + if (lf_tag_compare(earliest_tag_from_upstream, t_d) < 0) { + t_d = earliest_tag_from_upstream; + } } } return t_d; @@ -289,24 +273,28 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne // Even though there was no grant, mark the tag as if there was. e->last_granted = next_event_tag; } + + update_min_delays(); // Check downstream scheduling_nodes to see whether they should now be granted a TAG. - update_all_downstreams(e); - for (int i = 0; i < e->num_all_downstreams; i++) { - scheduling_node_t* downstream = rti_common->scheduling_nodes[e->all_downstreams[i]]; - notify_advance_grant_if_safe(downstream); + int n = rti_common->number_of_scheduling_nodes; + for (int j = 0; j < n; j++) { + if (lf_tag_compare(rti_common->min_delays[e->id * n + j], FOREVER_TAG) != 0) { + // The node j is a downstream node of e. + scheduling_node_t* downstream = rti_common->scheduling_nodes[j]; + notify_advance_grant_if_safe(downstream); + } } if (rti_common->dnet_enabled) { // Send DNET to the node e's upstream federates if needed - for (int i = 0; i < e->num_all_upstreams; i++) { - int target_upstream_id = e->all_upstreams[i]; - if (target_upstream_id == e->id) { - continue; - } - scheduling_node_t* node = rti_common->scheduling_nodes[target_upstream_id]; - tag_t dnet = downstream_next_event_tag(node, e->id); - if (lf_tag_compare(node->last_DNET, dnet) != 0 && lf_tag_compare(node->next_event, dnet) <= 0) { - notify_downstream_next_event_tag(node, dnet); + for (int i = 0; i < n; i++) { + if (lf_tag_compare(rti_common->min_delays[i * n + e->id], FOREVER_TAG) != 0 && i != e->id) { + // The node i is an upstream node of e. + scheduling_node_t* upstream = rti_common->scheduling_nodes[i]; + tag_t dnet = downstream_next_event_tag(upstream, e->id); + if (lf_tag_compare(upstream->last_DNET, dnet) != 0 && lf_tag_compare(upstream->next_event, dnet) <= 0) { + notify_downstream_next_event_tag(upstream, dnet); + } } } } @@ -381,80 +369,42 @@ static void _update_min_delays_upstream(scheduling_node_t* end, scheduling_node_ } } -void update_min_delays_upstream(scheduling_node_t* node) { +void update_min_delays() { + int n = rti_common->number_of_scheduling_nodes; // Check whether cached result is valid. - if (node->all_upstreams == NULL) { - - // This is not Dijkstra's algorithm, but rather one optimized for sparse upstream nodes. - // There must be a name for this algorithm. - - // Array of results on the stack: - tag_t path_delays[rti_common->number_of_scheduling_nodes]; - // This will be the number of non-FOREVER entries put into path_delays. - size_t count = 0; - - for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) { - path_delays[i] = FOREVER_TAG; - } - _update_min_delays_upstream(node, NULL, path_delays, &count); - - // Put the results onto the matrix. - node->num_all_upstreams = count; - node->all_upstreams = (uint16_t*)calloc(count, sizeof(uint16_t)); - LF_PRINT_DEBUG("++++ Node %hu is in ZDC: %d", node->id, is_in_zero_delay_cycle(node)); - int k = 0; - for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) { - if (lf_tag_compare(path_delays[i], FOREVER_TAG) < 0) { - // Node i is upstream. - if (k >= count) { - lf_print_error_and_exit("Internal error! Count of upstream nodes %zu for node %d is wrong!", count, i); - } - rti_common->min_delays[node->id + i * rti_common->number_of_scheduling_nodes] = path_delays[i]; - node->all_upstreams[k++] = i; - // N^2 debug statement could be a problem with large benchmarks. - // LF_PRINT_DEBUG("++++ Node %hu is upstream with delay" PRINTF_TAG "\n", i, path_delays[i].time, - // path_delays[i].microstep); - } - } - } -} - -void update_all_downstreams(scheduling_node_t* node) { - if (node->all_downstreams == NULL) { - bool visited[rti_common->number_of_scheduling_nodes]; - for (int i = 0; i < rti_common->number_of_scheduling_nodes; i++) { - visited[i] = false; - } - - uint16_t queue[rti_common->number_of_scheduling_nodes]; - int front = 0, rear = 0; - - visited[node->id] = true; - queue[rear++] = node->id; - - size_t count = 0; - while (front != rear) { - int current_id = queue[front++]; - scheduling_node_t* current_node = rti_common->scheduling_nodes[current_id]; - for (uint16_t i = 0; i < current_node->num_immediate_downstreams; i++) { - uint16_t downstream_id = current_node->immediate_downstreams[i]; - if (visited[downstream_id] == false) { - visited[downstream_id] = true; - queue[rear++] = downstream_id; - count++; - } + if (rti_common->min_delays == NULL) { + // Allocate memory for the array of min_delays. + rti_common->min_delays = (tag_t*)calloc((n * n), sizeof(tag_t)); + + for (int j = 0; j < n; j++) { + // This is not Dijkstra's algorithm, but rather one optimized for sparse upstream nodes. + // There must be a name for this algorithm. + + scheduling_node_t* node = rti_common->scheduling_nodes[j]; + // Array of results on the stack: + tag_t path_delays[n]; + // This will be the number of non-FOREVER entries put into path_delays. + size_t count = 0; + + for (int i = 0; i < n; i++) { + path_delays[i] = FOREVER_TAG; } - } - - int k = 0; - node->all_downstreams = (uint16_t*)calloc(count, sizeof(uint16_t)); - node->num_all_downstreams = count; - for (uint16_t i = 0; i < rti_common->number_of_scheduling_nodes; i++) { - if (visited[i] == true && i != node->id) { - if (k >= count) { - lf_print_error_and_exit("Internal error! Count of downstream nodes %zu for node %d is wrong!", count, i); + _update_min_delays_upstream(node, NULL, path_delays, &count); + + // Put the results onto the matrix. + LF_PRINT_DEBUG("++++ Node %hu is in ZDC: %d", node->id, is_in_zero_delay_cycle(node)); + int k = 0; + for (int i = 0; i < n; i++) { + rti_common->min_delays[i * n + j] = path_delays[i]; + if (lf_tag_compare(path_delays[i], FOREVER_TAG) < 0) { + // Node i is upstream. + if (k >= count) { + lf_print_error_and_exit("Internal error! Count of upstream nodes %zu for node %d is wrong!", count, i); + } + // N^2 debug statement could be a problem with large benchmarks. + LF_PRINT_DEBUG("++++ Node %hu is upstream with delay" PRINTF_TAG "\n", i, path_delays[i].time, + path_delays[i].microstep); } - node->all_downstreams[k++] = i; } } } @@ -492,37 +442,45 @@ tag_t get_dnet_candidate(tag_t next_event_tag, tag_t minimum_delay) { return result; } -tag_t downstream_next_event_tag(scheduling_node_t* node, uint16_t node_sending_new_net_id) { - if (is_in_zero_delay_cycle(node)) { +tag_t downstream_next_event_tag(scheduling_node_t* target_node, uint16_t node_sending_new_NET_id) { + if (is_in_zero_delay_cycle(target_node)) { return NEVER_TAG; } tag_t result = FOREVER_TAG; - scheduling_node_t* node_sending_new_net = rti_common->scheduling_nodes[node_sending_new_net_id]; - if (is_in_zero_delay_cycle(node_sending_new_net)) { + scheduling_node_t* node_sending_new_NET = rti_common->scheduling_nodes[node_sending_new_NET_id]; + if (is_in_zero_delay_cycle(node_sending_new_NET)) { return NEVER_TAG; } - int index = node->id * rti_common->number_of_scheduling_nodes + node_sending_new_net_id; - tag_t candidate = get_dnet_candidate(node_sending_new_net->next_event, rti_common->min_delays[index]); - - if (lf_tag_compare(node->last_DNET, candidate) >= 0) { + int n = rti_common->number_of_scheduling_nodes; + int index = target_node->id * n + node_sending_new_NET_id; + tag_t candidate = get_dnet_candidate(node_sending_new_NET->next_event, rti_common->min_delays[index]); + + if (lf_tag_compare(target_node->last_DNET, candidate) >= 0) { + // This function is called because a downstream node of target_node sent a new NET. + // If the candidate computed by that downstream node is earlier than or equal to the last DNET, + // this candidate must be the minimum among every candidate. + // Else if this candidate is later than the last DNET, we need to loop up every downstream node + // to compute the DNET value. result = candidate; } else { - for (int i = 0; i < node->num_all_downstreams; i++) { - uint16_t target_downstream_id = node->all_downstreams[i]; - scheduling_node_t* target_dowstream = rti_common->scheduling_nodes[target_downstream_id]; - - if (is_in_zero_delay_cycle(target_dowstream)) { - // This node is an upstream of ZDC. Do not send DNET to this node. - return NEVER_TAG; - } - - index = node->id * rti_common->number_of_scheduling_nodes + target_downstream_id; - candidate = get_dnet_candidate(target_dowstream->next_event, rti_common->min_delays[index]); - - if (lf_tag_compare(result, candidate) > 0) { - result = candidate; + for (int j = 0; j < n; j++) { + if (lf_tag_compare(rti_common->min_delays[target_node->id * n + j], FOREVER_TAG) != 0) { + // The node j is a downstream node. + scheduling_node_t* target_dowstream = rti_common->scheduling_nodes[j]; + // if (is_in_zero_delay_cycle(target_dowstream)) { + // // The target node is an upstream of ZDC. Do not send DNET to this node. + // return NEVER_TAG; + // } + + // Minimum tag increment between the node and its downstream node. + tag_t delay = rti_common->min_delays[target_node->id * n + j]; + candidate = get_dnet_candidate(target_dowstream->next_event, delay); + + if (lf_tag_compare(result, candidate) > 0) { + result = candidate; + } } } } @@ -536,12 +494,12 @@ tag_t downstream_next_event_tag(scheduling_node_t* node, uint16_t node_sending_n } bool is_in_zero_delay_cycle(scheduling_node_t* node) { - update_min_delays_upstream(node); + update_min_delays(); return node->flags & IS_IN_ZERO_DELAY_CYCLE; } bool is_in_cycle(scheduling_node_t* node) { - update_min_delays_upstream(node); + update_min_delays(); return node->flags & IS_IN_CYCLE; } diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 9db3f541b..418f324e1 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -62,10 +62,6 @@ typedef struct scheduling_node_t { uint16_t* immediate_downstreams; // Array of immediate downstream scheduling node ids. uint16_t num_immediate_downstreams; // Size of the array of immediate downstream scheduling nodes. execution_mode_t mode; // FAST or REALTIME. - uint16_t* all_upstreams; // Array of all upstream scheduling node ids. - uint16_t num_all_upstreams; // Size of the array of all upstream scheduling nodes and delays. - uint16_t* all_downstreams; // Array of all downstream scheduling node ids. - uint16_t num_all_downstreams; // Size of the array of all downstream scheduling nodes. int flags; // One of IS_IN_ZERO_DELAY_CYCLE, IS_IN_CYCLE } scheduling_node_t; @@ -258,13 +254,12 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e); tag_t eimt_strict(scheduling_node_t* e); /** - * For the given scheduling node (enclave or federate), if necessary, update the `min_delays`, - * `all_upstreams`, `num_all_upstreams`, and the fields that indicate cycles. These fields will be - * updated only if they have not been previously updated or if invalidate_min_delays + * If necessary, update the `min_delays` and the fields that indicate cycles. + * These fields will be updated only if they have not been previously updated or if invalidate_min_delays * has been called since they were last updated. * @param node The node. */ -void update_min_delays_upstream(scheduling_node_t* node); +void update_min_delays(); /** * For the given scheduling node (enclave or federate), if necessary, update the `all_downstreams` and diff --git a/core/federated/RTI/test/rti_common_test.c b/core/federated/RTI/test/rti_common_test.c index d7354d2aa..d93a7dc31 100644 --- a/core/federated/RTI/test/rti_common_test.c +++ b/core/federated/RTI/test/rti_common_test.c @@ -7,7 +7,7 @@ #include "tag.h" // The RTI under test. -static rti_common_t test_rti; +static rti_common_t test_RTI; /******************************************Start of Utility * Functions******************************************************/ @@ -26,6 +26,7 @@ void delete_scheduling_node(scheduling_node_t* node) { if (node->immediate_downstreams != NULL) { free(node->immediate_downstreams); } + free(node); } /** @@ -42,26 +43,26 @@ void delete_scheduling_node(scheduling_node_t* node) { void set_scheduling_node(int id, int num_immediate_upstreams, int num_immediate_downstreams, int* immediate_upstreams, interval_t* immediate_upstream_delays, int* immediate_downstreams) { // Save the number of immediate upstream and immediate downstream nodes. - test_rti.scheduling_nodes[id]->num_immediate_upstreams = num_immediate_upstreams; - test_rti.scheduling_nodes[id]->num_immediate_downstreams = num_immediate_downstreams; + test_RTI.scheduling_nodes[id]->num_immediate_upstreams = num_immediate_upstreams; + test_RTI.scheduling_nodes[id]->num_immediate_downstreams = num_immediate_downstreams; // If there is any immediate upstream nodes, store IDs and delays from the upstream nodes into the structure. - if (test_rti.scheduling_nodes[id]->num_immediate_upstreams > 0) { - test_rti.scheduling_nodes[id]->immediate_upstreams = - (uint16_t*)calloc(test_rti.scheduling_nodes[id]->num_immediate_upstreams, sizeof(uint16_t)); - test_rti.scheduling_nodes[id]->immediate_upstream_delays = - (interval_t*)calloc(test_rti.scheduling_nodes[id]->num_immediate_upstreams, sizeof(interval_t)); - for (int i = 0; i < test_rti.scheduling_nodes[id]->num_immediate_upstreams; i++) { - test_rti.scheduling_nodes[id]->immediate_upstreams[i] = immediate_upstreams[i]; - test_rti.scheduling_nodes[id]->immediate_upstream_delays[i] = immediate_upstream_delays[i]; + if (test_RTI.scheduling_nodes[id]->num_immediate_upstreams > 0) { + test_RTI.scheduling_nodes[id]->immediate_upstreams = + (uint16_t*)calloc(test_RTI.scheduling_nodes[id]->num_immediate_upstreams, sizeof(uint16_t)); + test_RTI.scheduling_nodes[id]->immediate_upstream_delays = + (interval_t*)calloc(test_RTI.scheduling_nodes[id]->num_immediate_upstreams, sizeof(interval_t)); + for (int i = 0; i < test_RTI.scheduling_nodes[id]->num_immediate_upstreams; i++) { + test_RTI.scheduling_nodes[id]->immediate_upstreams[i] = immediate_upstreams[i]; + test_RTI.scheduling_nodes[id]->immediate_upstream_delays[i] = immediate_upstream_delays[i]; } } // If there is any immediate downstream nodes, store IDs of the downstream nodes into the structure. - if (test_rti.scheduling_nodes[id]->num_immediate_downstreams > 0) { - test_rti.scheduling_nodes[id]->immediate_downstreams = - (uint16_t*)calloc(test_rti.scheduling_nodes[id]->num_immediate_downstreams, sizeof(uint16_t)); - for (int i = 0; i < test_rti.scheduling_nodes[id]->num_immediate_downstreams; i++) { - test_rti.scheduling_nodes[id]->immediate_downstreams[i] = immediate_downstreams[i]; + if (test_RTI.scheduling_nodes[id]->num_immediate_downstreams > 0) { + test_RTI.scheduling_nodes[id]->immediate_downstreams = + (uint16_t*)calloc(test_RTI.scheduling_nodes[id]->num_immediate_downstreams, sizeof(uint16_t)); + for (int i = 0; i < test_RTI.scheduling_nodes[id]->num_immediate_downstreams; i++) { + test_RTI.scheduling_nodes[id]->immediate_downstreams[i] = immediate_downstreams[i]; } } } @@ -73,14 +74,13 @@ void set_scheduling_node(int id, int num_immediate_upstreams, int num_immediate_ void reset_common_RTI() { invalidate_min_delays(); // For every scheduling nodes, delete them and free themselves, too. - for (uint16_t i = 0; i < test_rti.number_of_scheduling_nodes; i++) { - delete_scheduling_node(test_rti.scheduling_nodes[i]); - free(test_rti.scheduling_nodes[i]); + for (uint16_t i = 0; i < test_RTI.number_of_scheduling_nodes; i++) { + delete_scheduling_node(test_RTI.scheduling_nodes[i]); } // Free the array of scheduling nodes either. This will be re-created // in set_common_RTI(). - if (test_rti.scheduling_nodes != NULL) { - free(test_rti.scheduling_nodes); + if (test_RTI.scheduling_nodes != NULL) { + free(test_RTI.scheduling_nodes); } } @@ -90,18 +90,18 @@ void reset_common_RTI() { * @param num_nodes The number of scheduling nodes. */ void set_common_RTI(uint16_t num_nodes) { - reset_common_RTI(); - - test_rti.number_of_scheduling_nodes = num_nodes; + test_RTI.number_of_scheduling_nodes = num_nodes; // Allocate memory for the scheduling nodes - test_rti.scheduling_nodes = - (scheduling_node_t**)calloc(test_rti.number_of_scheduling_nodes, sizeof(scheduling_node_t*)); - test_rti.min_delays = (tag_t*)calloc((num_nodes * num_nodes), sizeof(tag_t)); - for (uint16_t i = 0; i < test_rti.number_of_scheduling_nodes; i++) { + test_RTI.scheduling_nodes = + (scheduling_node_t**)calloc(test_RTI.number_of_scheduling_nodes, sizeof(scheduling_node_t*)); + + test_RTI.min_delays = NULL; + + for (uint16_t i = 0; i < test_RTI.number_of_scheduling_nodes; i++) { scheduling_node_t* scheduling_node = (scheduling_node_t*)malloc(sizeof(scheduling_node_t)); initialize_scheduling_node(scheduling_node, i); - test_rti.scheduling_nodes[i] = scheduling_node; + test_RTI.scheduling_nodes[i] = scheduling_node; } } @@ -111,8 +111,8 @@ void set_common_RTI(uint16_t num_nodes) { * @param state The state that every scheduling node will have. */ void set_state_of_nodes(scheduling_node_state_t state) { - for (uint16_t i = 0; i < test_rti.number_of_scheduling_nodes; i++) { - test_rti.scheduling_nodes[i]->state = state; + for (uint16_t i = 0; i < test_RTI.number_of_scheduling_nodes; i++) { + test_RTI.scheduling_nodes[i]->state = state; } } /******************************************End of Utility @@ -120,6 +120,7 @@ void set_state_of_nodes(scheduling_node_state_t state) { void valid_cache() { set_common_RTI(2); + uint16_t n = test_RTI.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --> node[1] @@ -128,15 +129,19 @@ void valid_cache() { set_state_of_nodes(GRANTED); + test_RTI.min_delays = (tag_t*)calloc((n * n), sizeof(tag_t)); + test_RTI.min_delays[0] = (tag_t){.time = NSEC(1), .microstep = 0}; + // If min_delays is not null (the cached data is valid), nothing should be changed. - test_rti.scheduling_nodes[1]->num_all_upstreams = 1; - test_rti.scheduling_nodes[1]->all_upstreams = (uint16_t*)calloc(1, sizeof(uint16_t)); - update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 1); + update_min_delays(); + assert(lf_tag_compare(test_RTI.min_delays[0], (tag_t){.time = NSEC(1), .microstep = 0}) == 0); + + reset_common_RTI(); } void not_connected() { set_common_RTI(2); + uint16_t n = test_RTI.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --> node[1] @@ -145,14 +150,20 @@ void not_connected() { set_state_of_nodes(NOT_CONNECTED); - // If the nodes are not connected, num_all_upstreams should not be changed. - update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 0); + // If the nodes are not connected, the matrix should be filled with FOREVER_TAG. + update_min_delays(); + for (uint16_t i = 0; i < n; i++) { + for (uint16_t j = 0; j < n; j++) { + assert(lf_tag_compare(test_RTI.min_delays[i * n + j], FOREVER_TAG) == 0); + } + } + + reset_common_RTI(); } static void two_nodes_no_delay() { set_common_RTI(2); - uint16_t n = test_rti.number_of_scheduling_nodes; + uint16_t n = test_RTI.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --> node[1] @@ -161,21 +172,22 @@ static void two_nodes_no_delay() { set_state_of_nodes(GRANTED); - // Test update_min_delays_upstream - update_min_delays_upstream(test_rti.scheduling_nodes[0]); - assert(test_rti.scheduling_nodes[0]->num_all_upstreams == 0); // node[0] has no upstream nodes. + update_min_delays(); + // The min_delay from 0 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 0 to 1 should be ZERO_TAG which means no delay. + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 1], ZERO_TAG) == 0); + // The min_delay from 1 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 1 to 1 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 1], FOREVER_TAG) == 0); - update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 1); // node[1] has one upstream nodes. - assert(test_rti.scheduling_nodes[1]->all_upstreams[0] == 0); // node[1]'s upstream node is node[0]. - // The min_delay between them is node[0] and node[1] which means no delay. - // assert(lf_tag_compare(test_rti.scheduling_nodes[1]->min_delays[0].min_delay, ZERO_TAG) == 0); - assert(lf_tag_compare(test_rti.min_delays[0 * n + 1], ZERO_TAG) == 0); + reset_common_RTI(); } static void two_nodes_zero_delay() { set_common_RTI(2); - uint16_t n = test_rti.number_of_scheduling_nodes; + uint16_t n = test_RTI.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --/0/--> node[1] @@ -184,27 +196,22 @@ static void two_nodes_zero_delay() { set_state_of_nodes(GRANTED); - // Test update_min_delays_upstream - update_min_delays_upstream(test_rti.scheduling_nodes[0]); - assert(test_rti.scheduling_nodes[0]->num_all_upstreams == 0); // node[0] has no upstream nodes. - - update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 1); // node[1] has one upstream nodes. - assert(test_rti.scheduling_nodes[1]->all_upstreams[0] == 0); // node[1]'s upstream node is node[0]. - // The min_delay between node[0] and node[1] is (0, 1) which means zero delay. - assert(lf_tag_compare(test_rti.min_delays[0 * n + 1], (tag_t){.time = 0, .microstep = 1}) == 0); - - // Test update_all_downstreams - update_all_downstreams(test_rti.scheduling_nodes[0]); - assert(test_rti.scheduling_nodes[0]->num_all_downstreams == 1); // node[0] has one downstream nodes. + update_min_delays(); + // The min_delay from 0 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 0 to 1 should be (0, 1). + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 1], (tag_t){.time = 0, .microstep = 1}) == 0); + // The min_delay from 1 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 1 to 1 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 1], FOREVER_TAG) == 0); - update_all_downstreams(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_all_downstreams == 0); // node[1] has no downstream nodes. + reset_common_RTI(); } static void two_nodes_normal_delay() { set_common_RTI(2); - uint16_t n = test_rti.number_of_scheduling_nodes; + uint16_t n = test_RTI.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --/1 nsec/--> node[1] @@ -213,19 +220,81 @@ static void two_nodes_normal_delay() { set_state_of_nodes(GRANTED); - update_min_delays_upstream(test_rti.scheduling_nodes[0]); - assert(test_rti.scheduling_nodes[0]->num_all_upstreams == 0); // node[0] has no upstream nodes. + update_min_delays(); + // The min_delay from 0 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 0 to 1 should be (1, 0). + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 1], (tag_t){.time = 1, .microstep = 0}) == 0); + // The min_delay from 1 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 1 to 1 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 1], FOREVER_TAG) == 0); + + reset_common_RTI(); +} + +static void two_nodes_cycle() { + set_common_RTI(2); + uint16_t n = test_RTI.number_of_scheduling_nodes; + + // Construct the structure illustrated below. + // node[0] --/1 nsec/--> node[1] --> node[0] + set_scheduling_node(0, 1, 1, (int[]){1}, (interval_t[]){NEVER}, (int[]){1}); + set_scheduling_node(1, 1, 1, (int[]){0}, (interval_t[]){NSEC(1)}, (int[]){0}); + + set_state_of_nodes(GRANTED); + + update_min_delays(); + // The min_delay from 0 to 0 should be (1, 0). + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 0], (tag_t){.time = 1, .microstep = 0}) == 0); + // The min_delay from 0 to 1 should be (1, 0). + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 1], (tag_t){.time = 1, .microstep = 0}) == 0); + // The min_delay from 1 to 0 should be ZERO_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 0], ZERO_TAG) == 0); + // The min_delay from 1 to 1 should be (1, 0). + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 1], (tag_t){.time = 1, .microstep = 0}) == 0); + + // Both of them are in a cycle. + assert(is_in_cycle(test_RTI.scheduling_nodes[0]) == 1); + assert(is_in_cycle(test_RTI.scheduling_nodes[1]) == 1); + // Both of them are in a zero delay cycle. + assert(is_in_zero_delay_cycle(test_RTI.scheduling_nodes[0]) == 0); + assert(is_in_zero_delay_cycle(test_RTI.scheduling_nodes[1]) == 0); - update_min_delays_upstream(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_all_upstreams == 1); // node[1] has one upstream nodes. - assert(test_rti.scheduling_nodes[1]->all_upstreams[0] == 0); // node[1]'s upstream node is node[0]. - // The min_delay between node[0] and node[1] is (1 nsec, 0). - assert(lf_tag_compare(test_rti.min_delays[0 * n + 1], (tag_t){.time = NSEC(1), .microstep = 0}) == 0); + reset_common_RTI(); +} + +static void two_nodes_ZDC() { + set_common_RTI(2); + uint16_t n = test_RTI.number_of_scheduling_nodes; + + // Construct the structure illustrated below. + // node[0] --> node[1] --> node[0] + set_scheduling_node(0, 1, 1, (int[]){1}, (interval_t[]){NEVER}, (int[]){1}); + set_scheduling_node(1, 1, 1, (int[]){0}, (interval_t[]){NEVER}, (int[]){0}); + + set_state_of_nodes(GRANTED); + + update_min_delays(); + // The min_delay from 0 to 0 should be ZERO_TAG. + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 0], ZERO_TAG) == 0); + // The min_delay from 0 to 1 should be ZERO_TAG. + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 1], ZERO_TAG) == 0); + // The min_delay from 1 to 0 should be ZERO_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 0], ZERO_TAG) == 0); + // The min_delay from 1 to 1 should be ZERO_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 1], ZERO_TAG) == 0); + + // Both of them are in a zero delay cycle. + assert(is_in_zero_delay_cycle(test_RTI.scheduling_nodes[0]) == 1); + assert(is_in_zero_delay_cycle(test_RTI.scheduling_nodes[1]) == 1); + + reset_common_RTI(); } static void multiple_nodes() { set_common_RTI(4); - uint16_t n = test_rti.number_of_scheduling_nodes; + uint16_t n = test_RTI.number_of_scheduling_nodes; // Construct the structure illustrated below. // node[0] --/1 nsec/--> node[1] --/0/--> node[2] --/2 nsec/--> node[3] @@ -236,49 +305,56 @@ static void multiple_nodes() { set_state_of_nodes(GRANTED); - // Test update_min_delays_upstream - update_min_delays_upstream(test_rti.scheduling_nodes[2]); - assert(test_rti.scheduling_nodes[2]->num_all_upstreams == 2); // node[2] has two upstream nodes. - assert(test_rti.scheduling_nodes[2]->all_upstreams[0] == 0); // node[0] is an upstream node of node[2]. - // The min_delay between node[0] and node[2] is (1 nsec, 1) = 1 nsec + zero delay. - assert(lf_tag_compare(test_rti.min_delays[0 * n + 2], (tag_t){NSEC(1), 1}) == 0); - assert(test_rti.scheduling_nodes[2]->all_upstreams[1] == 1); // node[1] is an upstream node of node[2]. - // The min_delay between node[1] and node[2] is (0, 1), which denotes zero delay. - assert(lf_tag_compare(test_rti.min_delays[1 * n + 2], (tag_t){0, 1}) == 0); - - update_min_delays_upstream(test_rti.scheduling_nodes[3]); - assert(test_rti.scheduling_nodes[3]->num_all_upstreams == 3); // node[3] has three upstream nodes. - assert(test_rti.scheduling_nodes[3]->all_upstreams[0] == 0); // node[0] is an upstream node of node[3]. - // The min_delay between node[0] and node[3] is (3 nsec, 0) = 1 nsec + zero_delay + 2 nsec. - assert(lf_tag_compare(test_rti.min_delays[0 * n + 3], (tag_t){NSEC(3), 0}) == 0); - assert(test_rti.scheduling_nodes[3]->all_upstreams[1] == 1); // node[1] is an upstream node of node[3]. - // The min_delay between node[1] and node[3] is (2 nsec, 0) = zero_delay + 2 nsec. - assert(lf_tag_compare(test_rti.min_delays[1 * n + 3], (tag_t){NSEC(2), 0}) == 0); - assert(test_rti.scheduling_nodes[3]->all_upstreams[2] == 2); // node[2] is an upstream node of node[3]. - // The min_delay between node[2] and node[3] is (2 nsec, 0). - assert(lf_tag_compare(test_rti.min_delays[2 * n + 3], (tag_t){NSEC(2), 0}) == 0); - - // Test update_all_downstreams - update_all_downstreams(test_rti.scheduling_nodes[0]); - assert(test_rti.scheduling_nodes[0]->num_all_downstreams == 3); // node[0] has three downstream nodes. - assert(test_rti.scheduling_nodes[0]->all_downstreams[0] == 1); // node[1] is a downstream node of node[3]. - assert(test_rti.scheduling_nodes[0]->all_downstreams[1] == 2); // node[2] is a downstream node of node[3]. - assert(test_rti.scheduling_nodes[0]->all_downstreams[2] == 3); // node[3] is a downstream node of node[3]. - - update_all_downstreams(test_rti.scheduling_nodes[1]); - assert(test_rti.scheduling_nodes[1]->num_all_downstreams == 2); // node[1] has two downstream nodes. - assert(test_rti.scheduling_nodes[1]->all_downstreams[0] == 2); // node[2] is a downstream node of node[3]. - assert(test_rti.scheduling_nodes[1]->all_downstreams[1] == 3); // node[3] is a downstream node of node[3]. + update_min_delays(); + // The min_delay from 0 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 0 to 1 should be (1, 0). + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 1], (tag_t){.time = 1, .microstep = 0}) == 0); + // The min_delay from 0 to 2 should be (1, 1). + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 2], (tag_t){.time = 1, .microstep = 1}) == 0); + // The min_delay from 0 to 3 should be (3, 0). + assert(lf_tag_compare(test_RTI.min_delays[0 * n + 3], (tag_t){.time = 3, .microstep = 0}) == 0); + + // The min_delay from 1 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 1 to 1 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 1], FOREVER_TAG) == 0); + // The min_delay from 1 to 2 should be (0, 1). + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 2], (tag_t){.time = 0, .microstep = 1}) == 0); + // The min_delay from 1 to 3 should be (2, 0). + assert(lf_tag_compare(test_RTI.min_delays[1 * n + 3], (tag_t){.time = 2, .microstep = 0}) == 0); + + // The min_delay from 2 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[2 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 2 to 1 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[2 * n + 1], FOREVER_TAG) == 0); + // The min_delay from 2 to 2 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[2 * n + 2], FOREVER_TAG) == 0); + // The min_delay from 2 to 3 should be (2, 0). + assert(lf_tag_compare(test_RTI.min_delays[2 * n + 3], (tag_t){.time = 2, .microstep = 0}) == 0); + + // The min_delay from 3 to 0 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[3 * n + 0], FOREVER_TAG) == 0); + // The min_delay from 3 to 1 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[3 * n + 1], FOREVER_TAG) == 0); + // The min_delay from 3 to 2 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[3 * n + 2], FOREVER_TAG) == 0); + // The min_delay from 3 to 3 should be FOREVER_TAG. + assert(lf_tag_compare(test_RTI.min_delays[3 * n + 3], FOREVER_TAG) == 0); + + reset_common_RTI(); } int main() { - initialize_rti_common(&test_rti); + initialize_rti_common(&test_RTI); - // Tests for the function update_min_delays_upstream() and update_all_downstreams() + // Tests for the function update_min_delays valid_cache(); not_connected(); two_nodes_no_delay(); two_nodes_zero_delay(); two_nodes_normal_delay(); + two_nodes_cycle(); + two_nodes_ZDC(); multiple_nodes(); } From adc0a1d4250bba7b29a39cb4dce635b0db7e5e0c Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Fri, 18 Oct 2024 16:45:09 -0700 Subject: [PATCH 35/43] Clang format --- core/federated/RTI/rti_common.c | 5 ++--- core/federated/RTI/rti_common.h | 10 +--------- core/federated/RTI/rti_remote.c | 2 -- core/federated/RTI/test/rti_common_test.c | 2 +- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 1837a30f6..777eb14b3 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -111,8 +111,7 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e) { // by (0,1). If the time part of the delay is greater than 0, then we want to ignore // the microstep in upstream->next_event because that microstep will have been lost. // Otherwise, we want preserve it and add to it. This is handled by lf_tag_add(). - tag_t earliest_tag_from_upstream = - lf_tag_add(upstream->next_event, rti_common->min_delays[i * n + e->id]); + tag_t earliest_tag_from_upstream = lf_tag_add(upstream->next_event, rti_common->min_delays[i * n + e->id]); /* Following debug message is too verbose for normal use: LF_PRINT_DEBUG("RTI: Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".", @@ -403,7 +402,7 @@ void update_min_delays() { } // N^2 debug statement could be a problem with large benchmarks. LF_PRINT_DEBUG("++++ Node %hu is upstream with delay" PRINTF_TAG "\n", i, path_delays[i].time, - path_delays[i].microstep); + path_delays[i].microstep); } } } diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index 418f324e1..b1bf828ab 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -254,21 +254,13 @@ tag_t earliest_future_incoming_message_tag(scheduling_node_t* e); tag_t eimt_strict(scheduling_node_t* e); /** - * If necessary, update the `min_delays` and the fields that indicate cycles. + * If necessary, update the `min_delays` and the fields that indicate cycles. * These fields will be updated only if they have not been previously updated or if invalidate_min_delays * has been called since they were last updated. * @param node The node. */ void update_min_delays(); -/** - * For the given scheduling node (enclave or federate), if necessary, update the `all_downstreams` and - * `num_all_downstreams` fields. These fields will be updated only if they have not been previously updated - * or if invalidate_min_delays has been called since they were last updated. - * @param node The node. - */ -void update_all_downstreams(scheduling_node_t* node); - /** * Find the tag g that is the latest tag that satisfies lf_tag_add(g, minimum_delay) < next_event_tag. * This function behaves like the tag subtraction, next_event_tag - minimum_delay. diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index 81b3645a3..5fdb0dcba 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -1782,11 +1782,9 @@ void free_scheduling_nodes(scheduling_node_t** scheduling_nodes, uint16_t number if (node->immediate_upstreams != NULL) { free(node->immediate_upstreams); free(node->immediate_upstream_delays); - // free(node->all_upstreams); } if (node->immediate_downstreams != NULL) { free(node->immediate_downstreams); - // free(node->all_downstreams); } free(node); } diff --git a/core/federated/RTI/test/rti_common_test.c b/core/federated/RTI/test/rti_common_test.c index d93a7dc31..63affda5a 100644 --- a/core/federated/RTI/test/rti_common_test.c +++ b/core/federated/RTI/test/rti_common_test.c @@ -95,7 +95,7 @@ void set_common_RTI(uint16_t num_nodes) { // Allocate memory for the scheduling nodes test_RTI.scheduling_nodes = (scheduling_node_t**)calloc(test_RTI.number_of_scheduling_nodes, sizeof(scheduling_node_t*)); - + test_RTI.min_delays = NULL; for (uint16_t i = 0; i < test_RTI.number_of_scheduling_nodes; i++) { From c2f5316458bac61074285ca34d92a87c8511eaf9 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Sat, 19 Oct 2024 15:10:50 -0700 Subject: [PATCH 36/43] Turn on DNET signals by default and make the option to turn off the feature --- core/federated/RTI/main.c | 6 +++--- core/federated/RTI/rti_common.c | 2 +- core/federated/RTI/rti_common.h | 2 +- core/federated/RTI/rti_remote.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c index 4853491cd..f936bd292 100644 --- a/core/federated/RTI/main.c +++ b/core/federated/RTI/main.c @@ -136,7 +136,7 @@ void usage(int argc, const char* argv[]) { lf_print(" clock sync attempt (default is 10). Applies to 'init' and 'on'.\n"); lf_print(" -a, --auth Turn on HMAC authentication options.\n"); lf_print(" -t, --tracing Turn on tracing.\n"); - lf_print(" -d, --dnet Turn on DNET signals for reducing network messages.\n"); + lf_print(" -d, --dnet_disabled Turn off DNET signals.\n"); lf_print("Command given:"); for (int i = 0; i < argc; i++) { @@ -264,8 +264,8 @@ int process_args(int argc, const char* argv[]) { rti.authentication_enabled = true; } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) { rti.base.tracing_enabled = true; - } else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--dnet") == 0) { - rti.base.dnet_enabled = true; + } else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--dnet_disabled") == 0) { + rti.base.dnet_disabled = true; } else if (strcmp(argv[i], " ") == 0) { // Tolerate spaces continue; diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 777eb14b3..437e8870e 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -284,7 +284,7 @@ void update_scheduling_node_next_event_tag_locked(scheduling_node_t* e, tag_t ne } } - if (rti_common->dnet_enabled) { + if (!rti_common->dnet_disabled) { // Send DNET to the node e's upstream federates if needed for (int i = 0; i < n; i++) { if (lf_tag_compare(rti_common->min_delays[i * n + e->id], FOREVER_TAG) != 0 && i != e->id) { diff --git a/core/federated/RTI/rti_common.h b/core/federated/RTI/rti_common.h index b1bf828ab..3d87dc301 100644 --- a/core/federated/RTI/rti_common.h +++ b/core/federated/RTI/rti_common.h @@ -93,7 +93,7 @@ typedef struct rti_common_t { bool tracing_enabled; // Boolean indicating that DNET is enabled. - bool dnet_enabled; + bool dnet_disabled; // The RTI mutex for making thread-safe access to the shared state. lf_mutex_t* mutex; diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index 5fdb0dcba..19724955c 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -1765,7 +1765,7 @@ void initialize_RTI(rti_remote_t* rti) { rti_remote->clock_sync_exchanges_per_interval = 10; rti_remote->authentication_enabled = false; rti_remote->base.tracing_enabled = false; - rti_remote->base.dnet_enabled = false; + rti_remote->base.dnet_disabled = false; rti_remote->stop_in_progress = false; } From 7f81c7caff716ca66480159ea0fd175305dfc6a2 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Mon, 28 Oct 2024 14:48:55 -0700 Subject: [PATCH 37/43] Exclude the target node itself when computing DNET --- core/federated/RTI/rti_common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 437e8870e..420b79fc3 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -465,8 +465,8 @@ tag_t downstream_next_event_tag(scheduling_node_t* target_node, uint16_t node_se result = candidate; } else { for (int j = 0; j < n; j++) { - if (lf_tag_compare(rti_common->min_delays[target_node->id * n + j], FOREVER_TAG) != 0) { - // The node j is a downstream node. + if (target_node->id != j && (lf_tag_compare(rti_common->min_delays[target_node->id * n + j], FOREVER_TAG) != 0)) { + // The node j is a downstream node and not the target node itself. scheduling_node_t* target_dowstream = rti_common->scheduling_nodes[j]; // if (is_in_zero_delay_cycle(target_dowstream)) { // // The target node is an upstream of ZDC. Do not send DNET to this node. From 05f5acc92498d1bae572d41b445afb3267a7b2cc Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 7 Jan 2025 10:52:14 -0700 Subject: [PATCH 38/43] Send NET every time when DNET is disabled --- core/federated/federate.c | 7 +++++-- include/core/federated/federate.h | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/federated/federate.c b/core/federated/federate.c index 93f2547fc..0a467ca38 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -86,6 +86,7 @@ federate_instance_t _fed = {.socket_TCP_RTI = -1, .is_last_TAG_provisional = false, .has_upstream = false, .has_downstream = false, + .received_any_DNET = false, .last_DNET = {.time = NEVER, .microstep = 0u}, .received_stop_request_from_rti = false, .last_sent_LTC = {.time = NEVER, .microstep = 0u}, @@ -1471,6 +1472,7 @@ static void handle_downstream_next_event_tag() { tracepoint_federate_from_rti(receive_DNET, _lf_my_fed_id, &DNET); LF_PRINT_LOG("Received Downstream Next Event Tag (DNET): " PRINTF_TAG ".", DNET.time - start_time, DNET.microstep); + _fed.received_any_DNET = true; environment_t* env; _lf_get_environments(&env); @@ -2264,8 +2266,9 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) LF_PRINT_DEBUG("Granted tag " PRINTF_TAG " because TAG or PTAG has been received.", _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); - // In case a downstream federate needs the NET of this tag, send NET. - if (lf_tag_compare(_fed.last_DNET, tag) < 0 && lf_tag_compare(_fed.last_DNET, _fed.last_sent_NET) >= 0) { + // In case a downstream federate needs the NET of this tag or has not received any DNET, send NET. + if (!_fed.received_any_DNET || + (lf_tag_compare(_fed.last_DNET, tag) < 0 && lf_tag_compare(_fed.last_DNET, _fed.last_sent_NET) >= 0)) { send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag); _fed.last_sent_NET = tag; _fed.last_skipped_NET = NEVER_TAG; diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h index bfbd24314..e7697f259 100644 --- a/include/core/federated/federate.h +++ b/include/core/federated/federate.h @@ -173,6 +173,11 @@ typedef struct federate_instance_t { */ tag_t last_skipped_NET; + /** + * Indicator of whether this federate has received any DNET (downstream next event tag) signal. + */ + bool received_any_DNET; + /** * A record of the most recent DNET (downstream next event tag) signal. */ From e22ca8d1a07329e7ab088faee51c69ae9cdf41ee Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Tue, 7 Jan 2025 15:05:30 -0700 Subject: [PATCH 39/43] Record the skipped NET properly --- core/federated/federate.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/core/federated/federate.c b/core/federated/federate.c index 0a467ca38..f7f52e37a 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2272,9 +2272,14 @@ tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag); _fed.last_sent_NET = tag; _fed.last_skipped_NET = NEVER_TAG; - LF_PRINT_LOG("Sent the last skipped next event tag (NET) " PRINTF_TAG - " to RTI based on the last DNET " PRINTF_TAG ".", - _fed.last_DNET.time - start_time, _fed.last_DNET.microstep, tag.time - start_time, tag.microstep); + LF_PRINT_LOG("Sent a next event tag (NET) " PRINTF_TAG " to RTI based on the last DNET " PRINTF_TAG ".", + tag.time - start_time, tag.microstep, _fed.last_DNET.time - start_time, _fed.last_DNET.microstep); + } else { + _fed.last_skipped_NET = tag; + LF_PRINT_LOG("Skip sending a next event tag (NET) " PRINTF_TAG " to RTI based on the last DNET " PRINTF_TAG + " and the last sent NET" PRINTF_TAG ".", + tag.time - start_time, tag.microstep, _fed.last_DNET.time - start_time, _fed.last_DNET.microstep, + _fed.last_sent_NET.time - start_time, _fed.last_sent_NET.microstep); } return _fed.last_TAG; } From 463d49051dbe78fd7a7438f291517e60f5c99ac5 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 8 Jan 2025 13:55:17 -0700 Subject: [PATCH 40/43] Temporarily disable LTC and TAG optimization --- core/federated/RTI/rti_common.c | 4 ++-- core/federated/federate.c | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 420b79fc3..21611af31 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -225,8 +225,8 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) { "(adjusted by after delay). Granting tag advance (TAG) for " PRINTF_TAG, e->id, t_d.time - lf_time_start(), t_d.microstep, e->next_event.time - lf_time_start(), e->next_event.microstep); - // result.tag = e->next_event; - result.tag = lf_tag_latest_earlier(t_d); + result.tag = e->next_event; + // result.tag = lf_tag_latest_earlier(t_d); } else if ( // Scenario (2) above lf_tag_compare(t_d, e->next_event) == 0 // EIMT equal to NET && is_in_zero_delay_cycle(e) // The node is part of a ZDC diff --git a/core/federated/federate.c b/core/federated/federate.c index f7f52e37a..1843ff3ac 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2131,12 +2131,12 @@ void lf_latest_tag_confirmed(tag_t tag_to_send) { return; // Already sent this or later tag. } _lf_get_environments(&env); - if (!env->need_to_send_LTC) { - LF_PRINT_LOG("Skip sending Latest Tag Confirmed (LTC) to the RTI because there was no tagged message with the " - "tag " PRINTF_TAG " that this federate has received.", - tag_to_send.time - start_time, tag_to_send.microstep); - return; - } + // if (!env->need_to_send_LTC) { + // LF_PRINT_LOG("Skip sending Latest Tag Confirmed (LTC) to the RTI because there was no tagged message with the " + // "tag " PRINTF_TAG " that this federate has received.", + // tag_to_send.time - start_time, tag_to_send.microstep); + // return; + // } LF_PRINT_LOG("Sending Latest Tag Confirmed (LTC) " PRINTF_TAG " to the RTI.", tag_to_send.time - start_time, tag_to_send.microstep); send_tag(MSG_TYPE_LATEST_TAG_CONFIRMED, tag_to_send); From d32b2255db0e428e3323531c7c7cf7ae2715f04f Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Wed, 8 Jan 2025 14:00:27 -0700 Subject: [PATCH 41/43] Revert the commit 463d4905 --- core/federated/RTI/rti_common.c | 3 +-- core/federated/federate.c | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/core/federated/RTI/rti_common.c b/core/federated/RTI/rti_common.c index 21611af31..f99edb543 100644 --- a/core/federated/RTI/rti_common.c +++ b/core/federated/RTI/rti_common.c @@ -225,8 +225,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(scheduling_node_t* e) { "(adjusted by after delay). Granting tag advance (TAG) for " PRINTF_TAG, e->id, t_d.time - lf_time_start(), t_d.microstep, e->next_event.time - lf_time_start(), e->next_event.microstep); - result.tag = e->next_event; - // result.tag = lf_tag_latest_earlier(t_d); + result.tag = lf_tag_latest_earlier(t_d); } else if ( // Scenario (2) above lf_tag_compare(t_d, e->next_event) == 0 // EIMT equal to NET && is_in_zero_delay_cycle(e) // The node is part of a ZDC diff --git a/core/federated/federate.c b/core/federated/federate.c index 1843ff3ac..f7f52e37a 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2131,12 +2131,12 @@ void lf_latest_tag_confirmed(tag_t tag_to_send) { return; // Already sent this or later tag. } _lf_get_environments(&env); - // if (!env->need_to_send_LTC) { - // LF_PRINT_LOG("Skip sending Latest Tag Confirmed (LTC) to the RTI because there was no tagged message with the " - // "tag " PRINTF_TAG " that this federate has received.", - // tag_to_send.time - start_time, tag_to_send.microstep); - // return; - // } + if (!env->need_to_send_LTC) { + LF_PRINT_LOG("Skip sending Latest Tag Confirmed (LTC) to the RTI because there was no tagged message with the " + "tag " PRINTF_TAG " that this federate has received.", + tag_to_send.time - start_time, tag_to_send.microstep); + return; + } LF_PRINT_LOG("Sending Latest Tag Confirmed (LTC) " PRINTF_TAG " to the RTI.", tag_to_send.time - start_time, tag_to_send.microstep); send_tag(MSG_TYPE_LATEST_TAG_CONFIRMED, tag_to_send); From df786d36b3c40112b153c4cf6d6d421366d71fa0 Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Thu, 16 Jan 2025 17:21:15 -0700 Subject: [PATCH 42/43] Minor fix --- core/federated/RTI/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c index f936bd292..c08890ac7 100644 --- a/core/federated/RTI/main.c +++ b/core/federated/RTI/main.c @@ -136,7 +136,7 @@ void usage(int argc, const char* argv[]) { lf_print(" clock sync attempt (default is 10). Applies to 'init' and 'on'.\n"); lf_print(" -a, --auth Turn on HMAC authentication options.\n"); lf_print(" -t, --tracing Turn on tracing.\n"); - lf_print(" -d, --dnet_disabled Turn off DNET signals.\n"); + lf_print(" -d, --disable_dnet Turn off the use of DNET signals.\n"); lf_print("Command given:"); for (int i = 0; i < argc; i++) { From 31d28e2eb85dde8ce628b6a204356a7bc6b5466d Mon Sep 17 00:00:00 2001 From: Byeonggil Jun Date: Fri, 24 Jan 2025 12:18:22 -0700 Subject: [PATCH 43/43] Point to lingua-franca master --- lingua-franca-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt index 8ba7da0a2..8b25206ff 100644 --- a/lingua-franca-ref.txt +++ b/lingua-franca-ref.txt @@ -1 +1 @@ -rti-DNET +master \ No newline at end of file