DPI implemented - connections in the RoCE-stack still missing

driver/coyote_dev.h

@@ -367,6 +367,9 @@ extern char *config_fname;
 #define IRQ_NOTIFY 4
 #define IRQ_RCNFG 5
 
+// Enter a new IRQ for DPI-flagging 
+#define IRQ_DPI 6
+
 /**
  * @brief Cdev
  * 

driver/fpga_mmu.c

@@ -111,6 +111,22 @@ irqreturn_t fpga_isr(int irq, void *dev_id)
             pr_err("could not enqueue a workqueue, notify ISR");
         }
         break;
+
+    case IRQ_DPI:
+        dbg_info("(irq=%d) notify, vFPGA %d\n", irq, d->id);
+        dbg_info("Packet was flagged by DPI and transmission was stopped. Application will be stopped now.")
+        irq_not = kzalloc(sizeof(struct fpga_irq_notify), GFP_KERNEL);
+        BUG_ON(!irq_not);
+
+        irq_not->d = d;
+        fpga_read_irq_notify(d, irq_not);
+
+        INIT_WORK(&irq_not->work_notify, fpga_notify_handler);
+
+        if(!queue_work(d->wqueue_notify, &irq_not->work_notify)) {
+            pr_err("could not enqueue a workqueue, notify ISR");
+        }
+        break;
 
     default:
         break;

examples_hw/CMakeLists.txt

@@ -183,6 +183,11 @@ if(EXAMPLE STREQUAL "rdma_aes")
     create_hw()
 endif()
 
+# Network example, RDMA
+# @brief: RDMA for microbenchmarks 
+#
+# @note: Add a target device (FDEV_NAME)
+#
 if(EXAMPLE STREQUAL "rdma_perf")
     message("** RDMA perf test")
     set(SHELL_PROBE 6)
@@ -200,6 +205,26 @@ if(EXAMPLE STREQUAL "rdma_perf")
     create_hw()
 endif()
 
+# Network example, RDMA 
+# @brief: RDMA with DPI
+# 
+# @note: Add a target device (FDEV_NAME)
+#
+if(EXAMPLE STREQUAL "rdma_dpi")
+    message("** RDMA DPI test")
+    set(SHELL_PROBE 8)
+    set(N_REGIONS 1)
+    set(EN_STRM 1)
+    set(EN_RDMA 1)
+    set(N_STRM_AXI 2)
+
+    validation_checks_hw()
+
+    load_apps (
+        VFPGA_C0_0 "apps/rdma_dpi"
+    )
+endif()
+
 # Network example, TCP iperf
 # @brief: TCP iperf
 #

examples_hw/apps/rdma_dpi/vfpga_top.svh

@@ -0,0 +1,118 @@
+/**
+ * VFPGA TOP FOR DPI 
+ *
+ * Catch write commands to check if an IRQ needs to be raised or not
+ *
+ */ 
+
+// Direct comb of the read-interface, we're only looking at the write-commands for IRQs 
+always_comb begin 
+    // Read ops 
+    sq_rd.valid = rq_rd.valid;
+    rq_rd.ready = sq_rd.ready;
+    sq_rd.data = rq_rd.data;
+    // OW
+    sq_rd.data.strm = STRM_HOST;
+    sq_rd.data.dest = 1;
+end 
+
+// Introduce additional pipeline stage that allows to modify the write commands if it's actually malicious 
+req_t sq_wr_data_intermediate; 
+logic sq_wr_valid_intermediate; 
+
+irq_not_t notify_data_intermediate; 
+logic notify_valid_intermediate; 
+
+logic interrupt_raised; 
+
+// Do the required assignments to the intermediate pipeline-stage
+assign sq_wr.valid = sq_wr_valid_intermediate; 
+assign sq_wr.data = sq_wr_data_intermediate;
+assign rq_rd.ready = sq_wr.ready; 
+
+assign notify.valid = notify_valid_intermediate; 
+assign notify.data = notify_data_intermediate; 
+
+// Synchronous block to catch incoming commands and generate IRQs if required 
+always_ff @(posedge aclk) begin 
+    if(!aresetn) begin
+        // Reset all intermediate registers 
+        sq_wr_data_intermediate <= 512'b0; 
+        sq_wr_valid_intermediate <= 1'b0;  
+
+        notify_valid_intermediate <= 1'b0; 
+        notify_data_intermediate <= 38'b0; 
+
+        // Reset the interrupt-raised register 
+        interrupt_raised <= 1'b0; 
+    end else begin
+        // Wait for incoming command
+        if(rq_wr.valid && sq_wr.ready) begin
+            // Check the opcode of this command 
+            if(rq_wr.data.opcode == RC_ROCE_DPI_IRQ) begin 
+                // If the incoming command-opcode is the specialized DPI_IRQ, stop the transmission of the command and raise an actual interrupt 
+                sq_wr_data_intermediate <= 512'b0; 
+                sq_wr_valid_intermediate <= 1'b0; 
+
+                // For the interrupt, forward the pid from the incoming command 
+                notify_valid_intermediate <= 1'b1;
+                notify_data_intermediate.pid <= rq_wr.data.pid; 
+                notify_data_intermediate.value <= IRQ_DPI; 
+
+                // Set the bit that indicates that the interrupt was written 
+                interrupt_raised <= 1; 
+            end else begin 
+
+            end 
+        end else begin 
+            // If there's no command coming in, just reset the intermediate command stage 
+            sq_wr_data_intermediate <= 512'b0; 
+            sq_wr_valid_intermediate <= 1'b0; 
+        end 
+
+        // Check if interrupt was raised and picked up (ready-signal!) If so, it can be deasserted. 
+        if(interrupt_raised && notify.ready) begin 
+            interrupt_raised <= 1'b0; 
+            notify_data_intermediate <= 38'b0; 
+            notify_valid_intermediate <= 1'b0; 
+        end 
+    end 
+end 
+
+`AXISR_ASSIGN(axis_host_recv[0], axis_rreq_send[0])
+`AXISR_ASSIGN(axis_rreq_recv[0], axis_host_send[0])
+`AXISR_ASSIGN(axis_host_recv[1], axis_rrsp_send[0])
+`AXISR_ASSIGN(axis_rrsp_recv[0], axis_host_send[1])
+
+ila_0 inst_ila (
+    .clk(aclk),
+    .probe0(axis_host_recv[0].tvalid),
+    .probe1(axis_host_recv[0].tready),
+    .probe2(axis_host_recv[0].tlast),
+
+    .probe3(axis_host_recv[1].tvalid),
+    .probe4(axis_host_recv[1].tready),
+    .probe5(axis_host_recv[1].tlast),
+
+    .probe6(axis_host_send[0].tvalid),
+    .probe7(axis_host_send[0].tready),
+    .probe8(axis_host_send[0].tlast),
+
+    .probe9(axis_host_send[1].tvalid),
+    .probe10(axis_host_send[1].tready),
+    .probe11(axis_host_send[1].tlast),
+
+    .probe12(sq_wr.valid),
+    .probe13(sq_wr.ready),
+    .probe14(sq_wr.data), // 128
+    .probe15(sq_rd.valid),
+    .probe16(sq_rd.ready),
+    .probe17(sq_rd.data), // 128
+    .probe18(cq_rd.valid),
+    .probe19(cq_wr.valid)
+);
+
+// Tie-off unused 
+always_comb axi_ctrl.tie_off_s();
+always_comb cq_rd.tie_off_s();
+always_comb cq_wr.tie_off_s();

examples_sw/apps/rdma_service_no_daemon/client/main.cpp

@@ -104,7 +104,7 @@ int main(int argc, char *argv[])
         ("verbose,v", boost::program_options::value<bool>(), "Printout of single messages")
         ("encryption,e", boost::program_options::value<bool>(), "Encryption required")
         ("compression,c", boost::program_options::value<bool>(), "Compression required")
-        ("dpi,d", boost::program_options::value<bool>(), "DPI required");
+        ("dpi,p", boost::program_options::value<bool>(), "DPI required");
 
     boost::program_options::variables_map commandLineArgs;
     boost::program_options::store(boost::program_options::parse_command_line(argc, argv, programDescription), commandLineArgs);

examples_sw/apps/rdma_service_no_daemon/server/main.cpp

@@ -96,7 +96,7 @@ int main(int argc, char *argv[])
         ("verbose,v", boost::program_options::value<bool>(), "Printout of single messages")
         ("encryption,e", boost::program_options::value<bool>(), "Encryption required")
         ("compression,c", boost::program_options::value<bool>(), "Compression required")
-        ("dpi,d", boost::program_options::value<bool>(), "DPI required");
+        ("dpi,p", boost::program_options::value<bool>(), "DPI required");
 
     boost::program_options::variables_map commandLineArgs;
     boost::program_options::store(boost::program_options::parse_command_line(argc, argv, programDescription), commandLineArgs);

hw/hdl/network/rdma/dpi_transmission_dropper.sv

@@ -0,0 +1,185 @@
+// Module that sits on the output data stream from the HLS-stack, reads incoming data and DPI-decisions and can then either drop the payload or not 
+
+
+// Import the lynxTypes to be able to reference the datatypes 
+import lynxTypes::*; 
+
+module dpi_transmission_dropper(
+    // Incoming clock and reset signal 
+    input logic nclk, 
+    input logic nresetn, 
+
+    // Incoming data stream from the HLS-stack 
+    AXI4S.s s_axis_rdma_wr, 
+
+    // Outgoing data stream to the user 
+    AXI4S.m m_axis_rdma_wr, 
+
+    // Incoming user commands from the HLS-stack 
+    metaIntf.s s_rdma_wr_req, 
+
+    // Outgoing user commands to the user 
+    metaIntf.m m_rdma_wr_req, 
+
+    // Incoming intrusion decision input 
+    metaIntf.s s_intrusion_decision_in
+); 
+
+///////////////////////////////////////////////////////////////
+//
+// Definition of registers required for managing accesses
+//
+///////////////////////////////////////////////////////////////
+
+// Signal-array to cache two subsequent incoming DPI-decisions 
+logic dpi_decision[2]; 
+
+// Signal to store which of the two DPI-decisions is currently active 
+logic dpi_currently_active; 
+
+// Signal to store which of the two DPI-fields to load next 
+logic dpi_load_next;
+
+// Signal to show the currently active DPI-decision 
+logic current_dpi_acceptable; 
+
+// Register stage with the prepared AXI-data signals that need to be forwarded 
+logic [511:0] axis_rdma_wr_data_inter; 
+logic [63:0] axis_rdma_wr_keep_inter; 
+logic axis_rdma_wr_valid_inter; 
+logic axis_rdma_wr_last_inter; 
+
+// Register stage with the prepared control-signals that need to be forwarded 
+logic rdma_wr_req_valid_inter; 
+req_t rdma_wr_req_data_inter;
+
+
+//////////////////////////////////////////////////////////////////////
+//
+// Combinatorial logic: Assign ready-signals 
+//
+/////////////////////////////////////////////////////////////////////
+
+// ready signal will always be assigned - there's no blocking in this module 
+assign m_axis_rdma_wr.tready = s_axis_rdma_wr.tready; 
+assign m_rdma_wr_req.ready = s_rdma_wr_req.ready; 
+
+// Forwarding the data interface 
+assign m_rdma_wr_req.valid = rdma_wr_req_valid_inter; 
+assign m_rdma_wr_req.data = rdma_wr_req_data_inter; 
+
+// Assign the currently active dpi-acceptable signal 
+assign current_dpi_acceptable = dpi_currently_active ? dpi_decision[1].acceptable : dpi_decision[0].acceptable; 
+
+
+/////////////////////////////////////////////////////////////////////
+//
+// Sequential logic 
+//
+////////////////////////////////////////////////////////////////////
+
+always_ff @(posedge nclk) begin 
+    if(!nresetn) begin
+        // RESET: Assign 0-values to all internal registers 
+        dpi_decision[0] <= 0; 
+        dpi_decision[1] <= 0; 
+        dpi_currently_active <= 0; 
+        dpi_load_next <= 0; 
+
+        // RESET: Assign 0-values to all intermediate registers  
+        axis_rdma_wr_data_inter <= 512'b0; 
+        axis_rdma_wr_keep_inter <= 64'b0; 
+        axis_rdma_wr_valid_inter <= 1'b0; 
+        axis_rdma_wr_last_inter <= 1'b0; 
+
+        rdma_wr_req_valid_inter <= 1'b0; 
+        rdma_wr_req_data_inter <= 128'b0; 
+
+    end else begin 
+
+        // Wait for incoming DPI-decision 
+        if(s_intrusion_decision_in.valid) begin 
+            // Based on the load_next-signal, load the DPI-into the correct register 
+            if(!dpi_load_next) begin
+                dpi_decision[0] <= s_intrusion_decision_in.data.acceptable; 
+            end else begin 
+                dpi_decision[1] <= s_intrusion_decision_in.data.acceptable; 
+            end 
+
+            // Change the dpi_load_next so that the next DPI-decision can be stored in the other register 
+            dpi_load_next <= !dpi_load_next; 
+        end 
+
+        // Wait for incoming command 
+        if(s_rdma_wr_req.valid) begin 
+            // Check if the command is either a WRITE or a READ_RESPONSE. Everything else doesn't need treatment here
+            if(is_opcode_rd_resp(s_rdma_wr_req.data.opcode) || is_opcode_wr(s_rdma_wr_req.data.opcode)) begin 
+                // Check the currenty active DPI-decision, then based on that decide whether to forward the original command or the one modified to raise an IRQ in the vFPGA 
+                if(current_dpi_acceptable) begin
+                    // If the current DPI-decision indicates that the packet is acceptable, just forward the command 
+                    rdma_wr_req_data_inter <= s_rdma_wr_req.data;
+                    rdma_wr_req_valid_inter <= s_rdma_wr_req.valid; 
+                end else begin 
+                    // If the current DPI-decision indicates that the packet is not acceptable, forward a modified command with an opcode to raise an IRQ next 
+                    rdma_wr_req_valid_inter <= s_rdma_wr_req.valid; 
+
+                    rdma_wr_req_data_inter.opcode <= RC_ROCE_DPI_IRQ;  // Change opcode so that it's raising an IRQ in the vFPGA
+                    rdma_wr_req_data_inter.strm <= s_rdma_wr_req.data.strm; 
+                    rdma_wr_req_data_inter.mode <= s_rdma_wr_req.data.mode; 
+                    rdma_wr_req_data_inter.rdma <= s_rdma_wr_req.data.rdma; 
+                    rdma_wr_req_data_inter.remote <= s_rdma_wr_req.data.remote; 
+                    rdma_wr_req_data_inter.vfid <= s_rdma_wr_req.data.vfid; 
+                    rdma_wr_req_data_inter.pid <= s_rdma_wr_req.data.pid; 
+                    rdma_wr_req_data_inter.dest <= s_rdma_wr_req.data.dest; 
+                    rdma_wr_req_data_inter.last <= s_rdma_wr_req.data.last; 
+                    rdma_wr_req_data_inter.vaddr <= s_rdma_wr_req.data.vaddr; 
+                    rdma_wr_req_data_inter.len <= s_rdma_wr_req.data.len; 
+                    rdma_wr_req_data_inter.actv <= s_rdma_wr_req.data.actv; 
+                    rdma_wr_req_data_inter.host <= s_rdma_wr_req.data.host; 
+                    rdma_wr_req_data_inter.offs <= s_rdma_wr_req.data.offs; 
+                    rdma_wr_req_data_inter.rsrvd <= s_rdma_wr_req.data.rsrvd; 
+                end 
+
+            end else begin 
+                // If it's not a WRITE or READ_RESPONSE, just forward the commands on the corresponding interface 
+                rdma_wr_req_data_inter <= s_rdma_wr_req.data; 
+                rdma_wr_req_valid_inter <= s_rdma_wr_req.valid; 
+            end 
+        end else begin 
+            // If there's no active command, set the intermediate register to 0 
+            rdma_wr_req_data_inter <= 128'b0; 
+            rdma_wr_req_valid_inter <= 1'b0; 
+        end 
+    end
+
+
+    // Wait for incoming data 
+    if(s_axis_rdma_wr.tvalid) begin
+        // Check the currently active DPI-decision.
+        if(current_dpi_acceptable) begin
+            // If the current transmission is acceptable, just write the values in the intermediate register 
+            axis_rdma_wr_data_inter <= s_axis_rdma_wr.tdata; 
+            axis_rdma_wr_keep_inter <= s_axis_rdma_wr.tkeep; 
+            axis_rdma_wr_last_inter <= s_axis_rdma_wr.tlast; 
+            axis_rdma_wr_valid_inter <= s_axis_rdma_wr.tvalid; 
+        end else begin 
+            // If the current transmission is not acceptable, set the intermediate stage to 0 and thus drop the data 
+            axis_rdma_wr_data_inter <= 512'b0; 
+            axis_rdma_wr_keep_inter <= 64'b0; 
+            axis_rdma_wr_last_inter <= 1'b0; 
+            axis_rdma_wr_valid_inter <= 1'b0; 
+        end 
+
+        // If a tlast is set, switch the pointer for the current DPI-decision 
+        if(s_axis_rdma_wr.tlast) begin 
+            dpi_currently_active <= !dpi_currently_active; 
+        end 
+
+    end else begin 
+        // If there's no active transmission right now, set the intermediate register stage to all 0
+        axis_rdma_wr_data_inter <= 512'b0; 
+        axis_rdma_wr_keep_inter <= 64'b0; 
+        axis_rdma_wr_last_inter <= 1'b0; 
+        axis_rdma_wr_valid_inter <= 1'b0; 
+    end  
+end