flow: Add tests for excluding pkt recursion from flow

tests/flow-pkt-recursion/README.md

@@ -0,0 +1,25 @@
+# Test Purpose
+
+Tests comparing flows with and without recursion level set. Ignoring
+recursion level in flows is useful for devices that run inline IPS and
+terminate an unencrypted tunnel, like an IPv6 tunnel. Terminating the
+tunnel causes ingress request and reply traffic to have different
+headers. e.g.
+
+request:  IPv4]ICMP] -> |IPS| -> IPv6]IPv4]ICMP]
+reply:               <- |IPS| <- IPv6]IPv4]ICMP]
+
+There are tests for both IDS and IPS.
+
+The (ids|ips)-tunnel tests are checking when Suricata is an inline device
+that is terminating a tunnel.
+In this case, the request and reply traffic will have different recursion
+levels, due to the tunneling headers.
+
+The (ids|ips)-middleware tests check when the suricata device is analysing tunneled
+packets and is not a tunnel terminator.
+This case should not be affected by recursion level in flows.
+
+## PCAP
+
+This PCAP was generated with scapy.

tests/flow-pkt-recursion/ids-middleware-pkt-flows-recursion-excluded/test.yaml

@@ -0,0 +1,16 @@
+requires:
+  min-version: 8
+
+pcap: ../middleware-pkt-flows.pcap
+
+args:
+- --set decoder.recursion-level.use-for-tracking=false
+
+checks:
+  - filter:
+      count: 1
+      match:
+        event_type: flow
+        proto: ICMP
+        flow.pkts_toserver: 1
+        flow.pkts_toclient: 1

tests/flow-pkt-recursion/ids-middleware-pkt-flows-recursion-included/test.yaml

@@ -0,0 +1,13 @@
+requires:
+  min-version: 8
+
+pcap: ../middleware-pkt-flows.pcap
+
+checks:
+  - filter:
+      count: 1
+      match:
+        event_type: flow
+        proto: ICMP
+        flow.pkts_toserver: 1
+        flow.pkts_toclient: 1

tests/flow-pkt-recursion/ids-tunnel-pkt-flows-recursion-excluded/test.yaml

@@ -0,0 +1,17 @@
+requires:
+  min-version: 8
+
+pcap: ../tunnel-pkt-flows.pcap
+
+args:
+- --set decoder.recursion-level.use-for-tracking=false
+
+checks:
+  # All packets should be caught as being in one flow
+  - filter:
+      count: 2
+      match:
+        event_type: flow
+        proto: ICMP
+        flow.pkts_toserver: 1
+        flow.pkts_toclient: 1

tests/flow-pkt-recursion/ids-tunnel-pkt-flows-recursion-included/test.yaml

@@ -0,0 +1,14 @@
+requires:
+  min-version: 8
+
+pcap: ../tunnel-pkt-flows.pcap
+
+checks:
+  # None of the flows are joined due to different recursion levels
+  - filter:
+      count: 4
+      match:
+        event_type: flow
+        proto: ICMP
+        flow.pkts_toserver: 1
+        flow.pkts_toclient: 0

tests/flow-pkt-recursion/ips-middleware-pkt-flows-recursion-excluded/test.yaml

@@ -0,0 +1,16 @@
+requires:
+  min-version: 8
+
+pcap: ../middleware-pkt-flows.pcap
+
+args:
+-  --simulate-ips --set decoder.recursion-level.use-for-tracking=false
+
+checks:
+  - filter:
+      count: 1
+      match:
+        event_type: flow
+        proto: ICMP
+        flow.pkts_toserver: 1
+        flow.pkts_toclient: 1

tests/flow-pkt-recursion/ips-middleware-pkt-flows-recursion-included/test.yaml

@@ -0,0 +1,16 @@
+requires:
+  min-version: 8
+
+pcap: ../middleware-pkt-flows.pcap
+
+args:
+- --simulate-ips
+
+checks:
+  - filter:
+      count: 1
+      match:
+        event_type: flow
+        proto: ICMP
+        flow.pkts_toserver: 1
+        flow.pkts_toclient: 1

tests/flow-pkt-recursion/ips-tunnel-pkt-flows-recursion-excluded/test.yaml

@@ -0,0 +1,16 @@
+requires:
+  min-version: 8
+
+pcap: ../tunnel-pkt-flows.pcap
+
+args:
+- --simulate-ips --set decoder.recursion-level.use-for-tracking=false
+
+checks:
+  - filter:
+      count: 2
+      match:
+        event_type: flow
+        proto: ICMP
+        flow.pkts_toserver: 1
+        flow.pkts_toclient: 1

tests/flow-pkt-recursion/ips-tunnel-pkt-flows-recursion-included/test.yaml

@@ -0,0 +1,17 @@
+requires:
+  min-version: 8
+
+pcap: ../tunnel-pkt-flows.pcap
+
+args:
+- --simulate-ips
+
+checks:
+  # None of the flows are joined due to different recursion levels
+  - filter:
+      count: 4
+      match:
+        event_type: flow
+        proto: ICMP
+        flow.pkts_toserver: 1
+        flow.pkts_toclient: 0

tests/flow-pkt-recursion/test.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+
+from scapy.all import ICMP, IP, Ether, IPv6, PcapWriter, Raw
+
+mac_1, mac_2 = 'cb:cf:2b:50:a7:61', '49:a2:25:1a:07:4a'
+
+request = Ether(src=mac_1, dst=mac_2)
+reply = Ether(src=mac_2, dst=mac_1)
+
+ip_1, ip_2 = '1.1.1.1', '2.2.2.2'
+ipv6_1, ipv6_2 = 'fd01::1.1.1.1', 'fd02::2.2.2.2'
+
+payload = Raw(b'#JSb[abR^79aV(kDAN,(C\n\\A+p V+MF7\rd9Z&&9D31.;T%\x0ct.#')
+icmp_echo = ICMP(type=8, seq=1) / payload
+icmp_reply = ICMP(type=0, seq=1) / payload
+
+middleware_pcap = Path.cwd() / 'middleware-pkt-flows.pcap'
+with PcapWriter(str(middleware_pcap)) as pcap:
+    # Flow of IPv6 tunneled packets in both directions
+    pcap.write(request / IPv6(src=ipv6_1, dst=ipv6_2) / IP(src=ip_1, dst=ip_2) / icmp_echo)
+    pcap.write(reply / IPv6(src=ipv6_2, dst=ipv6_1) / IP(src=ip_2, dst=ip_1) / icmp_reply)
+
+terminated_pcap = Path.cwd() / 'tunnel-pkt-flows.pcap'
+with PcapWriter(str(terminated_pcap)) as pcap:
+    # Flow of tunnel terminated on Suricata device, echo originates
+    # from Suricata device
+    pcap.write(request / IP(src=ip_1, dst=ip_2) / icmp_echo)
+    pcap.write(reply / IPv6(src=ipv6_2, dst=ipv6_1) / IP(src=ip_2, dst=ip_1) / icmp_reply)
+
+    # Flow of tunnel terminated on Suricata device, reply originates
+    # from Suricata device
+    pcap.write(reply / IPv6(src=ipv6_2, dst=ipv6_1) / IP(src=ip_2, dst=ip_1) / icmp_echo)
+    pcap.write(request / IP(src=ip_1, dst=ip_2) / icmp_reply)