[goflow2] Add GoFlow2 integration (elastic#10561)

- Add the GoFlow2 integration to monitor goflow2 logs
- Add initial sflow data stream to ingest sflow logs from goflow2
- Add system and pipeline tests
---------

Co-authored-by: Christian Hilgers <[email protected]>
Co-authored-by: Mario Schäfer <[email protected]>

.github/CODEOWNERS

@@ -206,6 +206,7 @@
 /packages/github @elastic/security-service-integrations
 /packages/gitlab @elastic/security-service-integrations
 /packages/golang @elastic/obs-infraobs-integrations
+/packages/goflow2 @elastic/sec-deployment-and-devices
 /packages/google_cloud_storage @elastic/security-service-integrations
 /packages/google_scc @elastic/security-service-integrations
 /packages/google_workspace @elastic/security-service-integrations

packages/goflow2/_dev/build/build.yml

@@ -0,0 +1,4 @@
+dependencies:
+  ecs:
+    reference: [email protected]
+    import_mappings: true

packages/goflow2/_dev/build/docs/README.md

@@ -0,0 +1,60 @@
+# GoFlow2
+
+The GoFlow2 integration allows you to import logs generated by goflow2.
+
+The only protocol/normalisation of goflow2 that is supported in this integration is sFlow.
+The normalisation of IPFIX and/or NetFlow is not yet support.
+
+## Data streams
+### sflow
+The Goflow2 sFlow integration collects one type of data streams: logs
+
+#### Sample Event
+{{ event "sflow" }}
+
+## Requirements
+
+You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it.
+You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
+
+You need GoFlow2 to create log files for sFlow traffic.
+https://github.com/netsampler/goflow2
+
+## Setup
+
+- Install integration and role out elastic agent
+- Install GoFlow2 for sFlow logging
+
+Please use the following GoFlow2 mapping.yaml file:
+
+```
+# File: /etc/goflow2/mapping.yaml
+formatter:
+    fields: # list of fields to format in JSON
+        - type
+        - time_flow_start_ns
+        - sampler_address
+        - sequence_num
+        - in_if
+        - out_if
+        - src_addr
+        - dst_addr
+        - etype
+        - proto
+        - src_port
+        - dst_port
+        - src_vlan
+        - dst_vlan
+        - sampling_rate
+        - bytes
+```
+
+The output sflow transport files must be stored in the directory ```/var/log/sflow/goflow2/```
+
+Full command to run GoFlow2 for sflow traffic:
+```shell
+goflow2 -format json -listen "sflow://:6343" -mapping /etc/goflow2/mapping.yaml -transport.file /var/log/sflow/goflow2/goflow2.log
+```
+
+## Fields
+{{ fields "sflow" }}

packages/goflow2/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,7 @@
+services:
+  goflow2-sflow-filestream:
+    image: alpine
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+      - ${SERVICE_LOGS_DIR}:/var/log/sflow/goflow2/
+    command: /bin/sh -c "cp /sample_logs/* /var/log/sflow/goflow2/"

packages/goflow2/_dev/deploy/docker/sample_logs/test-goflow2-sflow-sample.log

@@ -0,0 +1,9 @@
+{"type":"SFLOW_5","time_flow_start_ns":1722384059314899647,"sampler_address":"67.43.156.1","sequence_num":44555,"in_if":563,"out_if":573,"src_addr":"216.160.83.57","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":10876,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":1000,"bytes":70}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"89.160.20.129","sequence_num":27481,"in_if":637,"out_if":742,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":80,"dst_port":55319,"src_vlan":500,"dst_vlan":500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"67.43.156.1","sequence_num":27481,"in_if":637,"out_if":609,"src_addr":"216.160.83.59","dst_addr":"216.160.83.60","etype":"IPv4","proto":"ESP","src_port":0,"dst_port":0,"src_vlan":500,"dst_vlan":500,"sampling_rate":500,"bytes":142}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.60","dst_addr":"216.160.83.59","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":531,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"UDP","src_port":1122,"dst_port":6097,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":49031,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":31385,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":561,"out_if":531,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":70}

packages/goflow2/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+  changes:
+    - description: Initial version of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561

packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log

@@ -0,0 +1,12 @@
+{"type":"SFLOW_5","time_flow_start_ns":1722384059314899647,"sampler_address":"67.43.156.1","sequence_num":44555,"in_if":563,"out_if":573,"src_addr":"216.160.83.57","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":10876,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":1000,"bytes":70}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"89.160.20.129","sequence_num":27481,"in_if":637,"out_if":742,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":80,"dst_port":55319,"src_vlan":500,"dst_vlan":500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"67.43.156.1","sequence_num":27481,"in_if":637,"out_if":609,"src_addr":"216.160.83.59","dst_addr":"216.160.83.60","etype":"IPv4","proto":"ESP","src_port":0,"dst_port":0,"src_vlan":500,"dst_vlan":500,"sampling_rate":500,"bytes":142}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.60","dst_addr":"216.160.83.59","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":531,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"UDP","src_port":1122,"dst_port":6097,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":49031,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":31385,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":561,"out_if":531,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":70}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","in_if":561,"out_if":531,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":""}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"sampling_rate":111,"bytes":3321}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"src_addr":"","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"sampling_rate":111,"bytes":3321}

packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log-config.yml

@@ -0,0 +1,7 @@
+fields:
+  tags:
+    - preserve_original_event
+    - forwarded
+    - sflow
+  event:
+    timezone: "+00:00"