diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1642e68e4cf..829dfcb759d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -206,6 +206,7 @@ /packages/github @elastic/security-service-integrations /packages/gitlab @elastic/security-service-integrations /packages/golang @elastic/obs-infraobs-integrations +/packages/goflow2 @elastic/sec-deployment-and-devices /packages/google_cloud_storage @elastic/security-service-integrations /packages/google_scc @elastic/security-service-integrations /packages/google_workspace @elastic/security-service-integrations diff --git a/packages/goflow2/_dev/build/build.yml b/packages/goflow2/_dev/build/build.yml new file mode 100644 index 00000000000..2ef4015ccd9 --- /dev/null +++ b/packages/goflow2/_dev/build/build.yml @@ -0,0 +1,4 @@ +dependencies: + ecs: + reference: git@v8.11.0 + import_mappings: true \ No newline at end of file diff --git a/packages/goflow2/_dev/build/docs/README.md b/packages/goflow2/_dev/build/docs/README.md new file mode 100644 index 00000000000..5fe541444ed --- /dev/null +++ b/packages/goflow2/_dev/build/docs/README.md @@ -0,0 +1,60 @@ +# GoFlow2 + +The GoFlow2 integration allows you to import logs generated by goflow2. + +The only protocol/normalisation of goflow2 that is supported in this integration is sFlow. +The normalisation of IPFIX and/or NetFlow is not yet support. + +## Data streams +### sflow +The Goflow2 sFlow integration collects one type of data streams: logs + +#### Sample Event +{{ event "sflow" }} + +## Requirements + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + +You need GoFlow2 to create log files for sFlow traffic. +https://github.com/netsampler/goflow2 + +## Setup + +- Install integration and role out elastic agent +- Install GoFlow2 for sFlow logging + +Please use the following GoFlow2 mapping.yaml file: + +``` +# File: /etc/goflow2/mapping.yaml +formatter: + fields: # list of fields to format in JSON + - type + - time_flow_start_ns + - sampler_address + - sequence_num + - in_if + - out_if + - src_addr + - dst_addr + - etype + - proto + - src_port + - dst_port + - src_vlan + - dst_vlan + - sampling_rate + - bytes +``` + +The output sflow transport files must be stored in the directory ```/var/log/sflow/goflow2/``` + +Full command to run GoFlow2 for sflow traffic: +```shell +goflow2 -format json -listen "sflow://:6343" -mapping /etc/goflow2/mapping.yaml -transport.file /var/log/sflow/goflow2/goflow2.log +``` + +## Fields +{{ fields "sflow" }} diff --git a/packages/goflow2/_dev/deploy/docker/docker-compose.yml b/packages/goflow2/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..63ac0969a32 --- /dev/null +++ b/packages/goflow2/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,7 @@ +services: + goflow2-sflow-filestream: + image: alpine + volumes: + - ./sample_logs:/sample_logs:ro + - ${SERVICE_LOGS_DIR}:/var/log/sflow/goflow2/ + command: /bin/sh -c "cp /sample_logs/* /var/log/sflow/goflow2/" diff --git a/packages/goflow2/_dev/deploy/docker/sample_logs/test-goflow2-sflow-sample.log b/packages/goflow2/_dev/deploy/docker/sample_logs/test-goflow2-sflow-sample.log new file mode 100644 index 00000000000..3abe9e4862c --- /dev/null +++ b/packages/goflow2/_dev/deploy/docker/sample_logs/test-goflow2-sflow-sample.log @@ -0,0 +1,9 @@ +{"type":"SFLOW_5","time_flow_start_ns":1722384059314899647,"sampler_address":"67.43.156.1","sequence_num":44555,"in_if":563,"out_if":573,"src_addr":"216.160.83.57","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":10876,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":1000,"bytes":70} +{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"89.160.20.129","sequence_num":27481,"in_if":637,"out_if":742,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":80,"dst_port":55319,"src_vlan":500,"dst_vlan":500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"67.43.156.1","sequence_num":27481,"in_if":637,"out_if":609,"src_addr":"216.160.83.59","dst_addr":"216.160.83.60","etype":"IPv4","proto":"ESP","src_port":0,"dst_port":0,"src_vlan":500,"dst_vlan":500,"sampling_rate":500,"bytes":142} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.60","dst_addr":"216.160.83.59","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":531,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"UDP","src_port":1122,"dst_port":6097,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":49031,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":31385,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":561,"out_if":531,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":70} diff --git a/packages/goflow2/changelog.yml b/packages/goflow2/changelog.yml new file mode 100644 index 00000000000..5cc386f1d8b --- /dev/null +++ b/packages/goflow2/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.1.0" + changes: + - description: Initial version of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/10561 diff --git a/packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log b/packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log new file mode 100644 index 00000000000..7f9b08df749 --- /dev/null +++ b/packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log @@ -0,0 +1,12 @@ +{"type":"SFLOW_5","time_flow_start_ns":1722384059314899647,"sampler_address":"67.43.156.1","sequence_num":44555,"in_if":563,"out_if":573,"src_addr":"216.160.83.57","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":10876,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":1000,"bytes":70} +{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"89.160.20.129","sequence_num":27481,"in_if":637,"out_if":742,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":80,"dst_port":55319,"src_vlan":500,"dst_vlan":500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"67.43.156.1","sequence_num":27481,"in_if":637,"out_if":609,"src_addr":"216.160.83.59","dst_addr":"216.160.83.60","etype":"IPv4","proto":"ESP","src_port":0,"dst_port":0,"src_vlan":500,"dst_vlan":500,"sampling_rate":500,"bytes":142} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.60","dst_addr":"216.160.83.59","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":531,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"UDP","src_port":1122,"dst_port":6097,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":49031,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":31385,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":561,"out_if":531,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":70} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","in_if":561,"out_if":531,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":""} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"sampling_rate":111,"bytes":3321} +{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"src_addr":"","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"sampling_rate":111,"bytes":3321} diff --git a/packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log-config.yml b/packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log-config.yml new file mode 100644 index 00000000000..70e5e766bdc --- /dev/null +++ b/packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log-config.yml @@ -0,0 +1,7 @@ +fields: + tags: + - preserve_original_event + - forwarded + - sflow + event: + timezone: "+00:00" diff --git a/packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log-expected.json b/packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log-expected.json new file mode 100644 index 00000000000..09cf71dc0c8 --- /dev/null +++ b/packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log-expected.json @@ -0,0 +1,1158 @@ +{ + "expected": [ + { + "@timestamp": "2024-07-31T00:00:59.314Z", + "destination": { + "address": [ + "216.160.83.58" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.58", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059314899647,\"sampler_address\":\"67.43.156.1\",\"sequence_num\":44555,\"in_if\":563,\"out_if\":573,\"src_addr\":\"216.160.83.57\",\"dst_addr\":\"216.160.83.58\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":10876,\"dst_port\":443,\"src_vlan\":1500,\"dst_vlan\":1500,\"sampling_rate\":1000,\"bytes\":70}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 70000, + "packets": 1000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "573" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "563" + }, + "vlan": { + "id": "1500" + } + }, + "ip": [ + "67.43.156.1" + ] + }, + "related": { + "ip": [ + "216.160.83.57", + "216.160.83.58" + ] + }, + "sflow": { + "bytes": 70, + "sample_rate": 1000, + "sequence_num": 44555 + }, + "source": { + "address": [ + "216.160.83.57" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.57", + "port": 10876 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.333Z", + "destination": { + "address": [ + "81.2.69.194" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.194", + "port": 55319 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059333197201,\"sampler_address\":\"89.160.20.129\",\"sequence_num\":27481,\"in_if\":637,\"out_if\":742,\"src_addr\":\"81.2.69.193\",\"dst_addr\":\"81.2.69.194\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":80,\"dst_port\":55319,\"src_vlan\":500,\"dst_vlan\":500,\"sampling_rate\":2000,\"bytes\":1518}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 3036000, + "packets": 2000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "742" + }, + "vlan": { + "id": "500" + } + }, + "ingress": { + "interface": { + "id": "637" + }, + "vlan": { + "id": "500" + } + }, + "ip": [ + "89.160.20.129" + ] + }, + "related": { + "ip": [ + "81.2.69.193", + "81.2.69.194" + ] + }, + "sflow": { + "bytes": 1518, + "sample_rate": 2000, + "sequence_num": 27481 + }, + "source": { + "address": [ + "81.2.69.193" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.193", + "port": 80 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.333Z", + "destination": { + "address": [ + "216.160.83.60" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.60", + "port": 0 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059333197201,\"sampler_address\":\"67.43.156.1\",\"sequence_num\":27481,\"in_if\":637,\"out_if\":609,\"src_addr\":\"216.160.83.59\",\"dst_addr\":\"216.160.83.60\",\"etype\":\"IPv4\",\"proto\":\"ESP\",\"src_port\":0,\"dst_port\":0,\"src_vlan\":500,\"dst_vlan\":500,\"sampling_rate\":500,\"bytes\":142}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 71000, + "packets": 500, + "transport": "esp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "609" + }, + "vlan": { + "id": "500" + } + }, + "ingress": { + "interface": { + "id": "637" + }, + "vlan": { + "id": "500" + } + }, + "ip": [ + "67.43.156.1" + ] + }, + "related": { + "ip": [ + "216.160.83.59", + "216.160.83.60" + ] + }, + "sflow": { + "bytes": 142, + "sample_rate": 500, + "sequence_num": 27481 + }, + "source": { + "address": [ + "216.160.83.59" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.59", + "port": 0 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.483Z", + "destination": { + "address": [ + "216.160.83.59" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.59", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059483524068,\"sampler_address\":\"67.43.156.1\",\"sequence_num\":1022,\"in_if\":0,\"out_if\":561,\"src_addr\":\"216.160.83.60\",\"dst_addr\":\"216.160.83.59\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":19156,\"dst_port\":443,\"src_vlan\":0,\"dst_vlan\":1500,\"sampling_rate\":2000,\"bytes\":1518}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 3036000, + "packets": 2000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "561" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "0" + }, + "vlan": { + "id": "0" + } + }, + "ip": [ + "67.43.156.1" + ] + }, + "related": { + "ip": [ + "216.160.83.60", + "216.160.83.59" + ] + }, + "sflow": { + "bytes": 1518, + "sample_rate": 2000, + "sequence_num": 1022 + }, + "source": { + "address": [ + "216.160.83.60" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.60", + "port": 19156 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.483Z", + "destination": { + "address": [ + "216.160.83.58" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.58", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059483524068,\"sampler_address\":\"67.43.156.1\",\"sequence_num\":1022,\"in_if\":0,\"out_if\":561,\"src_addr\":\"216.160.83.59\",\"dst_addr\":\"216.160.83.58\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":19156,\"dst_port\":443,\"src_vlan\":0,\"dst_vlan\":1500,\"sampling_rate\":2000,\"bytes\":1518}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 3036000, + "packets": 2000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "561" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "0" + }, + "vlan": { + "id": "0" + } + }, + "ip": [ + "67.43.156.1" + ] + }, + "related": { + "ip": [ + "216.160.83.59", + "216.160.83.58" + ] + }, + "sflow": { + "bytes": 1518, + "sample_rate": 2000, + "sequence_num": 1022 + }, + "source": { + "address": [ + "216.160.83.59" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.59", + "port": 19156 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.483Z", + "destination": { + "address": [ + "216.160.83.58" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.58", + "port": 6097 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059483524068,\"sampler_address\":\"67.43.156.1\",\"sequence_num\":1022,\"in_if\":531,\"out_if\":561,\"src_addr\":\"216.160.83.59\",\"dst_addr\":\"216.160.83.58\",\"etype\":\"IPv4\",\"proto\":\"UDP\",\"src_port\":1122,\"dst_port\":6097,\"src_vlan\":1500,\"dst_vlan\":1500,\"sampling_rate\":2000,\"bytes\":1518}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 3036000, + "packets": 2000, + "transport": "udp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "561" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "531" + }, + "vlan": { + "id": "1500" + } + }, + "ip": [ + "67.43.156.1" + ] + }, + "related": { + "ip": [ + "216.160.83.59", + "216.160.83.58" + ] + }, + "sflow": { + "bytes": 1518, + "sample_rate": 2000, + "sequence_num": 1022 + }, + "source": { + "address": [ + "216.160.83.59" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.59", + "port": 1122 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.483Z", + "destination": { + "address": [ + "81.2.69.194" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.194", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059483524068,\"sampler_address\":\"89.160.20.129\",\"sequence_num\":1022,\"in_if\":0,\"out_if\":561,\"src_addr\":\"81.2.69.193\",\"dst_addr\":\"81.2.69.194\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":49031,\"dst_port\":443,\"src_vlan\":0,\"dst_vlan\":1500,\"sampling_rate\":2000,\"bytes\":1518}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 3036000, + "packets": 2000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "561" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "0" + }, + "vlan": { + "id": "0" + } + }, + "ip": [ + "89.160.20.129" + ] + }, + "related": { + "ip": [ + "81.2.69.193", + "81.2.69.194" + ] + }, + "sflow": { + "bytes": 1518, + "sample_rate": 2000, + "sequence_num": 1022 + }, + "source": { + "address": [ + "81.2.69.193" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.193", + "port": 49031 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.483Z", + "destination": { + "address": [ + "81.2.69.194" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.194", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059483524068,\"sampler_address\":\"89.160.20.129\",\"sequence_num\":1022,\"in_if\":0,\"out_if\":561,\"src_addr\":\"81.2.69.193\",\"dst_addr\":\"81.2.69.194\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":31385,\"dst_port\":443,\"src_vlan\":0,\"dst_vlan\":1500,\"sampling_rate\":2000,\"bytes\":1518}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 3036000, + "packets": 2000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "561" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "0" + }, + "vlan": { + "id": "0" + } + }, + "ip": [ + "89.160.20.129" + ] + }, + "related": { + "ip": [ + "81.2.69.193", + "81.2.69.194" + ] + }, + "sflow": { + "bytes": 1518, + "sample_rate": 2000, + "sequence_num": 1022 + }, + "source": { + "address": [ + "81.2.69.193" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.193", + "port": 31385 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.483Z", + "destination": { + "address": [ + "81.2.69.194" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.194", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059483524068,\"sampler_address\":\"89.160.20.129\",\"sequence_num\":1022,\"in_if\":561,\"out_if\":531,\"src_addr\":\"81.2.69.193\",\"dst_addr\":\"81.2.69.194\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":6097,\"dst_port\":443,\"src_vlan\":1500,\"dst_vlan\":1500,\"sampling_rate\":2000,\"bytes\":70}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 140000, + "packets": 2000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "531" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "561" + }, + "vlan": { + "id": "1500" + } + }, + "ip": [ + "89.160.20.129" + ] + }, + "related": { + "ip": [ + "81.2.69.193", + "81.2.69.194" + ] + }, + "sflow": { + "bytes": 70, + "sample_rate": 2000, + "sequence_num": 1022 + }, + "source": { + "address": [ + "81.2.69.193" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.193", + "port": 6097 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.483Z", + "destination": { + "address": [ + "81.2.69.194" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.194", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059483524068,\"sampler_address\":\"89.160.20.129\",\"in_if\":561,\"out_if\":531,\"src_addr\":\"81.2.69.193\",\"dst_addr\":\"81.2.69.194\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":6097,\"dst_port\":443,\"src_vlan\":1500,\"dst_vlan\":1500,\"sampling_rate\":2000,\"bytes\":\"\"}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "packets": 2000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "531" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "561" + }, + "vlan": { + "id": "1500" + } + }, + "ip": [ + "89.160.20.129" + ] + }, + "related": { + "ip": [ + "81.2.69.193", + "81.2.69.194" + ] + }, + "sflow": { + "sample_rate": 2000 + }, + "source": { + "address": [ + "81.2.69.193" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.193", + "port": 6097 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.483Z", + "destination": { + "address": [ + "81.2.69.194" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.194", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059483524068,\"src_addr\":\"81.2.69.193\",\"dst_addr\":\"81.2.69.194\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":6097,\"dst_port\":443,\"sampling_rate\":111,\"bytes\":3321}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 368631, + "packets": 111, + "transport": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "81.2.69.193", + "81.2.69.194" + ] + }, + "sflow": { + "bytes": 3321, + "sample_rate": 111 + }, + "source": { + "address": [ + "81.2.69.193" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.193", + "port": 6097 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + }, + { + "@timestamp": "2024-07-31T00:00:59.483Z", + "destination": { + "address": [ + "81.2.69.194" + ], + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.194", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059483524068,\"src_addr\":\"\",\"dst_addr\":\"81.2.69.194\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":6097,\"dst_port\":443,\"sampling_rate\":111,\"bytes\":3321}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 368631, + "packets": 111, + "transport": "tcp", + "type": "ipv4" + }, + "related": { + "ip": [ + "81.2.69.194" + ] + }, + "sflow": { + "bytes": 3321, + "sample_rate": 111 + }, + "source": { + "port": 6097 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] + } + ] +} \ No newline at end of file diff --git a/packages/goflow2/data_stream/sflow/_dev/test/system/test-filestream-config.yml b/packages/goflow2/data_stream/sflow/_dev/test/system/test-filestream-config.yml new file mode 100644 index 00000000000..5f96eea2ea7 --- /dev/null +++ b/packages/goflow2/data_stream/sflow/_dev/test/system/test-filestream-config.yml @@ -0,0 +1,10 @@ +service: goflow2-sflow-filestream +input: filestream +wait_for_data_timeout: 1m +data_stream: + vars: + preserve_original_event: true + paths: + - "{{{SERVICE_LOGS_DIR}}}/*goflow2-sflow*.log" +assert: + hit_count: 9 diff --git a/packages/goflow2/data_stream/sflow/agent/stream/filestream.yml.hbs b/packages/goflow2/data_stream/sflow/agent/stream/filestream.yml.hbs new file mode 100644 index 00000000000..87d16c52999 --- /dev/null +++ b/packages/goflow2/data_stream/sflow/agent/stream/filestream.yml.hbs @@ -0,0 +1,34 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +{{#if exclude_files.length}} +exclude_files: +{{#each exclude_files as |exclude_file i|}} + - {{exclude_file}} +{{/each}} +{{/if}} +{{#if tags.length}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{else}} +{{#if preserve_original_event}} +tags: + - preserve_original_event +{{/if}} +{{/if}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} +{{#if ignore_older}} +ignore_older: {{ignore_older}} +{{/if}} \ No newline at end of file diff --git a/packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml b/packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..fad54500462 --- /dev/null +++ b/packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,277 @@ +--- +description: Process goflow2 sflow data +processors: +- set: + field: event.kind + value: event + tag: set_event_kind +- append: + field: event.category + value: + - network + tag: set_event_category +- append: + field: event.type + value: + - connection + tag: append_event_type +- set: + field: ecs.version + value: 8.11.0 + tag: set_ecs_version +- json: + field: message + target_field: goflow2 + tag: json_message_to_goflow2 +- set: + tag: set_event_original + field: event.original + copy_from: message +- remove: + tag: remove_message + field: message +- script: + source: ctx.goflow2.time_flow_start_ns = (ctx.goflow2?.time_flow_start_ns / 1000000); + tag: script_calculate_time_flow_start_ns +- script: + source: ctx.goflow2.flow_size = ctx.goflow2?.bytes * ctx.goflow2?.sampling_rate; + if: ctx.goflow2?.bytes != null && ctx.goflow2?.bytes != '' && ctx.goflow2?.sampling_rate != null && ctx.goflow2?.sampling_rate != '' + tag: script_calculate_flow_size +- date: + field: goflow2.time_flow_start_ns + target_field: "@timestamp" + formats: + - UNIX_MS + timezone: UTC + ignore_failure: false + tag: date_parse_time_flow_start_ns +- rename: + field: goflow2.type + target_field: event.action + if: ctx.goflow2?.type != null && ctx.goflow2?.type != '' + tag: rename_type_to_event_action +- append: + field: observer.ip + value: + - "{{{goflow2.sampler_address}}}" + if: ctx.goflow2?.sampler_address != null && ctx.goflow2?.sampler_address != '' + tag: append_sampler_address_to_observer_ip +- rename: + field: goflow2.sequence_num + target_field: sflow.sequence_num + if: ctx.goflow2?.sequence_num != null && ctx.goflow2?.sequence_num != '' + tag: rename_sequence_num_to_sflow_sequence_num +- rename: + field: goflow2.in_if + target_field: observer.ingress.interface.id + if: ctx.goflow2?.in_if != null && ctx.goflow2?.in_if != '' + tag: rename_in_if_to_observer_ingress_interface_id +- convert: + field: observer.ingress.interface.id + type: string + ignore_missing: true + tag: convert_observer_ingress_interface_id_to_string +- rename: + field: goflow2.out_if + target_field: observer.egress.interface.id + if: ctx.goflow2?.out_if != null && ctx.goflow2?.out_if != '' + tag: rename_out_if_to_observer_egress_interface_id +- convert: + field: observer.egress.interface.id + type: string + ignore_missing: true + tag: convert_observer_egress_interface_id_to_string +- convert: + field: goflow2.src_addr + type: ip + if: ctx.goflow2?.src_addr != null && ctx.goflow2?.src_addr != '' + tag: convert_src_addr_to_ip +- rename: + field: goflow2.src_addr + target_field: source.ip + if: ctx.goflow2?.src_addr != null && ctx.goflow2?.src_addr != '' + tag: rename_src_addr_to_source_ip +- convert: + field: goflow2.dst_addr + type: ip + if: ctx.goflow2?.dst_addr != null && ctx.goflow2?.dst_addr != '' + tag: convert_dst_addr_to_ip +- rename: + field: goflow2.dst_addr + target_field: destination.ip + if: ctx.goflow2?.dst_addr != null && ctx.goflow2?.dst_addr != '' + tag: rename_dst_addr_to_destination_ip +- rename: + field: goflow2.etype + target_field: network.type + if: ctx.goflow2?.etype != null && ctx.goflow2?.etype != '' + tag: rename_etype_to_network_type +- rename: + field: goflow2.proto + target_field: network.transport + if: ctx.goflow2?.proto != null && ctx.goflow2?.proto != '' + tag: rename_proto_to_network_transport +- rename: + field: goflow2.src_port + target_field: source.port + if: ctx.goflow2?.src_port != null && ctx.goflow2?.src_port != '' + tag: rename_src_port_to_source_port +- rename: + field: goflow2.dst_port + target_field: destination.port + if: ctx.goflow2?.dst_port != null && ctx.goflow2?.dst_port != '' + tag: rename_dst_port_to_destination_port +- rename: + field: goflow2.src_vlan + target_field: observer.ingress.vlan.id + if: ctx.goflow2?.src_vlan != null && ctx.goflow2?.src_vlan != '' + tag: rename_src_vlan_to_observer_ingress_vlan_id +- convert: + field: observer.ingress.vlan.id + type: string + ignore_missing: true + tag: convert_observer_ingress_vlan_id_to_string +- rename: + field: goflow2.dst_vlan + target_field: observer.egress.vlan.id + if: ctx.goflow2?.dst_vlan != null && ctx.goflow2?.dst_vlan != '' + tag: rename_dst_vlan_to_observer_egress_vlan_id +- convert: + field: observer.egress.vlan.id + type: string + ignore_missing: true + tag: convert_observer_egress_vlan_id_to_string +- rename: + field: goflow2.sampling_rate + target_field: network.packets + if: ctx.goflow2?.sampling_rate != null && ctx.goflow2?.sampling_rate != '' + tag: rename_sampling_rate_to_network_packets +- rename: + field: goflow2.bytes + target_field: sflow.bytes + if: ctx.goflow2?.bytes != null && ctx.goflow2?.bytes != '' + tag: rename_bytes_to_sflow_bytes +- rename: + field: goflow2.flow_size + target_field: network.bytes + ignore_missing: true + tag: rename_flow_size_to_network_bytes +- geoip: + if: ctx.source?.geo == null + field: source.ip + target_field: source.geo + ignore_missing: true + tag: geoip_source_ip_to_source_geo +- geoip: + if: ctx.destination?.geo == null + field: destination.ip + target_field: destination.geo + ignore_missing: true + tag: geoip_destination_ip_to_destination_geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + tag: geoip_source_ip_to_source_as +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + tag: geoip_destination_ip_to_destination_as +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true + tag: rename_source_as_asn_to_source_as_number +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true + tag: rename_source_as_organization_name_to_source_as_organization_name +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + tag: rename_destination_as_asn_to_destination_as_number +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + tag: rename_destination_as_organization_name_to_destination_as_organization_name +- remove: + field: + - goflow2.time_flow_start_ns + - goflow2.proto + - goflow2.etype + - goflow2.dst_addr + - goflow2.src_addr + - goflow2.sampler_address + - goflow2.bytes + ignore_missing: true + ignore_failure: true + tag: remove_unused_fields +- remove: + field: + - goflow2 + ignore_missing: true + ignore_failure: true + if: ctx.goflow2?.size() == 0 + tag: remove_goflow2_array_if_empty +- lowercase: + field: network.transport + ignore_missing: true + tag: lowercase_network_transport +- lowercase: + field: network.type + ignore_missing: true + tag: lowercase_network_type +- set: + field: sflow.sample_rate + copy_from: network.packets + ignore_empty_value: true + tag: set_sflow_sample_rate +- append: + field: destination.address + value: + - "{{{destination.ip}}}" + if: ctx.destination?.ip != null && ctx.destination?.ip != '' + tag: append_destination_ip_to_destination_address +- append: + field: source.address + value: + - "{{{source.ip}}}" + if: ctx.source?.ip != null && ctx.source?.ip != '' + tag: append_source_ip_to_source_address +- append: + field: related.ip + value: + - "{{{source.ip}}}" + if: ctx.source?.ip != null && ctx.source?.ip != '' + tag: append_source_ip_to_related_ip +- append: + field: related.ip + value: + - "{{{destination.ip}}}" + if: ctx.destination?.ip != null && ctx.destination?.ip != '' + tag: append_destination_ip_to_related_ip +- remove: + field: event.original + if: ctx?.tags == null || !(ctx.tags.contains('preserve_original_event')) + ignore_failure: true + ignore_missing: true + tag: remove_event_orginial_if_not_tags_preserve_original_event +on_failure: +- set: + field: error.message + value: Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag + }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message + }} - Source {{_source}} + tag: set_error_message_on_failure diff --git a/packages/goflow2/data_stream/sflow/fields/agent.yml b/packages/goflow2/data_stream/sflow/fields/agent.yml new file mode 100644 index 00000000000..a07e2ce0b2b --- /dev/null +++ b/packages/goflow2/data_stream/sflow/fields/agent.yml @@ -0,0 +1,6 @@ +- name: input.type + type: keyword + description: Input type +- name: log.offset + type: long + description: Log offset diff --git a/packages/goflow2/data_stream/sflow/fields/base-fields.yml b/packages/goflow2/data_stream/sflow/fields/base-fields.yml new file mode 100644 index 00000000000..4241322b2f5 --- /dev/null +++ b/packages/goflow2/data_stream/sflow/fields/base-fields.yml @@ -0,0 +1,18 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: log.file.device_id + description: Device Id of the log file this event came from. + type: keyword +- name: log.file.inode + type: keyword + description: Inode of the log file this event came from. diff --git a/packages/goflow2/data_stream/sflow/fields/fields.yml b/packages/goflow2/data_stream/sflow/fields/fields.yml new file mode 100644 index 00000000000..0708dda341c --- /dev/null +++ b/packages/goflow2/data_stream/sflow/fields/fields.yml @@ -0,0 +1,14 @@ +- name: sflow + title: sFlow + description: Fields specifically related to sFlows + type: group + fields: + - name: bytes + type: long + description: Original size in bytes of the sample packet. + - name: sample_rate + type: long + description: sample rate. + - name: sequence_num + type: long + description: flow sequence number. diff --git a/packages/goflow2/data_stream/sflow/manifest.yml b/packages/goflow2/data_stream/sflow/manifest.yml new file mode 100644 index 00000000000..2a4ad560aea --- /dev/null +++ b/packages/goflow2/data_stream/sflow/manifest.yml @@ -0,0 +1,61 @@ +title: "Goflow2 sFlow" +type: logs +streams: + - input: filestream + template_path: filestream.yml.hbs + title: sFlow logs + description: Collect sFlow logs form GoFlow2 + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/sflow/goflow2/*.log + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original`. + type: bool + multi: false + default: false + - name: ignore_older + type: text + title: Ignore events older than + default: 72h + required: false + show_user: false + multi: false + description: >- + If this option is specified, events that are older than the specified amount of time are ignored. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + - name: exclude_files + type: text + title: Exclude files + multi: true + required: false + show_user: false + description: Regular expression patterns in [RE2 syntax](https://github.com/google/re2/wiki/Syntax) matching files to exclude from input. See [exclude_files](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#filebeat-input-log-exclude-files) for details. + default: + - \.gz$ + - name: tags + type: text + title: Tags + multi: true + required: false + show_user: false + default: + - sflow + - forwarded + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + default: | + - add_locale: ~ diff --git a/packages/goflow2/data_stream/sflow/sample_event.json b/packages/goflow2/data_stream/sflow/sample_event.json new file mode 100644 index 00000000000..25a24a15aea --- /dev/null +++ b/packages/goflow2/data_stream/sflow/sample_event.json @@ -0,0 +1,105 @@ +{ + "@timestamp": "2024-07-31T00:00:59.314Z", + "destination": { + "address": [ + "216.160.83.58" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.58", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059314899647,\"sampler_address\":\"67.43.156.1\",\"sequence_num\":44555,\"in_if\":563,\"out_if\":573,\"src_addr\":\"216.160.83.57\",\"dst_addr\":\"216.160.83.58\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":10876,\"dst_port\":443,\"src_vlan\":1500,\"dst_vlan\":1500,\"sampling_rate\":1000,\"bytes\":70}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 70000, + "packets": 1000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "573" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "563" + }, + "vlan": { + "id": "1500" + } + }, + "ip": [ + "67.43.156.1" + ] + }, + "related": { + "ip": [ + "216.160.83.57", + "216.160.83.58" + ] + }, + "sflow": { + "bytes": 70, + "sample_rate": 1000, + "sequence_num": 44555 + }, + "source": { + "address": [ + "216.160.83.57" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.57", + "port": 10876 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] +} diff --git a/packages/goflow2/docs/README.md b/packages/goflow2/docs/README.md new file mode 100644 index 00000000000..d0d6923a090 --- /dev/null +++ b/packages/goflow2/docs/README.md @@ -0,0 +1,184 @@ +# GoFlow2 + +The GoFlow2 integration allows you to import logs generated by goflow2. + +The only protocol/normalisation of goflow2 that is supported in this integration is sFlow. +The normalisation of IPFIX and/or NetFlow is not yet support. + +## Data streams +### sflow +The Goflow2 sFlow integration collects one type of data streams: logs + +#### Sample Event +An example event for `sflow` looks as following: + +```json +{ + "@timestamp": "2024-07-31T00:00:59.314Z", + "destination": { + "address": [ + "216.160.83.58" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.58", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SFLOW_5", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"type\":\"SFLOW_5\",\"time_flow_start_ns\":1722384059314899647,\"sampler_address\":\"67.43.156.1\",\"sequence_num\":44555,\"in_if\":563,\"out_if\":573,\"src_addr\":\"216.160.83.57\",\"dst_addr\":\"216.160.83.58\",\"etype\":\"IPv4\",\"proto\":\"TCP\",\"src_port\":10876,\"dst_port\":443,\"src_vlan\":1500,\"dst_vlan\":1500,\"sampling_rate\":1000,\"bytes\":70}", + "timezone": "+00:00", + "type": [ + "connection" + ] + }, + "network": { + "bytes": 70000, + "packets": 1000, + "transport": "tcp", + "type": "ipv4" + }, + "observer": { + "egress": { + "interface": { + "id": "573" + }, + "vlan": { + "id": "1500" + } + }, + "ingress": { + "interface": { + "id": "563" + }, + "vlan": { + "id": "1500" + } + }, + "ip": [ + "67.43.156.1" + ] + }, + "related": { + "ip": [ + "216.160.83.57", + "216.160.83.58" + ] + }, + "sflow": { + "bytes": 70, + "sample_rate": 1000, + "sequence_num": 44555 + }, + "source": { + "address": [ + "216.160.83.57" + ], + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.57", + "port": 10876 + }, + "tags": [ + "preserve_original_event", + "forwarded", + "sflow" + ] +} + +``` + +## Requirements + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + +You need GoFlow2 to create log files for sFlow traffic. +https://github.com/netsampler/goflow2 + +## Setup + +- Install integration and role out elastic agent +- Install GoFlow2 for sFlow logging + +Please use the following GoFlow2 mapping.yaml file: + +``` +# File: /etc/goflow2/mapping.yaml +formatter: + fields: # list of fields to format in JSON + - type + - time_flow_start_ns + - sampler_address + - sequence_num + - in_if + - out_if + - src_addr + - dst_addr + - etype + - proto + - src_port + - dst_port + - src_vlan + - dst_vlan + - sampling_rate + - bytes +``` + +The output sflow transport files must be stored in the directory ```/var/log/sflow/goflow2/``` + +Full command to run GoFlow2 for sflow traffic: +```shell +goflow2 -format json -listen "sflow://:6343" -mapping /etc/goflow2/mapping.yaml -transport.file /var/log/sflow/goflow2/goflow2.log +``` + +## Fields +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| input.type | Input type | keyword | +| log.file.device_id | Device Id of the log file this event came from. | keyword | +| log.file.inode | Inode of the log file this event came from. | keyword | +| log.offset | Log offset | long | +| sflow.bytes | Original size in bytes of the sample packet. | long | +| sflow.sample_rate | sample rate. | long | +| sflow.sequence_num | flow sequence number. | long | + diff --git a/packages/goflow2/img/goflow2-logo.svg b/packages/goflow2/img/goflow2-logo.svg new file mode 100644 index 00000000000..7203756394a --- /dev/null +++ b/packages/goflow2/img/goflow2-logo.svg @@ -0,0 +1,435 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/goflow2/manifest.yml b/packages/goflow2/manifest.yml new file mode 100644 index 00000000000..b2d8af2fe8a --- /dev/null +++ b/packages/goflow2/manifest.yml @@ -0,0 +1,29 @@ +format_version: 3.2.1 +name: goflow2 +title: "GoFlow2 logs" +version: 0.1.0 +description: "Collect logs from goflow2 with Elastic Agent." +type: integration +categories: + - network +conditions: + kibana: + version: "^8.11.0" + elastic: + subscription: "basic" +icons: + - src: /img/goflow2-logo.svg + title: goflow2 logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: goflow2 + title: goflow2 logs + description: Collect logs generated with goflow2 + inputs: + - type: filestream + title: Collect logs via log file + description: Collecting logs via log file +owner: + github: elastic/sec-deployment-and-devices + type: community