From 3445a3edfdd7d7b826509f03fd66feef9f127e1d Mon Sep 17 00:00:00 2001 From: Panos Koutsovasilis Date: Thu, 29 Aug 2024 18:47:28 +0300 Subject: [PATCH] [netflow]: disable event normalisation (#40635) * feat: netflow disable event normalisation * fix: add kibana in docker-compose.yml used for agentbeat integration tests * fix: switch to normalize and use an active voice for normaliseIPFields --- CHANGELOG.next.asciidoc | 1 + x-pack/agentbeat/docker-compose.yml | 9 + x-pack/filebeat/docker-compose.yml | 9 + x-pack/filebeat/input/netflow/convert.go | 31 +- x-pack/filebeat/input/netflow/input.go | 2 +- .../input/netflow/integration_test.go | 125 +- .../netflow/testdata/integration/test.md | 1547 +++++++++++++++++ .../netflow/testdata/integration/test.pcap | Bin 0 -> 3362 bytes 8 files changed, 1683 insertions(+), 41 deletions(-) create mode 100644 x-pack/filebeat/input/netflow/testdata/integration/test.md create mode 100644 x-pack/filebeat/input/netflow/testdata/integration/test.pcap diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b0f3817b18dc..97e057af2837 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -291,6 +291,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Improve logging in Okta Entity Analytics provider. {issue}40106[40106] {pull}40347[40347] - Document `winlog` input. {issue}40074[40074] {pull}40462[40462] - Added retry logic to websocket connections in the streaming input. {issue}40271[40271] {pull}40601[40601] +- Disable event normalization for netflow input {pull}40635[40635] *Auditbeat* diff --git a/x-pack/agentbeat/docker-compose.yml b/x-pack/agentbeat/docker-compose.yml index 5c98c9b459c0..b4dbda4d86f6 100644 --- a/x-pack/agentbeat/docker-compose.yml +++ b/x-pack/agentbeat/docker-compose.yml @@ -6,6 +6,7 @@ services: image: busybox depends_on: elasticsearch: { condition: service_healthy } + kibana: { condition: service_healthy } cometd: { condition: service_healthy } elasticsearch: @@ -24,3 +25,11 @@ services: hostname: cometd ports: - 8080:8080 + + kibana: + extends: + file: ${ES_BEATS}/testing/environments/${STACK_ENVIRONMENT}.yml + service: kibana + healthcheck: + test: [ "CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'" ] + retries: 600 diff --git a/x-pack/filebeat/docker-compose.yml b/x-pack/filebeat/docker-compose.yml index 5c98c9b459c0..b4dbda4d86f6 100644 --- a/x-pack/filebeat/docker-compose.yml +++ b/x-pack/filebeat/docker-compose.yml @@ -6,6 +6,7 @@ services: image: busybox depends_on: elasticsearch: { condition: service_healthy } + kibana: { condition: service_healthy } cometd: { condition: service_healthy } elasticsearch: @@ -24,3 +25,11 @@ services: hostname: cometd ports: - 8080:8080 + + kibana: + extends: + file: ${ES_BEATS}/testing/environments/${STACK_ENVIRONMENT}.yml + service: kibana + healthcheck: + test: [ "CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'" ] + retries: 600 diff --git a/x-pack/filebeat/input/netflow/convert.go b/x-pack/filebeat/input/netflow/convert.go index 87bbf94788b6..9e39133fd0d4 100644 --- a/x-pack/filebeat/input/netflow/convert.go +++ b/x-pack/filebeat/input/netflow/convert.go @@ -24,13 +24,38 @@ import ( ) func toBeatEvent(flow record.Record, internalNetworks []string) (event beat.Event) { + var e beat.Event switch flow.Type { case record.Flow: - return flowToBeatEvent(flow, internalNetworks) + e = flowToBeatEvent(flow, internalNetworks) case record.Options: - return optionsToBeatEvent(flow) + e = optionsToBeatEvent(flow) default: - return toBeatEventCommon(flow) + e = toBeatEventCommon(flow) + } + + normaliseIPFields(e.Fields) + return e +} + +// normaliseIPFields normalizes net.IP fields in the given map from []byte to string. +// This function mutates the map and assumes every net.IP field is a direct entry. +// Fields that don't adhere to this convention (e.g. part of a struct) are not +// normalized. +func normaliseIPFields(fields mapstr.M) { + for key, value := range fields { + switch valueType := value.(type) { + case net.IP: + fields[key] = valueType.String() + case []net.IP: + stringIPs := make([]string, len(valueType)) + for i, ip := range valueType { + stringIPs[i] = ip.String() + } + fields[key] = stringIPs + case mapstr.M: + normaliseIPFields(valueType) + } } } diff --git a/x-pack/filebeat/input/netflow/input.go b/x-pack/filebeat/input/netflow/input.go index 96d3503b1ec1..f65ab6e2bc9d 100644 --- a/x-pack/filebeat/input/netflow/input.go +++ b/x-pack/filebeat/input/netflow/input.go @@ -155,7 +155,7 @@ func (n *netflowInput) Run(env v2.Context, connector beat.PipelineConnector) err client, err := connector.ConnectWith(beat.ClientConfig{ PublishMode: beat.DefaultGuarantees, Processing: beat.ProcessingConfig{ - EventNormalization: boolPtr(true), + EventNormalization: boolPtr(false), }, EventListener: nil, }) diff --git a/x-pack/filebeat/input/netflow/integration_test.go b/x-pack/filebeat/input/netflow/integration_test.go index 3cdb87de2f23..b7bb2a3203d4 100644 --- a/x-pack/filebeat/input/netflow/integration_test.go +++ b/x-pack/filebeat/input/netflow/integration_test.go @@ -7,6 +7,7 @@ package netflow_test import ( + "bytes" "context" "encoding/json" "errors" @@ -22,7 +23,6 @@ import ( "golang.org/x/time/rate" - "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/tests/integration" filebeat "github.com/elastic/beats/v7/x-pack/filebeat/cmd" "github.com/elastic/elastic-agent-client/v7/pkg/client/mock" @@ -35,7 +35,7 @@ import ( ) const ( - waitFor = 10 * time.Second + waitFor = 20 * time.Second tick = 200 * time.Millisecond ) @@ -50,16 +50,26 @@ func TestNetFlowIntegration(t *testing.T) { outputHost := fmt.Sprintf("%s://%s:%s", esConnectionDetails.Scheme, esConnectionDetails.Hostname(), esConnectionDetails.Port()) outputHosts := []interface{}{outputHost} + kibanaURL, kibanaUser := integration.GetKibana(t) + kibanaUsername := kibanaUser.Username() + kibanaPassword, ok := kibanaUser.Password() + require.True(t, ok, "kibana user should have a password") + + // since beat is managed by a mocked elastic-agent we need to install the netflow package + // through the Kibana API + err := installNetflowPackage(ctx, kibanaURL.String(), kibanaUsername, kibanaPassword) + require.NoError(t, err, "failed to install netflow package") + // we are going to need admin access to query ES about the logs-netflow.log-default data_stream outputUsername := os.Getenv("ES_SUPERUSER_USER") - require.NotEmpty(t, outputUsername) + require.NotEmpty(t, outputUsername, "ES_SUPERUSER_USER env var must be set") outputPassword := os.Getenv("ES_SUPERUSER_PASS") - require.NotEmpty(t, outputPassword) + require.NotEmpty(t, outputPassword, "ES_SUPERUSER_PASS env var must be set") outputProtocol := esConnectionDetails.Scheme deleted, err := DeleteDataStream(ctx, outputUsername, outputPassword, outputHost, "logs-netflow.log-default") - require.NoError(t, err) - require.True(t, deleted) + require.NoError(t, err, "failed to delete data stream") + require.True(t, deleted, "failed to delete data stream") // construct expected Agent units allStreams := []*proto.UnitExpected{ @@ -129,7 +139,7 @@ func TestNetFlowIntegration(t *testing.T) { "queue_size": 2 * 4 * 1600, "detect_sequence_reset": true, "max_message_size": "10KiB", - "workers": 100, + "workers": 8, }), }, }, @@ -190,38 +200,31 @@ func TestNetFlowIntegration(t *testing.T) { case err := <-beatRunErr: t.Fatalf("beat run err: %v", err) case <-time.After(waitFor): - t.Fatalf("timed out waiting for beat to become healthy") + t.Fatalf("timed out waiting for filebeat to report healthy") } registry := monitoring.GetNamespace("dataset").GetRegistry().GetRegistry("netflow_integration_test") discardedEventsTotalVar, ok := registry.Get("discarded_events_total").(*monitoring.Uint) - require.True(t, ok) + require.True(t, ok, "failed to get discarded_events_total metric") receivedEventTotalVar, ok := registry.Get("received_events_total").(*monitoring.Uint) - require.True(t, ok) + require.True(t, ok, "failed to get received_events_total metric") udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:6006") - require.NoError(t, err) + require.NoError(t, err, "failed to resolve UDP address") conn, err := net.DialUDP("udp", nil, udpAddr) - require.NoError(t, err) - - data, err := os.ReadFile("testdata/golden/ipfix_cisco.reversed.pcap.golden.json") - require.NoError(t, err) - - var expectedFlows struct { - Flows []beat.Event `json:"events,omitempty"` - } - err = json.Unmarshal(data, &expectedFlows) - require.NoError(t, err) + require.NoError(t, err, "failed to open UDP connection") - f, err := pcap.OpenOffline("testdata/pcap/ipfix_cisco.reversed.pcap") - require.NoError(t, err) + // for more info look testdata/integration/test.md + f, err := pcap.OpenOffline("testdata/integration/test.pcap") + require.NoError(t, err, "failed to open pcap file") defer f.Close() + expectedEventsNumbers := 32 var totalBytes, totalPackets int - rateLimit := 10000 + rateLimit := 3000 limiter := rate.NewLimiter(rate.Limit(rateLimit), rateLimit) packetSource := gopacket.NewPacketSource(f, f.LinkType()) @@ -229,33 +232,35 @@ func TestNetFlowIntegration(t *testing.T) { if totalPackets%rateLimit == 0 { err = limiter.WaitN(ctx, rateLimit) - require.NoError(t, err) + require.NoError(t, err, "failed to wait for rate limiter") } payloadData := pkt.TransportLayer().LayerPayload() n, err := conn.Write(payloadData) - require.NoError(t, err) + require.NoError(t, err, "failed to write payload to UDP connection") totalBytes += n totalPackets++ } - require.Zero(t, discardedEventsTotalVar.Get()) + require.Zero(t, discardedEventsTotalVar.Get(), "expected no discarded events") require.Eventually(t, func() bool { return receivedEventTotalVar.Get() == uint64(totalPackets) - }, waitFor, tick) + }, waitFor, tick, "expected all events to be received") require.Eventually(t, func() bool { return HasDataStream(ctx, outputUsername, outputPassword, outputHost, "logs-netflow.log-default") == nil - }, waitFor, tick) + }, waitFor, tick, "expected netflow data stream to be created") require.Eventually(t, func() bool { - eventsCount, err := DataStreamEventsCount(ctx, outputUsername, outputPassword, outputHost, "logs-netflow.log-default") - require.NoError(t, err) - return eventsCount == uint64(len(expectedFlows.Flows)) - }, waitFor, tick) + streamEventsCount, err := DataStreamEventsCount(ctx, outputUsername, outputPassword, outputHost, "logs-netflow.log-default") + if err != nil { + return false + } + return streamEventsCount == uint64(expectedEventsNumbers) + }, waitFor, tick, fmt.Sprintf("expected netflow data stream to have %d events", expectedEventsNumbers)) } type unitPayload map[string]interface{} @@ -299,7 +304,7 @@ type DataStreamResult struct { } func HasDataStream(ctx context.Context, username string, password string, url string, name string) error { - resultBytes, err := request(ctx, http.MethodGet, username, password, fmt.Sprintf("%s/_data_stream/%s", url, name)) + resultBytes, err := request(ctx, http.MethodGet, username, password, fmt.Sprintf("%s/_data_stream/%s", url, name), nil, nil) if err != nil { return err } @@ -339,7 +344,7 @@ type CountResults struct { } func DataStreamEventsCount(ctx context.Context, username string, password string, url string, name string) (uint64, error) { - resultBytes, err := request(ctx, http.MethodGet, username, password, fmt.Sprintf("%s/%s/_count?q=!_ignored:*+AND+!event.message:*", url, name)) + resultBytes, err := request(ctx, http.MethodGet, username, password, fmt.Sprintf("%s/%s/_count?q=!_ignored:*+AND+!event.message:*", url, name), nil, nil) if err != nil { return 0, err } @@ -362,7 +367,7 @@ type DeleteResults struct { } func DeleteDataStream(ctx context.Context, username string, password string, url string, name string) (bool, error) { - _, err := request(ctx, http.MethodDelete, username, password, fmt.Sprintf("%s/_data_stream/%s", url, name)) + _, err := request(ctx, http.MethodDelete, username, password, fmt.Sprintf("%s/_data_stream/%s", url, name), nil, nil) if err != nil { return false, err } @@ -370,13 +375,56 @@ func DeleteDataStream(ctx context.Context, username string, password string, url return true, nil } -func request(ctx context.Context, httpMethod string, username string, password string, url string) ([]byte, error) { +func installNetflowPackage(ctx context.Context, url string, username string, password string) error { + + type Response struct { + Item struct { + Version string `json:"version"` + } `json:"item"` + } + + resp, err := request(ctx, http.MethodGet, username, password, fmt.Sprintf("%s/api/fleet/epm/packages/netflow?prerelease=true", url), nil, nil) + if err != nil { + return err + } + + var results Response + err = json.Unmarshal(resp, &results) + if err != nil { + return err + } + + version := results.Item.Version + + resp, err = request(ctx, http.MethodPost, username, password, fmt.Sprintf("%s/api/fleet/epm/packages/netflow/%s", url, version), map[string]string{ + "kbn-xsrf": "true", + }, []byte(`{"force":true}`)) + if err != nil { + return err + } + + if resp == nil { + return errors.New("http not found error") + } + + return nil +} + +func request(ctx context.Context, httpMethod string, username string, password string, url string, headers map[string]string, reqBody []byte) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, httpMethod, url, nil) if err != nil { return nil, err } req.SetBasicAuth(username, password) + for k, v := range headers { + req.Header.Set(k, v) + } + + if reqBody != nil { + req.Body = io.NopCloser(bytes.NewReader(reqBody)) + } + res, err := http.DefaultClient.Do(req) if err != nil { return nil, err @@ -384,7 +432,10 @@ func request(ctx context.Context, httpMethod string, username string, password s defer res.Body.Close() if res.StatusCode == http.StatusNotFound { return nil, nil + } else if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", res.StatusCode) } + resultBytes, err := io.ReadAll(res.Body) if err != nil { return nil, err diff --git a/x-pack/filebeat/input/netflow/testdata/integration/test.md b/x-pack/filebeat/input/netflow/testdata/integration/test.md new file mode 100644 index 000000000000..2a6dcc4dc38a --- /dev/null +++ b/x-pack/filebeat/input/netflow/testdata/integration/test.md @@ -0,0 +1,1547 @@ +[integration/test.pcap](test.pcap) is a packet capture file that contains all the necessary Flow templates and records +in the correct sequence so that either with a single worker or multi workers (enabled template LRU) netflow input +produces the same number of `32` events. A snapshot of those extracted with netflow v2.18.0 integration installed is +shown below. The reason for relying only on checking the number of events in the integration test is to reduce the test +flaky-ness by making it less error-prone to future integration field changes. + +```json +{ + "took": 3, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 32, + "relation": "eq" + }, + "max_score": 1.0, + "hits": [ + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "YvpB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "type": "filebeat", + "version": "8.16.0" + }, + "destination": { + "port": 61137, + "ip": "10.100.11.14", + "locality": "internal", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 993, + "bytes": 298, + "ip": "17.42.251.56", + "locality": "external", + "mac": "56-E0-32-C1-82-07", + "packets": 4 + }, + "network": { + "community_id": "1:/kw4vWmSSwJD+zp6pUP//kxYUR8=", + "bytes": 298, + "transport": "tcp", + "type": "ipv4", + "iana_number": "6", + "packets": 4, + "direction": "inbound" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "input": { + "type": "netflow" + }, + "netflow": { + "packet_delta_count": 4, + "protocol_identifier": 6, + "vlan_id": 0, + "source_mac_address": "56-E0-32-C1-82-07", + "flow_start_sys_up_time": 2383112, + "egress_interface": 4, + "octet_delta_count": 298, + "type": "netflow_flow", + "destination_ipv4_address": "10.100.11.14", + "source_ipv4_address": "17.42.251.56", + "delta_flow_count": 0, + "exporter": { + "uptime_millis": 2446901, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:24.000Z" + }, + "tcp_control_bits": 24, + "ip_version": 4, + "ip_class_of_service": 0, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2383171, + "source_transport_port": 993, + "destination_transport_port": 61137 + }, + "@timestamp": "2024-06-13T23:29:24.000Z", + "related": { + "ip": [ + "10.100.11.14", + "17.42.251.56" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 59000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "kind": "event", + "created": "2024-07-28T21:31:36.104Z", + "start": "2024-06-13T23:28:20.211Z", + "action": "netflow_flow", + "end": "2024-06-13T23:28:20.270Z", + "category": [ + "network" + ], + "type": [ + "connection" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "kOQywjrRMT4" + } + } + }, + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "Y_pB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "type": "filebeat", + "version": "8.16.0" + }, + "destination": { + "port": 65058, + "ip": "10.100.11.10", + "locality": "internal", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 5223, + "bytes": 846, + "ip": "17.57.147.5", + "locality": "external", + "mac": "56-E0-32-C1-82-07", + "packets": 2 + }, + "network": { + "community_id": "1:JVogtRH3XanHiXtN+KyNkFU75VI=", + "bytes": 846, + "transport": "tcp", + "type": "ipv4", + "iana_number": "6", + "packets": 2, + "direction": "inbound" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "input": { + "type": "netflow" + }, + "netflow": { + "protocol_identifier": 6, + "packet_delta_count": 2, + "vlan_id": 0, + "flow_start_sys_up_time": 2383988, + "source_mac_address": "56-E0-32-C1-82-07", + "octet_delta_count": 846, + "egress_interface": 4, + "type": "netflow_flow", + "destination_ipv4_address": "10.100.11.10", + "source_ipv4_address": "17.57.147.5", + "delta_flow_count": 0, + "exporter": { + "uptime_millis": 2446901, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:24.000Z" + }, + "tcp_control_bits": 24, + "ip_class_of_service": 0, + "ip_version": 4, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2384006, + "source_transport_port": 5223, + "destination_transport_port": 65058 + }, + "@timestamp": "2024-06-13T23:29:24.000Z", + "ecs": { + "version": "8.11.0" + }, + "related": { + "ip": [ + "10.100.11.10", + "17.57.147.5" + ] + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 18000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "created": "2024-07-28T21:31:36.104Z", + "kind": "event", + "start": "2024-06-13T23:28:21.087Z", + "action": "netflow_flow", + "end": "2024-06-13T23:28:21.105Z", + "type": [ + "connection" + ], + "category": [ + "network" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "EXqS-Ey8D6o" + } + } + }, + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "ZPpB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "type": "filebeat", + "version": "8.16.0" + }, + "destination": { + "port": 45884, + "ip": "10.100.8.34", + "locality": "internal", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 853, + "bytes": 3273, + "ip": "1.0.0.1", + "locality": "external", + "packets": 16, + "mac": "56-E0-32-C1-82-07" + }, + "network": { + "community_id": "1:7zyW7exctP8D685WBGJPtxtT1xs=", + "bytes": 3273, + "transport": "tcp", + "type": "ipv4", + "iana_number": "6", + "packets": 16, + "direction": "inbound" + }, + "input": { + "type": "netflow" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "netflow": { + "packet_delta_count": 16, + "protocol_identifier": 6, + "vlan_id": 0, + "flow_start_sys_up_time": 2381818, + "source_mac_address": "56-E0-32-C1-82-07", + "egress_interface": 4, + "octet_delta_count": 3273, + "type": "netflow_flow", + "destination_ipv4_address": "10.100.8.34", + "source_ipv4_address": "1.0.0.1", + "exporter": { + "uptime_millis": 2446901, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:24.000Z" + }, + "delta_flow_count": 0, + "tcp_control_bits": 30, + "ip_version": 4, + "ip_class_of_service": 0, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2434554, + "source_transport_port": 853, + "destination_transport_port": 45884 + }, + "@timestamp": "2024-06-13T23:29:24.000Z", + "related": { + "ip": [ + "1.0.0.1", + "10.100.8.34" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 52736000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "created": "2024-07-28T21:31:38.073Z", + "kind": "event", + "start": "2024-06-13T23:28:18.917Z", + "action": "netflow_flow", + "end": "2024-06-13T23:29:11.653Z", + "category": [ + "network" + ], + "type": [ + "connection" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "1F9ai5fhOyY" + } + } + }, + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "ZfpB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "type": "filebeat", + "version": "8.16.0" + }, + "destination": { + "port": 56290, + "ip": "10.100.8.34", + "locality": "internal", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 853, + "bytes": 3402, + "ip": "1.0.0.1", + "locality": "external", + "mac": "56-E0-32-C1-82-07", + "packets": 15 + }, + "network": { + "community_id": "1:JiM5h5WwM8mL6KbDc2FcKTrW6l0=", + "bytes": 3402, + "transport": "tcp", + "type": "ipv4", + "iana_number": "6", + "packets": 15, + "direction": "inbound" + }, + "input": { + "type": "netflow" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "netflow": { + "protocol_identifier": 6, + "packet_delta_count": 15, + "vlan_id": 0, + "source_mac_address": "56-E0-32-C1-82-07", + "flow_start_sys_up_time": 2402156, + "octet_delta_count": 3402, + "egress_interface": 4, + "type": "netflow_flow", + "destination_ipv4_address": "10.100.8.34", + "source_ipv4_address": "1.0.0.1", + "exporter": { + "uptime_millis": 2446901, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:24.000Z" + }, + "delta_flow_count": 0, + "tcp_control_bits": 30, + "ip_version": 4, + "ip_class_of_service": 0, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2434554, + "source_transport_port": 853, + "destination_transport_port": 56290 + }, + "@timestamp": "2024-06-13T23:29:24.000Z", + "related": { + "ip": [ + "1.0.0.1", + "10.100.8.34" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "family": "debian", + "type": "linux", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 32398000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "created": "2024-07-28T21:31:38.073Z", + "kind": "event", + "start": "2024-06-13T23:28:39.255Z", + "action": "netflow_flow", + "end": "2024-06-13T23:29:11.653Z", + "type": [ + "connection" + ], + "category": [ + "network" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "t2rofv-PTS8" + } + } + }, + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "ZvpB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "type": "filebeat", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "version": "8.16.0" + }, + "destination": { + "port": 48478, + "ip": "10.100.8.38", + "locality": "internal", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 443, + "bytes": 7177, + "ip": "54.160.25.132", + "locality": "external", + "mac": "56-E0-32-C1-82-07", + "packets": 13 + }, + "network": { + "community_id": "1:dPba87I3GhRefpqzpLtFV1FmSCc=", + "bytes": 7177, + "transport": "tcp", + "type": "ipv4", + "iana_number": "6", + "packets": 13, + "direction": "inbound" + }, + "input": { + "type": "netflow" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "netflow": { + "packet_delta_count": 13, + "protocol_identifier": 6, + "vlan_id": 0, + "source_mac_address": "56-E0-32-C1-82-07", + "flow_start_sys_up_time": 2382399, + "octet_delta_count": 7177, + "egress_interface": 4, + "type": "netflow_flow", + "destination_ipv4_address": "10.100.8.38", + "source_ipv4_address": "54.160.25.132", + "delta_flow_count": 0, + "exporter": { + "uptime_millis": 2446901, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:24.000Z" + }, + "tcp_control_bits": 27, + "ip_version": 4, + "ip_class_of_service": 0, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2385519, + "destination_transport_port": 48478, + "source_transport_port": 443 + }, + "@timestamp": "2024-06-13T23:29:24.000Z", + "related": { + "ip": [ + "10.100.8.38", + "54.160.25.132" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 3120000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "created": "2024-07-28T21:31:38.073Z", + "kind": "event", + "start": "2024-06-13T23:28:19.498Z", + "action": "netflow_flow", + "end": "2024-06-13T23:28:22.618Z", + "type": [ + "connection" + ], + "category": [ + "network" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "faXLmv8YuVA" + } + } + }, + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "Z_pB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "type": "filebeat", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "version": "8.16.0" + }, + "destination": { + "port": 50020, + "ip": "71.191.210.227", + "locality": "external", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 50020, + "bytes": 1775780, + "ip": "64.225.12.142", + "locality": "external", + "packets": 5439, + "mac": "56-E0-32-C1-82-07" + }, + "network": { + "community_id": "1:5adPyESITZct6QsulanflJ1zzGw=", + "bytes": 1775780, + "transport": "udp", + "type": "ipv4", + "iana_number": "17", + "packets": 5439, + "direction": "external" + }, + "input": { + "type": "netflow" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "netflow": { + "packet_delta_count": 5439, + "protocol_identifier": 17, + "vlan_id": 0, + "flow_start_sys_up_time": 2386212, + "source_mac_address": "56-E0-32-C1-82-07", + "egress_interface": 0, + "octet_delta_count": 1775780, + "type": "netflow_flow", + "destination_ipv4_address": "71.191.210.227", + "source_ipv4_address": "64.225.12.142", + "delta_flow_count": 0, + "exporter": { + "uptime_millis": 2447278, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:25.000Z" + }, + "tcp_control_bits": 0, + "ip_version": 4, + "ip_class_of_service": 0, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2447242, + "source_transport_port": 50020, + "destination_transport_port": 50020 + }, + "@timestamp": "2024-06-13T23:29:25.000Z", + "related": { + "ip": [ + "64.225.12.142", + "71.191.210.227" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "containerized": false, + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 61030000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "created": "2024-07-28T21:31:38.073Z", + "kind": "event", + "start": "2024-06-13T23:28:23.934Z", + "action": "netflow_flow", + "end": "2024-06-13T23:29:24.964Z", + "category": [ + "network" + ], + "type": [ + "connection" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "Z43o3EB3dqc" + } + } + }, + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "aPpB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "type": "filebeat", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "version": "8.16.0" + }, + "destination": { + "port": 49212, + "ip": "10.100.11.10", + "locality": "internal", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 9243, + "bytes": 238429, + "ip": "3.215.12.84", + "locality": "external", + "packets": 169, + "mac": "56-E0-32-C1-82-07" + }, + "network": { + "community_id": "1:SboTt968e79D3MOYTPiGam4R7e4=", + "bytes": 238429, + "transport": "tcp", + "type": "ipv4", + "packets": 169, + "iana_number": "6", + "direction": "inbound" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "input": { + "type": "netflow" + }, + "netflow": { + "packet_delta_count": 169, + "protocol_identifier": 6, + "vlan_id": 0, + "source_mac_address": "56-E0-32-C1-82-07", + "flow_start_sys_up_time": 2386462, + "octet_delta_count": 238429, + "egress_interface": 4, + "type": "netflow_flow", + "destination_ipv4_address": "10.100.11.10", + "source_ipv4_address": "3.215.12.84", + "exporter": { + "uptime_millis": 2447278, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:25.000Z" + }, + "delta_flow_count": 0, + "tcp_control_bits": 24, + "ip_class_of_service": 0, + "ip_version": 4, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2447094, + "source_transport_port": 9243, + "destination_transport_port": 49212 + }, + "@timestamp": "2024-06-13T23:29:25.000Z", + "related": { + "ip": [ + "3.215.12.84", + "10.100.11.10" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "containerized": false, + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 60632000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "kind": "event", + "created": "2024-07-28T21:31:38.073Z", + "start": "2024-06-13T23:28:24.184Z", + "action": "netflow_flow", + "end": "2024-06-13T23:29:24.816Z", + "category": [ + "network" + ], + "type": [ + "connection" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "5dOC8RdfsHE" + } + } + }, + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "afpB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "type": "filebeat", + "version": "8.16.0" + }, + "destination": { + "port": 53284, + "ip": "10.100.8.98", + "locality": "internal", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 443, + "bytes": 8301, + "ip": "34.234.143.15", + "locality": "external", + "mac": "56-E0-32-C1-82-07", + "packets": 18 + }, + "network": { + "community_id": "1:sgZy+IyajSLfG50OHXQKQiTmQ7s=", + "bytes": 8301, + "transport": "tcp", + "type": "ipv4", + "packets": 18, + "iana_number": "6", + "direction": "inbound" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "input": { + "type": "netflow" + }, + "netflow": { + "packet_delta_count": 18, + "protocol_identifier": 6, + "vlan_id": 0, + "flow_start_sys_up_time": 2371639, + "source_mac_address": "56-E0-32-C1-82-07", + "egress_interface": 4, + "octet_delta_count": 8301, + "type": "netflow_flow", + "destination_ipv4_address": "10.100.8.98", + "source_ipv4_address": "34.234.143.15", + "delta_flow_count": 0, + "exporter": { + "uptime_millis": 2447278, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:25.000Z" + }, + "tcp_control_bits": 27, + "ip_version": 4, + "ip_class_of_service": 0, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2386709, + "source_transport_port": 443, + "destination_transport_port": 53284 + }, + "@timestamp": "2024-06-13T23:29:25.000Z", + "related": { + "ip": [ + "10.100.8.98", + "34.234.143.15" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "containerized": false, + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 15070000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "created": "2024-07-28T21:31:38.073Z", + "kind": "event", + "start": "2024-06-13T23:28:09.361Z", + "action": "netflow_flow", + "end": "2024-06-13T23:28:24.431Z", + "category": [ + "network" + ], + "type": [ + "connection" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "TXluQ-JewIU" + } + } + }, + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "avpB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "type": "filebeat", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "version": "8.16.0" + }, + "destination": { + "port": 59242, + "ip": "10.100.8.36", + "locality": "internal", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 443, + "bytes": 7068, + "ip": "54.80.119.44", + "locality": "external", + "mac": "56-E0-32-C1-82-07", + "packets": 11 + }, + "network": { + "community_id": "1:WORfT191rnMqMqwKSM+88k8ngso=", + "bytes": 7068, + "transport": "tcp", + "type": "ipv4", + "iana_number": "6", + "packets": 11, + "direction": "inbound" + }, + "input": { + "type": "netflow" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "netflow": { + "packet_delta_count": 11, + "protocol_identifier": 6, + "vlan_id": 0, + "source_mac_address": "56-E0-32-C1-82-07", + "flow_start_sys_up_time": 2383262, + "octet_delta_count": 7068, + "egress_interface": 4, + "type": "netflow_flow", + "destination_ipv4_address": "10.100.8.36", + "source_ipv4_address": "54.80.119.44", + "exporter": { + "uptime_millis": 2447278, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:25.000Z" + }, + "delta_flow_count": 0, + "tcp_control_bits": 27, + "ip_class_of_service": 0, + "ip_version": 4, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2386300, + "destination_transport_port": 59242, + "source_transport_port": 443 + }, + "@timestamp": "2024-06-13T23:29:25.000Z", + "related": { + "ip": [ + "10.100.8.36", + "54.80.119.44" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "containerized": false, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 3038000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "created": "2024-07-28T21:31:38.073Z", + "kind": "event", + "start": "2024-06-13T23:28:20.984Z", + "action": "netflow_flow", + "end": "2024-06-13T23:28:24.022Z", + "type": [ + "connection" + ], + "category": [ + "network" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "woOLyw0G8JI" + } + } + }, + { + "_index": ".ds-logs-netflow.log-default-2024.07.28-000001", + "_id": "a_pB-5ABDjNvB3XCq_Q2", + "_score": 1.0, + "_source": { + "agent": { + "name": "lima-linux", + "id": "6b1366c7-923c-4385-b168-7731c563d78e", + "type": "filebeat", + "ephemeral_id": "e9cd68c6-0c95-40f5-a20f-7af8dfe49ffe", + "version": "8.16.0" + }, + "destination": { + "port": 64047, + "ip": "10.100.11.10", + "locality": "internal", + "mac": "B4-FB-E4-D0-EA-7B" + }, + "source": { + "port": 443, + "bytes": 312, + "ip": "34.71.180.220", + "locality": "external", + "mac": "56-E0-32-C1-82-07", + "packets": 6 + }, + "network": { + "community_id": "1:dghJP1SzJKaNyBEJsW2sueeiDGw=", + "bytes": 312, + "transport": "tcp", + "type": "ipv4", + "iana_number": "6", + "packets": 6, + "direction": "inbound" + }, + "input": { + "type": "netflow" + }, + "observer": { + "ip": [ + "127.0.0.1" + ] + }, + "netflow": { + "protocol_identifier": 6, + "packet_delta_count": 6, + "vlan_id": 0, + "flow_start_sys_up_time": 2373708, + "source_mac_address": "56-E0-32-C1-82-07", + "octet_delta_count": 312, + "egress_interface": 4, + "type": "netflow_flow", + "destination_ipv4_address": "10.100.11.10", + "source_ipv4_address": "34.71.180.220", + "exporter": { + "uptime_millis": 2449304, + "address": "127.0.0.1:45715", + "source_id": 0, + "version": 9, + "timestamp": "2024-06-13T23:29:27.000Z" + }, + "delta_flow_count": 0, + "tcp_control_bits": 16, + "ip_class_of_service": 0, + "ip_version": 4, + "flow_direction": 0, + "mpls_label_stack_length": 3, + "ingress_interface": 3, + "destination_mac_address": "B4-FB-E4-D0-EA-7B", + "flow_end_sys_up_time": 2449057, + "source_transport_port": 443, + "destination_transport_port": 64047 + }, + "@timestamp": "2024-06-13T23:29:27.000Z", + "related": { + "ip": [ + "10.100.11.10", + "34.71.180.220" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "netflow.log" + }, + "host": { + "hostname": "lima-linux", + "os": { + "kernel": "6.8.0-39-generic", + "codename": "noble", + "name": "Ubuntu", + "type": "linux", + "family": "debian", + "version": "24.04 LTS (Noble Numbat)", + "platform": "ubuntu" + }, + "ip": [ + "192.168.5.15", + "fe80::5055:55ff:fed3:c3fa", + "192.168.64.11", + "fd7a:388c:fc26:8787:5055:55ff:fe57:4f30", + "fe80::5055:55ff:fe57:4f30", + "172.17.0.1", + "172.18.0.1", + "fe80::42:6eff:fef8:156a", + "172.19.0.1", + "fe80::42:23ff:fe61:a7e5", + "fe80::dc46:b8ff:feeb:7268", + "fe80::70be:2dff:fef5:9648", + "fe80::dc52:efff:feac:491c" + ], + "containerized": false, + "name": "lima-linux", + "id": "2efcdad36f7542db9d7212e71eafeebe", + "mac": [ + "02-42-23-61-A7-E5", + "02-42-33-CC-48-A9", + "02-42-6E-F8-15-6A", + "52-55-55-57-4F-30", + "52-55-55-D3-C3-FA", + "72-BE-2D-F5-96-48", + "DE-46-B8-EB-72-68", + "DE-52-EF-AC-49-1C" + ], + "architecture": "aarch64" + }, + "event": { + "duration": 75349000000, + "agent_id_status": "auth_metadata_missing", + "ingested": "2024-07-28T21:31:43Z", + "created": "2024-07-28T21:31:38.074Z", + "kind": "event", + "start": "2024-06-13T23:28:11.404Z", + "action": "netflow_flow", + "end": "2024-06-13T23:29:26.753Z", + "type": [ + "connection" + ], + "category": [ + "network" + ], + "dataset": "netflow.log" + }, + "flow": { + "locality": "external", + "id": "iZPv2ko_Ghs" + } + } + } + ] + } +} +``` diff --git a/x-pack/filebeat/input/netflow/testdata/integration/test.pcap b/x-pack/filebeat/input/netflow/testdata/integration/test.pcap new file mode 100644 index 0000000000000000000000000000000000000000..60ad6f116c6085968968ee0691fdca6916394fe6 GIT binary patch literal 3362 zcmeHJe@s%%k#RSZRk_ZLNS~|3ah(-KS zW)|sKONp7be#}}~)9#wLVJ)jQtIM#WA1F<_P+6j5xLMD4xNv`T4x_CuHyQDntZMA`gj7os33 z%NAE{RcPX~LAzHO$xlTrED&Wk)zmlpfatA(H?O3RWTep5OqxQIZe3i(ag{(@z_p2c zR+^0#!@YP~H1{nULV=SjH|=KbpWrjVO0)7Gytjn!oKMRSzSla{c;}cAMzkP{ROW991eE3+*;Yw#&v5jtHD`rT=Ls9N(N z)Kvw*>{vpl1)+H_UD-mfa}>`9m9Xo11-jWU$CFLCjv-2mWI@UBpV#_jD3k@o*77r1 zFhi6dV!w2=U$zk|SvbeNC!yA=u&2HZq!h?_#(oR?Y&(mNeRZa*x~rPi)nlRTBGkGx znD$o?@@g;@Q;%(htfy-Cy7cR*)~7(7I}gZsfY2F1sKfuvbjDt#cuq1VJr>kjDZ+h# z{Y66KgU~$p<{Uy*#dEe!7urMiI=7lM{pf3G(4nz+A(TD%3jP|?&^_&BuZQe4({VsU zm7P$Xw^J}qZdroCIEoJUN(vOuXf6_U^LW?{_5NRh^ydgo3qk`z5Q^v7Ufn#PZe%0X zv=J3UXo?^yUD^SvOp#{ zi{xya>J0mT=8l~`|6=BjSIfBL2TQbS?F_4yZAgiz7AppYrx-ha%|SNMwUlcf_pUHg z$tGDxNWGfA7QT^T#lyDMj^Rbx1aFkL}AV&Kkf&o?oX zes=9QbWI&7z1({Q>f^k$tm9vDD6IhBLT83~&6L7DCfh|{xW#iBbOul?#b*zL3YqIuUPGvRFGOKW z;FRM58A8(>>GhK)1E^HWeN@Up_O!G}%`5Fgn>3}I4^v9Z$3IXh{HC5z%D2QyrL2Rv zvm2u5PoB27a*x6d2}Y~g3pRsbAWCOJEaRZbmY<8^#5?47Lf-~MXk|P^30vjOqlD^A z3{RaARPA^TYFjx+8XHwtpVfrw>vQc2h`Uz-Zh4}I*G$!hO@onFQ9BG#afT