Merge branch 'main' into k8s-metadata-processor

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -33,6 +33,7 @@ steps:
     steps:
       - label: "Run check/update"
         command: |
+          set -eo pipefail
           make -C auditbeat check update
           make check-no-changes
         retry:

.buildkite/filebeat/filebeat-pipeline.yml

@@ -32,6 +32,7 @@ steps:
     steps:
       - label: "Run check/update"
         command: |
+          set -eo pipefail
           make -C filebeat check update
           make check-no-changes
         retry:

.buildkite/heartbeat/heartbeat-pipeline.yml

@@ -26,6 +26,34 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "Heartbeat Check/Update"
+    key: "heartbeat-check-update"
+
+    steps:
+      - label: "Run check/update"
+        command: |
+          set -eo pipefail
+          make -C heartbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "hearbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "heartbeat-check-update"
+
   - group: "Heartbeat Mandatory Testing"
     key: "heartbeat-mandatory-tests"
 

.buildkite/libbeat/pipeline.libbeat.yml

@@ -21,6 +21,7 @@ steps:
     steps:
       - label: "Run check/update"
         command: |
+          set -eo pipefail
           make -C libbeat check update
           make check-no-changes
         retry:

.buildkite/metricbeat/pipeline.yml

@@ -30,6 +30,33 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "Metricbeat Check/Update"
+    key: "metricbeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          set -eo pipefail
+          make -C metricbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "metricbeat: check/update"
+
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "metricbeat-check-update"
+
   - group: "Metricbeat Mandatory Tests"
     key: "metricbeat-mandatory-tests"
 

.buildkite/packetbeat/pipeline.packetbeat.yml

@@ -4,9 +4,11 @@ name: "beats-packetbeat"
 env:
   AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9"
@@ -27,9 +29,11 @@ env:
 steps:
   - group: "Packetbeat Check/Update"
     key: "packetbeat-check-update"
+
     steps:
-      - label: "Run check/update"
+      - label: "Packetbeat Run check/update"
         command: |
+          set -eo pipefail
           make -C packetbeat check update
           make check-no-changes
         retry:
@@ -43,7 +47,7 @@ steps:
         notify:
           - github_commit_status:
               context: "packetbeat: check/update"
-      
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
@@ -53,8 +57,9 @@ steps:
 
   - group: "packetbeat Mandatory Tests"
     key: "packetbeat-mandatory-tests"
+
     steps:
-      - label: ":linux: Ubuntu Unit Tests"
+      - label: ":ubuntu: Packetbeat Linux x86_64 Unit Tests"
         command: |
           cd packetbeat
           mage build unitTest
@@ -70,9 +75,9 @@ steps:
           - "packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "packetbeat: Ubuntu Unit Tests"
+              context: "packetbeat: Linux x86_64 Unit Tests"
 
-      - label: ":rhel: RHEL9 Unit Tests"
+      - label: ":rhel: Packetbeat RHEL9 Unit Tests"
         command: |
           cd packetbeat
           mage build unitTest
@@ -90,7 +95,7 @@ steps:
           - github_commit_status:
               context: "packetbeat: RHEL9 Unit Tests"
 
-      - label: ":windows: Windows 2016 Unit Tests"
+      - label: ":windows: Packetbeat Win-2016 Unit Tests"
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
@@ -108,9 +113,9 @@ steps:
           - "packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "packetbeat: Windows 2016 Unit Tests"
+              context: "packetbeat: Win-2016 Unit Tests"
 
-      - label: ":windows: Windows 2022 Unit Tests"
+      - label: ":windows: Packetbeat Win-2022 Unit Tests"
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
@@ -128,13 +133,14 @@ steps:
           - "packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "packetbeat: Windows 2022 Unit Tests"
+              context: "packetbeat: Win-2022 Unit Tests"
 
-  - group: "Extended Windows Tests"
+  - group: " Packetbeat Extended Windows Tests"
     key: "packetbeat-extended-windows-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
     steps:
-      - label: ":windows: Windows 10 Unit Tests"
+      - label: ":windows: Packetbeat Win 10 Unit Tests"
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
@@ -154,7 +160,7 @@ steps:
           - github_commit_status:
               context: "packetbeat: Extended Windows 10 Unit Tests"
 
-      - label: ":windows: Windows 11 Unit Tests"
+      - label: ":windows: Packetbeat Win 11 Unit Tests"
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
@@ -173,9 +179,9 @@ steps:
           - "packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "packetbeat: Extended Windows 11 Unit Tests"
+              context: "packetbeat: Win 11 Unit Tests"
 
-      - label: ":windows: Windows 2019 Unit Tests"
+      - label: ":windows: Packetbeat Win-2019 Unit Tests"
         command: |
           Set-Location -Path packetbeat
           mage build unitTest
@@ -194,12 +200,13 @@ steps:
           - "packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "packetbeat: Extended Windows 2019 Unit Tests"
+              context: "packetbeat: Win-2019 Unit Tests"
 
-  - group: "Extended Tests"
+  - group: "Packetbeat Extended Tests"
     key: "extended-tests"
+
     steps:
-      - label: ":mac: MacOS x86_64 Unit Tests"
+      - label: ":mac: Packetbeat macOS x86_64 Unit Tests"
         key: "macos-x86-64-unit-tests-extended"
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
@@ -218,9 +225,9 @@ steps:
           - "packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "packetbeat: Extended MacOS x86_64 Unit Tests"
+              context: "packetbeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: MacOS arm64 Unit Tests"
+      - label: ":mac: Packetbeat macOS arm64 Unit Tests"
         key: "macos-arm64-unit-tests-extended"
         if: build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
         command: |
@@ -241,9 +248,11 @@ steps:
           - github_commit_status:
               context: "packetbeat: Extended MacOS arm64 Unit Tests"
 
-      - label: ":linux: Ubuntu ARM Unit Tests"
+      - label: ":ubuntu: Linux arm64 Unit Tests"
         key: "linux-arm64-unit-tests-extended"
-        command: "cd packetbeat && mage build unitTest"
+        command: |
+          cd packetbeat
+          mage build unitTest
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         retry:
           automatic:
@@ -257,7 +266,7 @@ steps:
           - "packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "packetbeat: Extended Ubuntu ARM Unit Tests"
+              context: "packetbeat: Linux arm64 Unit Tests"
 
   - wait: ~
     # with PRs, we want to run packaging only if mandatory tests succeed
@@ -269,7 +278,7 @@ steps:
   - group: "Packetbeat Packaging"
     key: "packaging"
     steps:
-      - label: ":linux: Packaging Linux"
+      - label: ":ubuntu: Packetbeat Packaging Linux"
         key: "packaging-linux"
         command: |
           cd packetbeat
@@ -290,7 +299,7 @@ steps:
           - github_commit_status:
               context: "packetbeat: Packaging Linux"
 
-      - label: ":linux: Packaging ARM"
+      - label: ":ubuntu: Packaging Linux arm64"
         key: "packaging-arm"
         command: |
           cd packetbeat
@@ -308,4 +317,4 @@ steps:
           PACKAGES: "docker"
         notify:
           - github_commit_status:
-              context: "packetbeat: Packaging Linux ARM"
+              context: "packetbeat: Packaging Linux arm64"

.buildkite/winlogbeat/pipeline.winlogbeat.yml

@@ -25,9 +25,11 @@ env:
 steps:
   - group: "Winlogbeat Check/Update"
     key: "winlogbeat-check-update"
+
     steps:
-      - label: "Run check/update"
+      - label: "Winlogbeat Run check/update"
         command: |
+          set -eo pipefail
           make -C winlogbeat check update
           make check-no-changes
         retry:
@@ -41,7 +43,7 @@ steps:
         notify:
           - github_commit_status:
               context: "winlogbeat: check/update"
-      
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
@@ -68,7 +70,7 @@ steps:
           - "winlogbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Winlogbeat: Crosscompile"
+              context: "winlogbeat: Crosscompile"
 
       - label: ":windows: Winlogbeat Win-2016 Unit Tests"
         command: |
@@ -89,7 +91,7 @@ steps:
           - "winlogbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Winlogbeat: Win-2016 Unit Tests"
+              context: "winlogbeat: Win-2016 Unit Tests"
 
       - label: ":windows: Winlogbeat Win-2019 Unit Tests"
         command: |
@@ -110,7 +112,7 @@ steps:
           - "winlogbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Winlogbeat: Win-2019 Unit Tests"
+              context: "winlogbeat: Win-2019 Unit Tests"
 
       - label: ":windows: Winlogbeat Win-2022 Unit Tests"
         command: |
@@ -157,7 +159,7 @@ steps:
           - "winlogbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Winlogbeat: Win-10 Unit Tests"
+              context: "winlogbeat: Win-10 Unit Tests"
 
       - label: ":windows: Winlogbeat Win-11 Unit Tests"
         command: |
@@ -178,7 +180,7 @@ steps:
           - "winlogbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "Winlogbeat: Win-11 Unit Tests"
+              context: "winlogbeat: Win-11 Unit Tests"
 
   - wait: ~
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
@@ -187,7 +189,7 @@ steps:
   - group: "Winlogbeat Packaging"
     key: "winlogbeat-packaging"
     steps:
-      - label: ":ubuntu: Winlogbeat Packaging Ubuntu x86_64"
+      - label: ":ubuntu: Winlogbeat Packaging Linux"
         key: "packaging-linux"
         command: |
           cd winlogbeat
@@ -204,4 +206,4 @@ steps:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
         notify:
           - github_commit_status:
-              context: "Winlogbeat: Packaging Ubuntu x86_64"
+              context: "winlogbeat: Packaging Linux"

.buildkite/x-pack/pipeline.xpack.auditbeat.yml

@@ -33,6 +33,7 @@ steps:
     steps:
       - label: "Run check/update"
         command: |
+          set -eo pipefail
           make -C x-pack/auditbeat check update
           make check-no-changes
         retry:

.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml

@@ -23,6 +23,7 @@ steps:
     steps:
       - label: "Run check/update"
         command: |
+          set -eo pipefail
           make -C x-pack/dockerlogbeat check update
           make check-no-changes
         retry:

.buildkite/x-pack/pipeline.xpack.filebeat.yml

@@ -31,6 +31,7 @@ steps:
     steps:
       - label: "Run check/update"
         command: |
+          set -eo pipefail
           make -C x-pack/filebeat check update
           make check-no-changes
         retry: