Merge branch 'main' into e-parth-pathak-patch-1

.agent-versions.json

@@ -1,10 +1,10 @@
 {
   "testVersions": [
-    "8.13.0-SNAPSHOT",
-    "8.12.3-SNAPSHOT",
-    "8.12.2",
-    "8.12.1",
-    "7.17.19-SNAPSHOT",
-    "7.17.18"
+    "8.14.1",
+    "8.14.1-SNAPSHOT",
+    "8.13.5-SNAPSHOT",
+    "8.13.4",
+    "7.17.22-SNAPSHOT",
+    "7.17.21"
   ]
 }

.buildkite/hooks/pre-command

@@ -46,7 +46,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   # Set GCP credentials
   export GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH})
   echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json

.buildkite/hooks/pre-exit

@@ -2,19 +2,16 @@
 
 set -eo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   if [[ -z "${WORKSPACE-""}" ]]; then
       WORKSPACE=$(git rev-parse --show-toplevel)
   fi
   source "${WORKSPACE}/.buildkite/scripts/common.sh"
 
   # Perform cleanup of integration tests resources
   echo "--- Cleaning up integration test resources"
-  if [[ "$BUILDKITE_STEP_KEY" == "serverless-integration-tests" ]]; then
-    STACK_PROVISIONER=serverless SNAPSHOT=true mage integration:clean
-  else
-    SNAPSHOT=true mage integration:clean
-  fi
+  STACK_PROVISIONER=serverless SNAPSHOT=true mage integration:clean
+  SNAPSHOT=true mage integration:clean
 fi
 
 if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then

.buildkite/integration.pipeline.yml

@@ -0,0 +1,76 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+
+steps:
+  - label: "Serverless integration test"
+    key: "serverless-integration-tests"
+    concurrency_group: elastic-agent-extended-testing/serverless-integration
+    concurrency: 8
+    env:
+      # we run each step in a different data center to spread the load
+      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
+    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    notify:
+      - github_commit_status:
+          context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
+
+  - label: "Extended runtime leak tests"
+    key: "extended-integration-tests"
+    concurrency_group: elastic-agent-extended-testing/leak-tests
+    concurrency: 8
+    env:
+      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
+    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    notify:
+      - github_commit_status:
+          context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
+
+  - label: "Integration tests"
+    key: "integration-tests"
+    concurrency_group: elastic-agent-extended-testing/integration
+    concurrency: 8
+    env:
+      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-f"
+    command: ".buildkite/scripts/steps/integration_tests.sh stateful"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    notify:
+      - github_commit_status:
+          context: "buildkite/elastic-agent-extended-testing - Integration tests"
+
+  - label: "Serverless Beats Tests"
+    key: "serverless-beats-integration-tests"
+    concurrency_group: elastic-agent-extended-testing/beats-integration
+    concurrency: 8
+    env:
+      TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
+    command: ".buildkite/scripts/steps/beats_tests.sh"
+    # if: "build.env('CRON') == 'yes'"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+    retry:
+      manual:
+        allowed: true
+    notify:
+      - github_commit_status:
+          context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"

.buildkite/misc/gce-cleanup.yml

@@ -18,7 +18,7 @@ scanners:
         filters:
           - type: "<"
             pointer: "/extra/creationTimestamp"
-            param: "${CREATION_DATE}"
+            param: "${DELETE_CREATED_AFTER_DATE}"
             converters:
               param: "date"
               value: "date"

.buildkite/pipeline.elastic-agent-package.yml

@@ -10,10 +10,10 @@ env:
 
 steps:
   - input: "Build parameters"
-    if: build.env("ManifestURL") == null
+    if: build.env("MANIFEST_URL") == null
     fields:
-      - text: "ManifestURL"
-        key: "ManifestURL"
+      - text: "MANIFEST_URL"
+        key: "MANIFEST_URL"
         default: ""
         required: true
         hint: "Link to the build manifest URL."
@@ -51,7 +51,7 @@ steps:
         hint: "If the DRA release manager script would actually publish anything or just print"
 
   - wait: ~
-    if: build.env("ManifestURL") == null
+    if: build.env("MANIFEST_URL") == null
 
   - group: ":Packaging Artefacts"
     key: "package"
@@ -63,10 +63,10 @@ steps:
           machineType: "c2-standard-16"
           diskSizeGb: 400
         command: |
-          if [[ -z "${ManifestURL}" ]]; then
-            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
-            if [[ -z "${ManifestURL}" ]]; then
-              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+          if [[ -z "${MANIFEST_URL}" ]]; then
+            export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "")
+            if [[ -z "${MANIFEST_URL}" ]]; then
+              echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided"
               exit 1
             fi
           fi
@@ -86,10 +86,10 @@ steps:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         command: |
-          if [[ -z "${ManifestURL}" ]]; then
-            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
-            if [[ -z "${ManifestURL}" ]]; then
-              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+          if [[ -z "${MANIFEST_URL}" ]]; then
+            export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "")
+            if [[ -z "${MANIFEST_URL}" ]]; then
+              echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided"
               exit 1
             fi
           fi
@@ -134,6 +134,8 @@ steps:
     depends_on: package
     agents:
       provider: "gcp"
+      machineType: "n2-standard-8"
+      diskSizeGb: 250
     env:
       DRA_PROJECT_ID: "elastic-agent-package"
       DRA_PROJECT_ARTIFACT_ID: "agent-package"

.buildkite/pipeline.yml

@@ -213,47 +213,6 @@ steps:
       manual:
         allowed: true
 
-  - label: "Serverless integration test"
-    key: "serverless-integration-tests"
-    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-
-  - label: "Extended runtime leak tests"
-    key: "extended-integration-tests"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-
-  - label: "Integration tests"
-    key: "integration-tests"
-    command: ".buildkite/scripts/steps/integration_tests.sh stateful"
-    artifact_paths:
-      - "build/TEST-**"
-      - "build/diagnostics/*"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-
-  - label: "Serverless Beats Tests"
-    key: "serverless-beats-integration-tests"
-    command: ".buildkite/scripts/steps/beats_tests.sh"
-    if: "build.env('CRON') == 'yes'"
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
-    retry:
-      manual:
-        allowed: true
-
   # Triggers a dynamic step: Sync K8s
   # Runs only on main and if k8s files are changed
   - label: "Trigger k8s sync"
@@ -273,3 +232,51 @@ steps:
                   image: "family/core-ubuntu-2204"
                 env:
                   - GH_VERSION=2.4.0
+
+  # Trigger for pull requests
+  - label: "Trigger Integration tests for Pull request"
+    if: |
+      (build.pull_request.id != null && !build.env("GITHUB_PR_LABELS") =~ /skip-it/) ||
+      build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/
+
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - internal/
+                - dev-tools/
+                - pkg/
+                - testing/
+                - version/
+                - specs/
+                - .agent-versions.json
+                - .go-version
+                - .package-version
+                - go.mod
+                - go.sum
+                - magefile.go
+                - main.go
+
+                - .buildkite/integration.pipeline.yml
+                - .buildkite/pipeline.yml
+                - .buildkite/scripts/
+                - .buildkite/hooks/
+
+              config:
+                trigger: "elastic-agent-extended-testing"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  # Trigger for branches
+  - label: "Triggering Integration tests for branches"
+    if: build.pull_request.id == null
+    trigger: "elastic-agent-extended-testing"
+    build:
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"

.buildkite/pull-requests.json

@@ -1,27 +1,27 @@
 {
     "jobs": [
         {
-            "enabled": true,
+            "enabled": true,            
             "pipelineSlug": "elastic-agent",
             "allow_org_users": true,
             "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": ["dependabot[bot]", "mergify[bot]"],
+            "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"],
             "set_commit_status": true,
             "build_on_commit": true,
             "build_on_comment": true,
-            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
-            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
-            "skip_ci_labels": [  ],
+            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)",
+            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)",
+            "skip_ci_labels": [ "skip-ci" ],
             "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
+            "skip_ci_on_only_changed": [ "^.ci/", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
             "always_require_ci_on_changed": [ ]
         },
         {
             "enabled": true,
             "pipelineSlug": "elastic-agent-package",
             "allow_org_users": true,
             "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [],
+            "allowed_list": ["dependabot[bot]", "mergify[bot]"],
             "set_commit_status": false,
             "build_on_commit": false,
             "build_on_comment": true,
@@ -37,7 +37,7 @@
             "pipelineSlug": "elastic-agent-binary-dra",
             "allow_org_users": true,
             "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [],
+            "allowed_list": ["dependabot[bot]", "mergify[bot]"],
             "set_commit_status": false,
             "build_on_commit": false,
             "build_on_comment": true,

.buildkite/scripts/steps/gce-cleanup.sh

@@ -5,11 +5,11 @@ set -euo pipefail
 export ACCOUNT_KEY_SECRET=$(vault kv get -field=client_email $VAULT_PATH)
 export ACCOUNT_SECRET=$(vault kv get -field=private_key $VAULT_PATH)
 export ACCOUNT_PROJECT_SECRET=$(vault kv get -field=project_id $VAULT_PATH)
-export CREATION_DATE=$(date -Is -d "24 hours ago")
+export DELETE_CREATED_AFTER_DATE=$(date -Is -d "5 hours ago")
 
 docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
   -e ACCOUNT_SECRET="$ACCOUNT_SECRET" \
   -e ACCOUNT_KEY="$ACCOUNT_KEY_SECRET" \
   -e ACCOUNT_PROJECT=$ACCOUNT_PROJECT_SECRET \
-  -e CREATION_DATE=$CREATION_DATE \
+  -e DELETE_CREATED_AFTER_DATE=$DELETE_CREATED_AFTER_DATE \
   ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper --config /etc/cloud-reaper/config.yml destroy --confirm

.buildkite/scripts/steps/integration_tests.sh

@@ -9,9 +9,9 @@ MAGE_SUBTARGET="${3:-""}"
 
 
 # Override the agent package version using a string with format <major>.<minor>.<patch>
-# NOTE: use only after version bump when the new version is not yet available, for example:
-# OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3" otherwise OVERRIDE_AGENT_PACKAGE_VERSION="".
-OVERRIDE_AGENT_PACKAGE_VERSION=""
+# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
+# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
+OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)"
 
 if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then
   OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"

.buildkite/scripts/steps/package.sh

@@ -5,19 +5,19 @@ set -euo pipefail
 _SELF=$(dirname $0)
 source "${_SELF}/../common.sh"
 
-if test -z "${ManifestURL=:""}"; then
-  echo "Missing variable ManifestURL, export it before use."
+if test -z "${MANIFEST_URL=:""}"; then
+  echo "Missing variable MANIFEST_URL, export it before use."
   exit 2
 fi
 
 export AGENT_DROP_PATH=build/elastic-agent-drop
 mkdir -p $AGENT_DROP_PATH
 
-# Download the components from the ManifestURL and then package those downloaded into the $AGENT_DROP_PATH
-mage clean downloadManifest package ironbank fixDRADockerArtifacts
+# Download the components from the MANIFEST_URL and then package those downloaded into the $AGENT_DROP_PATH
+mage clean downloadManifest packageUsingDRA ironbank fixDRADockerArtifacts
 
 echo  "+++ Generate dependencies report"
-BEAT_VERSION_FULL=$(curl -s -XGET "${ManifestURL}" |jq '.version' -r )
+BEAT_VERSION_FULL=$(curl -s -XGET "${MANIFEST_URL}" |jq '.version' -r )
 bash "${_SELF}/../../../dev-tools/dependencies-report"
 mkdir -p build/distributions/reports
 mv dependencies.csv "build/distributions/reports/dependencies-${BEAT_VERSION_FULL}.csv"