New daily job for beats AWS tests (elastic#40222)

As a short-term solution added AWS daily job (runs twice a day).

.buildkite/aws-tests-pipeline.yml

@@ -0,0 +1,64 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+
+  #  TEST_TAGS should be reviewed and updated: https://github.com/elastic/ingest-dev/issues/3476
+  TEST_TAGS: "aws"
+  MODULE: "aws"
+
+  # Other deps
+  ASDF_TERRAFORM_VERSION: 1.0.2
+  ASDF_MAGE_VERSION: 1.15.0
+
+  # Unit tests
+  RACE_DETECTOR: "true"
+  TEST_COVERAGE: "true"
+
+steps:
+  - label: ":ubuntu: x-pack/filebeat: AWS Tests"
+    key: "x-pack-filebeat-extended-cloud-test"
+    skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467"
+    # Related issue: https://github.com/elastic/ingest-dev/issues/3467
+    env:
+      MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"
+    command: |
+      set -euo pipefail
+      source .buildkite/scripts/initCloudEnv.sh
+      echo "~~~ Running tests"
+      cd x-pack/filebeat
+      mage build test goIntegTest
+    agents:
+      provider: "aws"
+      imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+      instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+    artifact_paths:
+      - "x-pack/filebeat/build/*.xml"
+      - "x-pack/filebeat/build/*.json"
+    notify:
+      - github_commit_status:
+          context: "x-pack/filebeat: AWS Tests"
+
+  - label: ":ubuntu: x-pack/metricbeat: AWS Tests"
+    key: "x-pack-metricbeat-extended-cloud-test"
+    env:
+      MODULE_DIR: "x-pack/metricbeat/module/aws"
+    command: |
+      set -euo pipefail
+      source .buildkite/scripts/initCloudEnv.sh
+      echo "~~~ Running tests"
+      cd x-pack/metricbeat
+      mage build test goIntegTest
+    agents:
+      provider: "aws"
+      imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+      instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+    artifact_paths:
+      - "x-pack/metricbeat/build/*.xml"
+      - "x-pack/metricbeat/build/*.json"
+    notify:
+      - github_commit_status:
+          context: "x-pack/metricbeat: AWS Tests"
+
+

catalog-info.yaml

@@ -1115,6 +1115,40 @@ spec:
         everyone:
           access_level: BUILD_AND_READ
 
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: beats-aws-tests
+  description: 'Runs of Beats AWS tests'
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/beats-aws-tests
+spec:
+  type: buildkite-pipeline
+  owner: group:ingest-fp
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: beats-aws-tests
+      description: 'Runs of Beats AWS tests'
+    spec:
+      repository: elastic/beats
+      pipeline_file: ".buildkite/aws-tests-pipeline.yml"
+      cancel_intermediate_builds: false
+      provider_settings:
+        trigger_mode: none
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        release-eng:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -1146,6 +1180,12 @@ spec:
           message: Daily trigger of Iron Bank validation Pipeline per branch
           env:
             PIPELINES_TO_TRIGGER: 'beats-ironbank-validation'
+        Daily run of AWS tests:
+          branch: main
+          cronline: 0 */12 * * *
+          message: Daily trigger of Beats AWS tests
+          env:
+            PIPELINES_TO_TRIGGER: 'beats-aws-tests'
       skip_intermediate_builds: true
       provider_settings:
         trigger_mode: none