Merge remote-tracking branch 'origin/main' into data-streams/improve-…

…jest-tests

.buildkite/ftr_oblt_stateful_configs.yml

@@ -1,15 +1,15 @@
 disabled:
   # Cypress configs, for now these are still run manually
-  - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_open.ts
-  - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_runner.ts
-  - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config_run.ts
-  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts
-  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts
-  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_open.ts
+  - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_runner.ts
+  - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config_run.ts
+  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config_run.ts
+  - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_open.ts
+  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_runner.ts
+  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config.ts
 
   #FTR configs
   - x-pack/solutions/observability/plugins/uptime/e2e/config.ts
@@ -38,7 +38,6 @@ enabled:
   - x-pack/test/apm_api_integration/rules/config.ts
   - x-pack/test/apm_api_integration/trial/config.ts
   - x-pack/test/dataset_quality_api_integration/basic/config.ts
-  - x-pack/test/functional/apps/observability_logs_explorer/config.ts
   - x-pack/test/functional/apps/dataset_quality/config.ts
   - x-pack/test/functional/apps/slo/embeddables/config.ts
   - x-pack/test/functional/apps/uptime/config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -178,6 +178,7 @@ enabled:
   - x-pack/test/banners_functional/config.ts
   - x-pack/test/cases_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/cases_api_integration/security_and_spaces/config_trial.ts
+  - x-pack/test/cases_api_integration/security_and_spaces/config_trial_common.ts
   - x-pack/test/cases_api_integration/security_and_spaces/config_no_public_base_url.ts
   - x-pack/test/cases_api_integration/spaces_only/config.ts
   - x-pack/test/disable_ems/config.ts

.buildkite/pipeline-resource-definitions/kibana-es-forward-testing-9-dot-0.yml

@@ -27,6 +27,7 @@ spec:
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_forward_9_dot_0.yml # Note: this file exists in 8.x only and should be moved into 8.18 once the branch is cut
+      skip_intermediate_builds: false
       provider_settings:
         prefix_pull_request_fork_branch_names: false
         trigger_mode: none

.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml

@@ -27,6 +27,7 @@ spec:
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_forward.yml # Note: this file exists in 7.17 only
+      skip_intermediate_builds: false
       provider_settings:
         prefix_pull_request_fork_branch_names: false
         trigger_mode: none

.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.17 8.16 8.15 7.17
+      branch_configuration: main 8.x 8.17 8.16 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/build.yml
@@ -95,7 +95,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.17 8.16 8.15 7.17
+      branch_configuration: main 8.x 8.17 8.16 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/promote.yml
@@ -144,7 +144,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.17 8.16 8.15 7.17
+      branch_configuration: main 8.x 8.17 8.16 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/verify.yml

.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-unsupported-ftrs-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.17 8.16 8.15 7.17
+      branch_configuration: main 8.x 8.17 8.16 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge_unsupported_ftrs.yml

.buildkite/pipeline-resource-definitions/kibana-on-merge.yml

@@ -25,7 +25,7 @@ spec:
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.x 8.17 8.16 8.15 7.17
+      branch_configuration: main 8.x 8.17 8.16 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge.yml

.buildkite/pipeline-resource-definitions/kibana-package-registry.yml

@@ -2,10 +2,10 @@
 apiVersion: backstage.io/v1alpha1
 kind: Resource
 metadata:
-  name: bk-kibana-package-registry-promote
+  name: bk-kibana-package-registry-verify-and-promote
   description: Promote package-registry/distribution:lite
   links:
-    - url: 'https://buildkite.com/elastic/kibana-package-registry-promote'
+    - url: 'https://buildkite.com/elastic/kibana-package-registry-verify-and-promote'
       title: Pipeline link
 spec:
   type: buildkite-pipeline
@@ -15,18 +15,23 @@ spec:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      name: kibana / package registry promote
-      description: Promote package-registry/distribution:lite
+      name: kibana / package registry verify and promote
+      description: Verify and promote docker.elastic.co/package-registry/distribution:lite
     spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: "#kibana-operations-alerts"
-        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "false"
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
+        REPORT_FAILED_TESTS_TO_GITHUB: "true"
+        FLEET_PACKAGE_REGISTRY_DOCKER_IMAGE: "docker.elastic.co/package-registry/distribution:lite"
       repository: elastic/kibana
       branch_configuration: main
       default_branch: main
       pipeline_file: ".buildkite/pipelines/fleet/package_registry.yml"
       provider_settings:
         trigger_mode: none
+      schedules:
+        Daily promotion:
+          cronline: 0 22 * * * America/New_York
       teams:
         everyone:
           access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-pr.yml

@@ -20,7 +20,7 @@ spec:
     spec:
       env:
         ELASTIC_PR_COMMENTS_ENABLED: 'true'
-        GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
+        ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
         GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-ci
       allow_rebuilds: true
       branch_configuration: ''

.buildkite/pipeline-utils/buildkite/emitPipeline.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export function emitPipeline(pipelineSteps: string[]) {
+  const pipelineStr = [...new Set(pipelineSteps)].join('\n');
+  console.log(pipelineStr);
+}

.buildkite/pipeline-utils/buildkite/index.ts

@@ -9,3 +9,4 @@
 
 export * from './client';
 export * from './types';
+export * from './emitPipeline';

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -485,7 +485,10 @@ export async function pickTestGroupRunOrder() {
             parallelism: unit.count,
             timeout_in_minutes: 120,
             key: 'jest',
-            agents: expandAgentQueue('n2-4-spot'),
+            agents: {
+              ...expandAgentQueue('n2-4-spot'),
+              diskSizeGb: 75,
+            },
             retry: {
               automatic: [
                 { exit_status: '-1', limit: 3 },

.buildkite/pipelines/artifacts.yml

@@ -6,7 +6,7 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: c2-standard-16
-      diskSizeGb: 125
+      diskSizeGb: 150
     timeout_in_minutes: 120
     retry:
       automatic:

.buildkite/pipelines/build_api_docs.yml

@@ -18,7 +18,7 @@ steps:
       image: family/kibana-ubuntu-2004
       imageProject: elastic-images-prod
       provider: gcp
-      machineType: n2-standard-4
+      machineType: n2-highmem-4
       preemptible: true
     key: build_api_docs
     timeout_in_minutes: 50

.buildkite/pipelines/chrome_forward_testing.yml

@@ -97,7 +97,7 @@ steps:
     depends_on:
       - build
     timeout_in_minutes: 60
-    parallelism: 8
+    parallelism: 10
     retry:
       automatic:
         - exit_status: '-1'
@@ -153,7 +153,7 @@ steps:
     depends_on:
       - build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '-1'
@@ -181,7 +181,7 @@ steps:
     depends_on:
       - build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '-1'
@@ -209,7 +209,7 @@ steps:
     depends_on:
       - build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 3
     retry:
       automatic:
         - exit_status: '-1'
@@ -265,7 +265,7 @@ steps:
     depends_on:
       - build
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 3
     retry:
       automatic:
         - exit_status: '-1'
@@ -279,7 +279,7 @@ steps:
     depends_on:
       - build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 8
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -107,7 +107,7 @@ steps:
           preemptible: true
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 8
+        parallelism: 10
         retry:
           automatic:
             - exit_status: "-1"
@@ -175,7 +175,7 @@ steps:
           preemptible: true
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 2
+        parallelism: 4
         retry:
           automatic:
             - exit_status: "-1"

.buildkite/pipelines/fleet/package_registry.yml

@@ -1,2 +1,106 @@
+env:
+  IGNORE_SHIP_CI_STATS_ERROR: 'true'
 steps:
-  - command: echo "Placeholder"
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+
+  - wait
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+      preemptible: true
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+    label: 'Pick Test Group Run Order'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+    timeout_in_minutes: 10
+    env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
+      LIMIT_CONFIG_TYPE: integration,functional
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows.sh
+    label: 'Defend Workflows Cypress Tests'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      enableNestedVirtualization: true
+      machineType: n2-standard-4
+    timeout_in_minutes: 60
+    parallelism: 20
+    key: defend-workflows-stateful
+    depends_on: build
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
+    label: 'Defend Workflows Cypress Tests on Serverless'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      enableNestedVirtualization: true
+      machineType: n2-standard-4
+    timeout_in_minutes: 60
+    parallelism: 14
+    key: defend-workflows-serverless
+    depends_on: build
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/fleet/promote_package_registry.sh
+    label: Promote docker.elastic.co/package-registry/distribution:lite
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+    depends_on:
+      - ftr-configs
+      - jest-integration
+      - defend-workflows-stateful
+      - defend-workflows-serverless
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build
+    timeout_in_minutes: 10
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2