Merge branch 'main' into move-uptime-rule-params

.buildkite/ftr_oblt_stateful_configs.yml

@@ -3,13 +3,13 @@ disabled:
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_open.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_runner.ts
   - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config_run.ts
-  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts
-  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts
-  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config_run.ts
+  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config_run.ts
+  - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_open.ts
+  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_runner.ts
+  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config.ts
 
   #FTR configs
   - x-pack/solutions/observability/plugins/uptime/e2e/config.ts

.buildkite/pipeline-resource-definitions/kibana-package-registry.yml

@@ -2,10 +2,10 @@
 apiVersion: backstage.io/v1alpha1
 kind: Resource
 metadata:
-  name: bk-kibana-package-registry-promote
+  name: bk-kibana-package-registry-verify-and-promote
   description: Promote package-registry/distribution:lite
   links:
-    - url: 'https://buildkite.com/elastic/kibana-package-registry-promote'
+    - url: 'https://buildkite.com/elastic/kibana-package-registry-verify-and-promote'
       title: Pipeline link
 spec:
   type: buildkite-pipeline
@@ -15,18 +15,21 @@ spec:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      name: kibana / package registry promote
-      description: Promote package-registry/distribution:lite
+      name: kibana / package registry verify and promote
+      description: Verify and promote docker.elastic.co/package-registry/distribution:lite
     spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: "#kibana-operations-alerts"
-        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "false"
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
       repository: elastic/kibana
       branch_configuration: main
       default_branch: main
       pipeline_file: ".buildkite/pipelines/fleet/package_registry.yml"
       provider_settings:
         trigger_mode: none
+      schedules:
+        Daily promotion:
+          cronline: 0 22 * * * America/New_York
       teams:
         everyone:
           access_level: BUILD_AND_READ

.buildkite/pipelines/fleet/package_registry.yml

@@ -1,2 +1,106 @@
+env:
+  IGNORE_SHIP_CI_STATS_ERROR: 'true'
 steps:
-  - command: echo "Placeholder"
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+
+  - wait
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+      preemptible: true
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+    label: 'Pick Test Group Run Order'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+    timeout_in_minutes: 10
+    env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
+      LIMIT_CONFIG_TYPE: integration,functional
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows.sh
+    label: 'Defend Workflows Cypress Tests'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      enableNestedVirtualization: true
+      machineType: n2-standard-4
+    timeout_in_minutes: 60
+    parallelism: 20
+    key: defend-workflows-stateful
+    depends_on: build
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
+    label: 'Defend Workflows Cypress Tests on Serverless'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      enableNestedVirtualization: true
+      machineType: n2-standard-4
+    timeout_in_minutes: 60
+    parallelism: 14
+    key: defend-workflows-serverless
+    depends_on: build
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/fleet/promote_package_registry.sh
+    label: Promote docker.elastic.co/package-registry/distribution:lite
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+    depends_on:
+      - ftr-configs
+      - jest-integration
+      - defend-worfklows-stateful
+      - defend-workflows-serverless
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build
+    timeout_in_minutes: 10
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -78,7 +78,7 @@ const getPipeline = (filename: string, removeSteps = true) => {
 
     if (
       (await doAnyChangesMatch([
-        /^x-pack\/plugins\/observability_solution\/apm/,
+        /^x-pack\/solutions\/observability\/plugins\/apm/,
         /^packages\/kbn-apm-synthtrace/,
       ])) ||
       GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
@@ -88,7 +88,7 @@ const getPipeline = (filename: string, removeSteps = true) => {
 
     if (
       (await doAnyChangesMatch([
-        /^x-pack\/plugins\/observability_solution\/inventory/,
+        /^x-pack\/solutions\/observability\/plugins\/inventory/,
         /^packages\/kbn-apm-synthtrace/,
       ])) ||
       GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
@@ -109,7 +109,7 @@ const getPipeline = (filename: string, removeSteps = true) => {
     }
 
     if (
-      (await doAnyChangesMatch([/^x-pack\/plugins\/observability_solution\/profiling/])) ||
+      (await doAnyChangesMatch([/^x-pack\/solutions\/observability\/plugins\/profiling/])) ||
       GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/profiling_cypress.yml'));
@@ -253,7 +253,7 @@ const getPipeline = (filename: string, removeSteps = true) => {
         /^package.json/,
         /^packages\/kbn-securitysolution-.*/,
         /^x-pack\/platform\/plugins\/shared\/alerting/,
-        /^x-pack\/plugins\/data_views\/common/,
+        /^x-pack\/platform\/plugins\/shared\/data_views\/common/,
         /^x-pack\/solutions\/security\/plugins\/lists/,
         /^x-pack\/platform\/plugins\/shared\/rule_registry\/common/,
         /^x-pack\/solutions\/security\/plugins\/security_solution/,
@@ -296,48 +296,48 @@ const getPipeline = (filename: string, removeSteps = true) => {
     if (
       (await doAnyChangesMatch([
         /^package.json/,
-        /^packages\/kbn-discover-utils/,
+        /^src\/platform\/packages\/shared\/kbn-discover-utils/,
         /^packages\/kbn-doc-links/,
         /^packages\/kbn-dom-drag-drop/,
-        /^packages\/kbn-es-query/,
+        /^src\/platform\/packages\/shared\/kbn-es-query/,
         /^packages\/kbn-i18n/,
         /^packages\/kbn-i18n-react/,
         /^src\/platform\/packages\/shared\/kbn-grouping/,
-        /^packages\/kbn-resizable-layout/,
-        /^packages\/kbn-rison/,
+        /^src\/platform\/packages\/shared\/kbn-resizable-layout/,
+        /^src\/platform\/packages\/shared\/kbn-rison/,
         /^src\/platform\/packages\/shared\/kbn-rule-data-utils/,
         /^packages\/kbn-safer-lodash-set/,
-        /^packages\/kbn-search-types/,
+        /^src\/platform\/packages\/shared\/kbn-search-types/,
         /^packages\/kbn-securitysolution-.*/,
         /^src\/platform\/packages\/shared\/kbn-securitysolution-ecs/,
         /^x-pack\/solutions\/security\/packages\/kbn-securitysolution-io-ts-alerting-types/,
         /^x-pack\/solutions\/security\/packages\/kbn-securitysolution-io-ts-list-types/,
         /^x-pack\/solutions\/security\/packages\/kbn-securitysolution-list-hooks/,
         /^x-pack\/solutions\/security\/packages\/kbn-securitysolution-t-grid/,
-        /^packages\/kbn-ui-theme/,
+        /^src\/platform\/packages\/shared\/kbn-ui-theme/,
         /^packages\/kbn-utility-types/,
         /^packages\/react/,
         /^packages\/shared-ux/,
         /^src\/core/,
         /^src\/plugins\/charts/,
         /^src\/platform\/plugins\/shared\/controls/,
         /^src\/plugins\/data/,
-        /^src\/plugins\/data_views/,
-        /^src\/plugins\/discover/,
-        /^src\/plugins\/field_formats/,
+        /^src\/platform\/plugins\/shared\/data_views/,
+        /^src\/platform\/plugins\/shared\/discover/,
+        /^src\/platform\/plugins\/shared\/field_formats/,
         /^src\/platform\/plugins\/shared\/inspector/,
         /^src\/plugins\/kibana_react/,
         /^src\/plugins\/kibana_utils/,
-        /^src\/plugins\/saved_search/,
+        /^src\/platform\/plugins\/shared\/saved_search/,
         /^src\/plugins\/ui_actions/,
-        /^src\/plugins\/unified_histogram/,
+        /^src\/platform\/plugins\/shared\/unified_histogram/,
         /^src\/plugins\/unified_search/,
         /^x-pack\/platform\/packages\/shared\/kbn-elastic-assistant/,
         /^x-pack\/platform\/packages\/shared\/kbn-elastic-assistant-common/,
         /^x-pack\/solutions\/security\/packages/,
         /^x-pack\/platform\/plugins\/shared\/alerting/,
         /^x-pack\/platform\/plugins\/shared\/cases/,
-        /^x-pack\/plugins\/data_views\/common/,
+        /^x-pack\/platform\/plugins\/shared\/data_views\/common/,
         /^x-pack\/solutions\/security\/plugins\/elastic_assistant/,
         /^x-pack\/solutions\/security\/plugins\/lists/,
         /^x-pack\/platform\/plugins\/shared\/rule_registry\/common/,
@@ -391,7 +391,7 @@ const getPipeline = (filename: string, removeSteps = true) => {
 
     if (
       (await doAnyChangesMatch([
-        /^x-pack\/plugins\/discover_enhanced\/ui_tests/,
+        /^x-pack\/platform\/plugins\/private\/discover_enhanced\/ui_tests/,
         /^packages\/kbn-scout/,
       ])) ||
       GITHUB_PR_LABELS.includes('ci:scout-ui-tests')

.buildkite/scripts/steps/fleet/promote_package_registry.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -euo pipefail
+
+PACKAGE_REGISTRY_BASE_IMAGE='docker.elastic.co/package-registry/distribution:lite'
+PACKAGE_REGISTRY_TARGET_IMAGE='docker.elastic.co/kibana-ci/package-registry-distribution:lite'
+
+if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
+  docker buildx imagetools create -t "$PACKAGE_REGISTRY_TARGET_IMAGE" "$PACKAGE_REGISTRY_BASE_IMAGE"
+  cat << EOF | buildkite-agent pipeline upload
+steps:
+  - label: "Update cache for EPR image"
+    trigger: kibana-vm-images
+    async: true
+    build:
+      env:
+        IMAGES_CONFIG: "kibana/images.yml"
+        RETRY: "1"
+EOF
+else
+  echo "Skipping promotion for untracked branch $BUILDKITE_BRANCH"
+fi

.buildkite/scripts/steps/functional/apm_cypress.sh

@@ -31,6 +31,6 @@ echo "--- APM Cypress Tests"
 
 cd "$XPACK_DIR"
 
-node plugins/observability_solution/apm/scripts/test/e2e.js \
+node solutions/observability/plugins/apm/scripts/test/e2e.js \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
   $CYPRESS_ARGS

.buildkite/scripts/steps/functional/inventory_cypress.sh

@@ -14,5 +14,5 @@ echo "--- Observability Inventory Cypress Tests"
 
 cd "$XPACK_DIR"
 
-node plugins/observability_solution/inventory/scripts/test/e2e.js \
+node solutions/observability/plugins/inventory/scripts/test/e2e.js \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \

.buildkite/scripts/steps/functional/profiling_cypress.sh

@@ -14,5 +14,5 @@ echo "--- Profiling Cypress Tests"
 
 cd "$XPACK_DIR"
 
-NODE_OPTIONS=--openssl-legacy-provider node plugins/observability_solution/profiling/scripts/test/e2e.js \
+NODE_OPTIONS=--openssl-legacy-provider node solutions/observability/plugins/profiling/scripts/test/e2e.js \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \

.buildkite/scripts/steps/functional/scout_ui_tests.sh

@@ -6,7 +6,7 @@ source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-scout-ui-tests
 
-TEST_CONFIG="x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts"
+TEST_CONFIG="x-pack/platform/plugins/private/discover_enhanced/ui_tests/playwright.config.ts"
 KIBANA_DIR="$KIBANA_BUILD_LOCATION"
 
 declare -A TESTS=(