diff --git a/.buildkite/hooks/post-checkout.ps1 b/.buildkite/hooks/post-checkout.ps1
new file mode 100644
index 00000000000..f2c590378fa
--- /dev/null
+++ b/.buildkite/hooks/post-checkout.ps1
@@ -0,0 +1,60 @@
+# Set error handling
+$ErrorActionPreference = "Stop"
+
+# Define a function to checkout and merge
+function Checkout-Merge {
+    param (
+        [string]$targetBranch,
+        [string]$prCommit,
+        [string]$mergeBranch
+    )
+
+    if (-not $targetBranch) {
+        Write-Host "No pull request target branch"
+        exit 1
+    }
+
+    git fetch -v origin $targetBranch
+    git checkout FETCH_HEAD
+    Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Create a temporary branch to merge the PR with the target branch
+    git checkout -b $mergeBranch
+    Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Set author identity so it can be used for git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit $prCommit
+
+    if ($LASTEXITCODE -ne 0) {
+        $mergeResult = $LASTEXITCODE
+        Write-Host "Merge failed: $mergeResult"
+        git merge --abort
+        exit $mergeResult
+    }
+}
+
+$pullRequest = $env:BUILDKITE_PULL_REQUEST
+
+if ($pullRequest -eq "false") {
+    Write-Host "Not a pull request, skipping"
+    exit 0
+}
+
+$targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH
+$prCommit = $env:BUILDKITE_COMMIT
+$prId = $env:BUILDKITE_PULL_REQUEST
+$mergeBranch = "pr_merge_$prId"
+
+Checkout-Merge $targetBranch $prCommit $mergeBranch
+
+Write-Host "Commit information"
+git --no-pager log --format=%B -n 1
+
+Write-Host "Fixing CRLF in git checkout --"
+git config core.autocrlf true
+
+# Ensure Buildkite groups are rendered
+Write-Host ""
diff --git a/.buildkite/hooks/pre-command.ps1 b/.buildkite/hooks/pre-command.ps1
new file mode 100644
index 00000000000..ca5b34bc920
--- /dev/null
+++ b/.buildkite/hooks/pre-command.ps1
@@ -0,0 +1,25 @@
+# Install gcc TODO: Move to the VM image
+choco install mingw
+Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1
+refreshenv
+
+# Install gvm and go
+# TODO: Move GVM download to the base VM image
+$env:GvmVersion = "0.5.2"
+[Net.ServicePointManager]::SecurityProtocol = "tls12"
+$env:GoVersion = Get-Content -Path .go-version
+Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v$env:GvmVersion/gvm-windows-amd64.exe -Outfile C:\Windows\System32\gvm.exe
+gvm --format=powershell $env:GoVersion | Invoke-Expression
+go version
+
+$GOPATH = $(go env GOPATH)
+$env:Path = "$GOPATH\bin;" + $env:Path
+[Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
+[Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
+
+# Install tools
+go install github.com/magefile/mage
+go install github.com/elastic/go-licenser
+go install golang.org/x/tools/cmd/goimports
+go install github.com/jstemmer/go-junit-report
+go install gotest.tools/gotestsum
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index edd8b7ff301..ae5c8239fbd 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -4,6 +4,151 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
   DOCKER_REGISTRY: "docker.elastic.co"
 steps:
+<<<<<<< HEAD
+=======
+  - label: "Unit tests - Ubuntu 22.04"
+    key: "unit-tests-2204"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-ubuntu-2204"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Ubuntu 22.04 ARM64"
+    key: "unit-tests-2204-arm64"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "aws"
+      imagePrefix: "core-ubuntu-2204-aarch64"
+      diskSizeGb: 200
+      instanceType: "m6g.4xlarge"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Windows 2022"
+    key: "unit-tests-win2022"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2022"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - Windows 2016"
+    key: "unit-tests-win2016"
+    command: ".\\.buildkite\\scripts\\steps\\unit-tests.ps1"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: "gcp"
+      image: "family/core-windows-2016"
+      machine_type: "n2-standard-8"
+      disk_size: 200
+      disk_type: "pd-ssd"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - MacOS 13 ARM"
+    key: "unit-tests-macos-13-arm"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: orka
+      imagePrefix: generic-13-ventura-arm
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Unit tests - MacOS 13"
+    key: "unit-tests-macos-13"
+    command: ".buildkite/scripts/steps/unit-tests.sh"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+      - "coverage.out"
+    agents:
+      provider: orka
+      imagePrefix: generic-13-ventura-x64
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Merge coverage reports"
+    key: "merge-coverage"
+    env:
+      BUILDKITE_REPO: ""
+    command: "
+      .buildkite/scripts/steps/merge.sh
+      unit-tests-2204
+      unit-tests-2204-arm64
+      unit-tests-win2016
+      unit-tests-win2022
+      unit-tests-macos-13
+      unit-tests-macos-13-arm
+      "
+    artifact_paths:
+      - "build/TEST-**"
+    agents:
+      image: "golang:1.20.10"
+    depends_on:
+      - unit-tests-2204
+      - unit-tests-2204-arm64
+      - unit-tests-win2022
+      - unit-tests-win2016
+      - unit-tests-macos-13
+      - unit-tests-macos-13-arm
+    allow_dependency_failure: true
+
+  - label: ":sonarqube: Continuous Code Inspection"
+    env:
+      VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/elastic-agent/sonar-analyze-token"
+    agents:
+      image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest"
+    command:
+      - "buildkite-agent artifact download --step merge-coverage build/TEST-go-unit.cov ."
+      - "/scan-source-code.sh"
+    depends_on:
+      - "merge-coverage"
+    retry:
+      manual:
+        allowed: true
+
+  - label: "Serverless integration test"
+    key: "serverless-integration-tests"
+    command: ".buildkite/scripts/steps/integration_tests.sh serverless integration:single TestMonitoringLogsShipped" #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+
+>>>>>>> bf78cc1e14 ([CI] Pull requests migration to buildkite (#3573))
   - label: "Integration tests"
     key: "integration-tests"
     command: ".buildkite/scripts/integration_tests.sh"
@@ -13,3 +158,15 @@ steps:
     agents:
       provider: "gcp"
       machineType: "n1-standard-8"
+<<<<<<< HEAD
+=======
+
+  - wait: ~
+    continue_on_failure: true
+  - label: "Processing test results"
+    agents:
+      provider: "gcp"
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: build/TEST-go-integration*.xml
+>>>>>>> bf78cc1e14 ([CI] Pull requests migration to buildkite (#3573))
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
new file mode 100644
index 00000000000..16d32fb77ab
--- /dev/null
+++ b/.buildkite/scripts/common.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ -z "${WORKSPACE-""}" ]]; then
+    WORKSPACE=$(git rev-parse --show-toplevel)
+    export WORKSPACE
+fi
+PIPELINE="${WORKSPACE}/.buildkite/pipeline.elastic-agent-package.yml"
+if [[ -z "${SETUP_MAGE_VERSION-""}" ]]; then
+    SETUP_MAGE_VERSION=$(grep -oe "SETUP_MAGE_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" )
+fi
+if [[ -z "${SETUP_GVM_VERSION-""}" ]]; then
+    SETUP_GVM_VERSION=$(grep -oe "SETUP_GVM_VERSION\: [\"'].*[\"']" "$PIPELINE" | awk '{print $2}' | sed "s/'//g" )
+fi
+BEAT_VERSION=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' "${WORKSPACE}/version/version.go")
+export BEAT_VERSION
+
+getOSOptions() {
+  case $(uname | tr '[:upper:]' '[:lower:]') in
+    linux*)
+      export AGENT_OS_NAME=linux
+      ;;
+    darwin*)
+      export AGENT_OS_NAME=darwin
+      ;;
+    msys*)
+      export AGENT_OS_NAME=windows
+      ;;
+    *)
+      export AGENT_OS_NAME=notset
+      ;;
+  esac
+  case $(uname -m | tr '[:upper:]' '[:lower:]') in
+    aarch64*)
+      export AGENT_OS_ARCH=arm64
+      ;;
+    arm64*)
+      export AGENT_OS_ARCH=arm64
+      ;;
+    amd64*)
+      export AGENT_OS_ARCH=amd64
+      ;;
+    x86_64*)
+      export AGENT_OS_ARCH=amd64
+      ;;
+    *)
+      export AGENT_OS_ARCH=notset
+      ;;
+  esac
+}
+
+# Wrapper function for executing mage
+mage() {
+    go version
+    if ! [ -x "$(type -p mage | sed 's/mage is //g')" ];
+    then
+        echo "installing mage ${SETUP_MAGE_VERSION}"
+        make mage
+    fi
+    pushd "$WORKSPACE"
+    command "mage" "$@"
+    ACTUAL_EXIT_CODE=$?
+    popd
+    return $ACTUAL_EXIT_CODE
+}
+
+# Wrapper function for executing go
+go(){
+    # Search for the go in the Path
+    if ! [ -x "$(type -p go | sed 's/go is //g')" ];
+    then
+        getOSOptions
+        echo "installing golang "${GO_VERSION}" for "${AGENT_OS_NAME}/${AGENT_OS_ARCH}" "
+        local _bin="${WORKSPACE}/bin"
+        mkdir -p "${_bin}"
+        retry 5 curl -sL -o "${_bin}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${AGENT_OS_NAME}-${AGENT_OS_ARCH}"
+        chmod +x "${_bin}/gvm"
+        eval "$(command "${_bin}/gvm" "${GO_VERSION}" )"
+        export GOPATH=$(command go env GOPATH)
+        export PATH="${PATH}:${GOPATH}/bin"
+    fi
+    pushd "$WORKSPACE"
+    command go "$@"
+    ACTUAL_EXIT_CODE=$?
+    popd
+    return $ACTUAL_EXIT_CODE
+}
+
+google_cloud_auth() {
+    local keyFile=$1
+
+    gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null
+
+    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
+}
+
+retry() {
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}
diff --git a/.buildkite/scripts/steps/merge.sh b/.buildkite/scripts/steps/merge.sh
new file mode 100755
index 00000000000..4eda3e2fd3b
--- /dev/null
+++ b/.buildkite/scripts/steps/merge.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Downloads and merges coverage files from multiple steps into a single file (build/TEST-go-unit.cov).
+# Usage: merge.sh <step1> <step2> ... Where <step> is the id of the step that contains the coverage artifact.#  
+
+set -exuo pipefail
+
+COV_ARTIFACT="coverage.out"
+MERGED_COV_FILE="build/TEST-go-unit.cov"
+# Space separated list of paths to coverage files
+COV_PATHS=""
+
+go install github.com/wadey/gocovmerge@latest
+mkdir -p build
+
+for STEP_ID in "$@"; do
+  mkdir -p $STEP_ID 
+  buildkite-agent artifact download --step $STEP_ID $COV_ARTIFACT $STEP_ID
+  COV_PATHS="${COV_PATHS} $STEP_ID/$COV_ARTIFACT"
+done
+
+gocovmerge $COV_PATHS > $MERGED_COV_FILE
+echo "Merged coverage file: $MERGED_COV_FILE. See artifacts"
\ No newline at end of file
diff --git a/.buildkite/scripts/steps/unit-tests.ps1 b/.buildkite/scripts/steps/unit-tests.ps1
new file mode 100644
index 00000000000..e1645554532
--- /dev/null
+++ b/.buildkite/scripts/steps/unit-tests.ps1
@@ -0,0 +1,28 @@
+$ErrorActionPreference = "Stop"
+
+Write-Host "-- Fixing CRLF in git checkout --"
+git config core.autocrlf input
+git rm --quiet --cached -r .
+git reset --quiet --hard
+
+$env:GOTMPDIR = "$env:BUILDKITE_BUILD_CHECKOUT_PATH"
+
+Write-Host "--- Build"
+mage build
+
+if ($LASTEXITCODE -ne 0) {
+  exit 1 
+}
+
+Write-Host "--- Unit tests"
+$env:TEST_COVERAGE = $true
+$env:RACE_DETECTOR = $true
+mage unitTest
+# Copy coverage file to build directory so it can be downloaded as an artifact
+cp .\build\TEST-go-unit.cov coverage.out
+
+if ($LASTEXITCODE -ne 0) {
+  exit 1 
+}
+
+
diff --git a/.buildkite/scripts/steps/unit-tests.sh b/.buildkite/scripts/steps/unit-tests.sh
new file mode 100644
index 00000000000..6df75a26a08
--- /dev/null
+++ b/.buildkite/scripts/steps/unit-tests.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+source .buildkite/scripts/common.sh
+
+echo "--- Unit tests"
+RACE_DETECTOR=true TEST_COVERAGE=true mage unitTest
+# Copy coverage file to build directory so it can be downloaded as an artifact
+cp build/TEST-go-unit.cov coverage.out
\ No newline at end of file
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 0b7d459834e..07bf19d7bf1 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -95,57 +95,7 @@ pipeline {
             values 'ubuntu-22 && immutable', 'aws && aarch64 && gobld/diskSizeGb:200', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64'
           }
         }
-        stages {
-          stage('build'){
-            steps {
-              withGithubNotify(context: "Build-${PLATFORM}") {
-                deleteDir()
-                unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-                withMageEnv(){
-                  dir("${BASE_DIR}"){
-                    cmd(label: 'Go build', script: 'mage build')
-                  }
-                }
-              }
-            }
-          }
-          stage('Test') {
-            steps {
-              withGithubNotify(context: "Test-${PLATFORM}") {
-                withMageEnv(){
-                  dir("${BASE_DIR}"){
-                    withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
-                      cmd(label: 'Go unitTest', script: 'mage unitTest')
-                    }
-                  }
-                }
-              }
-            }
-            post {
-              always {
-                junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
-                whenTrue(isCodeCoverageEnabled()) {
-                  coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml')
-                }
-              }
-            }
-          }
-          stage('Integration tests') {
-            when {
-              // Always when running builds on branches/tags
-              // Enable if integration test related changes.
-              beforeAgent true
-              anyOf {
-                not { changeRequest() }
-                expression { return params.integration_tests_ci }
-                expression { return env.GITHUB_COMMENT?.contains('integration tests') }
-                expression { matchesPrLabel(label: 'ci:integration') }
-              }
-            }
-            steps {
-              echo 'TBD'
-            }
-          }
+        stages {        
           stage('K8s') {
             when {
               beforeAgent true
@@ -160,7 +110,13 @@ pipeline {
               }
             }
             steps {
+<<<<<<< HEAD
               runK8s(k8sVersion: 'v1.26.0', kindVersion: 'v0.17.0', context: "K8s-${PLATFORM}")
+=======
+              deleteDir()
+              unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+              runK8s(k8sVersion: 'v1.27.3', kindVersion: 'v0.20.0', context: "K8s-${PLATFORM}")
+>>>>>>> bf78cc1e14 ([CI] Pull requests migration to buildkite (#3573))
             }
           }
           stage('Package') {
@@ -348,77 +304,7 @@ pipeline {
           }
         }
       }
-    }
-    stage('m1') {
-      agent { label 'orka && darwin && aarch64' }
-      options { skipDefaultCheckout() }
-      when {
-        // Always when running builds on branches/tags
-        // Enable if extended M1 support related changes.
-        beforeAgent true
-        anyOf {
-          not { changeRequest() }
-          expression { return isExtendedM1Enabled() && env.ONLY_DOCS == "false"}
-        }
-      }
-      stages {
-        stage('build'){
-          steps {
-            withGithubNotify(context: "Build-darwin-aarch64") {
-              deleteDir()
-              unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-              withMageEnv(){
-                dir("${BASE_DIR}"){
-                  cmd(label: 'Go build', script: 'mage build')
-                }
-              }
-            }
-          }
-        }
-        stage('Test') {
-          steps {
-            withGithubNotify(context: "Test-darwin-aarch64") {
-              withMageEnv(){
-                dir("${BASE_DIR}"){
-                  withEnv(["RACE_DETECTOR=true", "TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
-                    cmd(label: 'Go unitTest', script: 'mage unitTest')
-                  }
-                }
-              }
-            }
-          }
-          post {
-            always {
-              junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
-              whenTrue(isCodeCoverageEnabled()) {
-                coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml')
-              }
-            }
-          }
-        }
-      }
-    }
-    stage('e2e tests') {
-      when {
-        // Disable until we fix the e2e test.
-        // This should be running when there is a comment or the e2e label
-        beforeAgent true
-        anyOf {
-          // package artifacts are not generated if ONLY_DOCS, therefore e2e should not run if ONLY_DOCS
-          expression { return isE2eEnabled() && env.ONLY_DOCS == "false"}
-        }
-      }
-      steps {
-        runE2E(testMatrixFile: '.ci/.e2e-tests-for-elastic-agent.yaml',
-               beatVersion: "${env.BEAT_VERSION}-SNAPSHOT",
-               elasticAgentVersion: "${env.BEAT_VERSION}-SNAPSHOT",
-               gitHubCheckName: "e2e-tests",
-               gitHubCheckRepo: env.REPO,
-               gitHubCheckSha1: env.GIT_BASE_COMMIT,
-               propagate: true,
-               wait: true)
-      }
-    }
+    }        
   }
   post {
     cleanup {
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
deleted file mode 100644
index 23a7c6a4da8..00000000000
--- a/.github/workflows/macos.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: macos
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-      - 8.*
-
-jobs:
-  macos:
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-go@v3
-      with:
-        go-version-file: .go-version
-    - name: Install dependencies
-      run:  go install github.com/magefile/mage
-    - name: Run build
-      run: mage build
-    - name: Run test
-      run: mage unitTest
diff --git a/internal/pkg/agent/application/upgrade/upgrade_test.go b/internal/pkg/agent/application/upgrade/upgrade_test.go
index 84a2977e84b..456a115b4db 100644
--- a/internal/pkg/agent/application/upgrade/upgrade_test.go
+++ b/internal/pkg/agent/application/upgrade/upgrade_test.go
@@ -93,7 +93,7 @@ func Test_CopyFile(t *testing.T) {
 			}
 
 			err := copyDir(l, tc.From, tc.To, tc.IgnoreErr)
-			require.Equal(t, tc.ExpectedErr, err != nil)
+			require.Equal(t, tc.ExpectedErr, err != nil, err)
 		})
 	}
 }