From cead3935db91306d61db7b08f75a0bb1abccfb3f Mon Sep 17 00:00:00 2001
From: sharbuz <87968844+sharbuz@users.noreply.github.com>
Date: Wed, 31 Jan 2024 10:56:36 +0200
Subject: [PATCH] migrate DRA gcp bucket (#3242)

* migrate DRA gcp bucket

* add test trigger step

* test trigger step

* test

* test

* test

* chenge the credentials

* remove public-read ACL

* rollback the test changes

(cherry picked from commit 73bb556c88b3643b26a75e8afef495df33acdbf9)
---
 .buildkite/hooks/pre-command | 16 ++++++++--------
 .buildkite/scripts/common.sh | 25 +++++++------------------
 2 files changed, 15 insertions(+), 26 deletions(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 1b3a53983..b8735e5f2 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -6,9 +6,9 @@ source .buildkite/scripts/common.sh
 
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 EC_KEY_SECRET_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
-PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials"
+PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
-JOB_GCS_BUCKET_SECRET_PATH="kv/ci-shared/platform-ingest/fleet_gcs_bucket"
+JOB_GCS_BUCKET="ingest-buildkite-ci"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 GITHUB_REPO_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
 
@@ -41,14 +41,14 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
-  export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
-  export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
+  export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+  export JOB_GCS_BUCKET
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" || "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
-    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
-    export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
+    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+    export JOB_GCS_BUCKET
   fi
 fi
 
@@ -66,7 +66,7 @@ fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" ]]; then
-    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
-    export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
+    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
+    export JOB_GCS_BUCKET
   fi
 fi
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
index a339398c2..5018d0210 100755
--- a/.buildkite/scripts/common.sh
+++ b/.buildkite/scripts/common.sh
@@ -104,32 +104,21 @@ google_cloud_auth() {
 
 upload_packages_to_gcp_bucket() {
     local pattern=${1}
-    local baseUri="gs://${JOB_GCS_BUCKET}/${REPO}"              #TODO: needs to add the "/buildkite" for rollback
-    local bucketUriCommit="${baseUri}"/commits/${BUILDKITE_COMMIT}
-    local bucketUriDefault="${baseUri}"/snapshots
+    local baseUri="gs://${JOB_GCS_BUCKET}/${REPO}"
+    local bucketUriCommit="${baseUri}/commits/${BUILDKITE_COMMIT}"
+    local bucketUriDefault="${baseUri}/snapshots"
 
     if [[ ${BUILDKITE_PULL_REQUEST} != "false" ]]; then
-        bucketUriDefault="${baseUri}"/pull-requests/pr-${GITHUB_PR_NUMBER}
+        bucketUriDefault="${baseUri}/pull-requests/pr-${GITHUB_PR_NUMBER}"
     fi
     for bucketUri in "${bucketUriCommit}" "${bucketUriDefault}"; do
-        gsutil -m -q cp -a public-read -r ${pattern} "${bucketUri}"
+        gsutil -m -q cp -r ${pattern} "${bucketUri}"
     done
 }
 
 get_bucket_uri() {
     local type=${1}
-    local baseUri="gs://${JOB_GCS_BUCKET}/jobs"              #TODO: needs to add the "/buildkite" for rollback
-    if [[ ${type} == "snapshot" ]]; then
-        local folder="commits"
-    else
-        local folder="${type}"
-    fi
-    bucketUri="${baseUri}/${folder}/${BUILDKITE_COMMIT}"
-}
-
-get_bucket_uri() {
-    local type=${1}
-    local baseUri="gs://${JOB_GCS_BUCKET}/jobs"               #TODO: needs to add the "/buildkite" for rollback
+    local baseUri="gs://${JOB_GCS_BUCKET}/jobs"
     if [[ ${type} == "snapshot" ]]; then
         local folder="commits"
     else
@@ -142,7 +131,7 @@ upload_mbp_packages_to_gcp_bucket() {
     local pattern=${1}
     local type=${2}
     get_bucket_uri "${type}"
-    gsutil -m -q cp -a public-read -r ${pattern} ${bucketUri}
+    gsutil -m -q cp -r ${pattern} ${bucketUri}
 }
 
 download_mbp_packages_from_gcp_bucket() {