Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.17](backport #3242) migrate DRA gcp bucket #3244

Merged
merged 1 commit into from
Jan 31, 2024
Merged

[7.17](backport #3242) migrate DRA gcp bucket #3244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .buildkite/hooks/pre-command
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ source .buildkite/scripts/common.sh

DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
EC_KEY_SECRET_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials"
PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
JOB_GCS_BUCKET_SECRET_PATH="kv/ci-shared/platform-ingest/fleet_gcs_bucket"
JOB_GCS_BUCKET="ingest-buildkite-ci"
GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
GITHUB_REPO_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})

Expand Down Expand Up @@ -41,14 +41,14 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" || "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET
fi
fi

Expand All @@ -66,7 +66,7 @@ fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" ]]; then
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET
fi
fi
25 changes: 7 additions & 18 deletions .buildkite/scripts/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,32 +104,21 @@ google_cloud_auth() {

upload_packages_to_gcp_bucket() {
local pattern=${1}
local baseUri="gs://${JOB_GCS_BUCKET}/${REPO}" #TODO: needs to add the "/buildkite" for rollback
local bucketUriCommit="${baseUri}"/commits/${BUILDKITE_COMMIT}
local bucketUriDefault="${baseUri}"/snapshots
local baseUri="gs://${JOB_GCS_BUCKET}/${REPO}"
local bucketUriCommit="${baseUri}/commits/${BUILDKITE_COMMIT}"
local bucketUriDefault="${baseUri}/snapshots"

if [[ ${BUILDKITE_PULL_REQUEST} != "false" ]]; then
bucketUriDefault="${baseUri}"/pull-requests/pr-${GITHUB_PR_NUMBER}
bucketUriDefault="${baseUri}/pull-requests/pr-${GITHUB_PR_NUMBER}"
fi
for bucketUri in "${bucketUriCommit}" "${bucketUriDefault}"; do
gsutil -m -q cp -a public-read -r ${pattern} "${bucketUri}"
gsutil -m -q cp -r ${pattern} "${bucketUri}"
done
}

get_bucket_uri() {
local type=${1}
local baseUri="gs://${JOB_GCS_BUCKET}/jobs" #TODO: needs to add the "/buildkite" for rollback
if [[ ${type} == "snapshot" ]]; then
local folder="commits"
else
local folder="${type}"
fi
bucketUri="${baseUri}/${folder}/${BUILDKITE_COMMIT}"
}

get_bucket_uri() {
local type=${1}
local baseUri="gs://${JOB_GCS_BUCKET}/jobs" #TODO: needs to add the "/buildkite" for rollback
local baseUri="gs://${JOB_GCS_BUCKET}/jobs"
if [[ ${type} == "snapshot" ]]; then
local folder="commits"
else
Expand All @@ -142,7 +131,7 @@ upload_mbp_packages_to_gcp_bucket() {
local pattern=${1}
local type=${2}
get_bucket_uri "${type}"
gsutil -m -q cp -a public-read -r ${pattern} ${bucketUri}
gsutil -m -q cp -r ${pattern} ${bucketUri}
}

download_mbp_packages_from_gcp_bucket() {
Expand Down
Loading