Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

migrate DRA gcp bucket #3242

Merged
merged 9 commits into from
Jan 31, 2024
Merged

migrate DRA gcp bucket #3242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
25b82fe
migrate DRA gcp bucket
Jan 30, 2024
41f7235
add test trigger step
Jan 30, 2024
1e60beb
test trigger step
Jan 30, 2024
0ce131f
test
Jan 30, 2024
b0bde9f
test
Jan 30, 2024
a33cbab
test
Jan 30, 2024
b107b2e
chenge the credentials
Jan 30, 2024
e80405a
remove public-read ACL
Jan 30, 2024
8890de3
rollback the test changes
Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .buildkite/hooks/pre-command
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ source .buildkite/scripts/common.sh

DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
EC_KEY_SECRET_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials"
PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
JOB_GCS_BUCKET_SECRET_PATH="kv/ci-shared/platform-ingest/fleet_gcs_bucket"
JOB_GCS_BUCKET="ingest-buildkite-ci"
GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
GITHUB_REPO_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})

Expand Down Expand Up @@ -55,14 +55,14 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET
fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" || "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET
fi
fi

Expand All @@ -80,7 +80,7 @@ fi

if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" ]]; then
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET=$(retry 5 vault kv get -field plaintext ${JOB_GCS_BUCKET_SECRET_PATH})
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export JOB_GCS_BUCKET
fi
fi
25 changes: 7 additions & 18 deletions .buildkite/scripts/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,32 +104,21 @@ google_cloud_auth() {

upload_packages_to_gcp_bucket() {
local pattern=${1}
local baseUri="gs://${JOB_GCS_BUCKET}/${REPO}" #TODO: needs to add the "/buildkite" for rollback
local bucketUriCommit="${baseUri}"/commits/${BUILDKITE_COMMIT}
local bucketUriDefault="${baseUri}"/snapshots
local baseUri="gs://${JOB_GCS_BUCKET}/${REPO}"
local bucketUriCommit="${baseUri}/commits/${BUILDKITE_COMMIT}"
local bucketUriDefault="${baseUri}/snapshots"

if [[ ${BUILDKITE_PULL_REQUEST} != "false" ]]; then
bucketUriDefault="${baseUri}"/pull-requests/pr-${GITHUB_PR_NUMBER}
bucketUriDefault="${baseUri}/pull-requests/pr-${GITHUB_PR_NUMBER}"
fi
for bucketUri in "${bucketUriCommit}" "${bucketUriDefault}"; do
gsutil -m -q cp -a public-read -r ${pattern} "${bucketUri}"
gsutil -m -q cp -r ${pattern} "${bucketUri}"
done
}

get_bucket_uri() {
local type=${1}
local baseUri="gs://${JOB_GCS_BUCKET}/jobs" #TODO: needs to add the "/buildkite" for rollback
if [[ ${type} == "snapshot" ]]; then
local folder="commits"
else
local folder="${type}"
fi
bucketUri="${baseUri}/${folder}/${BUILDKITE_COMMIT}"
}

get_bucket_uri() {
local type=${1}
local baseUri="gs://${JOB_GCS_BUCKET}/jobs" #TODO: needs to add the "/buildkite" for rollback
local baseUri="gs://${JOB_GCS_BUCKET}/jobs"
if [[ ${type} == "snapshot" ]]; then
local folder="commits"
else
Expand All @@ -142,7 +131,7 @@ upload_mbp_packages_to_gcp_bucket() {
local pattern=${1}
local type=${2}
get_bucket_uri "${type}"
gsutil -m -q cp -a public-read -r ${pattern} ${bucketUri}
gsutil -m -q cp -r ${pattern} ${bucketUri}
}

download_mbp_packages_from_gcp_bucket() {
Expand Down
Loading