Skip to content

Commit

Permalink
Postmigration script cleanup (elastic#39915)
Browse files Browse the repository at this point in the history
Removed unused/unnecessary scripts and updated its' usage in other corresponding scripts
  • Loading branch information
oakrizan authored Jul 1, 2024
1 parent b3e52e3 commit 351cabb
Show file tree
Hide file tree
Showing 12 changed files with 30 additions and 346 deletions.
13 changes: 8 additions & 5 deletions .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,18 @@

set -euo pipefail

source .buildkite/env-scripts/util.sh

export KUBECONFIG="${WORKSPACE}/kubecfg"
export BIN="${WORKSPACE}/bin"
add_bin_path

echo "--- Installing kind & kubectl"
echo "--- Add ${BIN} to PATH"
if [[ ! -d "${BIN}" ]]; then
mkdir -p "${BIN}"
fi
export PATH="${PATH}:${BIN}"

echo "~~~ Installing kind & kubectl"
asdf plugin add kind
asdf install kind $ASDF_KIND_VERSION
asdf install kind "$ASDF_KIND_VERSION"

echo "~~~ Setting up kind"
max_retries=3
Expand Down
59 changes: 0 additions & 59 deletions .buildkite/env-scripts/env.sh

This file was deleted.

104 changes: 0 additions & 104 deletions .buildkite/env-scripts/util.sh

This file was deleted.

7 changes: 0 additions & 7 deletions .buildkite/env-scripts/win-env.sh

This file was deleted.

53 changes: 2 additions & 51 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -2,56 +2,9 @@

set -euo pipefail

source .buildkite/env-scripts/util.sh

# Secrets must be redacted
# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"

if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" ]]; then
source .buildkite/env-scripts/env.sh
if [[ -z "${GO_VERSION-""}" ]]; then
export GO_VERSION=$(cat "${WORKSPACE}/.go-version")
fi

# Images with prefix "platform-ingest-beats-*" has Go pre-setup.
# Image itself takes care of Go version download/installation based on .go-version file
if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
if [[ -z "${GO_VERSION-""}" ]]; then
GO_VERSION=$(cat "${WORKSPACE}/.go-version")
export GO_VERSION
fi

# To prevent "OSError: [Errno 24] Too many open files"
ulimit -Sn 150000
echo "--- Ulimit: $(ulimit)"

echo "--- Setting up environment"
add_bin_path
with_go
with_mage
fi

if [[ "$BUILDKITE_STEP_KEY" == *packaging* ]]; then
DOCKER_USERNAME_SECRET=$(retry_with_count 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
DOCKER_PASSWORD_SECRET=$(retry_with_count 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
GITHUB_TOKEN_SECRET=$(retry_with_count 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})

export DOCKER_USERNAME_SECRET
export DOCKER_PASSWORD_SECRET
export GITHUB_TOKEN_SECRET

docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null

github_username=$(retry_with_count 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
github_email=$(retry_with_count 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})

git config user.name "$github_username"
git config user.email "$github_email"
fi
fi

ENABLED_BEATS_PIPELINES_SLUGS=(
"auditbeat"
Expand All @@ -74,10 +27,8 @@ for slug in "${ENABLED_BEATS_PIPELINES_SLUGS[@]}"; do
fi
done

if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry_with_count 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP_KEY" == *"system-tests"* ]]; then
PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry -t 5 -- vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
export PRIVATE_CI_GCS_CREDENTIALS_SECRET
fi
fi

26 changes: 18 additions & 8 deletions .buildkite/hooks/pre-exit
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,24 @@

set -euo pipefail

source .buildkite/hooks/scripts/util.sh
unset_secrets () {
for var in $(printenv | sed 's;=.*;;' | sort); do
if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
unset "$var"
fi
done
}

if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then
if [[ "$BUILDKITE_STEP_KEY" == package* ]]; then
docker logout "${DOCKER_REGISTRY}"
cleanup() {
if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then
echo "Deleting temporary files..."
if [[ -n "${BIN:-}" ]]; then
rm -rf "${BIN}"
fi
echo "Done."
fi
}

unset_secrets
cleanup

# Ensure that any temporal files created during any step are removed
cleanup
unset_secrets
fi
35 changes: 0 additions & 35 deletions .buildkite/hooks/scripts/util.sh

This file was deleted.

17 changes: 0 additions & 17 deletions .buildkite/scripts/generate_pipeline.sh

This file was deleted.

4 changes: 2 additions & 2 deletions .buildkite/scripts/initCloudEnv.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ exportAwsSecrets() {
local awsSecretKey
local awsAccessKey

awsSecretKey=$(retry -t 5 -- vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
awsAccessKey=$(retry -t 5 -- vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
awsSecretKey=$(retry -t 5 -- vault kv get -field secret_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")
awsAccessKey=$(retry -t 5 -- vault kv get -field access_key "${AWS_SERVICE_ACCOUNT_SECRET_PATH}")

echo "~~~ Exporting AWS secrets"
export AWS_ACCESS_KEY_ID=$awsAccessKey
Expand Down
5 changes: 0 additions & 5 deletions .buildkite/scripts/install-msitools.sh

This file was deleted.

Loading

0 comments on commit 351cabb

Please sign in to comment.