From f48648ac160d320f5bd7876106edb48fd84829ab Mon Sep 17 00:00:00 2001 From: Konrad Ohms <40577406+konrad-ohms@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:08:25 +0100 Subject: [PATCH] ci: Use reslock for resource locking (#233) * ci: Use reslock for resource locking * ci: Add wait in reslock script --- ci/pipeline.yaml | 97 +++++++++++++++++++++++++++++-------- ci/pr-pipeline.yml | 108 ++++++++++++++++++++++++++++++------------ ci/scripts/reslock.sh | 36 ++++++++++++++ 3 files changed, 192 insertions(+), 49 deletions(-) create mode 100755 ci/scripts/reslock.sh diff --git a/ci/pipeline.yaml b/ci/pipeline.yaml index 79a90b69..29d244f2 100644 --- a/ci/pipeline.yaml +++ b/ci/pipeline.yaml @@ -49,6 +49,14 @@ resource_types: source: repository: cfcommunity/slack-notification-resource tag: latest + - name: metadata + type: registry-image + source: + repository: delivery.instana.io/int-docker-private-virtual/olhtbr/metadata-resource + tag: 2.0.1 + username: ((delivery-instana-io-internal-project-artifact-read-writer-creds.username)) + password: ((delivery-instana-io-internal-project-artifact-read-writer-creds.password)) + resources: - name: test-clusters @@ -168,7 +176,7 @@ resources: - name: e2e-test-base-image type: registry-image icon: cube - source: + source: &e2e-test-base-image repository: gcr.io/instana-agent-qa/agent-operator/e2e-test-base-image username: _json_key password: ((project-berlin-tests-gcp-instana-qa)) @@ -209,6 +217,9 @@ resources: start: 9:00 AM #UTC stop: 10:00 AM + - name: metadata + type: metadata + jobs: - name: self-update on_success: @@ -498,11 +509,23 @@ jobs: passed: [docker-build] - in_parallel: - do: - - put: gke-lowest - inputs: detect - resource: test-clusters - params: - claim: gke-lowest + - put: metadata + - task: reslock-claim-gke-lowest + timeout: 60m + config: + platform: linux + image_resource: + type: registry-image + source: *e2e-test-base-image + params: + RESLOCK_COMMAND: claim + RESLOCK_RESOURCE_NAME: gke-lowest + RESLOCK_GITHUB_TOKEN: ((ibm-ghe-tokens.concourse-other-teams)) + inputs: + - name: pipeline-source + - name: metadata + run: + path: pipeline-source/ci/scripts/reslock.sh - task: run-e2e-test-gke-lowest timeout: 30m attempts: 1 @@ -581,17 +604,41 @@ jobs: - name: pipeline-source run: path: pipeline-source/ci/scripts/cleanup-resources.sh - - put: test-clusters - inputs: detect - params: - release: gke-lowest + - task: reslock-release-gke-lowest + timeout: 5m + config: + platform: linux + image_resource: + type: registry-image + source: *e2e-test-base-image + params: + RESLOCK_COMMAND: release + RESLOCK_RESOURCE_NAME: gke-lowest + RESLOCK_GITHUB_TOKEN: ((ibm-ghe-tokens.concourse-other-teams)) + inputs: + - name: pipeline-source + - name: metadata + run: + path: pipeline-source/ci/scripts/reslock.sh - do: - - put: gke-latest - inputs: detect - resource: test-clusters - params: - claim: gke-latest + - put: metadata + - task: reslock-claim-gke-latest + timeout: 60m + config: + platform: linux + image_resource: + type: registry-image + source: *e2e-test-base-image + params: + RESLOCK_COMMAND: claim + RESLOCK_RESOURCE_NAME: gke-latest + RESLOCK_GITHUB_TOKEN: ((ibm-ghe-tokens.concourse-other-teams)) + inputs: + - name: pipeline-source + - name: metadata + run: + path: pipeline-source/ci/scripts/reslock.sh - task: run-e2e-test-gke-latest timeout: 30m attempts: 1 @@ -667,10 +714,22 @@ jobs: - name: pipeline-source run: path: pipeline-source/ci/scripts/cleanup-resources.sh - - put: test-clusters - inputs: detect - params: - release: gke-latest + - task: reslock-release-gke-latest + timeout: 5m + config: + platform: linux + image_resource: + type: registry-image + source: *e2e-test-base-image + params: + RESLOCK_COMMAND: release + RESLOCK_RESOURCE_NAME: gke-latest + RESLOCK_GITHUB_TOKEN: ((ibm-ghe-tokens.concourse-other-teams)) + inputs: + - name: pipeline-source + - name: metadata + run: + path: pipeline-source/ci/scripts/reslock.sh # - do: # - put: openshift-4.11 # inputs: detect diff --git a/ci/pr-pipeline.yml b/ci/pr-pipeline.yml index dca898ca..48b2884f 100644 --- a/ci/pr-pipeline.yml +++ b/ci/pr-pipeline.yml @@ -73,6 +73,14 @@ resource_types: username: ((delivery-instana-io-internal-project-artifact-read-writer-creds.username)) password: ((delivery-instana-io-internal-project-artifact-read-writer-creds.password)) + - name: metadata + type: registry-image + source: + repository: delivery.instana.io/int-docker-private-virtual/olhtbr/metadata-resource + tag: 2.0.1 + username: ((delivery-instana-io-internal-project-artifact-read-writer-creds.username)) + password: ((delivery-instana-io-internal-project-artifact-read-writer-creds.password)) + resources: - name: pipeline-source type: git @@ -162,24 +170,18 @@ resources: password: ((delivery-instana-io-internal-project-artifact-read-writer-creds.password)) tag: ((branch)) - - name: test-clusters - type: pool - source: - uri: https://github.ibm.com/instana/k8s-e2e-resources-coordination.git - username: ((ibm-ghe-tokens.concourse-other-teams)) - password: x-oauth-basic - branch: main - pool: k8s-clusters - - name: e2e-test-base-image type: registry-image icon: cube - source: + source: &e2e-test-base-image repository: gcr.io/instana-agent-qa/agent-operator/e2e-test-base-image username: _json_key password: ((project-berlin-tests-gcp-instana-qa)) tag: ((branch)) + - name: metadata + type: metadata + jobs: - name: self-update <<: *gh-status-hooks @@ -266,7 +268,6 @@ jobs: args: - -ceu - | - pushd golangci-lint-release LINTER_RPM=$(ls golangci-lint-*-linux-amd64.rpm) rpm -i $LINTER_RPM @@ -484,7 +485,6 @@ jobs: args: - -ceu - | - IMAGE_TAG=${GIT_COMMIT} set +x unset HISTFILE @@ -574,11 +574,23 @@ jobs: # - <<: *gh-status-set-pending-e2e-openshift - in_parallel: - do: - - put: gke-lowest - inputs: detect - resource: test-clusters - params: - claim: gke-lowest + - put: metadata + - task: reslock-claim-gke-lowest + timeout: 60m + config: + platform: linux + image_resource: + type: registry-image + source: *e2e-test-base-image + params: + RESLOCK_COMMAND: claim + RESLOCK_RESOURCE_NAME: gke-lowest + RESLOCK_GITHUB_TOKEN: ((ibm-ghe-tokens.concourse-other-teams)) + inputs: + - name: pipeline-source + - name: metadata + run: + path: pipeline-source/ci/scripts/reslock.sh - task: run-e2e-test-gke-lowest timeout: 30m attempts: 1 @@ -661,17 +673,41 @@ jobs: - name: pipeline-source run: path: pipeline-source/ci/scripts/cleanup-resources.sh - - put: test-clusters - inputs: detect - params: - release: gke-lowest + - task: reslock-release-gke-lowest + timeout: 5m + config: + platform: linux + image_resource: + type: registry-image + source: *e2e-test-base-image + params: + RESLOCK_COMMAND: release + RESLOCK_RESOURCE_NAME: gke-lowest + RESLOCK_GITHUB_TOKEN: ((ibm-ghe-tokens.concourse-other-teams)) + inputs: + - name: pipeline-source + - name: metadata + run: + path: pipeline-source/ci/scripts/reslock.sh - do: - - put: gke-latest - inputs: detect - resource: test-clusters - params: - claim: gke-latest + - put: metadata + - task: reslock-claim-gke-latest + timeout: 60m + config: + platform: linux + image_resource: + type: registry-image + source: *e2e-test-base-image + params: + RESLOCK_COMMAND: claim + RESLOCK_RESOURCE_NAME: gke-latest + RESLOCK_GITHUB_TOKEN: ((ibm-ghe-tokens.concourse-other-teams)) + inputs: + - name: pipeline-source + - name: metadata + run: + path: pipeline-source/ci/scripts/reslock.sh - task: run-e2e-test-gke-latest timeout: 30m attempts: 1 @@ -751,10 +787,22 @@ jobs: - name: pipeline-source run: path: pipeline-source/ci/scripts/cleanup-resources.sh - - put: test-clusters - inputs: detect - params: - release: gke-latest + - task: reslock-release-gke-latest + timeout: 5m + config: + platform: linux + image_resource: + type: registry-image + source: *e2e-test-base-image + params: + RESLOCK_COMMAND: release + RESLOCK_RESOURCE_NAME: gke-latest + RESLOCK_GITHUB_TOKEN: ((ibm-ghe-tokens.concourse-other-teams)) + inputs: + - name: pipeline-source + - name: metadata + run: + path: pipeline-source/ci/scripts/reslock.sh # - do: # - put: openshift-4.11 # inputs: detect diff --git a/ci/scripts/reslock.sh b/ci/scripts/reslock.sh new file mode 100755 index 00000000..c0ade6cf --- /dev/null +++ b/ci/scripts/reslock.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# +# (c) Copyright IBM Corp. 2024 +# (c) Copyright Instana Inc. +# + +# Helper script to calculate CI Server specific metadata to be used for locking. +# When invoked, it generates a unique lock owner name from concourse meta-data and +# fetched the most recent go binary from github to execute the lock or release command. +set -e +set -o pipefail +set -u # fail if env var is not set + +echo "${RESLOCK_COMMAND} lock ${RESLOCK_RESOURCE_NAME}" + +echo "Reading concourse metada" +ls -lah metadata/ +# injecting concourse metadata to be added as lock owner +BUILD_PIPELINE_NAME="$(cat metadata/build_pipeline_name)" +BUILD_JOB_NAME="$(cat metadata/build_job_name)" +BUILD_NAME="$(cat metadata/build_name)" +BUILD_ID="$(cat metadata/build_id)" + +export RESLOCK_GITHUB_REPO_OWNER=instana +export RESLOCK_LOCK_OWNER="${BUILD_PIPELINE_NAME}/${BUILD_JOB_NAME}/${BUILD_NAME}/${BUILD_ID}" + +echo "RESLOCK_GITHUB_REPO_OWNER=${RESLOCK_GITHUB_REPO_OWNER}" +echo "RESLOCK_LOCK_OWNER=${RESLOCK_LOCK_OWNER}" + +curl -s "https://${RESLOCK_GITHUB_TOKEN}@raw.github.ibm.com/instana/reslock/main/run.sh" > run.sh +if [ "${RESLOCK_COMMAND}" == "claim" ]; then + bash run.sh claim k8s-clusters "${RESLOCK_RESOURCE_NAME}" -t 30m -w +else + bash run.sh "${RESLOCK_COMMAND}" k8s-clusters "${RESLOCK_RESOURCE_NAME}" +fi