From ba88b3d76be63c130ba52e51adf9e8733301e1cf Mon Sep 17 00:00:00 2001 From: Pedro Chambino Date: Wed, 14 Feb 2024 13:16:47 +0000 Subject: [PATCH] Cancel build if pipeline number is newer If pipeline number is newer, that means we have reach the API limit of running jobs, and newer pipeline already got front of the line, so we can cancel the current build as it is already out of date. In our use case this means we would try to deploy an older commit which would then get rejected since a newer one is already deployed. Also, reduce max wait time by default, we never want to wait for that long. Plus, drop "dont-quit" and "force-cancel-previous", we don't use them and not implemented. --- scripts/loop.bash | 38 ++++++++++++++-------------- src/commands/until_front_of_line.yml | 12 +-------- src/jobs/block_workflow.yml | 11 +------- 3 files changed, 21 insertions(+), 40 deletions(-) diff --git a/scripts/loop.bash b/scripts/loop.bash index 74ba0d8..b0ce7ab 100755 --- a/scripts/loop.bash +++ b/scripts/loop.bash @@ -182,6 +182,9 @@ cancel_build_num(){ echo "Cancelling build ${BUILD_NUM}" cancel_api_url_template="${CIRCLECI_BASE_URL}/api/v1.1/project/${VCS_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${BUILD_NUM}/cancel?circle-token=${CCI_TOKEN}" curl -s -X POST $cancel_api_url_template > /dev/null + + sleep 30 # wait for API to cancel this job, rather than showing as failure + exit 1 # but just in case, fail job } @@ -190,7 +193,7 @@ cancel_build_num(){ # # MAIN LOGIC STARTS HERE -# +# load_variables do_we_run #exit early if we can echo "Max Queue Time: ${max_time_seconds} seconds." @@ -207,6 +210,18 @@ while true; do echo "This Job's Pipeline #: $MY_PIPELINE_NUMBER" echo "Front of Queue (fifo) Pipeline #: $front_of_queue_pipeline_number" + if [ -z "$front_of_queue_pipeline_number" ]; then + # NOTE: This can happen if there are more jobs running than the ones we fetched from the API due to pagination limits. + echo "We failed to find the front of the queue, this means another pipeline probably already got front of the queue, so cancel this one..." + cancel_build_num $CIRCLE_BUILD_NUM + fi + + if [ $front_of_queue_pipeline_number -gt $MY_PIPELINE_NUMBER ]; then + # NOTE: This can happen if there are more jobs running than the ones we fetched from the API due to pagination limits. + echo "We found a newer pipeline as the front of the queue, this means another pipeline probably already got front of the queue, so cancel this one..." + cancel_build_num $CIRCLE_BUILD_NUM + fi + if [ "$front_of_queue_pipeline_number" = "$MY_PIPELINE_NUMBER" ]; then # recent-jobs API does not include pending, so it is possible we queried in between a workflow transition, and we're NOT really front of line. if [ $confidence -lt $CONFIDENCE_THRESHOLD ]; then @@ -221,28 +236,13 @@ while true; do else # If we fail, reset confidence confidence=0 - if [ -z "$front_of_queue_pipeline_number" ]; then - echo "This build (${CIRCLE_BUILD_NUM}), pipeline (${MY_PIPELINE_NUMBER}) is queued, waiting for total number of jobs to be under 100." - else - echo "This build (${CIRCLE_BUILD_NUM}), pipeline (${MY_PIPELINE_NUMBER}) is queued, waiting for build(${oldest_running_build_num}) pipeline (${front_of_queue_pipeline_number}) to complete." - fi + echo "This build (${CIRCLE_BUILD_NUM}), pipeline (${MY_PIPELINE_NUMBER}) is queued, waiting for build(${oldest_running_build_num}) pipeline (${front_of_queue_pipeline_number}) to complete." echo "Total Queue time: ${wait_time} seconds." fi if [ $wait_time -ge $max_time_seconds ]; then - echo "Max wait time exceeded, fail or force cancel..." - if [ "${DONT_QUIT}" = "1" ];then - echo "Orb parameter dont-quit is set to true, letting this job proceed!" - if [ "${FORCE_CANCEL_PREVIOUS}" = "1" ]; then - "FEATURE NOT IMPLEMENTED" - exit 1 - fi - exit 0 - else - cancel_build_num $CIRCLE_BUILD_NUM - sleep 5 # wait for API to cancel this job, rather than showing as failure - exit 1 # but just in case, fail job - fi + echo "Max wait time exceeded, cancel..." + cancel_build_num $CIRCLE_BUILD_NUM fi sleep $loop_time diff --git a/src/commands/until_front_of_line.yml b/src/commands/until_front_of_line.yml index 1b7ad38..fabf110 100644 --- a/src/commands/until_front_of_line.yml +++ b/src/commands/until_front_of_line.yml @@ -10,16 +10,8 @@ parameters: description: "If true, this job will block until no other workflows with ANY JOBS with an earlier timestamp are running. Typically used as first job." max-wait-time: type: integer - default: 10 + default: 5 description: "How many minutes to wait before giving up." - dont-quit: - type: boolean - default: false - description: "Quitting is for losers. Force job through once time expires instead of failing." - force-cancel-previous: - type: boolean - default: false - description: "No Mercy. Issue cancel commands for any previous competitors (only applies when dont-quit also true)" limit-branch-name: type: string default: "*" @@ -64,8 +56,6 @@ steps: ONLY_ON_BRANCH: <> BLOCK_WORKFLOW: <> MAX_TIME: <> - DONT_QUIT: <> - FORCE_CANCEL_PREVIOUS: <> FILTER_BRANCH: << parameters.this-branch-only >> ONLY_ON_WORKFLOW: <> CONFIDENCE_THRESHOLD: <> diff --git a/src/jobs/block_workflow.yml b/src/jobs/block_workflow.yml index 248f398..3c1f32b 100644 --- a/src/jobs/block_workflow.yml +++ b/src/jobs/block_workflow.yml @@ -10,16 +10,8 @@ parameters: description: "If true, this job will block until no other workflows with an earlier timestamp are running. Typically used as first job." max-wait-time: type: integer - default: 10 + default: 5 description: "How many minutes to wait before giving up." - dont-quit: - type: boolean - default: false - description: "Quitting is for losers. Force job through once time expires instead of failing." - force-cancel-previous: - type: boolean - default: false - description: "No Mercy. Issue cancel commands for any previous competitors (only applies when dont-quit also true)" limit-branch-name: type: string default: "*" @@ -64,7 +56,6 @@ steps: this-branch-only: <> block-workflow: <> max-wait-time: <> - dont-quit: <> limit-workflow-name: <> limit-branch-name: <> confidence: <>