From 95ec1c4d2cd676bb9769ad300cc6166fe686adf3 Mon Sep 17 00:00:00 2001 From: Samuel Beaulieu Date: Fri, 12 Mar 2021 11:49:55 -0600 Subject: [PATCH] (retry-scheduling) Retry logic when scheduling is not accepted Before this change the call to scheduleBuild would run a jenkins.getQueue().schedule2() and return the value of .isAccepted() but there was no logic around the case where it is not accepted. We have seen instances in production where the console logs would print 'Triggering XYZ' but that build would never make it to the queue, leading the code to then wait forever for the completion of an inexisting build. I assume queue contention when isAccepted() is false and that a retry will get around the problem. --- .../DefaultMatrixExecutionStrategyImpl.java | 23 ++++++++++--- ...CombinationFilterUsingBuildParamsTest.java | 34 ++++++------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/main/java/hudson/matrix/DefaultMatrixExecutionStrategyImpl.java b/src/main/java/hudson/matrix/DefaultMatrixExecutionStrategyImpl.java index 7d119756..75081922 100644 --- a/src/main/java/hudson/matrix/DefaultMatrixExecutionStrategyImpl.java +++ b/src/main/java/hudson/matrix/DefaultMatrixExecutionStrategyImpl.java @@ -15,7 +15,9 @@ import hudson.model.Result; import hudson.model.Run; import hudson.model.queue.CauseOfBlockage; +import org.kohsuke.stapler.DataBoundConstructor; +import javax.annotation.Nullable; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; @@ -24,8 +26,6 @@ import java.util.HashSet; import java.util.List; import java.util.TreeSet; -import javax.annotation.Nullable; -import org.kohsuke.stapler.DataBoundConstructor; /** * {@link MatrixExecutionStrategy} that captures historical behavior. @@ -233,18 +233,31 @@ private TreeSet createTreeSet(Collection items, Comparator sorter) * * This function schedule a build of a configuration passing all of the Matrixchild actions * that are present in the parent build. + * If scheduling the build in the queue fails, it will retry 5 times and wait 1/2 seconds + * before retrying. * * @param exec Matrix build that is the parent of the configuration * @param c Configuration to schedule */ private void scheduleConfigurationBuild(MatrixBuildExecution exec, MatrixConfiguration c) { MatrixBuild build = exec.getBuild(); - exec.getListener().getLogger().println(Messages.MatrixBuild_Triggering(ModelHyperlinkNote.encodeTo(c))); - // filter the parent actions for those that can be passed to the individual jobs. List childActions = new ArrayList(build.getActions(MatrixChildAction.class)); childActions.addAll(build.getActions(ParametersAction.class)); // used to implement MatrixChildAction - c.scheduleBuild(childActions, new UpstreamCause((Run)build)); + Boolean scheduled; + int retry = 5; + for(int i=0; i < retry; i++) { + exec.getListener().getLogger().println(Messages.MatrixBuild_Triggering(ModelHyperlinkNote.encodeTo(c))); + scheduled = c.scheduleBuild(childActions, new UpstreamCause((Run) build)); + if(scheduled == true) { + break; + } + try { + Thread.sleep(500); + } catch(InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } } private MatrixRun waitForCompletion(MatrixBuildExecution exec, MatrixConfiguration c) throws InterruptedException, IOException { diff --git a/src/test/java/hudson/matrix/CombinationFilterUsingBuildParamsTest.java b/src/test/java/hudson/matrix/CombinationFilterUsingBuildParamsTest.java index 4b260584..250febb5 100644 --- a/src/test/java/hudson/matrix/CombinationFilterUsingBuildParamsTest.java +++ b/src/test/java/hudson/matrix/CombinationFilterUsingBuildParamsTest.java @@ -23,33 +23,10 @@ */ package hudson.matrix; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyInt; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import hudson.ExtensionList; import hudson.matrix.MatrixBuild.MatrixBuildExecution; import hudson.matrix.listeners.MatrixBuildListener; -import hudson.model.AbstractItem; -import hudson.model.BuildListener; -import hudson.model.Cause; -import hudson.model.ParametersAction; -import hudson.model.Result; -import hudson.model.Run; -import hudson.model.StringParameterValue; - -import java.io.IOException; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - +import hudson.model.*; import org.apache.commons.io.output.ByteArrayOutputStream; import org.jenkinsci.plugins.scriptsecurity.sandbox.Whitelist; import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.BlanketWhitelist; @@ -65,6 +42,14 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.io.IOException; +import java.io.PrintStream; +import java.util.*; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Mockito.*; + /** * Make sure that the combination filter schedules correct builds in correct order * @@ -270,6 +255,7 @@ private MatrixConfiguration getConfiguration (final String axis) { when(conf.getDisplayName()).thenReturn(axis); when(conf.getUrl()).thenReturn(axis); when(conf.getBuildByNumber(anyInt())).thenReturn(run); + when(conf.scheduleBuild(anyList(),any())).thenReturn(true); return conf; }