diff --git a/src/main/java/hudson/plugins/ec2/EC2ComputerLauncher.java b/src/main/java/hudson/plugins/ec2/EC2ComputerLauncher.java index 4d95a7b01..6204f04a4 100644 --- a/src/main/java/hudson/plugins/ec2/EC2ComputerLauncher.java +++ b/src/main/java/hudson/plugins/ec2/EC2ComputerLauncher.java @@ -23,15 +23,29 @@ */ package hudson.plugins.ec2; +import com.amazonaws.services.ec2.model.SpotInstanceRequest; +import hudson.model.Action; +import hudson.model.Actionable; +import hudson.model.Executor; +import hudson.model.Queue; +import hudson.model.Result; +import hudson.model.Slave; import hudson.model.TaskListener; import hudson.slaves.ComputerLauncher; +import hudson.slaves.OfflineCause; import hudson.slaves.SlaveComputer; +import hudson.model.queue.SubTask; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import com.amazonaws.AmazonClientException; +import jenkins.model.CauseOfInterruption; + +import javax.annotation.Nonnull; /** * {@link ComputerLauncher} for EC2 that wraps the real user-specified {@link ComputerLauncher}. @@ -75,4 +89,95 @@ public void launch(SlaveComputer slaveComputer, TaskListener listener) { protected abstract void launchScript(EC2Computer computer, TaskListener listener) throws AmazonClientException, IOException, InterruptedException; + + /** + * This method is called after a node disconnects. See {@link ComputerLauncher#afterDisconnect(SlaveComputer, TaskListener)} + * This method is overriden to perform a check to see if the node that is disconnected is a spot instance and + * whether the disconnection is a spot interruption event. If it is a spot interruption event, the tasks that the + * node was processing will be resubmitted if a user selects the option to do so. + * @param computer + * @param listener + */ + @Override + public void afterDisconnect(SlaveComputer computer, TaskListener listener) { + if (computer == null) return; // potential edge case where computer is null + + Slave node = computer.getNode(); + if (node instanceof EC2SpotSlave) { + + // checking if its an unexpected disconnection + final boolean isUnexpectedDisconnection = computer.isOffline() && computer.getOfflineCause() + instanceof OfflineCause.ChannelTermination; + boolean shouldRestart = ((EC2SpotSlave) node).getRestartSpotInterruption(); + if (isUnexpectedDisconnection && shouldRestart) { + SpotInstanceRequest spotRequest = ((EC2SpotSlave) node).getSpotRequest(); + if (spotRequest == null) { + LOGGER.log(Level.WARNING, String.format("Could not get spot request for spot instance node %s", + node.getNodeName())); + return; + } + String code = spotRequest.getStatus().getCode(); + // list of status codes - https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html#spot-instance-bid-status-understand + if (code.equals("instance-stopped-by-price") || code.equals("instance-stopped-no-capacity") || + code.equals("instance-terminated-by-price") || code.equals("instance-terminated-no-capacity")) { + LOGGER.log(Level.INFO, String.format("Node %s was terminated due to spot interruption. Retriggering " + + "job", node.getNodeName())); + List executors = computer.getExecutors(); + for (Executor executor : executors) { + Queue.Executable currentExecutable = executor.getCurrentExecutable(); + if (currentExecutable !=null) { // interrupting all executables + executor.interrupt(Result.ABORTED, new EC2SpotInterruptedCause(node.getNodeName())); + SubTask subTask = currentExecutable.getParent(); + Queue.Task task = subTask.getOwnerTask(); + // Get actions (if any) + List actions = new ArrayList<>(); + if (currentExecutable instanceof Actionable) { + actions = ((Actionable) currentExecutable).getActions(Action.class); + } + LOGGER.log(Level.INFO, String.format("Spot instance for node %s was terminated. " + + "Resubmitting task %s with actions %s", node.getNodeName(), task, actions)); + Queue.getInstance().schedule2(task, 10, actions); + } + } + } + } + } + } + + /** + * This {@link CauseOfInterruption} is used when a Node is disconnected due to a Spot Interruption event + */ + static class EC2SpotInterruptedCause extends CauseOfInterruption { + + @Nonnull + private final String nodeName; + + public EC2SpotInterruptedCause(@Nonnull String nodeName) { + this.nodeName = nodeName; + } + + @Override + public String getShortDescription() { + return "EC2 spot instance for node " + nodeName + " was terminated"; + } + + @Override + public int hashCode() { + return nodeName.hashCode(); + } + + @Override + public String toString() { + return getShortDescription(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof EC2SpotInterruptedCause) { + return nodeName.equals(((EC2SpotInterruptedCause) obj).nodeName); + } else { + return false; + } + } + } } diff --git a/src/main/java/hudson/plugins/ec2/EC2SpotSlave.java b/src/main/java/hudson/plugins/ec2/EC2SpotSlave.java index 6a9fee8d2..958bd791a 100644 --- a/src/main/java/hudson/plugins/ec2/EC2SpotSlave.java +++ b/src/main/java/hudson/plugins/ec2/EC2SpotSlave.java @@ -33,6 +33,7 @@ public class EC2SpotSlave extends EC2AbstractSlave implements EC2Readiness { private static final Logger LOGGER = Logger.getLogger(EC2SpotSlave.class.getName()); private final String spotInstanceRequestId; + private boolean restartSpotInterruption; @Deprecated public EC2SpotSlave(String name, String spotInstanceRequestId, String templateDescription, String remoteFS, int numExecutors, Mode mode, String initScript, String tmpDir, String labelString, String remoteAdmin, String jvmopts, String idleTerminationMinutes, List tags, String cloudName, int launchTimeout, AMITypeData amiType) @@ -43,18 +44,25 @@ public EC2SpotSlave(String name, String spotInstanceRequestId, String templateDe @Deprecated public EC2SpotSlave(String name, String spotInstanceRequestId, String templateDescription, String remoteFS, int numExecutors, Mode mode, String initScript, String tmpDir, String labelString, String remoteAdmin, String jvmopts, String idleTerminationMinutes, List tags, String cloudName, boolean usePrivateDnsName, int launchTimeout, AMITypeData amiType) throws FormException, IOException { - this(templateDescription + " (" + name + ")", spotInstanceRequestId, templateDescription, remoteFS, numExecutors, mode, initScript, tmpDir, labelString, Collections.emptyList(), remoteAdmin, jvmopts, idleTerminationMinutes, tags, cloudName, launchTimeout, amiType, ConnectionStrategy.backwardsCompatible(usePrivateDnsName, false, false), -1); + this(templateDescription + " (" + name + ")", spotInstanceRequestId, templateDescription, remoteFS, numExecutors, mode, initScript, tmpDir, labelString, Collections.emptyList(), remoteAdmin, jvmopts, idleTerminationMinutes, tags, cloudName, launchTimeout, amiType, ConnectionStrategy.backwardsCompatible(usePrivateDnsName, false, false), -1, false); } - @DataBoundConstructor + @Deprecated public EC2SpotSlave(String name, String spotInstanceRequestId, String templateDescription, String remoteFS, int numExecutors, Mode mode, String initScript, String tmpDir, String labelString, List> nodeProperties, String remoteAdmin, String jvmopts, String idleTerminationMinutes, List tags, String cloudName, int launchTimeout, AMITypeData amiType, ConnectionStrategy connectionStrategy, int maxTotalUses) throws FormException, IOException { + this(name, spotInstanceRequestId, templateDescription, remoteFS, numExecutors, mode, initScript, tmpDir, labelString, nodeProperties, remoteAdmin, jvmopts, idleTerminationMinutes, tags, cloudName, launchTimeout, amiType, connectionStrategy, maxTotalUses, false); + } + + @DataBoundConstructor + public EC2SpotSlave(String name, String spotInstanceRequestId, String templateDescription, String remoteFS, int numExecutors, Mode mode, String initScript, String tmpDir, String labelString, List> nodeProperties, String remoteAdmin, String jvmopts, String idleTerminationMinutes, List tags, String cloudName, int launchTimeout, AMITypeData amiType, ConnectionStrategy connectionStrategy, int maxTotalUses, boolean restartSpotInterruption) + throws FormException, IOException { super(name, "", templateDescription, remoteFS, numExecutors, mode, labelString, amiType.isWindows() ? new EC2WindowsLauncher() : new EC2UnixLauncher(), new EC2RetentionStrategy(idleTerminationMinutes), initScript, tmpDir, nodeProperties, remoteAdmin, jvmopts, false, idleTerminationMinutes, tags, cloudName, false, launchTimeout, amiType, connectionStrategy, maxTotalUses); this.name = name; this.spotInstanceRequestId = spotInstanceRequestId; + this.restartSpotInterruption = restartSpotInterruption; } @Override @@ -190,6 +198,14 @@ public void onConnected() { pushLiveInstancedata(); } + /** + * Gets whether the node has the setting configured to restart all its tasks when a spot interruption event occurs + * @return true if the node's tasks should be restarted + */ + public boolean getRestartSpotInterruption() { + return restartSpotInterruption; + } + @Extension public static final class DescriptorImpl extends EC2AbstractSlave.DescriptorImpl { diff --git a/src/main/java/hudson/plugins/ec2/SlaveTemplate.java b/src/main/java/hudson/plugins/ec2/SlaveTemplate.java index 5656c79b7..d07eb5b10 100644 --- a/src/main/java/hudson/plugins/ec2/SlaveTemplate.java +++ b/src/main/java/hudson/plugins/ec2/SlaveTemplate.java @@ -1276,6 +1276,7 @@ protected EC2SpotSlave newSpotSlave(SpotInstanceRequest sir) throws FormExceptio .withAmiType(amiType) .withConnectionStrategy(connectionStrategy) .withMaxTotalUses(maxTotalUses) + .withRestartSpotInterruption(spotConfig.getRestartSpotInterruption()) .build(); return EC2AgentFactory.getInstance().createSpotAgent(config); } diff --git a/src/main/java/hudson/plugins/ec2/SpotConfiguration.java b/src/main/java/hudson/plugins/ec2/SpotConfiguration.java index 5b278d45f..0c7601a80 100644 --- a/src/main/java/hudson/plugins/ec2/SpotConfiguration.java +++ b/src/main/java/hudson/plugins/ec2/SpotConfiguration.java @@ -18,6 +18,7 @@ import java.util.Collection; import java.util.Date; import java.util.Objects; +import javax.annotation.Nonnull; import javax.servlet.ServletException; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; @@ -31,6 +32,7 @@ public final class SpotConfiguration extends AbstractDescribableImpl { + @Nonnull @Override public String getDisplayName() { return "spotConfig"; diff --git a/src/main/java/hudson/plugins/ec2/util/EC2AgentConfig.java b/src/main/java/hudson/plugins/ec2/util/EC2AgentConfig.java index d09f3bbcf..0450131a8 100644 --- a/src/main/java/hudson/plugins/ec2/util/EC2AgentConfig.java +++ b/src/main/java/hudson/plugins/ec2/util/EC2AgentConfig.java @@ -71,10 +71,12 @@ private OnDemand(OnDemandBuilder builder) { public static class Spot extends EC2AgentConfig { final String spotInstanceRequestId; + boolean restartSpotInterruption; private Spot(SpotBuilder builder) { super(builder); this.spotInstanceRequestId = builder.spotInstanceRequestId; + this.restartSpotInterruption = builder.restartSpotInterruption; } } @@ -265,12 +267,18 @@ public OnDemand build() { public static class SpotBuilder extends Builder { private String spotInstanceRequestId; + private boolean restartSpotInterruption; public SpotBuilder withSpotInstanceRequestId(String spotInstanceRequestId) { this.spotInstanceRequestId = spotInstanceRequestId; return this; } + public SpotBuilder withRestartSpotInterruption(boolean restartSpotInterruption) { + this.restartSpotInterruption = restartSpotInterruption; + return this; + } + @Override protected SpotBuilder self() { return this; diff --git a/src/main/java/hudson/plugins/ec2/util/EC2AgentFactoryImpl.java b/src/main/java/hudson/plugins/ec2/util/EC2AgentFactoryImpl.java index 9b3a8fca4..d29f942fb 100644 --- a/src/main/java/hudson/plugins/ec2/util/EC2AgentFactoryImpl.java +++ b/src/main/java/hudson/plugins/ec2/util/EC2AgentFactoryImpl.java @@ -17,6 +17,6 @@ public EC2OndemandSlave createOnDemandAgent(EC2AgentConfig.OnDemand config) @Override public EC2SpotSlave createSpotAgent(EC2AgentConfig.Spot config) throws Descriptor.FormException, IOException { - return new EC2SpotSlave(config.name, config.spotInstanceRequestId, config.description, config.remoteFS, config.numExecutors, config.mode, config.initScript, config.tmpDir, config.labelString, config.nodeProperties, config.remoteAdmin, config.jvmopts, config.idleTerminationMinutes, config.tags, config.cloudName, config.launchTimeout, config.amiType, config.connectionStrategy, config.maxTotalUses); + return new EC2SpotSlave(config.name, config.spotInstanceRequestId, config.description, config.remoteFS, config.numExecutors, config.mode, config.initScript, config.tmpDir, config.labelString, config.nodeProperties, config.remoteAdmin, config.jvmopts, config.idleTerminationMinutes, config.tags, config.cloudName, config.launchTimeout, config.amiType, config.connectionStrategy, config.maxTotalUses, config.restartSpotInterruption); } } diff --git a/src/main/resources/hudson/plugins/ec2/SpotConfiguration/config.jelly b/src/main/resources/hudson/plugins/ec2/SpotConfiguration/config.jelly index fa0701ac1..cb25a176a 100644 --- a/src/main/resources/hudson/plugins/ec2/SpotConfiguration/config.jelly +++ b/src/main/resources/hudson/plugins/ec2/SpotConfiguration/config.jelly @@ -35,4 +35,8 @@ THE SOFTWARE. + + + + diff --git a/src/main/resources/hudson/plugins/ec2/SpotConfiguration/help-restartSpotInterruption.html b/src/main/resources/hudson/plugins/ec2/SpotConfiguration/help-restartSpotInterruption.html new file mode 100644 index 000000000..b15eb18aa --- /dev/null +++ b/src/main/resources/hudson/plugins/ec2/SpotConfiguration/help-restartSpotInterruption.html @@ -0,0 +1,4 @@ +
+ If set to true, the tasks that a node was running will be restarted if a spot interruption event has happened. + Find out more information on Spot Interruption here +
\ No newline at end of file diff --git a/src/test/java/hudson/plugins/ec2/SlaveTemplateTest.java b/src/test/java/hudson/plugins/ec2/SlaveTemplateTest.java index c16af93cf..c07ec1f91 100644 --- a/src/test/java/hudson/plugins/ec2/SlaveTemplateTest.java +++ b/src/test/java/hudson/plugins/ec2/SlaveTemplateTest.java @@ -39,6 +39,7 @@ import com.amazonaws.services.ec2.model.InstanceState; import com.amazonaws.services.ec2.model.InstanceType; import com.amazonaws.services.ec2.model.KeyPair; +import com.amazonaws.services.ec2.model.RequestSpotInstancesRequest; import com.amazonaws.services.ec2.model.RunInstancesRequest; import com.amazonaws.services.ec2.model.RunInstancesResult; import com.amazonaws.services.ec2.model.SecurityGroup; @@ -60,6 +61,7 @@ import org.mockito.ArgumentCaptor; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -281,6 +283,39 @@ public void testSpotConfigWithFallback() throws Exception { r.assertEqualBeans(orig, received, "ami,zone,spotConfig,description,remoteFS,type,jvmopts,stopOnTerminate,securityGroups,subnetId,tags,connectionStrategy,hostKeyVerificationStrategy"); } + /** + * Test to make sure the slave created has been configured properly with the restart spot interruption configuration + * correctly set + * @throws Exception - Exception that can be thrown by the Jenkins test harness + */ + @Test + public void testRestartSpotInterruption() throws Exception { + String ami = "ami1"; + String description = "foo ami"; + + EC2Tag tag1 = new EC2Tag("name1", "value1"); + EC2Tag tag2 = new EC2Tag("name2", "value2"); + List tags = new ArrayList(); + tags.add(tag1); + tags.add(tag2); + + SpotConfiguration spotConfig = new SpotConfiguration(true); + spotConfig.setSpotMaxBidPrice("0.1"); + spotConfig.setFallbackToOndemand(true); + spotConfig.setSpotBlockReservationDuration(0); + spotConfig.setRestartSpotInterruption(true); + + SlaveTemplate slaveTemplate = new SlaveTemplate(ami, EC2AbstractSlave.TEST_ZONE, spotConfig, "default", "foo", InstanceType.M1Large, false, "ttt", Node.Mode.NORMAL, "foo ami", "bar", "bbb", "aaa", "10", "fff", null, "-Xmx1g", false, "subnet 456", tags, null, true, null, "", false, false, "", false, ""); + List templates = new ArrayList<>(); + templates.add(slaveTemplate); + + AmazonEC2Cloud ac = new AmazonEC2Cloud("us-east-1", false, "abc", "us-east-1", "ghi", "3", templates, null, null); + r.jenkins.clouds.add(ac); + + SlaveTemplate received = ((EC2Cloud) r.jenkins.clouds.iterator().next()).getTemplate(description); + r.assertEqualBeans(slaveTemplate, received, "ami,zone,spotConfig,description,remoteFS,type,jvmopts,stopOnTerminate,securityGroups,subnetId,tags,connectionStrategy,hostKeyVerificationStrategy"); + } + /** * Test to make sure the IAM Role is set properly. * @@ -480,8 +515,8 @@ public void testMinimumNumberOfInstancesActiveRangeConfig() throws Exception { Assert.assertEquals(true, stored.getMinimumNoInstancesActiveTimeRangeDays().get("tuesday")); } - @Test - public void provisionOndemandSetsAwsNetworkingOnEc2Request() throws Exception { + @Test + public void provisionOndemandSetsAwsNetworkingOnEc2Request() throws Exception { boolean associatePublicIp = false; String ami = "ami1"; String description = "foo ami"; @@ -521,10 +556,10 @@ public void provisionOndemandSetsAwsNetworkingOnEc2Request() throws Exception { assertEquals(actualRequest.getSecurityGroups(), Stream.of(securityGroups).collect(Collectors.toList())); } } - } + } - @Test - public void provisionOndemandSetsAwsNetworkingOnNetworkInterface() throws Exception { + @Test + public void provisionOndemandSetsAwsNetworkingOnNetworkInterface() throws Exception { boolean associatePublicIp = true; String ami = "ami1"; String description = "foo ami"; @@ -561,9 +596,9 @@ public void provisionOndemandSetsAwsNetworkingOnNetworkInterface() throws Except assertEquals(actualRequest.getSecurityGroupIds(), Collections.emptyList()); assertEquals(actualRequest.getSecurityGroups(), Collections.emptyList()); } - } + } - private AmazonEC2 setupTestForProvisioning(SlaveTemplate template) throws Exception { + private AmazonEC2 setupTestForProvisioning(SlaveTemplate template) throws Exception { AmazonEC2Cloud mockedCloud = mock(AmazonEC2Cloud.class); AmazonEC2 mockedEC2 = mock(AmazonEC2.class); EC2PrivateKey mockedPrivateKey = mock(EC2PrivateKey.class); @@ -614,5 +649,6 @@ private AmazonEC2 setupTestForProvisioning(SlaveTemplate template) throws Except when(mockedEC2.runInstances(any(RunInstancesRequest.class))).thenReturn(mockedResult); return mockedEC2; - } + } + } diff --git a/src/test/java/hudson/plugins/ec2/SpotResubmitTest.java b/src/test/java/hudson/plugins/ec2/SpotResubmitTest.java new file mode 100644 index 000000000..83ce65929 --- /dev/null +++ b/src/test/java/hudson/plugins/ec2/SpotResubmitTest.java @@ -0,0 +1,143 @@ +package hudson.plugins.ec2; + +import com.amazonaws.services.ec2.model.SpotInstanceRequest; +import com.amazonaws.services.ec2.model.SpotInstanceStatus; +import hudson.model.Action; +import hudson.model.Actionable; +import hudson.model.Descriptor; +import hudson.model.Executor; +import hudson.model.Queue; +import hudson.model.Result; +import hudson.model.TaskListener; +import hudson.plugins.ec2.ssh.EC2UnixLauncher; +import hudson.slaves.OfflineCause; +import hudson.slaves.SlaveComputer; +import jenkins.model.Jenkins; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import hudson.model.queue.SubTask; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.logging.ConsoleHandler; +import java.util.logging.Logger; + +import static org.mockito.Mockito.*; +import static org.powermock.api.mockito.PowerMockito.mockStatic; + +@PowerMockIgnore({"javax.crypto.*", "org.hamcrest.*", "javax.net.ssl.*", "com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*"}) +@RunWith(PowerMockRunner.class) +@PrepareForTest({Jenkins.class, Queue.class}) +public class SpotResubmitTest { + + @Mock + private SlaveComputer computer; + @Mock + private EC2SpotSlave ec2SpotSlave; + @Mock + private TaskListener taskListener; + @Mock + private Jenkins jenkins; + @Mock + private Queue queue; + @Mock + private Queue.Task task; + @Mock + private Action action; + @Mock + private Executor executor; + + /** + * Setups the queue and task + */ + @Before + public void setup() { + + // attaching console handler for debugging + Logger LOGGER = Logger.getLogger(EC2ComputerLauncher.class.getName()); + LOGGER.addHandler(new ConsoleHandler()); + + // Mocking the static classes + mockStatic(Jenkins.class); + when(Jenkins.get()).thenReturn(jenkins); + when(Jenkins.getInstance()).thenReturn(jenkins); + when(Jenkins.getInstanceOrNull()).thenReturn(jenkins); + when(Queue.getInstance()).thenReturn(queue); + // Setting computer offline due to channel termination - disconnection + when(computer.isOffline()).thenReturn(true); + when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null)); + // Mocking executor + List executors = Arrays.asList(executor); + when(computer.getExecutors()).thenReturn(executors); + // Mocking executable + Actionable executable = mock(Actionable.class, withSettings().extraInterfaces(Queue.Executable.class)); + + when(executor.getCurrentExecutable()).thenReturn((Queue.Executable) executable); + // Mocking task list + SubTask subTask = mock(SubTask.class); + when(((Queue.Executable) executable).getParent()).thenReturn(subTask); + when(subTask.getOwnerTask()).thenReturn(task); + List actions = Arrays.asList(action); + when(executable.getActions(Action.class)).thenReturn(actions); + } + + @Test + public void testNonSpotInstanceDisconnect() { + EC2OndemandSlave ec2OndemandSlave = mock(EC2OndemandSlave.class); + when(computer.getNode()).thenReturn(ec2OndemandSlave); // set node as ec2ondemandslave + when(ec2SpotSlave.getRestartSpotInterruption()).thenReturn(true); // setting restart spot instances to true + new EC2UnixLauncher().afterDisconnect(computer, taskListener); + verify(computer).getNode(); + verifyNoMoreInteractions(computer); + } + + @Test + public void testSpotInterruptionNoResubmit() { + + EC2SpotSlave ec2SpotSlave = mock(EC2SpotSlave.class); + when(ec2SpotSlave.getRestartSpotInterruption()).thenReturn(false); // setting is turned off + new EC2UnixLauncher().afterDisconnect(computer, taskListener); + PowerMockito.verifyZeroInteractions(queue); // verify that the queue has no tasks resubmitted + } + + @Test + public void testInterruptExecutors() { + + EC2SpotSlave ec2SpotSlave = mock(EC2SpotSlave.class); + when(computer.getNode()).thenReturn(ec2SpotSlave); + when(ec2SpotSlave.getNodeName()).thenReturn("mocked_node"); + when(ec2SpotSlave.getRestartSpotInterruption()).thenReturn(true); + SpotInstanceRequest spotInstanceRequest = mock(SpotInstanceRequest.class); + when(ec2SpotSlave.getSpotRequest()).thenReturn(spotInstanceRequest); + SpotInstanceStatus spotInstanceStatus = mock(SpotInstanceStatus.class); + when(spotInstanceRequest.getStatus()).thenReturn(spotInstanceStatus); + when(spotInstanceStatus.getCode()).thenReturn("instance-terminated-by-price"); + new EC2UnixLauncher().afterDisconnect(computer, taskListener); + verify(executor).interrupt(Result.ABORTED, new EC2ComputerLauncher.EC2SpotInterruptedCause(ec2SpotSlave.getNodeName())); + } + + @Test + public void testSpotInterruptionResubmitQueue() throws IOException, Descriptor.FormException { + + when(computer.getNode()).thenReturn(ec2SpotSlave); + when(ec2SpotSlave.getNodeName()).thenReturn("mocked_node"); + // Mocking the spot interruption settings and events + when(ec2SpotSlave.getRestartSpotInterruption()).thenReturn(true); + SpotInstanceRequest spotInstanceRequest = mock(SpotInstanceRequest.class); + when(ec2SpotSlave.getSpotRequest()).thenReturn(spotInstanceRequest); + SpotInstanceStatus spotInstanceStatus = mock(SpotInstanceStatus.class); + when(spotInstanceRequest.getStatus()).thenReturn(spotInstanceStatus); + when(spotInstanceStatus.getCode()).thenReturn("instance-terminated-by-price"); + + // Verifying that the queue task was rescheduled + new EC2UnixLauncher().afterDisconnect(computer, taskListener); + verify(queue).schedule2(eq(task), anyInt(), eq(Arrays.asList(action))); + } +} diff --git a/src/test/java/hudson/plugins/ec2/util/EC2AgentFactoryMockImpl.java b/src/test/java/hudson/plugins/ec2/util/EC2AgentFactoryMockImpl.java index cb2b1be96..e913ec90e 100644 --- a/src/test/java/hudson/plugins/ec2/util/EC2AgentFactoryMockImpl.java +++ b/src/test/java/hudson/plugins/ec2/util/EC2AgentFactoryMockImpl.java @@ -23,7 +23,7 @@ public EC2OndemandSlave createOnDemandAgent(EC2AgentConfig.OnDemand config) @Override public EC2SpotSlave createSpotAgent(EC2AgentConfig.Spot config) throws Descriptor.FormException, IOException { - return new MockEC2SpotSlave(config.name, config.spotInstanceRequestId, config.description, config.remoteFS, config.numExecutors, config.mode, config.initScript, config.tmpDir, config.labelString, config.nodeProperties, config.remoteAdmin, config.jvmopts, config.idleTerminationMinutes, config.tags, config.cloudName, config.launchTimeout, config.amiType, config.connectionStrategy, config.maxTotalUses); + return new MockEC2SpotSlave(config.name, config.spotInstanceRequestId, config.description, config.remoteFS, config.numExecutors, config.mode, config.initScript, config.tmpDir, config.labelString, config.nodeProperties, config.remoteAdmin, config.jvmopts, config.idleTerminationMinutes, config.tags, config.cloudName, config.launchTimeout, config.amiType, config.connectionStrategy, config.maxTotalUses, config.restartSpotInterruption); } private static class MockEC2OndemandSlave extends EC2OndemandSlave { @@ -49,9 +49,9 @@ public Computer createComputer() { private static class MockEC2SpotSlave extends EC2SpotSlave { private static final long serialVersionUID = 1L; - private MockEC2SpotSlave(String name, String spotInstanceRequestId, String description, String remoteFS, int numExecutors, Mode mode, String initScript, String tmpDir, String labelString, List> nodeProperties, String remoteAdmin, String jvmopts, String idleTerminationMinutes, List tags, String cloudName, int launchTimeout, AMITypeData amiType, ConnectionStrategy connectionStrategy, int maxTotalUses) + private MockEC2SpotSlave(String name, String spotInstanceRequestId, String description, String remoteFS, int numExecutors, Mode mode, String initScript, String tmpDir, String labelString, List> nodeProperties, String remoteAdmin, String jvmopts, String idleTerminationMinutes, List tags, String cloudName, int launchTimeout, AMITypeData amiType, ConnectionStrategy connectionStrategy, int maxTotalUses, boolean restartSpotInterruption) throws Descriptor.FormException, IOException { - super(name, spotInstanceRequestId, description, remoteFS, numExecutors, mode, initScript, tmpDir, labelString, nodeProperties, remoteAdmin, jvmopts, idleTerminationMinutes, tags, cloudName, launchTimeout, amiType, connectionStrategy, maxTotalUses); + super(name, spotInstanceRequestId, description, remoteFS, numExecutors, mode, initScript, tmpDir, labelString, nodeProperties, remoteAdmin, jvmopts, idleTerminationMinutes, tags, cloudName, launchTimeout, amiType, connectionStrategy, maxTotalUses, restartSpotInterruption); } @Override