Merge pull request #2996 from rata/k8s-prereq-timeout

K8s prereq timeout

plugins/kubernetes/README.md

@@ -125,6 +125,14 @@ Via [Template filler](/plugins/kubernetes/app/models/kubernetes/template_filler.
 Add a role with only a `Pod`, `metadata.annotations.samson/prerequisite: true`, and command to run a migrations.
 It will be executed before the rest is deployed.
 
+For default it waits for 10 minutes before timeout, you can change the timeout
+using KUBE_WAIT_FOR_PREREQ env variable (specified in seconds).
+
+### Deployment timeouts
+
+A deploy will wait for 10 minutes for pods to come alive. You can adjust this
+timeout using KUBE_WAIT_FOR_LIVE.
+
 ### StatefulSet
 
 On kubernetes <1.7 they can only be updated with `OnDelete` updateStrategy,

plugins/kubernetes/app/models/kubernetes/deploy_executor.rb

@@ -6,6 +6,7 @@
 module Kubernetes
   class DeployExecutor
     WAIT_FOR_LIVE = ENV.fetch('KUBE_WAIT_FOR_LIVE', 10).to_i.minutes
+    WAIT_FOR_PREREQ = ENV.fetch('KUBE_WAIT_FOR_PREREQ', 10).to_i.minutes
     STABILITY_CHECK_DURATION = 1.minute
     TICK = 2.seconds
     RESTARTED = "Restarted"
@@ -59,11 +60,11 @@ def execute(*)
       prerequisites, deploys = @release.release_docs.partition(&:prerequisite?)
       if prerequisites.any?
         @output.puts "First deploying prerequisite ..." if deploys.any?
-        return false unless deploy_and_watch(prerequisites)
+        return false unless deploy_and_watch(prerequisites, WAIT_FOR_PREREQ)
         @output.puts "Now deploying other roles ..." if deploys.any?
       end
       if deploys.any?
-        return false unless deploy_and_watch(deploys)
+        return false unless deploy_and_watch(deploys, WAIT_FOR_LIVE)
       end
       true
     end
@@ -72,7 +73,7 @@ def execute(*)
 
     # check all pods and see if they are running
     # once they are running check if they are stable (for apps only, since jobs are finished and will not change)
-    def wait_for_resources_to_complete(release_docs)
+    def wait_for_resources_to_complete(release_docs, timeout)
       raise "prerequisites should not check for stability" if @testing_for_stability
       @wait_start_time = Time.now
       @output.puts "Waiting for pods to be created"
@@ -100,7 +101,7 @@ def wait_for_resources_to_complete(release_docs)
             if stopped = not_ready.select(&:stop).presence
               unstable!('one or more pods stopped', stopped)
               return statuses
-            elsif seconds_waiting > WAIT_FOR_LIVE
+            elsif seconds_waiting > timeout
               @output.puts "TIMEOUT, pods took too long to get live"
               return statuses
             end
@@ -388,9 +389,9 @@ def deploy(release_docs)
       Samson::Parallelizer.map(resources, db: true, &:deploy)
     end
 
-    def deploy_and_watch(release_docs)
+    def deploy_and_watch(release_docs, timeout)
       deploy(release_docs)
-      result = wait_for_resources_to_complete(release_docs)
+      result = wait_for_resources_to_complete(release_docs, timeout)
       if result == true
         if blue_green = release_docs.select(&:blue_green_color).presence
           finish_blue_green_deployment(blue_green)

plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb

@@ -728,7 +728,7 @@ def create_previous_successful_release
 
       executor.expects(:wait_for_resources_to_complete).returns(true)
       executor.instance_variable_set(:@release, release)
-      assert executor.send(:deploy_and_watch, release.release_docs)
+      assert executor.send(:deploy_and_watch, release.release_docs, 60)
 
       out.must_equal <<~OUT
         Deploying BLUE resources for Pod1 role app-server
@@ -757,7 +757,7 @@ def create_previous_successful_release
 
       executor.expects(:wait_for_resources_to_complete).returns(true)
       executor.instance_variable_set(:@release, release)
-      assert executor.send(:deploy_and_watch, release.release_docs)
+      assert executor.send(:deploy_and_watch, release.release_docs, 60)
 
       out.must_equal <<~OUT
         Deploying BLUE resources for Pod1 role app-server
@@ -781,7 +781,7 @@ def create_previous_successful_release
       executor.expects(:wait_for_resources_to_complete).returns([])
       executor.expects(:print_resource_events)
       executor.instance_variable_set(:@release, release)
-      refute executor.send(:deploy_and_watch, release.release_docs)
+      refute executor.send(:deploy_and_watch, release.release_docs, 60)
 
       out.must_equal <<~OUT
         Deploying BLUE resources for Pod1 role app-server