deflakes test-grace-period-disabled by checking instance health from …

…responder state (#1636)
twosigma · Sep 2, 2022 · 8a0bd98 · 8a0bd98
1 parent d7360e1
commit 8a0bd98
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 14 deletions.
diff --git a/waiter/integration/waiter/basic_test.clj b/waiter/integration/waiter/basic_test.clj
@@ -1634,14 +1634,16 @@
 
 (deftest ^:parallel ^:integration-fast test-grace-period-disabled
   (testing-using-waiter-url
-    (let [{:keys [cookies request-headers service-id] :as response}
+    (let [base-cid (utils/unique-identifier)
+          {:keys [cookies request-headers service-id] :as response}
           (make-request-with-debug-info
-            {:x-waiter-cmd (kitchen-cmd "--enable-status-change -p $PORT0")
-             :x-waiter-grace-period-secs 0
-             :x-waiter-instance-expiry-mins 30
-             :x-waiter-max-instances 1
-             :x-waiter-name (rand-name)}
-            #(make-kitchen-request waiter-url % :method :get :path "/"))]
+            {"x-cid" (str base-cid "-1")
+             "x-waiter-cmd" (kitchen-cmd "--enable-status-change -p $PORT0")
+             "x-waiter-grace-period-secs" 0
+             "x-waiter-instance-expiry-mins" 30
+             "x-waiter-max-instances" 1
+             "x-waiter-name" (rand-name)}
+            #(make-kitchen-request waiter-url % :method :get :path "/hello-1"))]
       (with-service-cleanup
         service-id
         (assert-response-status response http-200-ok)
@@ -1651,14 +1653,16 @@
                                               :service-id service-id})))
         (assert-service-on-all-routers waiter-url service-id cookies)
         (let [request-headers (assoc request-headers
-                                :x-kitchen-default-status-timeout 600000
-                                :x-kitchen-default-status-value http-400-bad-request)
-              response (make-kitchen-request waiter-url request-headers :path "/hello")]
+                                "x-cid" (str base-cid "-2")
+                                "x-kitchen-default-status-timeout" 600000
+                                "x-kitchen-default-status-value" http-400-bad-request)
+              response (make-kitchen-request waiter-url request-headers :path "/hello-2")]
           (assert-response-status response http-400-bad-request))
         (assert-service-unhealthy-on-all-routers waiter-url service-id cookies)
         (let [request-headers (assoc request-headers
-                                :x-waiter-queue-timeout 5000)
-              response (make-kitchen-request waiter-url request-headers :path "/hello")]
+                                "x-cid" (str base-cid "-3")
+                                "x-waiter-queue-timeout" 5000)
+              response (make-kitchen-request waiter-url request-headers :path "/hello-3")]
           (assert-response-status response http-503-service-unavailable))))))
 
 (deftest  ^:parallel ^:integration-fast test-request-unsupported-headers

diff --git a/waiter/src/waiter/util/client_tools.clj b/waiter/src/waiter/util/client_tools.clj
@@ -1267,6 +1267,13 @@
     (map :service-id)
     set))
 
+(defn active-instances-from-state
+  "Returns the active instances for the given service-id from the /state endpoint."
+  [waiter-url service-id & {:keys [cookies] :or {cookies {}}}]
+  (some-> (service-state waiter-url service-id :cookies cookies)
+      (get-in [:state :responder-state :id->instance])
+      (vals)))
+
 (defn active-instances
   "Returns the active instances for the given service-id"
   [waiter-url service-id & {:keys [cookies] :or {cookies {}}}]
@@ -1306,7 +1313,7 @@
   `(let [service-id# ~service-id
          cookies# ~cookies]
      (doseq [[_# router-url#] (routers ~waiter-url)]
-       (is (wait-for #(let [instances# (active-instances router-url# service-id# :cookies cookies#)]
+       (is (wait-for #(let [instances# (active-instances-from-state router-url# service-id# :cookies cookies#)]
                         (log/info "instances:" instances#)
                         (and (seq instances#) (some :healthy? instances#)))
                      :interval 1 :timeout 30)))))
@@ -1316,7 +1323,7 @@
   `(let [service-id# ~service-id
          cookies# ~cookies]
      (doseq [[_# router-url#] (routers ~waiter-url)]
-       (is (wait-for #(let [instances# (active-instances router-url# service-id# :cookies cookies#)]
+       (is (wait-for #(let [instances# (active-instances-from-state router-url# service-id# :cookies cookies#)]
                         (log/info "instances:" instances#)
                         (and (seq instances#) (every? (comp not :healthy?) instances#)))
                      :interval 1 :timeout 30)))))