Use larger backoffs for registry test, and retry network removal (#32)

Cargo.toml

@@ -13,7 +13,7 @@ name = "passivized_docker_engine_client"
 readme = "README.md"
 repository = "https://github.com/iamjpotts/passivized_docker_engine_client"
 rust-version = "1.64"
-version = "0.0.6"
+version = "0.0.7-alpha"
 
 [dependencies]
 base64 = "0.13"
@@ -27,27 +27,28 @@ futures = "0.3"
 log = "0.4"
 time = { version = "0.3", features = ["parsing"] }
 thiserror = "1.0"
-tokio = { version = "1.21", features = ["macros", "rt-multi-thread"] }
+tokio = { version = "1.23", features = ["macros", "rt-multi-thread"] }
 url = "2.3"
 
 [dev-dependencies]
 backoff = { version = "0.4", features = ["futures", "tokio"] }
 http = "0.2"
 hyper = { version = "0.14", features = ["client", "http1", "http2"] }
 hyper-tls = "0.5"
+itertools = "0.10"
 log = "0.4"
 mockito = "0.31.0"
 native-tls = "0.2"
 openssl = "0.10"
 passivized_htpasswd = "0.0.3"
-passivized_test_support = "0.0.6"
+passivized_test_support = "0.0.7"
 rand = "0.8"
 serial_test = "0.9"
 simple_logger = { version = "4.0", default-features = false, features = ["timestamps", "threads"] }
 tar = "0.4"
 thiserror = "1.0"
 tempfile = "3.3"
-tokio = { version = "1.21", features = ["fs"] }
+tokio = { version = "1.23", features = ["fs"] }
 
 [target."cfg(unix)".dependencies.hyperlocal]
 version = "0.8"

examples/example_utils/lib.rs

@@ -1,4 +1,3 @@
 #![allow(dead_code)]  // Each example compiles separately; not all examples use all example utilities.
 
 pub mod errors;
-pub mod retry;

tests/test_registry.rs

@@ -1,8 +1,4 @@
-
-#[cfg(not(target_os = "macos"))]  // On Macs, containers run in a VM, and their network is inaccessible.
-#[cfg(not(windows))]  // No registry image available for Windows
-#[path = "../examples/example_utils/lib.rs"]
-mod example_utils;
+#![allow(unstable_name_collisions)]  // itertools::intersperse
 
 #[cfg(not(target_os = "macos"))]  // On Macs, containers run in a VM, and their network is inaccessible.
 #[cfg(not(windows))]  // No registry image available for Windows
@@ -31,6 +27,7 @@ async fn test_push_to_authenticated_registry() {
 
     use http::StatusCode;
     use hyper_tls::native_tls::{Certificate, Identity, TlsConnector};
+    use itertools::Itertools;
     use log::info;
     use openssl::pkey::{PKey, Private};
     use tar::{Archive, Builder, Header};
@@ -50,7 +47,7 @@ async fn test_push_to_authenticated_registry() {
     use passivized_htpasswd::Htpasswd;
     use passivized_test_support::http_status_tests::{equals, is_success};
     use passivized_test_support::logging;
-    use passivized_test_support::waiter::wait_for_https_server;
+    use passivized_test_support::waiter::{wait_for_https_server_with_backoff, wait_for_tcp_server_with_backoff};
 
     const HTPASSWD_USERNAME: &str = "foo";
     const HTPASSWD_PASSWORD: &str = "bar";
@@ -106,6 +103,29 @@ async fn test_push_to_authenticated_registry() {
         .await
         .unwrap();
 
+    let network_containers: Vec<String> = inspected_network
+        .containers
+        .iter()
+        .map(|(id, c)| c.
+            name
+            .as_ref()
+            .map(|n| n.to_string())
+            .unwrap_or(id.to_string())
+        )
+        .sorted()
+        .collect();
+
+    assert!(
+        network_containers.is_empty(),
+        "Network {} has running containers: {}",
+        NETWORK_NAME,
+        network_containers
+            .into_iter()
+            .sorted()
+            .intersperse(", ".into())
+            .collect::<String>()
+    );
+
     let subnet = inspected_network
         .ipam
         .config
@@ -215,7 +235,12 @@ async fn test_push_to_authenticated_registry() {
         .build()
         .unwrap();
 
-    wait_for_https_server(registry_url, registry_tls, is_success())
+    wait_for_https_server_with_backoff(
+        registry_url,
+        registry_tls,
+        is_success(),
+        imp::build_backoff_for_registry()
+    )
         .await
         .unwrap();
 
@@ -254,7 +279,7 @@ async fn test_push_to_authenticated_registry() {
 
     info!("dind url: {dind_url}");
 
-    example_utils::retry::wait_for_tcp_server(&dind_ip, dind::PORT)
+    wait_for_tcp_server_with_backoff(&dind_ip, dind::PORT, imp::build_backoff_for_dind())
         .await
         .unwrap();
 
@@ -315,7 +340,12 @@ async fn test_push_to_authenticated_registry() {
         .build()
         .unwrap();
 
-    wait_for_https_server(dind_url, tls.clone(), equals(StatusCode::NOT_FOUND)).await.unwrap();
+    wait_for_https_server_with_backoff(
+        dind_url,
+        tls.clone(),
+        equals(StatusCode::NOT_FOUND),
+        imp::build_backoff_for_dind()
+    ).await.unwrap();
 
     let private_url = format!("https://{}:{}", dind_ip, dind::PORT);
     let private = DockerEngineClient::with_tls_config(&private_url, tls.clone())
@@ -370,7 +400,7 @@ async fn test_push_to_authenticated_registry() {
     // Pull a public image so we can tag it and push it to the authenticated private registry.
     // Authentication is not needed so we won't use it.
 
-    private.images().pull(web::IMAGE, web::TAG)
+    private.images().pull_if_not_present(web::IMAGE, web::TAG)
         .await
         .unwrap();
 
@@ -488,7 +518,57 @@ async fn test_push_to_authenticated_registry() {
         .await
         .unwrap();
 
-    public.network(NETWORK_NAME).remove()
+    imp::remove_network(&public, NETWORK_NAME)
         .await
         .unwrap();
 }
+
+#[cfg(not(target_os = "macos"))]  // On Macs, containers run in a VM, and their network is inaccessible.
+#[cfg(not(windows))]  // No registry image available for Windows
+mod imp {
+    use backoff::backoff::Constant;
+    use backoff::future::retry_notify;
+    use log::warn;
+    use passivized_test_support::retry::Limit;
+    use passivized_docker_engine_client::DockerEngineClient;
+    use passivized_docker_engine_client::errors::DecUseError;
+
+    pub(super) fn build_backoff_for_dind() -> Limit {
+        use std::time::Duration;
+
+        let interval = Duration::from_secs(2);
+
+        Limit::new(10, Constant::new(interval))
+    }
+
+    pub(super) fn build_backoff_for_registry() -> Limit {
+        use std::time::Duration;
+
+        let interval = Duration::from_secs(2);
+
+        Limit::new(10, Constant::new(interval))
+    }
+
+    pub(super) fn build_backoff_for_network_removal() -> Limit {
+        use std::time::Duration;
+
+        let interval = Duration::from_secs(1);
+
+        Limit::new(5, Constant::new(interval))
+    }
+
+    pub(super) async fn remove_network(docker: &DockerEngineClient, id: &str) -> Result<(), DecUseError> {
+        retry_notify(
+            build_backoff_for_network_removal(),
+            || async {
+                (match docker.network(id).remove().await {
+                    Ok(()) => Ok(()),
+                    Err(DecUseError::NotFound { .. }) => Ok(()),
+                    Err(e) => Err(e)
+                })
+                    .map_err(backoff::Error::transient)
+            },
+            |error, _| warn!("Retrying after failure: {:?}", error)
+        ).await
+    }
+}