Extend agones test to check against previous versions as well (#1013)

build/Makefile

@@ -38,6 +38,7 @@ endif
 # Set this value if you want to use an external registry
 REPOSITORY ?= ""
 IMAGE_TAG ?= ${REPOSITORY}quilkin:$(package_version)
+PREV_IMAGE_TAG ?= us-docker.pkg.dev/quilkin/release/quilkin:0.9.0
 MINIKUBE_PROFILE ?= quilkin
 CARGO_TARGET_DIR ?= /workspace/target/build-image
 common_rust_args := -v $(project_path):/workspace -w /workspace \
@@ -214,6 +215,7 @@ test-agones:
 
 run-test-agones: ensure-kube-dirs
 run-test-agones:
+	docker image pull ${PREV_IMAGE_TAG}
 	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) -w /workspace/crates/agones  \
 			--entrypoint=cargo $(BUILD_IMAGE_TAG) clippy --tests -- -D warnings
 	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) -w /workspace/crates/agones \
@@ -222,7 +224,7 @@ run-test-agones:
 	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) -w /workspace/crates/agones \
 		--entrypoint=kubectl $(BUILD_IMAGE_TAG) get ns
 	docker run --rm $(DOCKER_RUN_ARGS) $(common_rust_args) $(kube_mount_args) -w /workspace/crates/agones \
-			-e "RUST_BACKTRACE=1" -e "IMAGE_TAG=${IMAGE_TAG}" --entrypoint=cargo $(BUILD_IMAGE_TAG) test $(ARGS)
+			-e "RUST_BACKTRACE=1" -e "IMAGE_TAG=${IMAGE_TAG}" -e "PREV_IMAGE_TAG=${PREV_IMAGE_TAG}" --entrypoint=cargo $(BUILD_IMAGE_TAG) test $(ARGS)
 
 # Convenience target to build and push quilkin images to a repository.
 # Use `REPOSITORY` arg to specify the repository to push to.

crates/agones/Cargo.toml

@@ -28,6 +28,7 @@ workspace = true
 
 [dependencies]
 base64.workspace = true
+either = "1.13"
 futures.workspace = true
 k8s-openapi.workspace = true
 kube = { workspace = true, features = ["openssl-tls", "client", "derive", "runtime"] }

crates/agones/src/lib.rs

@@ -60,10 +60,9 @@ mod provider;
 mod relay;
 mod sidecar;
 
-#[allow(dead_code)]
-static CLIENT: OnceCell<Client> = OnceCell::const_new();
-#[allow(dead_code)]
-const IMAGE_TAG: &str = "IMAGE_TAG";
+pub static CLIENT: OnceCell<Client> = OnceCell::const_new();
+pub const IMAGE_TAG: &str = "IMAGE_TAG";
+pub const PREV_IMAGE_TAG: &str = "PREV_IMAGE_TAG";
 const DELETE_DELAY_SECONDS: &str = "DELETE_DELAY_SECONDS";
 /// A simple udp server that returns packets that are sent to it.
 /// See: <https://github.com/googleforgames/agones/tree/main/examples/simple-game-server>
@@ -82,6 +81,7 @@ pub struct Client {
     pub namespace: String,
     /// The name and tag of the Quilkin image being tested
     pub quilkin_image: String,
+    pub prev_quilkin_image: String,
 }
 
 impl Client {
@@ -100,7 +100,8 @@ impl Client {
                 Client {
                     kubernetes: client.clone(),
                     namespace: setup_namespace(client).await,
-                    quilkin_image: env::var(IMAGE_TAG).unwrap(),
+                    quilkin_image: env::var(IMAGE_TAG).expect(IMAGE_TAG),
+                    prev_quilkin_image: env::var(PREV_IMAGE_TAG).expect(PREV_IMAGE_TAG),
                 }
             })
             .await
@@ -303,14 +304,15 @@ pub async fn create_agones_rbac_read_account(
     rbac_name.into()
 }
 
-/// Create a Deployment with a singular Quilkin proxy, and return it's address.
+/// Create a Deployment with a singular Quilkin proxy, and return its address.
 /// The `name` variable is used as role={name} for label lookup.
 pub async fn quilkin_proxy_deployment(
     client: &Client,
     deployments: Api<Deployment>,
     name: String,
     host_port: u16,
     management_server: String,
+    current: bool,
 ) -> SocketAddr {
     let pp = PostParams::default();
     let mut container = quilkin_container(
@@ -320,6 +322,7 @@ pub async fn quilkin_proxy_deployment(
             format!("--management-server={management_server}"),
         ]),
         None,
+        current,
     );
 
     // we'll use a host port, since spinning up a load balancer takes a long time.
@@ -580,10 +583,17 @@ pub fn quilkin_container(
     client: &Client,
     args: Option<Vec<String>>,
     volume_mount: Option<String>,
+    current: bool,
 ) -> Container {
+    let image = if current {
+        client.quilkin_image.clone()
+    } else {
+        client.prev_quilkin_image.clone()
+    };
+
     let mut container = Container {
         name: "quilkin".into(),
-        image: Some(client.quilkin_image.clone()),
+        image: Some(image),
         args,
         env: Some(vec![EnvVar {
             name: "RUST_LOG".to_string(),

crates/agones/src/pod.rs

@@ -38,7 +38,7 @@ mod tests {
                 ..Default::default()
             },
             spec: Some(PodSpec {
-                containers: vec![quilkin_container(&client, Some(cmds), None)],
+                containers: vec![quilkin_container(&client, Some(cmds), None, true)],
                 ..Default::default()
             }),
             status: None,

crates/agones/src/provider.rs

@@ -78,6 +78,7 @@ mod tests {
             PROXY_DEPLOYMENT.into(),
             7005,
             "http://quilkin-manage-agones:7800".into(),
+            true,
         )
         .await;
 
@@ -210,7 +211,7 @@ mod tests {
                         ..Default::default()
                     }),
                     spec: Some(PodSpec {
-                        containers: vec![quilkin_container(client, Some(args), None)],
+                        containers: vec![quilkin_container(client, Some(args), None, true)],
                         service_account_name: Some(rbac_name),
                         ..Default::default()
                     }),

crates/agones/src/relay.rs

@@ -51,12 +51,23 @@ mod tests {
         TOKEN_KEY,
     };
 
+    const SLOW: Duration = Duration::from_secs(30);
+
     #[tokio::test]
     #[serial]
     /// Test for Agones Provider integration. Since this will look at all GameServers in the namespace
     /// for this test, we should only run Agones integration test in a serial manner, since they
     /// could easily collide with each other.
     async fn agones_token_router() {
+        run_test(true, true, true, 0).await;
+        run_test(true, true, false, 1).await;
+        run_test(true, false, true, 2).await;
+        run_test(false, true, true, 3).await;
+    }
+
+    async fn run_test(proxy: bool, relay: bool, agent: bool, id: u8) {
+        println!("running agones_token_router {id}");
+
         let client = Client::new().await;
         let config_maps: Api<ConfigMap> = client.namespaced_api();
         let deployments: Api<Deployment> = client.namespaced_api();
@@ -67,15 +78,17 @@ mod tests {
         let dp = DeleteParams::default();
 
         let config_map = create_token_router_config(&config_maps).await;
-        agones_agent_deployment(&client, deployments.clone()).await;
+        let (relay_name, agent_name) =
+            agones_agent_deployment(&client, deployments.clone(), relay, agent, id).await;
 
-        let relay_proxy_name = "quilkin-relay-proxy";
+        let relay_proxy_name = format!("quilkin-relay-proxy-{id}");
         let proxy_address = quilkin_proxy_deployment(
             &client,
             deployments.clone(),
-            relay_proxy_name.into(),
+            relay_proxy_name.clone(),
             7005,
-            "http://quilkin-relay-agones:7800".into(),
+            format!("http://{relay_name}:7800"),
+            proxy,
         )
         .await;
 
@@ -98,8 +111,12 @@ mod tests {
 
         // Proxy Deployment should be ready, since there is now an endpoint
         if timeout(
-            Duration::from_secs(30),
-            await_condition(deployments.clone(), relay_proxy_name, is_deployment_ready()),
+            SLOW,
+            await_condition(
+                deployments.clone(),
+                &relay_proxy_name,
+                is_deployment_ready(),
+            ),
         )
         .await
         .is_err()
@@ -157,15 +174,77 @@ mod tests {
         }
         assert!(failed, "Packet should have failed");
 
+        println!("deleting resources...");
+        use either::Either;
+        let cm_name = config_map.name_unchecked();
         // cleanup
-        config_maps
-            .delete(&config_map.name_unchecked(), &dp)
+        match config_maps
+            .delete(&cm_name, &dp)
             .await
-            .unwrap();
+            .expect("failed to delete config map")
+        {
+            Either::Left(_) => {
+                timeout(
+                    SLOW,
+                    await_condition(
+                        deployments.clone(),
+                        &relay_proxy_name,
+                        kube::runtime::conditions::is_deleted(&cm_name),
+                    ),
+                )
+                .await
+                .expect("failed to delete config map within timeout")
+                .expect("failed to delete config map");
+                println!("...config map deleted");
+            }
+            Either::Right(_) => {
+                println!("config map deleted");
+            }
+        }
+
+        async fn delete_deployment(dp: &Api<Deployment>, name: &str) -> Result<(), kube::Error> {
+            async fn inner(dp: &Api<Deployment>, name: &str) -> Result<(), kube::Error> {
+                if let Either::Left(d) = dp.delete(name, &DeleteParams::default()).await? {
+                    await_condition(
+                        dp.clone(),
+                        name,
+                        kube::runtime::conditions::is_deleted(&d.uid().unwrap()),
+                    )
+                    .await
+                    .map_err(|err| kube::Error::Service(Box::new(err)))?;
+                }
+
+                Ok(())
+            }
+
+            timeout(SLOW, inner(dp, name)).await.map_err(|_err| {
+                kube::Error::Api(kube::error::ErrorResponse {
+                    message: format!("failed to delete deployment {name} within {SLOW:?}"),
+                    status: String::new(),
+                    reason: String::new(),
+                    code: 408,
+                })
+            })??;
+            println!("deployment {name} deleted");
+            Ok(())
+        }
+
+        tokio::try_join!(
+            delete_deployment(&deployments, &relay_proxy_name),
+            delete_deployment(&deployments, &agent_name),
+            delete_deployment(&deployments, &relay_name),
+        )
+        .expect("failed to delete deployment(s) within timeout");
     }
 
     /// Deploys the Agent and Relay Server Deployments and Services
-    async fn agones_agent_deployment(client: &Client, deployments: Api<Deployment>) {
+    async fn agones_agent_deployment(
+        client: &Client,
+        deployments: Api<Deployment>,
+        relay: bool,
+        agent: bool,
+        id: u8,
+    ) -> (String, String) {
         let service_accounts: Api<ServiceAccount> = client.namespaced_api();
         let cluster_roles: Api<ClusterRole> = Api::all(client.kubernetes.clone());
         let role_bindings: Api<RoleBinding> = client.namespaced_api();
@@ -177,6 +256,8 @@ mod tests {
             create_agones_rbac_read_account(client, service_accounts, cluster_roles, role_bindings)
                 .await;
 
+        let relay_name = format!("quilkin-relay-agones-{id}");
+
         // Setup the relay
         let args = [
             "relay",
@@ -189,7 +270,7 @@ mod tests {
         let labels = BTreeMap::from([("role".to_string(), "relay".to_string())]);
         let deployment = Deployment {
             metadata: ObjectMeta {
-                name: Some("quilkin-relay-agones".into()),
+                name: Some(relay_name.clone()),
                 labels: Some(labels.clone()),
                 ..Default::default()
             },
@@ -205,7 +286,7 @@ mod tests {
                         ..Default::default()
                     }),
                     spec: Some(PodSpec {
-                        containers: vec![quilkin_container(client, Some(args), None)],
+                        containers: vec![quilkin_container(client, Some(args), None, relay)],
                         service_account_name: Some(rbac_name.clone()),
                         ..Default::default()
                     }),
@@ -219,7 +300,7 @@ mod tests {
         // relay service
         let service = Service {
             metadata: ObjectMeta {
-                name: Some("quilkin-relay-agones".into()),
+                name: Some(relay_name.clone()),
                 ..Default::default()
             },
             spec: Some(ServiceSpec {
@@ -248,8 +329,8 @@ mod tests {
 
         let name = relay_deployment.name_unchecked();
         let result = timeout(
-            Duration::from_secs(30),
-            await_condition(deployments.clone(), name.as_str(), is_deployment_ready()),
+            SLOW,
+            await_condition(deployments.clone(), &name, is_deployment_ready()),
         )
         .await;
         if result.is_err() {
@@ -259,11 +340,13 @@ mod tests {
         }
         result.unwrap().expect("Should have a relay deployment");
 
+        let agent_name = format!("quilkin-agones-agent-{id}");
+
         // agent deployment
         let args = [
             "agent",
             "--relay",
-            "http://quilkin-relay-agones:7900",
+            &format!("http://{relay_name}:7900"),
             "agones",
             "--config-namespace",
             client.namespace.as_str(),
@@ -275,7 +358,7 @@ mod tests {
         let labels = BTreeMap::from([("role".to_string(), "agent".to_string())]);
         let deployment = Deployment {
             metadata: ObjectMeta {
-                name: Some("quilkin-agones-agent".into()),
+                name: Some(agent_name.clone()),
                 labels: Some(labels.clone()),
                 ..Default::default()
             },
@@ -291,7 +374,7 @@ mod tests {
                         ..Default::default()
                     }),
                     spec: Some(PodSpec {
-                        containers: vec![quilkin_container(client, Some(args), None)],
+                        containers: vec![quilkin_container(client, Some(args), None, agent)],
                         service_account_name: Some(rbac_name),
                         ..Default::default()
                     }),
@@ -303,7 +386,7 @@ mod tests {
         let agent_deployment = deployments.create(&pp, &deployment).await.unwrap();
         let name = agent_deployment.name_unchecked();
         let result = timeout(
-            Duration::from_secs(30),
+            SLOW,
             await_condition(deployments.clone(), name.as_str(), is_deployment_ready()),
         )
         .await;
@@ -312,5 +395,6 @@ mod tests {
             panic!("Agent Deployment should be ready");
         }
         result.unwrap().expect("Should have an agent deployment");
+        (relay_name, agent_name)
     }
 }

crates/agones/src/sidecar.rs

@@ -100,6 +100,7 @@ clusters:
             &client,
             Some(vec!["proxy".into()]),
             Some(mount_name.clone()),
+            true,
         ));
 
         template.volumes = Some(vec![Volume {