diff --git a/CODEOWNERS b/CODEOWNERS index d4ea6baa26..21e6c5797a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @markmandel @xampprocky @No-ops @rezvaneh @vallfors +* @markmandel @xampprocky @koslib diff --git a/Cargo.toml b/Cargo.toml index 3ffa910ffc..ec1c240ab2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,14 +53,14 @@ arc-swap = { version = "1.6.0", features = ["serde"] } async-stream = "0.3.5" base64.workspace = true base64-serde = "0.7.0" -bytes = { version = "1.4.0", features = ["serde"] } -cached = "0.45.0" -chrono = "0.4.28" -clap = { version = "4.4.2", features = ["cargo", "derive", "env"] } +bytes = { version = "1.5.0", features = ["serde"] } +cached = "0.46.0" +chrono = "0.4.31" +clap = { version = "4.4.6", features = ["cargo", "derive", "env"] } dashmap = { version = "5.5.3", features = ["serde"] } dirs2 = "3.0.1" either = "1.9.0" -enum-map = "2.6.1" +enum-map = "2.6.3" eyre = "0.6.8" futures.workspace = true hyper = { version = "0.14.27", features = ["http2"] } @@ -73,23 +73,23 @@ num_cpus = "1.16.0" once_cell = "1.18.0" parking_lot = "0.12.1" prometheus = { version = "0.13.3", default-features = false } -prost = "0.12.0" -prost-types = "0.12.0" +prost = "0.12.1" +prost-types = "0.12.1" rand = "0.8.5" -regex = "1.9.5" -schemars = { version = "0.8.13", features = ["chrono", "bytes", "url"] } +regex = "1.9.6" +schemars = { version = "0.8.15", features = ["chrono", "bytes", "url"] } serde = { version = "1.0.188", features = ["derive", "rc"] } -serde_json = "1.0.105" +serde_json = "1.0.107" serde_regex = "1.1.0" serde_stacker = "0.1.10" serde_yaml = "0.9.25" snap = "1.1.0" -socket2 = { version = "0.5.3", features = ["all"] } +socket2 = { version = "0.5.4", features = ["all"] } stable-eyre = "0.2.2" -thiserror = "1.0.48" +thiserror = "1.0.49" tokio.workspace = true tokio-stream = { version = "0.1.14", features = ["sync"] } -tonic = "0.10.0" +tonic = "0.10.2" tracing.workspace = true tracing-futures = { version = "0.2.5", features = ["futures-03"] } tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] } @@ -103,14 +103,14 @@ async-trait = "0.1.73" nom = "7.1.3" atty = "0.2.14" strum = "0.25.0" -strum_macros = "0.25" +strum_macros = "0.25.2" itertools = "0.11.0" [target.'cfg(target_os = "linux")'.dependencies] sys-info = "0.9.1" [dev-dependencies] -regex = "1.9.5" +regex = "1.9.6" criterion = { version = "0.5.1", features = ["html_reports"] } once_cell = "1.18.0" tracing-test = "0.2.4" @@ -119,9 +119,9 @@ tempfile = "3.8.0" rand = "0.8.5" [build-dependencies] -tonic-build = { version = "0.10.0", default_features = false, features = ["transport", "prost"] } -prost-build = "0.12.0" -built = { version = "0.6.1", features = ["git2"] } +tonic-build = { version = "0.10.2", default_features = false, features = ["transport", "prost"] } +prost-build = "0.12.1" +built = { version = "0.7.0", features = ["git2"] } protobuf-src = { version = "1.1.0", optional = true } [features] diff --git a/agones/Cargo.toml b/agones/Cargo.toml index e79de498ea..6eb3769db0 100644 --- a/agones/Cargo.toml +++ b/agones/Cargo.toml @@ -25,9 +25,10 @@ readme = "README.md" [dependencies] base64.workspace = true +futures.workspace = true k8s-openapi.workspace = true kube = { workspace = true, features = ["openssl-tls", "client", "derive", "runtime"] } quilkin = { path = "../" } +serial_test = "2.0.0" tokio.workspace = true -futures.workspace = true tracing.workspace = true diff --git a/agones/src/lib.rs b/agones/src/lib.rs index b9941ff214..5b96e0a9d5 100644 --- a/agones/src/lib.rs +++ b/agones/src/lib.rs @@ -17,30 +17,37 @@ use std::{ collections::BTreeMap, env, - time::{SystemTime, UNIX_EPOCH}, + net::SocketAddr, + time::{Duration, SystemTime, UNIX_EPOCH}, }; use futures::{AsyncBufReadExt, TryStreamExt}; use k8s_openapi::{ api::{ - apps::v1::Deployment, + apps::v1::{Deployment, DeploymentSpec}, core::v1::{ ConfigMap, Container, EnvVar, Event, HTTPGetAction, Namespace, Pod, PodSpec, PodTemplateSpec, Probe, ResourceRequirements, ServiceAccount, VolumeMount, }, - rbac::v1::{RoleBinding, RoleRef, Subject}, + core::v1::{ContainerPort, Node}, + rbac::{ + v1::PolicyRule, + v1::{ClusterRole, RoleBinding, RoleRef, Subject}, + }, }, apimachinery::pkg::{ - api::resource::Quantity, apis::meta::v1::ObjectMeta, util::intstr::IntOrString, + api::resource::Quantity, + apis::meta::v1::{LabelSelector, ObjectMeta}, + util::intstr::IntOrString, }, chrono, }; use kube::{ api::{DeleteParams, ListParams, LogParams, PostParams}, - runtime::wait::Condition, + runtime::wait::{await_condition, Condition}, Api, Resource, ResourceExt, }; -use tokio::sync::OnceCell; +use tokio::{sync::OnceCell, time::timeout}; use tracing::debug; use quilkin::config::providers::k8s::agones::{ @@ -49,8 +56,9 @@ use quilkin::config::providers::k8s::agones::{ }; mod pod; +mod provider; +mod relay; mod sidecar; -mod xds; #[allow(dead_code)] static CLIENT: OnceCell = OnceCell::const_new(); @@ -63,6 +71,9 @@ const DELETE_DELAY_SECONDS: &str = "DELETE_DELAY_SECONDS"; pub const GAMESERVER_IMAGE: &str = "us-docker.pkg.dev/agones-images/examples/simple-game-server:0.16"; +/// The dynamic metadata key for routing tokens +pub const TOKEN_KEY: &str = "quilkin.dev/tokens"; + #[derive(Clone)] pub struct Client { /// The Kubernetes client @@ -215,6 +226,230 @@ async fn add_agones_service_account(client: kube::Client, namespace: String) { let _ = role_bindings.create(&pp, &role_binding).await.unwrap(); } +/// Creates a Service account and related RBAC objects to enable a process to query Agones +/// and ConfigMap resources within a cluster +pub async fn create_agones_rbac_read_account( + client: &Client, + service_accounts: Api, + cluster_roles: Api, + role_bindings: Api, +) -> String { + let pp = PostParams::default(); + let rbac_name = "quilkin-agones"; + + // check if sevice account already exists, otherwise create it. + if service_accounts.get(rbac_name).await.is_ok() { + return rbac_name.into(); + } + + // create all the rbac rules + + let rbac_meta = ObjectMeta { + name: Some(rbac_name.into()), + ..Default::default() + }; + let service_account = ServiceAccount { + metadata: rbac_meta.clone(), + ..Default::default() + }; + service_accounts + .create(&pp, &service_account) + .await + .unwrap(); + + // Delete the cluster role if it already exists, since it's cluster wide. + match cluster_roles + .delete(rbac_name, &DeleteParams::default()) + .await + { + Ok(_) => {} + Err(err) => println!("Cluster role not found: {err}"), + }; + let cluster_role = ClusterRole { + metadata: rbac_meta.clone(), + rules: Some(vec![ + PolicyRule { + api_groups: Some(vec!["agones.dev".into()]), + resources: Some(vec!["gameservers".into()]), + verbs: ["get", "list", "watch"].map(String::from).to_vec(), + ..Default::default() + }, + PolicyRule { + api_groups: Some(vec!["".into()]), + resources: Some(vec!["configmaps".into()]), + verbs: ["get", "list", "watch"].map(String::from).to_vec(), + ..Default::default() + }, + ]), + ..Default::default() + }; + cluster_roles.create(&pp, &cluster_role).await.unwrap(); + + let binding = RoleBinding { + metadata: rbac_meta, + subjects: Some(vec![Subject { + kind: "User".into(), + name: format!("system:serviceaccount:{}:{rbac_name}", client.namespace), + api_group: Some("rbac.authorization.k8s.io".into()), + ..Default::default() + }]), + role_ref: RoleRef { + api_group: "rbac.authorization.k8s.io".into(), + kind: "ClusterRole".into(), + name: rbac_name.into(), + }, + }; + role_bindings.create(&pp, &binding).await.unwrap(); + rbac_name.into() +} + +/// Create a Deployment with a singular Quilkin proxy, and return it's address. +/// The `name` variable is used as role={name} for label lookup. +pub async fn quilkin_proxy_deployment( + client: &Client, + deployments: Api, + name: String, + host_port: u16, + management_server: String, +) -> SocketAddr { + let pp = PostParams::default(); + let mut container = quilkin_container( + client, + Some(vec![ + "proxy".into(), + format!("--management-server={management_server}"), + ]), + None, + ); + + // we'll use a host port, since spinning up a load balancer takes a long time. + // we know that port 7777 is open because this is an Agones cluster and it has associated + // firewall rules , and even if we conflict with a GameServer + // the k8s scheduler will move us to another node. + container.ports = Some(vec![ContainerPort { + container_port: 7777, + host_port: Some(host_port as i32), + protocol: Some("UDP".into()), + ..Default::default() + }]); + + let labels = BTreeMap::from([("role".to_string(), name.clone())]); + let deployment = Deployment { + metadata: ObjectMeta { + name: Some(name), + labels: Some(labels.clone()), + ..Default::default() + }, + spec: Some(DeploymentSpec { + replicas: Some(1), + selector: LabelSelector { + match_expressions: None, + match_labels: Some(labels.clone()), + }, + template: PodTemplateSpec { + metadata: Some(ObjectMeta { + labels: Some(labels.clone()), + ..Default::default() + }), + spec: Some(PodSpec { + containers: vec![container], + ..Default::default() + }), + }, + ..Default::default() + }), + ..Default::default() + }; + + let deployment = deployments.create(&pp, &deployment).await.unwrap(); + let name = deployment.name_unchecked(); + // should not be ready, since there are no endpoints, but let's wait 3 seconds, make sure it doesn't do something we don't expect + let result = timeout( + Duration::from_secs(3), + await_condition(deployments.clone(), name.as_str(), is_deployment_ready()), + ) + .await; + assert!(result.is_err()); + + // get the address to send data to + let pods = client.namespaced_api::(); + let list = pods + .list(&ListParams { + label_selector: Some(format!("role={name}")), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(1, list.items.len()); + + let nodes: Api = Api::all(client.kubernetes.clone()); + let name = list.items[0] + .spec + .as_ref() + .unwrap() + .node_name + .as_ref() + .unwrap(); + let node = nodes.get(name.as_str()).await.unwrap(); + let external_ip = node + .status + .unwrap() + .addresses + .unwrap() + .iter() + .find(|addr| addr.type_ == "ExternalIP") + .unwrap() + .address + .clone(); + + SocketAddr::new(external_ip.parse().unwrap(), host_port) +} + +/// Create a Fleet, and pick on it's GameServers and add the token to it. +/// Returns the details of the GameServer that has been selected. +pub async fn create_tokenised_gameserver( + fleets: Api, + gameservers: Api, + token: &str, +) -> GameServer { + let pp = PostParams::default(); + + // create a fleet so we can ensure that a packet is going to the GameServer we expect, and not + // any other. + let fleet = fleet(); + let fleet = fleets.create(&pp, &fleet).await.unwrap(); + let name = fleet.name_unchecked(); + timeout( + Duration::from_secs(30), + await_condition(fleets.clone(), name.as_str(), is_fleet_ready()), + ) + .await + .expect("Fleet should be ready") + .unwrap(); + + let lp = ListParams { + label_selector: Some(format!("agones.dev/fleet={}", fleet.name_unchecked())), + ..Default::default() + }; + let list = gameservers.list(&lp).await.unwrap(); + + let mut gs = list.items[0].clone(); + // add routing label to the GameServer + assert_eq!(3, token.as_bytes().len()); + gs.metadata + .annotations + .get_or_insert(Default::default()) + .insert( + TOKEN_KEY.into(), + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, token), + ); + gameservers + .replace(gs.name_unchecked().as_str(), &pp, &gs) + .await + .unwrap(); + gs +} + /// Returns a test GameServer with the UDP test binary that is used for /// Agones e2e tests. pub fn game_server() -> GameServer { @@ -371,7 +606,7 @@ pub fn quilkin_container( ..Default::default() }), initial_delay_seconds: Some(3), - period_seconds: Some(2), + period_seconds: Some(1), ..Default::default() }), ..Default::default() @@ -404,6 +639,30 @@ pub fn quilkin_config_map(config: &str) -> ConfigMap { } } +/// Return a ConfigMap that has a standard testing Token Router configuration +pub async fn create_token_router_config(config_maps: &Api) -> ConfigMap { + let pp = PostParams::default(); + + let config = r#" +version: v1alpha1 +filters: + - name: quilkin.filters.capture.v1alpha1.Capture # Capture and remove the authentication token + config: + suffix: + size: 3 + remove: true + - name: quilkin.filters.token_router.v1alpha1.TokenRouter +"#; + let mut config_map = quilkin_config_map(config); + config_map + .metadata + .labels + .get_or_insert(Default::default()) + .insert("quilkin.dev/configmap".into(), "true".into()); + + config_maps.create(&pp, &config_map).await.unwrap() +} + /// Convenience function to return the address with the first port of GameServer pub fn gameserver_address(gs: &GameServer) -> String { let status = gs.status.as_ref().unwrap(); diff --git a/agones/src/provider.rs b/agones/src/provider.rs new file mode 100644 index 0000000000..74c9d77217 --- /dev/null +++ b/agones/src/provider.rs @@ -0,0 +1,254 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, time::Duration}; + + use k8s_openapi::{ + api::{ + apps::v1::{Deployment, DeploymentSpec}, + core::v1::{ + ConfigMap, PodSpec, PodTemplateSpec, Service, ServiceAccount, ServicePort, + ServiceSpec, + }, + rbac::v1::{ClusterRole, RoleBinding}, + }, + apimachinery::pkg::{ + apis::meta::v1::{LabelSelector, ObjectMeta}, + util::intstr::IntOrString, + }, + }; + use kube::{ + api::{DeleteParams, PostParams}, + runtime::wait::await_condition, + Api, ResourceExt, + }; + use serial_test::serial; + use tokio::time::timeout; + + use quilkin::{ + config::providers::k8s::agones::{Fleet, GameServer}, + test_utils::TestHelper, + }; + + use crate::{ + create_agones_rbac_read_account, create_token_router_config, create_tokenised_gameserver, + debug_pods, is_deployment_ready, quilkin_container, quilkin_proxy_deployment, Client, + TOKEN_KEY, + }; + + const PROXY_DEPLOYMENT: &str = "quilkin-xds-proxies"; + + #[tokio::test] + #[serial] + /// Test for Agones Provider integration. Since this will look at all GameServers in the namespace + /// for this test, we should only run Agones integration test in a serial manner, since they + /// could easily collide with each other. + async fn agones_token_router() { + let client = Client::new().await; + + let deployments: Api = client.namespaced_api(); + let fleets: Api = client.namespaced_api(); + let gameservers: Api = client.namespaced_api(); + let config_maps: Api = client.namespaced_api(); + + let pp = PostParams::default(); + let dp = DeleteParams::default(); + + let config_map = create_token_router_config(&config_maps).await; + + agones_control_plane(&client, deployments.clone()).await; + let proxy_address = quilkin_proxy_deployment( + &client, + deployments.clone(), + PROXY_DEPLOYMENT.into(), + 7005, + "http://quilkin-manage-agones:7800".into(), + ) + .await; + + let token = "456"; // NDU2 + let gs = create_tokenised_gameserver(fleets, gameservers.clone(), token).await; + let gs_address = crate::gameserver_address(&gs); + // and allocate it such that we have an endpoint. + // let's allocate this specific game server + let mut t = TestHelper::default(); + let (mut rx, socket) = t.open_socket_and_recv_multiple_packets().await; + socket + .send_to("ALLOCATE".as_bytes(), gs_address) + .await + .unwrap(); + + let response = timeout(Duration::from_secs(30), rx.recv()) + .await + .expect("should receive packet from GameServer") + .unwrap(); + assert_eq!("ACK: ALLOCATE\n", response); + + // Proxy Deployment should be ready, since there is now an endpoint + if timeout( + Duration::from_secs(30), + await_condition(deployments.clone(), PROXY_DEPLOYMENT, is_deployment_ready()), + ) + .await + .is_err() + { + debug_pods(&client, format!("role={PROXY_DEPLOYMENT}")).await; + panic!("Quilkin proxy deployment should be ready"); + } + + // keep trying to send the packet to the proxy until it works, since distributed systems are eventually consistent. + let mut response: String = "not-found".into(); + for i in 0..30 { + println!("Connection Attempt: {i}"); + + // returns the nae of the GameServer. This proves we are routing the the allocated + // GameServer with the correct token attached. + socket + .send_to(format!("GAMESERVER{token}").as_bytes(), proxy_address) + .await + .unwrap(); + + let result = timeout(Duration::from_secs(1), rx.recv()).await; + if let Ok(Some(value)) = result { + response = value; + break; + } + } + assert_eq!(format!("NAME: {}\n", gs.name_unchecked()), response); + + // let's remove the token from the gameserver, which should remove access. + let mut gs = gameservers.get(gs.name_unchecked().as_str()).await.unwrap(); + let name = gs.name_unchecked(); + gs.metadata + .annotations + .as_mut() + .map(|annotations| annotations.remove(TOKEN_KEY).unwrap()); + gameservers.replace(name.as_str(), &pp, &gs).await.unwrap(); + // now we should send a packet, and not get a response. + let mut failed = false; + for i in 0..30 { + println!("Disconnection Attempt: {i}"); + socket + .send_to(format!("GAMESERVER{token}").as_bytes(), proxy_address) + .await + .unwrap(); + + let result = timeout(Duration::from_secs(1), rx.recv()).await; + if result.is_err() { + failed = true; + break; + } + } + if !failed { + debug_pods(&client, format!("role={PROXY_DEPLOYMENT}")).await; + debug_pods(&client, "role=xds".into()).await; + } + assert!(failed, "Packet should have failed"); + + // cleanup + config_maps + .delete(&config_map.name_unchecked(), &dp) + .await + .unwrap(); + } + + /// Creates Quilkin xDS management instance that is in the mode to watch Agones GameServers + /// in this test namespace + async fn agones_control_plane(client: &Client, deployments: Api) { + let services: Api = client.namespaced_api(); + let service_accounts: Api = client.namespaced_api(); + let cluster_roles: Api = Api::all(client.kubernetes.clone()); + let role_bindings: Api = client.namespaced_api(); + let pp = PostParams::default(); + + let rbac_name = + create_agones_rbac_read_account(client, service_accounts, cluster_roles, role_bindings) + .await; + + // Setup the xDS Agones provider server + let args = [ + "manage", + "agones", + "--config-namespace", + client.namespace.as_str(), + "--gameservers-namespace", + client.namespace.as_str(), + ] + .map(String::from) + .to_vec(); + let labels = BTreeMap::from([("role".to_string(), "xds".to_string())]); + let deployment = Deployment { + metadata: ObjectMeta { + name: Some("quilkin-manage-agones".into()), + labels: Some(labels.clone()), + ..Default::default() + }, + spec: Some(DeploymentSpec { + replicas: Some(1), + selector: LabelSelector { + match_expressions: None, + match_labels: Some(labels.clone()), + }, + template: PodTemplateSpec { + metadata: Some(ObjectMeta { + labels: Some(labels.clone()), + ..Default::default() + }), + spec: Some(PodSpec { + containers: vec![quilkin_container(client, Some(args), None)], + service_account_name: Some(rbac_name), + ..Default::default() + }), + }, + ..Default::default() + }), + ..Default::default() + }; + + let deployment = deployments.create(&pp, &deployment).await.unwrap(); + + let service = Service { + metadata: ObjectMeta { + name: Some("quilkin-manage-agones".into()), + ..Default::default() + }, + spec: Some(ServiceSpec { + selector: Some(labels), + ports: Some(vec![ServicePort { + protocol: Some("TCP".into()), + port: 7800, + target_port: Some(IntOrString::Int(7800)), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + }; + services.create(&pp, &service).await.unwrap(); + + // make sure the deployment and service are ready + let name = deployment.name_unchecked(); + timeout( + Duration::from_secs(30), + await_condition(deployments.clone(), name.as_str(), is_deployment_ready()), + ) + .await + .expect("xDS provider deployment should be ready") + .unwrap(); + } +} diff --git a/agones/src/relay.rs b/agones/src/relay.rs new file mode 100644 index 0000000000..45d6e4876d --- /dev/null +++ b/agones/src/relay.rs @@ -0,0 +1,316 @@ +/* + * Copyright 2023 Google LLC All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, time::Duration}; + + use k8s_openapi::{ + api::{ + apps::v1::{Deployment, DeploymentSpec}, + core::v1::{ + ConfigMap, PodSpec, PodTemplateSpec, Service, ServiceAccount, ServicePort, + ServiceSpec, + }, + rbac::v1::{ClusterRole, RoleBinding}, + }, + apimachinery::pkg::{ + apis::meta::v1::{LabelSelector, ObjectMeta}, + util::intstr::IntOrString, + }, + }; + use kube::{ + api::{DeleteParams, PostParams}, + runtime::wait::await_condition, + Api, ResourceExt, + }; + use serial_test::serial; + use tokio::time::timeout; + + use quilkin::{ + config::providers::k8s::agones::{Fleet, GameServer}, + test_utils::TestHelper, + }; + + use crate::{ + create_agones_rbac_read_account, create_token_router_config, create_tokenised_gameserver, + debug_pods, is_deployment_ready, quilkin_container, quilkin_proxy_deployment, Client, + TOKEN_KEY, + }; + + #[tokio::test] + #[serial] + /// Test for Agones Provider integration. Since this will look at all GameServers in the namespace + /// for this test, we should only run Agones integration test in a serial manner, since they + /// could easily collide with each other. + async fn agones_token_router() { + let client = Client::new().await; + let config_maps: Api = client.namespaced_api(); + let deployments: Api = client.namespaced_api(); + let fleets: Api = client.namespaced_api(); + let gameservers: Api = client.namespaced_api(); + + let pp = PostParams::default(); + let dp = DeleteParams::default(); + + let config_map = create_token_router_config(&config_maps).await; + agones_agent_deployment(&client, deployments.clone()).await; + + let relay_proxy_name = "quilkin-relay-proxy"; + let proxy_address = quilkin_proxy_deployment( + &client, + deployments.clone(), + relay_proxy_name.into(), + 7005, + "http://quilkin-relay-agones:7800".into(), + ) + .await; + + let token = "789"; + let gs = create_tokenised_gameserver(fleets, gameservers.clone(), token).await; + let gs_address = crate::gameserver_address(&gs); + + let mut t = TestHelper::default(); + let (mut rx, socket) = t.open_socket_and_recv_multiple_packets().await; + socket + .send_to("ALLOCATE".as_bytes(), gs_address) + .await + .unwrap(); + + let response = timeout(Duration::from_secs(30), rx.recv()) + .await + .expect("should receive packet from GameServer") + .unwrap(); + assert_eq!("ACK: ALLOCATE\n", response); + + // Proxy Deployment should be ready, since there is now an endpoint + if timeout( + Duration::from_secs(30), + await_condition(deployments.clone(), relay_proxy_name, is_deployment_ready()), + ) + .await + .is_err() + { + debug_pods(&client, format!("role={relay_proxy_name}")).await; + panic!("Quilkin proxy deployment should be ready"); + } + + // keep trying to send the packet to the proxy until it works, since distributed systems are eventually consistent. + let mut response: String = "not-found".into(); + for i in 0..30 { + println!("Connection Attempt: {i}"); + + // returns the nae of the GameServer. This proves we are routing the the allocated + // GameServer with the correct token attached. + socket + .send_to(format!("GAMESERVER{token}").as_bytes(), proxy_address) + .await + .unwrap(); + + let result = timeout(Duration::from_secs(1), rx.recv()).await; + if let Ok(Some(value)) = result { + response = value; + break; + } + } + assert_eq!(format!("NAME: {}\n", gs.name_unchecked()), response); + + // let's remove the token from the gameserver, which should remove access. + let mut gs = gameservers.get(gs.name_unchecked().as_str()).await.unwrap(); + let name = gs.name_unchecked(); + gs.metadata + .annotations + .as_mut() + .map(|annotations| annotations.remove(TOKEN_KEY).unwrap()); + gameservers.replace(name.as_str(), &pp, &gs).await.unwrap(); + // now we should send a packet, and not get a response. + let mut failed = false; + for i in 0..30 { + println!("Disconnection Attempt: {i}"); + socket + .send_to(format!("GAMESERVER{token}").as_bytes(), proxy_address) + .await + .unwrap(); + + let result = timeout(Duration::from_secs(1), rx.recv()).await; + if result.is_err() { + failed = true; + break; + } + } + if !failed { + debug_pods(&client, format!("role={relay_proxy_name}")).await; + debug_pods(&client, "role=xds".into()).await; + } + assert!(failed, "Packet should have failed"); + + // cleanup + config_maps + .delete(&config_map.name_unchecked(), &dp) + .await + .unwrap(); + } + + /// Deploys the Agent and Relay Server Deployents and Services + async fn agones_agent_deployment(client: &Client, deployments: Api) { + let service_accounts: Api = client.namespaced_api(); + let cluster_roles: Api = Api::all(client.kubernetes.clone()); + let role_bindings: Api = client.namespaced_api(); + let services: Api = client.namespaced_api(); + + let pp = PostParams::default(); + + let rbac_name = + create_agones_rbac_read_account(client, service_accounts, cluster_roles, role_bindings) + .await; + + // Setup the relay + let args = [ + "relay", + "agones", + "--config-namespace", + client.namespace.as_str(), + ] + .map(String::from) + .to_vec(); + let labels = BTreeMap::from([("role".to_string(), "relay".to_string())]); + let deployment = Deployment { + metadata: ObjectMeta { + name: Some("quilkin-relay-agones".into()), + labels: Some(labels.clone()), + ..Default::default() + }, + spec: Some(DeploymentSpec { + replicas: Some(1), + selector: LabelSelector { + match_expressions: None, + match_labels: Some(labels.clone()), + }, + template: PodTemplateSpec { + metadata: Some(ObjectMeta { + labels: Some(labels.clone()), + ..Default::default() + }), + spec: Some(PodSpec { + containers: vec![quilkin_container(client, Some(args), None)], + service_account_name: Some(rbac_name.clone()), + ..Default::default() + }), + }, + ..Default::default() + }), + ..Default::default() + }; + let relay_deployment = deployments.create(&pp, &deployment).await.unwrap(); + + // relay service + let service = Service { + metadata: ObjectMeta { + name: Some("quilkin-relay-agones".into()), + ..Default::default() + }, + spec: Some(ServiceSpec { + selector: Some(labels), + ports: Some(vec![ + ServicePort { + name: Some("ads".into()), + protocol: Some("TCP".into()), + port: 7800, + target_port: Some(IntOrString::Int(7800)), + ..Default::default() + }, + ServicePort { + name: Some("cpds".into()), + protocol: Some("TCP".into()), + port: 7900, + target_port: Some(IntOrString::Int(7900)), + ..Default::default() + }, + ]), + ..Default::default() + }), + ..Default::default() + }; + services.create(&pp, &service).await.unwrap(); + + let name = relay_deployment.name_unchecked(); + let result = timeout( + Duration::from_secs(30), + await_condition(deployments.clone(), name.as_str(), is_deployment_ready()), + ) + .await; + if result.is_err() { + debug_pods(client, "role=relay".into()).await; + + panic!("Relay Deployment should be ready"); + } + result.unwrap().expect("Should have a relay deployment"); + + // agent deployment + let args = [ + "agent", + "--relay", + "http://quilkin-relay-agones:7900", + "agones", + "--config-namespace", + client.namespace.as_str(), + "--gameservers-namespace", + client.namespace.as_str(), + ] + .map(String::from) + .to_vec(); + let labels = BTreeMap::from([("role".to_string(), "agent".to_string())]); + let deployment = Deployment { + metadata: ObjectMeta { + name: Some("quilkin-agones-agent".into()), + labels: Some(labels.clone()), + ..Default::default() + }, + spec: Some(DeploymentSpec { + replicas: Some(1), + selector: LabelSelector { + match_expressions: None, + match_labels: Some(labels.clone()), + }, + template: PodTemplateSpec { + metadata: Some(ObjectMeta { + labels: Some(labels.clone()), + ..Default::default() + }), + spec: Some(PodSpec { + containers: vec![quilkin_container(client, Some(args), None)], + service_account_name: Some(rbac_name), + ..Default::default() + }), + }, + ..Default::default() + }), + ..Default::default() + }; + let agent_deployment = deployments.create(&pp, &deployment).await.unwrap(); + let name = agent_deployment.name_unchecked(); + let result = timeout( + Duration::from_secs(30), + await_condition(deployments.clone(), name.as_str(), is_deployment_ready()), + ) + .await; + if result.is_err() { + debug_pods(client, "role=agent".into()).await; + panic!("Agent Deployment should be ready"); + } + result.unwrap().expect("Should have an agent deployment"); + } +} diff --git a/agones/src/sidecar.rs b/agones/src/sidecar.rs index 7aa54781c5..2acd6aa9e4 100644 --- a/agones/src/sidecar.rs +++ b/agones/src/sidecar.rs @@ -71,7 +71,7 @@ mod tests { let config = r#" version: v1alpha1 filters: - - name: quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes + - name: quilkin.filters.concatenate.v1alpha1.Concatenate config: on_read: APPEND on_write: DO_NOTHING diff --git a/agones/src/xds.rs b/agones/src/xds.rs deleted file mode 100644 index 48689dfdec..0000000000 --- a/agones/src/xds.rs +++ /dev/null @@ -1,447 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#[cfg(test)] -mod tests { - use std::net::SocketAddr; - use std::{collections::BTreeMap, time::Duration}; - - use k8s_openapi::{ - api::{ - apps::v1::{Deployment, DeploymentSpec}, - core::v1::{ - ConfigMap, ContainerPort, Node, Pod, PodSpec, PodTemplateSpec, Service, - ServiceAccount, ServicePort, ServiceSpec, - }, - rbac::v1::{ClusterRole, PolicyRule, RoleBinding, RoleRef, Subject}, - }, - apimachinery::pkg::{ - apis::meta::v1::{LabelSelector, ObjectMeta}, - util::intstr::IntOrString, - }, - }; - use kube::{ - api::{DeleteParams, ListParams, PostParams}, - runtime::wait::await_condition, - Api, ResourceExt, - }; - use tokio::time::timeout; - - use quilkin::{ - config::providers::k8s::agones::{Fleet, GameServer}, - test_utils::TestHelper, - }; - - use crate::{ - debug_pods, fleet, is_deployment_ready, is_fleet_ready, quilkin_config_map, - quilkin_container, Client, - }; - - const PROXY_DEPLOYMENT: &str = "quilkin-proxies"; - - #[tokio::test] - /// Test for Agones integration. Since this will look at all GameServers in the namespace - /// for this test, we should only have single Agones integration test in this suite, since they - /// could easily collide with each other. - async fn agones_token_router() { - let client = Client::new().await; - - let deployments: Api = client.namespaced_api(); - let fleets: Api = client.namespaced_api(); - let gameservers: Api = client.namespaced_api(); - let config_maps: Api = client.namespaced_api(); - - let pp = PostParams::default(); - - let config = r#" -version: v1alpha1 -filters: - - name: quilkin.filters.capture.v1alpha1.Capture # Capture and remove the authentication token - config: - suffix: - size: 3 - remove: true - - name: quilkin.filters.token_router.v1alpha1.TokenRouter -"#; - let mut config_map = quilkin_config_map(config); - config_map - .metadata - .labels - .get_or_insert(Default::default()) - .insert("quilkin.dev/configmap".into(), "true".into()); - - config_maps.create(&pp, &config_map).await.unwrap(); - - agones_control_plane(&client, deployments.clone()).await; - let proxy_address = quilkin_proxy_deployment(&client, deployments.clone()).await; - - // create a fleet so we can ensure that a packet is going to the GameServer we expect, and not - // any other. - let fleet = fleet(); - let fleet = fleets.create(&pp, &fleet).await.unwrap(); - let name = fleet.name_unchecked(); - timeout( - Duration::from_secs(30), - await_condition(fleets.clone(), name.as_str(), is_fleet_ready()), - ) - .await - .expect("Fleet should be ready") - .unwrap(); - - let lp = ListParams { - label_selector: Some(format!("agones.dev/fleet={}", fleet.name_unchecked())), - ..Default::default() - }; - let list = gameservers.list(&lp).await.unwrap(); - - // let's allocate this specific game server - let mut t = TestHelper::default(); - let (mut rx, socket) = t.open_socket_and_recv_multiple_packets().await; - - let mut gs = list.items[0].clone(); - let gs_address = crate::gameserver_address(&gs); - - // add routing label to the GameServer - let token = "456"; // NDU2 - assert_eq!(3, token.as_bytes().len()); - let token_key = "quilkin.dev/tokens"; - gs.metadata - .annotations - .get_or_insert(Default::default()) - .insert( - token_key.into(), - base64::Engine::encode(&base64::engine::general_purpose::STANDARD, token), - ); - gameservers - .replace(gs.name_unchecked().as_str(), &pp, &gs) - .await - .unwrap(); - // and allocate it such that we have an endpoint. - socket - .send_to("ALLOCATE".as_bytes(), gs_address) - .await - .unwrap(); - - let response = timeout(Duration::from_secs(30), rx.recv()) - .await - .expect("should receive packet from GameServer") - .unwrap(); - assert_eq!("ACK: ALLOCATE\n", response); - - // Proxy Deployment should be ready, since there is now an endpoint - if timeout( - Duration::from_secs(30), - await_condition(deployments.clone(), PROXY_DEPLOYMENT, is_deployment_ready()), - ) - .await - .is_err() - { - debug_pods(&client, "role=proxy".into()).await; - panic!("Quilkin proxy deployment should be ready"); - } - - // keep trying to send the packet to the proxy until it works, since distributed systems are eventually consistent. - let mut response: String = "not-found".into(); - for i in 0..30 { - println!("Connection Attempt: {i}"); - - // returns the nae of the GameServer. This proves we are routing the the allocated - // GameServer with the correct token attached. - socket - .send_to(format!("GAMESERVER{token}").as_bytes(), proxy_address) - .await - .unwrap(); - - let result = timeout(Duration::from_secs(1), rx.recv()).await; - if let Ok(Some(value)) = result { - response = value; - break; - } - } - assert_eq!(format!("NAME: {}\n", gs.name_unchecked()), response); - - // let's remove the token from the gameserver, which should remove access. - let mut gs = gameservers.get(gs.name_unchecked().as_str()).await.unwrap(); - let name = gs.name_unchecked(); - gs.metadata - .annotations - .as_mut() - .map(|annotations| annotations.remove(token_key).unwrap()); - gameservers.replace(name.as_str(), &pp, &gs).await.unwrap(); - // now we should send a packet, and not get a response. - let mut failed = false; - for i in 0..30 { - println!("Disconnection Attempt: {i}"); - socket - .send_to(format!("GAMESERVER{token}").as_bytes(), proxy_address) - .await - .unwrap(); - - let result = timeout(Duration::from_secs(1), rx.recv()).await; - if result.is_err() { - failed = true; - break; - } - } - if !failed { - debug_pods(&client, "role=proxy".into()).await; - debug_pods(&client, "role=xds".into()).await; - } - assert!(failed, "Packet should have failed"); - } - - /// Creates Quilkin xDS management instance that is in the mode to watch Agones GameServers - /// in this test namespace - async fn agones_control_plane(client: &Client, deployments: Api) { - let services: Api = client.namespaced_api(); - let service_accounts: Api = client.namespaced_api(); - let cluster_roles: Api = Api::all(client.kubernetes.clone()); - let role_bindings: Api = client.namespaced_api(); - let pp = PostParams::default(); - - // create all the rbac rules - let rbac_name = "quilkin-agones"; - let rbac_meta = ObjectMeta { - name: Some(rbac_name.into()), - ..Default::default() - }; - let service_account = ServiceAccount { - metadata: rbac_meta.clone(), - ..Default::default() - }; - service_accounts - .create(&pp, &service_account) - .await - .unwrap(); - - // Delete the cluster role if it already exists, since it's cluster wide. - match cluster_roles - .delete(rbac_name, &DeleteParams::default()) - .await - { - Ok(_) => {} - Err(err) => println!("Cluster role not found: {err}"), - }; - let cluster_role = ClusterRole { - metadata: rbac_meta.clone(), - rules: Some(vec![ - PolicyRule { - api_groups: Some(vec!["agones.dev".into()]), - resources: Some(vec!["gameservers".into()]), - verbs: ["get", "list", "watch"].map(String::from).to_vec(), - ..Default::default() - }, - PolicyRule { - api_groups: Some(vec!["".into()]), - resources: Some(vec!["configmaps".into()]), - verbs: ["get", "list", "watch"].map(String::from).to_vec(), - ..Default::default() - }, - ]), - ..Default::default() - }; - cluster_roles.create(&pp, &cluster_role).await.unwrap(); - - let binding = RoleBinding { - metadata: rbac_meta, - subjects: Some(vec![Subject { - kind: "User".into(), - name: format!("system:serviceaccount:{}:{rbac_name}", client.namespace), - api_group: Some("rbac.authorization.k8s.io".into()), - ..Default::default() - }]), - role_ref: RoleRef { - api_group: "rbac.authorization.k8s.io".into(), - kind: "ClusterRole".into(), - name: rbac_name.into(), - }, - }; - role_bindings.create(&pp, &binding).await.unwrap(); - - // Setup the xDS Agones provider server - let args = [ - "manage", - "agones", - "--config-namespace", - client.namespace.as_str(), - "--gameservers-namespace", - client.namespace.as_str(), - ] - .map(String::from) - .to_vec(); - let mut container = quilkin_container(client, Some(args), None); - container.ports = Some(vec![ContainerPort { - container_port: 7777, - ..Default::default() - }]); - let labels = BTreeMap::from([("role".to_string(), "xds".to_string())]); - let deployment = Deployment { - metadata: ObjectMeta { - name: Some("quilkin-manage-agones".into()), - labels: Some(labels.clone()), - ..Default::default() - }, - spec: Some(DeploymentSpec { - replicas: Some(1), - selector: LabelSelector { - match_expressions: None, - match_labels: Some(labels.clone()), - }, - template: PodTemplateSpec { - metadata: Some(ObjectMeta { - labels: Some(labels.clone()), - ..Default::default() - }), - spec: Some(PodSpec { - containers: vec![container], - service_account_name: Some(rbac_name.into()), - ..Default::default() - }), - }, - ..Default::default() - }), - ..Default::default() - }; - - let deployment = deployments.create(&pp, &deployment).await.unwrap(); - - let service = Service { - metadata: ObjectMeta { - name: Some("quilkin-manage-agones".into()), - ..Default::default() - }, - spec: Some(ServiceSpec { - selector: Some(labels), - ports: Some(vec![ServicePort { - protocol: Some("TCP".into()), - port: 7800, - target_port: Some(IntOrString::Int(7800)), - ..Default::default() - }]), - ..Default::default() - }), - ..Default::default() - }; - services.create(&pp, &service).await.unwrap(); - - // make sure the deployment and service are ready - let name = deployment.name_unchecked(); - timeout( - Duration::from_secs(30), - await_condition(deployments.clone(), name.as_str(), is_deployment_ready()), - ) - .await - .expect("xDS provider deployment should be ready") - .unwrap(); - } - - /// create a Deployment with a singular Quilkin proxy, that is configured - /// to be attached to the Quilkin Agones xDS server in `agones_control_plane()`. - async fn quilkin_proxy_deployment(client: &Client, deployments: Api) -> SocketAddr { - let pp = PostParams::default(); - let mut container = quilkin_container( - client, - Some(vec![ - "proxy".into(), - "--management-server=http://quilkin-manage-agones:7800".into(), - ]), - None, - ); - - // we'll use a host port, since spinning up a load balancer takes a long time. - // we know that port 7777 is open because this is an Agones cluster and it has associated - // firewall rules , and even if we conflict with a GameServer - // the k8s scheduler will move us to another node. - let host_port: u16 = 7005; - container.ports = Some(vec![ContainerPort { - container_port: 7777, - host_port: Some(host_port as i32), - protocol: Some("UDP".into()), - ..Default::default() - }]); - - let labels = BTreeMap::from([("role".to_string(), "proxy".to_string())]); - let deployment = Deployment { - metadata: ObjectMeta { - name: Some(PROXY_DEPLOYMENT.into()), - labels: Some(labels.clone()), - ..Default::default() - }, - spec: Some(DeploymentSpec { - replicas: Some(1), - selector: LabelSelector { - match_expressions: None, - match_labels: Some(labels.clone()), - }, - template: PodTemplateSpec { - metadata: Some(ObjectMeta { - labels: Some(labels.clone()), - ..Default::default() - }), - spec: Some(PodSpec { - containers: vec![container], - ..Default::default() - }), - }, - ..Default::default() - }), - ..Default::default() - }; - - let deployment = deployments.create(&pp, &deployment).await.unwrap(); - let name = deployment.name_unchecked(); - // should not be ready, since there are no endpoints, but let's wait 3 seconds, make sure it doesn't do something we don't expect - let result = timeout( - Duration::from_secs(3), - await_condition(deployments.clone(), name.as_str(), is_deployment_ready()), - ) - .await; - assert!(result.is_err()); - - // get the address to send data to - let pods = client.namespaced_api::(); - let list = pods - .list(&ListParams { - label_selector: Some("role=proxy".into()), - ..Default::default() - }) - .await - .unwrap(); - assert_eq!(1, list.items.len()); - - let nodes: Api = Api::all(client.kubernetes.clone()); - let name = list.items[0] - .spec - .as_ref() - .unwrap() - .node_name - .as_ref() - .unwrap(); - let node = nodes.get(name.as_str()).await.unwrap(); - let external_ip = node - .status - .unwrap() - .addresses - .unwrap() - .iter() - .find(|addr| addr.type_ == "ExternalIP") - .unwrap() - .address - .clone(); - - SocketAddr::new(external_ip.parse().unwrap(), host_port) - } -} diff --git a/benches/throughput.rs b/benches/throughput.rs index 6480075e78..2cd558d259 100644 --- a/benches/throughput.rs +++ b/benches/throughput.rs @@ -43,7 +43,8 @@ fn run_quilkin(port: u16, endpoint: SocketAddr) { runtime.block_on(async move { let (_shutdown_tx, shutdown_rx) = tokio::sync::watch::channel::<()>(()); - proxy.run(config, shutdown_rx).await.unwrap(); + let admin = quilkin::cli::Admin::Proxy(<_>::default()); + proxy.run(config, admin, shutdown_rx).await.unwrap(); }); }); } diff --git a/build.rs b/build.rs index c308464e61..93f8903e69 100644 --- a/build.rs +++ b/build.rs @@ -36,7 +36,7 @@ fn main() -> Result<(), Box> { "proto/quilkin/config/v1alpha1/config.proto", "proto/quilkin/filters/capture/v1alpha1/capture.proto", "proto/quilkin/filters/compress/v1alpha1/compress.proto", - "proto/quilkin/filters/concatenate_bytes/v1alpha1/concatenate_bytes.proto", + "proto/quilkin/filters/concatenate/v1alpha1/concatenate.proto", "proto/quilkin/filters/debug/v1alpha1/debug.proto", "proto/quilkin/filters/drop/v1alpha1/drop.proto", "proto/quilkin/filters/firewall/v1alpha1/firewall.proto", diff --git a/build/ci/github-bot/go.mod b/build/ci/github-bot/go.mod index f3615cf153..a7441cb015 100644 --- a/build/ci/github-bot/go.mod +++ b/build/ci/github-bot/go.mod @@ -37,13 +37,13 @@ require ( github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect go.opencensus.io v0.22.6 // indirect - golang.org/x/crypto v0.1.0 // indirect + golang.org/x/crypto v0.14.0 // indirect golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/tools v0.6.0 // indirect google.golang.org/api v0.39.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc v1.36.1 // indirect diff --git a/build/ci/github-bot/go.sum b/build/ci/github-bot/go.sum index d7c3a6f041..3471758a34 100644 --- a/build/ci/github-bot/go.sum +++ b/build/ci/github-bot/go.sum @@ -178,8 +178,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -214,8 +214,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -249,8 +249,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -273,6 +273,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -307,8 +308,8 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -317,8 +318,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -369,8 +370,8 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/docs/preprocessor.sh b/docs/preprocessor.sh index e3190927bf..fd55562c32 100755 --- a/docs/preprocessor.sh +++ b/docs/preprocessor.sh @@ -23,6 +23,7 @@ cargo run -q --manifest-path ../Cargo.toml &> ../target/quilkin.commands || true cargo run -q --manifest-path ../Cargo.toml -- proxy --help &> ../target/quilkin.proxy.commands || true cargo run -q --manifest-path ../Cargo.toml -- manage --help &> ../target/quilkin.manage.commands || true cargo run -q --manifest-path ../Cargo.toml -- relay --help &> ../target/quilkin.relay.commands || true +cargo run -q --manifest-path ../Cargo.toml -- agent --help &> ../target/quilkin.agent.commands || true # Credit: https://github.com/rust-lang/mdBook/issues/1462#issuecomment-778650045 jq -M -c .[1] <&0 diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index f41253666e..cadd6ade9a 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -7,6 +7,7 @@ - [Netcat](./deployment/quickstarts/netcat.md) - [Agones + Xonotic (Sidecar)](./deployment/quickstarts/agones-xonotic-sidecar.md) - [Agones + Xonotic (xDS)](./deployment/quickstarts/agones-xonotic-xds.md) +- [Agones + Xonotic (Relay)](./deployment/quickstarts/agones-xonotic-relay.md) # Services @@ -15,7 +16,7 @@ - [Filters](./services/proxy/filters.md) - [Capture](./services/proxy/filters/capture.md) - [Compress](./services/proxy/filters/compress.md) - - [Concatenate Bytes](./services/proxy/filters/concatenate_bytes.md) + - [Concatenate](./services/proxy/filters/concatenate.md) - [Debug](./services/proxy/filters/debug.md) - [Drop](./services/proxy/filters/drop.md) - [Firewall](./services/proxy/filters/firewall.md) @@ -41,7 +42,7 @@ - [Relay](./services/relay.md) - [Metrics]() - - [Providers]() + - [Agents](./services/agent.md) # SDKs diff --git a/docs/src/deployment/admin.md b/docs/src/deployment/admin.md index 4180f82f78..c2d34b02a0 100644 --- a/docs/src/deployment/admin.md +++ b/docs/src/deployment/admin.md @@ -34,15 +34,38 @@ The admin interface provides the following endpoints: This provides a liveness probe endpoint, most commonly used in [Kubernetes based systems](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-liveness-command). -Will return an HTTP status of 200 when all health checks pass. +Liveness is defined as "hasn't panicked", as long as the process has not +panicked quilkin is considered live. ### /ready This provides a readiness probe endpoint, most commonly used in [Kubernetes based systems](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-readiness-probes). -Depending on whether Quilkin is run in Proxy mode i.e. `quilkin proxy`, vs an xDS provider mode, such as `quilkin -manage agones`, will dictate how readiness is calculated: +Readiness is service and provider specific, so based on what you're running +there will be different criteria for a service to be considered ready. Here's +a list of the criteria for each service an provider. + +| Service | Readiness | +|---------|---------------------------------------------------------------------| +| Proxy | Management server is connected (or always true if config is static) AND if there is more than one endpoint configured| +| Manage | Provider is ready | +| Relay | Provider is ready | +| Agent | Provider is ready AND connected to relay | + +
+ +| Provider | Readiness | +|----------|--------------------------------------------| +| Agones | The service is connected to kube-api | +| File | The service has found and watches the file | + +When setting thresholds for your `proxy` probes, you generally want to set a low +check period (e.g. `periodSeconds=1`) and a low success threshold +(e.g. `successThreshold=1`), but a high `failureThreshold` +(e.g. `failureThreshold=60`) and `terminationGracePeriodSeconds` to allow for +backoff attempts and existing player sessions to continue without disruption. + #### Proxy Mode @@ -67,4 +90,4 @@ See the [xDS Metrics](../services/xds/metrics.md) documentation for what xDS met Returns a JSON representation of the cluster and filterchain configuration that the instance is running with at the time of invocation. -[log-docs]: https://docs.rs/env_logger/latest/env_logger/#enabling-logging \ No newline at end of file +[log-docs]: https://docs.rs/env_logger/latest/env_logger/#enabling-logging diff --git a/docs/src/deployment/quickstarts/agones-xonotic-relay.md b/docs/src/deployment/quickstarts/agones-xonotic-relay.md new file mode 100644 index 0000000000..18b88d5e20 --- /dev/null +++ b/docs/src/deployment/quickstarts/agones-xonotic-relay.md @@ -0,0 +1,317 @@ +# Quickstart: Quilkin with Agones and Xonotic (Relay) + +{{#include _agones.md}} +* A local copy of the [Quilkin Binary](https://github.com/googleforgames/quilkin/releases). + +## 1. Overview + +In this quickstart, we'll be setting up an example multi-cluster +[Xonotic](https://xonotic.org/) [Agones](https://agones.dev/) Fleet, that will +only be accessible through Quilkin, via utilising the [TokenRouter] Filter to +provide routing and access control to the Allocated `GameServer` instances. + +To do this, we'll take advantage of the Quilkin [Relay](../../services/relay.md) to provide +an out-of-the-box multi-cluster xDS control plane, and the [Agones Agent](../../services/agent.md) +to send information from the cluster(s) to the relay, which can be used as a +management server for each of the Quilkin [Proxy](../../services/proxy.md) instances. + +> While the application of `quilkin relay` is to ultimately provide a solution where multiple clusters feed +> configuration information into a single relay endpoint via a `quilkin agent`, in this example we'll +> use a single cluster for demonstrative purposes. + +## 2. Install Quilkin Relay and Agones Agent + +To install Quilkin as an Agones integrated relay control plane, we can create a deployment of Quilkin running as +`quilkin relay` with a corresponding Agones agent, `quilkin agent agones`, with the appropriate permissions. + +Run the following: + +```shell +kubectl apply -f https://raw.githubusercontent.com/googleforgames/quilkin/{{GITHUB_REF_NAME}}/examples/agones-xonotic-relay/relay-control-plane.yaml +``` + +This applies several resources to your cluster: + +1. A [ConfigMap] with a [Capture] and [TokenRouter] Filter set up to route packets to Endpoints, to be the base + configuration for all the Quilkin proxies. +2. Appropriate [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) permissions for the + `quilkin agent agones` process to inspect Agones resources. +3. A [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) that runs the + `quilkin relay` process, a matching Deployment for the `quilkin agent` process and a + [Service](https://kubernetes.io/docs/concepts/services-networking/service/) that the Quilkin agents can send configuration information to, and the Proxies can connect to, + to get their Filter and Endpoint configuration from. + +Now we can run `kubectl get pods` until we see that the Pod for the Deployment is up and running. + +```shell +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +quilkin-agones-agent-9dd6699bd-qh7cq 1/1 Running 0 6s +quilkin-relay-agones-55fbd69f5d-cdh9k 1/1 Running 0 6s +``` + +We can now run `kubectl get service quilkin-relay-agones` and see the +service that is generated in front of the above Quilkin Relay Deployment for our Quilkin Proxies to connect to and +receive their configuration information from. + +```shell +$ kubectl get service quilkin-relay-agones +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +quilkin-relay-agones ClusterIP 10.103.243.246 7800/TCP,7900/TCP 57s +``` + +## 3. Install Quilkin Proxy Pool + +To install the Quilkin Proxy pool which connects to the above Relay xDS provider, we can create a Deployment of Quilkin +proxy instances that point to the aforementioned Service, like so: + +```shell +kubectl apply -f https://raw.githubusercontent.com/googleforgames/quilkin/{{GITHUB_REF_NAME}}/examples/agones-xonotic-relay/proxy-pool.yaml +``` + +This will set up three instances of Quilkin running as `quilkin proxy --management-server +http://quilkin-relay-agones:7900` all of which are connected to the `quilkin-relay-agones` service. + +Now we can run `kubectl get pods` until we see that the Pods for the proxy Deployment is up and running. + +```shell +$ kubectl get pods +NAME READY STATUS RESTARTS AGE +quilkin-agones-agent-9dd6699bd-5brzf 1/1 Running 0 18s +quilkin-proxies-7d9bbbccdf-5mz4l 1/1 Running 0 7s +quilkin-proxies-7d9bbbccdf-9vd59 1/1 Running 0 7s +quilkin-proxies-7d9bbbccdf-vwn2f 1/1 Running 0 7s +quilkin-relay-agones-55fbd69f5d-k2n7b 1/1 Running 0 18s +``` + +Let's take this one step further, and check the configuration of the proxies that should have come from the `quilkin +agent agones` instance and passed through the `quilkin relay instance` into each of the proxies. + +In another terminal, run: `kubectl port-forward deployments/quilkin-proxies 8001:8000`, to port forward the +[admin endpoint](../admin.md) locally to port 8001, which we can then query. + +Go back to your original terminal and run `curl -s http://localhost:8001/config` + +> If you have [jq](https://stedolan.github.io/jq/) installed, run `curl -s http://localhost:8001/config | jq` for a +> nicely formatted JSON output. + +```shell +$ curl -s http://localhost:8001/config | jq +{ + "clusters": [ + { + "endpoints": [], + "locality": null + } + ], + "filters": [ + { + "name": "quilkin.filters.capture.v1alpha1.Capture", + "label": null, + "config": { + "metadataKey": "quilkin.dev/capture", + "suffix": { + "size": 3, + "remove": true + } + } + }, + { + "name": "quilkin.filters.token_router.v1alpha1.TokenRouter", + "label": null, + "config": null + } + ], + "id": "quilkin-proxies-7d9bbbccdf-9vd59", + "version": "v1alpha1" +} +``` + +This shows us the current configuration of the proxies coming from the xDS server created via `quilkin agent +agones`. The most interesting part that we see here, is that we have a matching set of +[Filters](../../services/proxy/filters.md) that are found in the `ConfigMap` in the +[relay-control-plane.yaml](https://github.com/googleforgames/quilkin/blob/{{GITHUB_REF_NAME}}/examples/agones-xonotic-relay/relay-control-plane.yaml) +we installed earlier. + +## 4. Create the Agones Fleet + +Now we will create an [Agones Fleet](https://agones.dev/site/docs/reference/fleet/) to spin up all our Xonotic +game servers. + +Thankfully, Agones Fleets require no specific configuration to work with Quilkin proxies, so this yaml is a +[standard Agones Fleet configuration](https://github.com/googleforgames/quilkin/blob/{{GITHUB_REF_NAME}}/examples/agones-xonotic-relay/fleet.yaml) + +```shell +kubectl apply -f https://raw.githubusercontent.com/googleforgames/quilkin/{{GITHUB_REF_NAME}}/examples/agones-xonotic-relay/fleet.yaml +``` + +Run `kubectl get gameservers` until all the `GameServer` records show that they are `Ready` and able to take players. + +```shell +$ kubectl get gs +NAME STATE ADDRESS PORT NODE AGE +xonotic-8ns7b-2lk5d Ready 39.168.219.72 7015 gke-agones-default-ad8cd7e5-3b12 1m +xonotic-8ns7b-hrc8j Ready 39.168.219.72 7298 gke-agones-default-ad8cd7e5-3b12 1m +xonotic-8ns7b-mldg6 Ready 39.168.219.72 7558 gke-agones-default-ad8cd7e5-3b12 1m +``` + +## 5. Allocate a `GameServer` + +To let the Quilkin Agones Agent know what token will route to which `GameServer` we need to apply the +`quilkin.dev/tokens` annotation to an allocated `GameServer`, with the token content as its value. + +> This token would normally get generated by some kind of player authentication service and passed to the client +> via the matchmaking service - but for demonstrative purposes, we've hardcoded it into the example +> `GameServerAllocation`. + +Since you can add annotations to `GameServers` at +[allocation time](https://agones.dev/site/docs/reference/gameserverallocation/), we can both allocate a `GameServer` +and apply the annotation at the same time! + +```shell +kubectl create -f https://raw.githubusercontent.com/googleforgames/quilkin/{{GITHUB_REF_NAME}}/examples/agones-xonotic-relay/gameserverallocation.yaml +``` + +If we check our `GameServers` now, we should see that one of them has moved to the `Allocated` state, marking it as +having players playing on it, and therefore it is protected by Agones until the game session ends. + +```shell +$ kubectl get gs +NAME STATE ADDRESS PORT NODE AGE +xonotic-8ns7b-2lk5d Allocated 39.168.219.72 7015 gke-agones-default-ad8cd7e5-3b12 17m +xonotic-8ns7b-hrc8j Ready 39.168.219.72 7298 gke-agones-default-ad8cd7e5-3b12 17m +xonotic-8ns7b-mldg6 Ready 39.168.219.72 7558 gke-agones-default-ad8cd7e5-3b12 17m +``` + +> Don't do this more than once, as then multiple allocated `GameServers` will have the same routing token! + +If we `kubectl describe gameserver ` and have a look at the annotations section, we +should see something similar to this: + +```shell +❯ kubectl describe gs xonotic-8ns7b-2lk5d +Name: xonotic-8ns7b-2lk5d +Namespace: default +Labels: agones.dev/fleet=xonotic + agones.dev/gameserverset=xonotic-8ns7b +Annotations: agones.dev/last-allocated: 2023-10-04T19:47:04.047026419Z + agones.dev/ready-container-id: containerd://b39d30965becdbc40336fd9aa642fe776421553615f642dd599e1b0d88c505b6 + agones.dev/sdk-version: 1.33.0 + quilkin.dev/tokens: NDU2 +API Version: agones.dev/v1 +Kind: GameServer +... +``` + +Where we can see that there is now an annotation of `quilkin.dev/tokens` with the base64 encoded version of `456` as +our authentication and routing token ("NDU2"). + +> You should use something more cryptographically random than `456` in your application. + +Let's run `curl -s http://localhost:8001/config` again, so we can see what has changed! + +```shell +❯ curl -s http://localhost:8001/config | jq +{ + "clusters": [ + { + "endpoints": [ + { + "address": "39.168.219.72:7015", + "metadata": { + "quilkin.dev": { + "tokens": [ + "NDU2" + ] + }, + "name": "xonotic-8ns7b-2lk5d" + } + } + ], + "locality": null + } + ], + "filters": [ + { + "name": "quilkin.filters.capture.v1alpha1.Capture", + "label": null, + "config": { + "metadataKey": "quilkin.dev/capture", + "suffix": { + "size": 3, + "remove": true + } + } + }, + { + "name": "quilkin.filters.token_router.v1alpha1.TokenRouter", + "label": null, + "config": null + } + ], + "id": "quilkin-proxies-7d9bbbccdf-9vd59", + "version": "v1alpha1" +} +``` + +Looking under `clusters` > `endpoints` we can see an address and token that matches up with the +`GameServer` record we created above! + +The Agones agent process saw that allocated `GameServer`, turned it into a Quilkin `Endpoint` and applied the set +routing token appropriately -- without you having to write a line of xDS compliant code! + +## Connecting Client Side + +Instead of connecting to Xonotic or an Agones `GameServer` directly, we'll want to grab the IP and exposed port of +the `Service` that fronts all our Quilkin proxies and connect to that instead -- but we'll have to append our +routing token `456` from before, to ensure our traffic gets routed to the correct Xonotic `GameServer` address. + +Run `kubectl get service quilkin-proxies` to get the `EXTERNAL-IP` of the Service you created. + +```shell +$ kubectl get service quilkin-proxies +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +quilkin-proxies LoadBalancer 10.109.0.12 35.246.94.14 7000:30174/UDP 3h22m +``` + +We have a [Quilkin config yaml](https://github.com/googleforgames/quilkin/blob/{{GITHUB_REF_NAME}}/examples/agones-xonotic-relay/client-token.yaml) +file all ready for you, that is configured to append the routing token `456` to each +packet that passes through it, via the power of a +[Concatenate](../../services/proxy/filters/concatenate.md) Filter. + +Download `client-token.yaml` locally, so you can edit it: + +```shell +curl https://raw.githubusercontent.com/googleforgames/quilkin/{{GITHUB_REF_NAME}}/examples/agones-xonotic-relay/client-token.yaml --output client-token.yaml +``` + +We then take the EXTERNAL-IP and port from the `quilkin-proxies` service, and replace the`${LOADBALANCER_IP}` +with it in `client-token.yaml`. + +Run this edited configuration locally with your quilkin binary as `quilkin -c ./client-token.yaml proxy`: + +```shell +$ ./quilkin --config ./client-token.yaml proxy +2023-10-04T20:09:07.320780Z INFO quilkin::cli: src/cli.rs: Starting Quilkin version="0.7.0-dev" commit="d42db7e14c2e0e758e9a6eb655ccf4184941066c" +2023-10-04T20:09:07.321711Z INFO quilkin::admin: src/admin.rs: Starting admin endpoint address=[::]:8000 +2023-10-04T20:09:07.322089Z INFO quilkin::cli::proxy: src/cli/proxy.rs: Starting port=7777 proxy_id="markmandel45" +2023-10-04T20:09:07.322576Z INFO quilkin::cli::proxy: src/cli/proxy.rs: Quilkin is ready +2023-10-04T20:09:07.322692Z INFO qcmp_task{v4_addr=0.0.0.0:7600 v6_addr=[::]:7600}: quilkin::protocol: src/protocol.rs: awaiting qcmp packets v4_addr=0.0.0.0:7600 v6_addr=[::]:7600 +``` + +Now connect to the local client proxy on "[::1]:7777" via the "Multiplayer > Address" field in the +Xonotic client, and Quilkin will take care of appending the routing token to all your UDP packets, which the Quilkin +proxies will route to the Allocated GameServer, and you can play a gamee! + +![xonotic-address-v6.png](xonotic-address-v6.png) + +...And you didn't have to change the client or the dedicated game server 🤸 + +## What's Next? + +* Check out the variety of [Filters](../../services/proxy/filters.md) that are possible with Quilkin. +* Read into the [xDS Management API](../../services/xds.md). + +[ConfigMap]: https://kubernetes.io/docs/concepts/configuration/configmap/ +[Capture]: ../../services/proxy/filters/capture.md +[TokenRouter]: ../../services/proxy/filters/token_router.md diff --git a/docs/src/deployment/quickstarts/agones-xonotic-sidecar.md b/docs/src/deployment/quickstarts/agones-xonotic-sidecar.md index 7647079994..f6a33a0d72 100644 --- a/docs/src/deployment/quickstarts/agones-xonotic-sidecar.md +++ b/docs/src/deployment/quickstarts/agones-xonotic-sidecar.md @@ -37,7 +37,7 @@ Usually with Agones you would Choose one of the listed `GameServer`s from the previous step, and connect to the IP and port of the Xonotic server via the "Multiplayer > Address" field in the Xonotic client in the format of {IP}:{PORT}. -![xonotic-address.png](xonotic-address.png) +![xonotic-address.png](xonotic-address-v6.png) You should now be playing a game of Xonotic against 4 bots! @@ -49,10 +49,10 @@ Grab the name of the GameServer you connected to before, and replace the `${game command. This will forward the [admin](../admin.md) interface to localhost. ```shell -kubectl port-forward ${gameserver} 9091 +kubectl port-forward ${gameserver} 8000 ``` -Then open a browser to [http://localhost:9091/metrics](http://localhost:9091/metrics) to see the +Then open a browser to [http://localhost:8000/metrics](http://localhost:9091/metrics) to see the [Prometheus](https://prometheus.io/) metrics that Quilkin exports. ## 5. Cleanup diff --git a/docs/src/deployment/quickstarts/agones-xonotic-xds.md b/docs/src/deployment/quickstarts/agones-xonotic-xds.md index fbf4cac28d..04aa949f2e 100644 --- a/docs/src/deployment/quickstarts/agones-xonotic-xds.md +++ b/docs/src/deployment/quickstarts/agones-xonotic-xds.md @@ -6,8 +6,8 @@ ## 1. Overview In this quickstart, we'll be setting up an example [Xonotic](https://xonotic.org/) [Agones](https://agones.dev/) -Fleet, that will only be accessible through Quilkin, via utilising the [TokenRouter] -Filter to provide routing and access control to the Allocated `GameServer` instances. +Fleet, that will only be accessible through Quilkin that is hosted within the same cluster, utilising the +[TokenRouter] Filter to provide routing and access control to the Allocated `GameServer` instances. To do this, we'll take advantage of the Quilkin [Agones xDS Provider](../../services/xds/providers/agones.md) to provide an out-of-the-box control plane for integration between Agones and [Quilkin's xDS configuration API](../../services/xds.md) with @@ -79,17 +79,44 @@ quilkin-proxies-78965c446d-m4rr7 1/1 Running 0 6s Let's take this one step further, and check the configuration of the proxies that should have come from the `quilkin manage agones` instance. -In another terminal, run: `kubectl port-forward deployments/quilkin-proxies 8000`, to port forward the -[admin endpoint](../admin.md) locally, which we can then query. +In another terminal, run: `kubectl port-forward deployments/quilkin-proxies 8001:8000`, to port forward the +[admin endpoint](../admin.md) locally to port 8001, which we can then query. -Go back to your original terminal and run `curl -s http://localhost:8000/config` +Go back to your original terminal and run `curl -s http://localhost:8001/config` -> If you have [jq](https://stedolan.github.io/jq/) installed, run `curl -s http://localhost:8000/config | jq` for a +> If you have [jq](https://stedolan.github.io/jq/) installed, run `curl -s http://localhost:8001/config | jq` for a > nicely formatted JSON output. ```shell -$ curl -s http://localhost:8000/config -{"admin":{"address":"0.0.0.0:8000"},"clusters":{},"filters":[{"name":"quilkin.filters.capture.v1alpha1.Capture","config":{"metadataKey":"quilkin.dev/capture","suffix":{"size":3,"remove":true}}},{"name":"quilkin.filters.token_router.v1alpha1.TokenRouter","config":null}],"id":"quilkin-proxies-78965c446d-dqvjg","management_servers":[{"address":"http://quilkin-manage-agones:80"}],"port":7000,"version":"v1alpha1","maxmind_db":null}% +$ curl -s http://localhost:8001/config | jq +{ + "clusters": [ + { + "endpoints": [], + "locality": null + } + ], + "filters": [ + { + "name": "quilkin.filters.capture.v1alpha1.Capture", + "label": null, + "config": { + "metadataKey": "quilkin.dev/capture", + "suffix": { + "size": 3, + "remove": true + } + } + }, + { + "name": "quilkin.filters.token_router.v1alpha1.TokenRouter", + "label": null, + "config": null + } + ], + "id": "quilkin-proxies-7d9bbbccdf-9vd59", + "version": "v1alpha1" +} ``` This shows us the current configuration of the proxies coming from the xDS server created via `quilkin manage @@ -161,7 +188,7 @@ Labels: agones.dev/fleet=xonotic agones.dev/gameserverset=xonotic-h5cfn Annotations: agones.dev/last-allocated: 2022-12-19T22:59:22.099818298Z agones.dev/ready-container-id: containerd://7b3d9e9dbda6f2e0381df7669f6117bf3e54171469cfacbce2670605a61ce4b8 - agones.dev/sdk-version: 1.24.0 + agones.dev/sdk-version: 1.33.0 quilkin.dev/tokens: NDU2 API Version: agones.dev/v1 Kind: GameServer @@ -173,17 +200,56 @@ our authentication and routing token ("NDU2"). > You should use something more cryptographically random than `456` in your application. -Let's run `curl -s http://localhost:8000/config` again, so we can see what has changed! +Let's run `curl -s http://localhost:8001/config` again, so we can see what has changed! ```shell -$ curl -s http://localhost:8000/config -{"admin":{"address":"0.0.0.0:8000"},"clusters": [{"locality":null,"endpoints":[{"address":"34.168.170.51:7226","metadata":{"quilkin.dev":{"tokens":["NDU2"]}}}]}],"filters":[{"name":"quilkin.filters.capture.v1alpha1.Capture","config":{"metadataKey":"quilkin.dev/capture","suffix":{"size":3,"remove":true}}},{"name":"quilkin.filters.token_router.v1alpha1.TokenRouter","config":null}],"id":"quilkin-proxies-78965c446d-tfgsj","management_servers":[{"address":"http://quilkin-manage-agones:80"}],"port":7000,"version":"v1alpha1","maxmind_db":null}% +❯ curl -s http://localhost:8001/config | jq +{ + "clusters": [ + { + "endpoints": [ + { + "address": "34.168.170.51:7226", + "metadata": { + "quilkin.dev": { + "tokens": [ + "NDU2" + ] + }, + "name": "xonotic-8ns7b-2lk5d" + } + } + ], + "locality": null + } + ], + "filters": [ + { + "name": "quilkin.filters.capture.v1alpha1.Capture", + "label": null, + "config": { + "metadataKey": "quilkin.dev/capture", + "suffix": { + "size": 3, + "remove": true + } + } + }, + { + "name": "quilkin.filters.token_router.v1alpha1.TokenRouter", + "label": null, + "config": null + } + ], + "id": "quilkin-proxies-7d9bbbccdf-9vd59", + "version": "v1alpha1" +} ``` Looking under `clusters` > `endpoints` we can see an address and token that matches up with the `GameServer` record we created above! -The xDS process saw that allocated `GameServer`, turned it into a Quilkin `Endpoint` and applied the set routing +The xDS process saw that allocated `GameServer`, turned it into a Quilkin `Endpoint` and applied the set the routing token appropriately -- without you having to write a line of xDS compliant code! ## Connecting Client Side @@ -203,7 +269,7 @@ quilkin-proxies LoadBalancer 10.109.0.12 35.246.94.14 7000:30174/UDP We have a [Quilkin config yaml](https://github.com/googleforgames/quilkin/blob/{{GITHUB_REF_NAME}}/examples/agones-xonotic-xds/client-token.yaml) file all ready for you, that is configured to append the routing token `456` to each packet that passes through it, via the power of a -[ConcatenateBytes](../../services/proxy/filters/concatenate_bytes.md) Filter. +[Concatenate](../../services/proxy/filters/concatenate.md) Filter. Download `client-token.yaml` locally, so you can edit it: @@ -217,18 +283,19 @@ with it in `client-token.yaml`. Run this edited configuration locally with your quilkin binary as `quilkin -c ./client-token.yaml proxy`: ```shell -$ quilkin -c ./client-token.yaml proxy -{"timestamp":"2022-10-07T22:10:47.257635Z","level":"INFO","fields":{"message":"Starting Quilkin","version":"0.4.0-dev","commit":"c77260a2526542c564829a2c66935c60f00adcd2"},"target":"quilkin::cli"} -{"timestamp":"2022-10-07T22:10:47.258273Z","level":"INFO","fields":{"message":"Starting","port":7000,"proxy_id":"markmandel45"},"target":"quilkin::proxy"} -{"timestamp":"2022-10-07T22:10:47.258321Z","level":"INFO","fields":{"message":"Starting admin endpoint","address":"[::]:9092"},"target":"quilkin::admin"} -{"timestamp":"2022-10-07T22:10:47.258812Z","level":"INFO","fields":{"message":"Quilkin is ready"},"target":"quilkin::proxy"} +$ ./quilkin --config ./client-token.yaml proxy +2023-10-04T20:09:07.320780Z INFO quilkin::cli: src/cli.rs: Starting Quilkin version="0.7.0-dev" commit="d42db7e14c2e0e758e9a6eb655ccf4184941066c" +2023-10-04T20:09:07.321711Z INFO quilkin::admin: src/admin.rs: Starting admin endpoint address=[::]:8000 +2023-10-04T20:09:07.322089Z INFO quilkin::cli::proxy: src/cli/proxy.rs: Starting port=7777 proxy_id="markmandel45" +2023-10-04T20:09:07.322576Z INFO quilkin::cli::proxy: src/cli/proxy.rs: Quilkin is ready +2023-10-04T20:09:07.322692Z INFO qcmp_task{v4_addr=0.0.0.0:7600 v6_addr=[::]:7600}: quilkin::protocol: src/protocol.rs: awaiting qcmp packets v4_addr=0.0.0.0:7600 v6_addr=[::]:7600 ``` -Now connect to the local client proxy on "127.0.0.1:7000" via the "Multiplayer > Address" field in the +Now connect to the local client proxy on "[::1]:7777" via the "Multiplayer > Address" field in the Xonotic client, and Quilkin will take care of appending the routing token to all your UDP packets, which the Quilkin proxies will route to the Allocated GameServer, and you can play a gamee! -![xonotic-address.png](xonotic-address.png) +![xonotic-address-v6.png](xonotic-address-v6.png) ...And you didn't have to change the client or the dedicated game server 🤸 diff --git a/docs/src/deployment/quickstarts/xonotic-address-v6.png b/docs/src/deployment/quickstarts/xonotic-address-v6.png new file mode 100644 index 0000000000..00519cc4e3 Binary files /dev/null and b/docs/src/deployment/quickstarts/xonotic-address-v6.png differ diff --git a/docs/src/deployment/quickstarts/xonotic-address.png b/docs/src/deployment/quickstarts/xonotic-address.png deleted file mode 100644 index 63dbdab6ee..0000000000 Binary files a/docs/src/deployment/quickstarts/xonotic-address.png and /dev/null differ diff --git a/docs/src/services/agent.md b/docs/src/services/agent.md index cc6a266621..ce5672dec8 100644 --- a/docs/src/services/agent.md +++ b/docs/src/services/agent.md @@ -1,8 +1,8 @@ -# Control Plane Relay +# Control Plane Agents -| services | ports | Protocol | -|----------|-------|-----------| -| QCMP | 7600 | UDP(IPv4 OR IPv6) | +| services | ports | Protocol | +|----------|-------|-------------------| +| QCMP | 7600 | UDP(IPv4 OR IPv6) | > **Note:** This service is currently in active experimentation and development so there may be bugs which cause it to be unusable for production, as always @@ -12,6 +12,9 @@ For multi-cluster integration, Quilkin provides a `agent` service, that can be deployed to a cluster to act as a beacon for QCMP pings and forward cluster configuration information to a `relay` service +Agent configuration and functionality matches that of Control Plane Providers, such as +[Filesystem](./xds/providers/filesystem.md) and [Agones](./xds/providers/agones.md). + To view all options for the `agent` subcommand, run: ```shell @@ -19,6 +22,9 @@ $ quilkin agent --help {{#include ../../../target/quilkin.agent.commands}} ``` +> Each sub-control planes (`file`, `agones`, etc) matches the `quilkin manage` providers capabilities. +> Have a look at each of the [Control Plane > Providers](../services/xds.md) documentation for integration details. + ## Quickstart The simplest version of the `agent` service is just running `quilkin agent`, this will setup just the QCMP service allowing the agent to be pinged for diff --git a/docs/src/services/proxy/filters.md b/docs/src/services/proxy/filters.md index 9f8f6a8e37..5319f0e9c0 100644 --- a/docs/src/services/proxy/filters.md +++ b/docs/src/services/proxy/filters.md @@ -92,8 +92,8 @@ Quilkin includes several filters out of the box. |----------------------------------------------------|-------------------------------------------------------------------------------------------------------------| | [Capture] | Capture specific bytes from a packet and store them in [filter dynamic metadata](#filter-dynamic-metadata). | | [Compress](./filters/compress.md) | Compress and decompress packets data. | -| [ConcatenateBytes](./filters/concatenate_bytes.md) | Add authentication tokens to packets. | -| [Debug](./filters/concatenate_bytes.md) | Logs every packet. | +| [Concatenate](./filters/concatenate.md) | Add authentication tokens to packets. | +| [Debug](./filters/debug.md) | Logs every packet. | | [Drop](./filters/drop.md) | Drop all packets | | [Firewall](./filters/firewall.md) | Allowing/blocking traffic by IP and port. | | [LoadBalancer](./filters/load_balancer.md) | Distributes downstream packets among upstream endpoints. | diff --git a/docs/src/services/proxy/filters/capture.md b/docs/src/services/proxy/filters/capture.md index a21a72ac90..19e989488b 100644 --- a/docs/src/services/proxy/filters/capture.md +++ b/docs/src/services/proxy/filters/capture.md @@ -5,7 +5,7 @@ The `Capture` filter's job is to find a series of bytes within a packet, and cap down the chain. This is often used as a way of retrieving authentication tokens from a packet, and used in combination with -[ConcatenateBytes](concatenate_bytes.md) and +[Concatenate](concatenate.md) and [TokenRouter](token_router.md) filter to provide common packet routing utilities. ## Capture strategies diff --git a/docs/src/services/proxy/filters/concatenate_bytes.md b/docs/src/services/proxy/filters/concatenate.md similarity index 59% rename from docs/src/services/proxy/filters/concatenate_bytes.md rename to docs/src/services/proxy/filters/concatenate.md index 76bc61eaeb..e036c6ef56 100644 --- a/docs/src/services/proxy/filters/concatenate_bytes.md +++ b/docs/src/services/proxy/filters/concatenate.md @@ -1,11 +1,11 @@ -# ConcatenateBytes +# Concatenate -The `ConcatenateBytes` filter's job is to add a byte packet to either the beginning or end of each UDP packet that passes +The `Concatenate` filter's job is to add a byte packet to either the beginning or end of each UDP packet that passes through. This is commonly used to provide an auth token to each packet, so they can be routed appropriately. ## Filter name ```text -quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes +quilkin.filters.concatenate.v1alpha1.Concatenate ``` ## Configuration Examples @@ -13,7 +13,7 @@ quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes # let yaml = " version: v1alpha1 filters: - - name: quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes + - name: quilkin.filters.concatenate.v1alpha1.Concatenate config: on_read: APPEND on_write: DO_NOTHING @@ -26,8 +26,8 @@ clusters: # assert_eq!(config.filters.load().len(), 1); ``` -## Configuration Options ([Rust Doc](../../../../api/quilkin/filters/concatenate_bytes/struct.Config.html)) +## Configuration Options ([Rust Doc](../../../../api/quilkin/filters/concatenate/struct.Config.html)) ```yaml -{{#include ../../../../../target/quilkin.filters.concatenate_bytes.v1alpha1.yaml}} +{{#include ../../../../../target/quilkin.filters.concatenate.v1alpha1.yaml}} ``` diff --git a/docs/src/services/proxy/filters/token_router.md b/docs/src/services/proxy/filters/token_router.md index ed56428eef..d92d26d9a8 100644 --- a/docs/src/services/proxy/filters/token_router.md +++ b/docs/src/services/proxy/filters/token_router.md @@ -99,7 +99,7 @@ clusters: # assert_eq!(config.filters.load().len(), 2); ``` -On the game client side the [ConcatenateBytes](concatenate_bytes.md) filter could also be used to add authentication +On the game client side the [Concatenate](concatenate.md) filter could also be used to add authentication tokens to outgoing packets. [filter-dynamic-metadata]: ../filters.md#filter-dynamic-metadata diff --git a/docs/src/services/relay.md b/docs/src/services/relay.md index 5ebeabff92..58104bc889 100644 --- a/docs/src/services/relay.md +++ b/docs/src/services/relay.md @@ -1,9 +1,9 @@ # Control Plane Relay -| services | ports | Protocol | -|----------|-------|-----------| -| ADS | 7800 | gRPC(IPv4) | -| CPDS | 7900 | gRPC(IPv4) | +| services | ports | Protocol | +|----------|-------|--------------------| +| ADS | 7800 | gRPC(IPv4 OR IPv6) | +| CPDS | 7900 | gRPC(IPv4 OR IPv6) | > **Note:** This service is currently in active experimentation and development so there may be bugs which cause it to be unusable for production, as always @@ -25,6 +25,8 @@ To view all options for the `relay` subcommand, run: $ quilkin relay --help {{#include ../../../target/quilkin.relay.commands}} ``` +> Each sub-control planes (`file`, `agones`, etc) matches the `quilkin manage` providers capabilities. +> Have a look at each of the [Control Plane > Providers](../services/xds.md) documentation for integration details. ## Quickstart To get started with the relay service we need to start the relay service, and diff --git a/docs/src/services/xds.md b/docs/src/services/xds.md index 358a1b9ba6..b2051d9a10 100644 --- a/docs/src/services/xds.md +++ b/docs/src/services/xds.md @@ -4,7 +4,7 @@ |----------|-------|---------------------| | xDS | 7800 | gRPC (IPv4 OR IPv6) | -For multi-cluster integration, Quilkin provides a `manage` service, that can be +For single-cluster integration, Quilkin provides a `manage` service, that can be used with a number of configuration discovery providers to provide cluster configuration multiple [`proxy`s](./proxy.md). With each provider automating the complexity of a full xDS management control plane via integrations with popular diff --git a/docs/src/services/xds/metrics.md b/docs/src/services/xds/metrics.md index 488a2b6a06..f8f78d25ec 100644 --- a/docs/src/services/xds/metrics.md +++ b/docs/src/services/xds/metrics.md @@ -64,4 +64,4 @@ The following metrics are exposed when Quilkin is running as an [xDS provider](. of connected proxies since snapshots for disconnected proxies are only periodically cleared from the cache. -[DiscoveryRequest]: https://www.envoyproxy.io/docs/envoy/v1.22.0/api-v3/service/discovery/v3/discovery.proto.html?highlight=discoveryrequest#service-discovery-v3-discoveryrequest +[DiscoveryRequest]: https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/discovery/v3/discovery.proto.html#service-discovery-v3-discoveryrequest diff --git a/examples/agones-xonotic-relay/README.md b/examples/agones-xonotic-relay/README.md new file mode 100644 index 0000000000..e2f50ba206 --- /dev/null +++ b/examples/agones-xonotic-relay/README.md @@ -0,0 +1,3 @@ +# Agones & Xonotic via Relay Example + +This is the code example for the "Quickstart: Quilkin with Agones and Xonotic (Relay)" Guide, linked via the homepage. diff --git a/examples/agones-xonotic-relay/client-token.yaml b/examples/agones-xonotic-relay/client-token.yaml new file mode 100644 index 0000000000..440eadb8c4 --- /dev/null +++ b/examples/agones-xonotic-relay/client-token.yaml @@ -0,0 +1,26 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: v1alpha1 +filters: + - name: quilkin.filters.concatenate.v1alpha1.Concatenate + config: + on_read: APPEND + on_write: DO_NOTHING + bytes: NDU2 # 456 +clusters: + - endpoints: + - address: ${LOADBALANCER_IP}:7777 diff --git a/examples/agones-xonotic-relay/fleet.yaml b/examples/agones-xonotic-relay/fleet.yaml new file mode 100644 index 0000000000..4be622a6d7 --- /dev/null +++ b/examples/agones-xonotic-relay/fleet.yaml @@ -0,0 +1,38 @@ +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usually you would define a Fleet rather than a GameServerSet +# directly. This is here mostly for testing purposes + +apiVersion: "agones.dev/v1" +kind: Fleet +metadata: + name: xonotic +spec: + replicas: 3 + strategy: + type: Recreate + template: + spec: + ports: + - name: default + containerPort: 26000 + health: + initialDelaySeconds: 30 + periodSeconds: 60 + template: + spec: + containers: + - name: xonotic + image: us-docker.pkg.dev/agones-images/examples/xonotic-example:1.2 diff --git a/examples/agones-xonotic-relay/gameserverallocation.yaml b/examples/agones-xonotic-relay/gameserverallocation.yaml new file mode 100644 index 0000000000..920fbf657f --- /dev/null +++ b/examples/agones-xonotic-relay/gameserverallocation.yaml @@ -0,0 +1,23 @@ +# Copyright 2023 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: "allocation.agones.dev/v1" +kind: GameServerAllocation +spec: + selectors: + - matchLabels: + agones.dev/fleet: xonotic + metadata: + annotations: + quilkin.dev/tokens: NDU2 # 456 diff --git a/examples/agones-xonotic-relay/proxy-pool.yaml b/examples/agones-xonotic-relay/proxy-pool.yaml new file mode 100644 index 0000000000..bddbb2fdca --- /dev/null +++ b/examples/agones-xonotic-relay/proxy-pool.yaml @@ -0,0 +1,81 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Pool of Quilkin proxies, tied to the +# Quilkin relay control plane in relay-control-plane.yaml +# + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + role: proxy + name: quilkin-proxies +spec: + replicas: 3 + selector: + matchLabels: + role: proxy + template: + metadata: + labels: + role: proxy + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "8000" + spec: + containers: + - name: quilkin + image: us-docker.pkg.dev/quilkin-mark-dev/release/quilkin:0.7.0 + args: ["proxy", "--management-server", "http://quilkin-relay-agones:7800"] + env: + - name: RUST_LOG + value: info #,quilkin=trace + livenessProbe: + periodSeconds: 5 + failureThreshold: 3 + httpGet: + path: /live + port: 8000 + scheme: HTTP + readinessProbe: + periodSeconds: 1 + failureThreshold: 900 + successThreshold: 1 + httpGet: + path: /ready + port: 8000 + scheme: HTTP + ports: + - containerPort: 7777 + protocol: UDP + +--- +apiVersion: v1 +kind: Service +metadata: + name: quilkin-proxies +spec: + ports: + - port: 7777 + protocol: UDP + targetPort: 7777 + selector: + role: proxy + type: LoadBalancer diff --git a/examples/agones-xonotic-relay/relay-control-plane.yaml b/examples/agones-xonotic-relay/relay-control-plane.yaml new file mode 100644 index 0000000000..46e59c4f6c --- /dev/null +++ b/examples/agones-xonotic-relay/relay-control-plane.yaml @@ -0,0 +1,203 @@ +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# Everything to setup the relay and agent Agones control plane +# + +--- +# ANCHOR: config-map +apiVersion: v1 +kind: ConfigMap +metadata: + name: quilkin-relay-filter-config + labels: + quilkin.dev/configmap: "true" +data: + quilkin.yaml: | + version: v1alpha1 + filters: + - name: quilkin.filters.capture.v1alpha1.Capture + config: + suffix: + size: 3 + remove: true + - name: quilkin.filters.token_router.v1alpha1.TokenRouter +# ANCHOR_END: config-map + +--- + +# RBAC Setup for Agones Relay Control Plane + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: quilkin-agones +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: quilkin-agones +rules: + - apiGroups: + - agones.dev + resources: + - gameservers + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: quilkin-agones +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: quilkin-agones +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:default:quilkin-agones +--- + +# Quilkin Relay server + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + role: xds + name: quilkin-relay-agones +spec: + replicas: 1 + selector: + matchLabels: + role: xds + template: + metadata: + labels: + role: xds + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "8000" + spec: + containers: + - name: quilkin + args: + - relay + - agones + env: + - name: RUST_LOG + value: info + image: us-docker.pkg.dev/quilkin-mark-dev/release/quilkin:0.7.0 + livenessProbe: + periodSeconds: 5 + failureThreshold: 3 + httpGet: + path: /live + port: 8000 + scheme: HTTP + readinessProbe: + periodSeconds: 1 + failureThreshold: 900 + successThreshold: 1 + httpGet: + path: /ready + port: 8000 + scheme: HTTP + ports: + - containerPort: 7800 + protocol: TCP + serviceAccountName: quilkin-agones +--- + +# Quilkin Agones Agent + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + role: xds + name: quilkin-agones-agent +spec: + replicas: 1 + selector: + matchLabels: + role: xds + template: + metadata: + labels: + role: xds + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "8000" + spec: + containers: + - name: quilkin + args: + - agent + - --relay + - http://quilkin-relay-agones:7900 + - agones + env: + - name: RUST_LOG + value: info + image: us-docker.pkg.dev/quilkin-mark-dev/release/quilkin:0.7.0 + livenessProbe: + periodSeconds: 5 + failureThreshold: 3 + httpGet: + path: /live + port: 8000 + scheme: HTTP + readinessProbe: + periodSeconds: 1 + failureThreshold: 900 + successThreshold: 1 + httpGet: + path: /ready + port: 8000 + scheme: HTTP + serviceAccountName: quilkin-agones +--- +apiVersion: v1 +kind: Service +metadata: + name: quilkin-relay-agones +spec: + ports: + - name: ads + port: 7800 + protocol: TCP + targetPort: 7800 + - name: cpds + port: 7900 + protocol: TCP + targetPort: 7900 + selector: + role: xds diff --git a/examples/agones-xonotic-sidecar/README.md b/examples/agones-xonotic-sidecar/README.md index 975117c2a4..57232ba699 100644 --- a/examples/agones-xonotic-sidecar/README.md +++ b/examples/agones-xonotic-sidecar/README.md @@ -1,44 +1,3 @@ -# Agones & Xonotic Example +# Agones & Xonotic via Sidecar Example -An example of running [Xonotic](https://xonotic.org/) with Quilkin on an [Agones](https://agones.dev/) cluster, -utilising the sidecar integration pattern. - -To interact with the demo, you will need to download the Xonotic client. - -## Sidecar with no filter - -Run `kubectl apply -f ./sidecar.yaml` -to create a Fleet of Xonotic dedicated game servers, with traffic processed through a Quilkin sidecar proxy. - -This is particularly useful if you want to take advantage of the inbuilt metrics that Quilkin provides without -having to alter your dedicated game server. - -Since the configuration is so simple, we don't need a configuration file and can utilise the command line arguements. - -Connect to the Agones hosted Xonotic server via the "Multiplayer > Address" field in the Xonotic client. - -## Sidecar with compression filter - -Run `kubectl apply -f ./sidecar-compress.yaml` -to create a Fleet of Xonotic dedicated game servers, with traffic processed through a Quilkin sidecar proxy, -that is configured to decompress UDP traffic with the [Snappy](https://crates.io/crates/snap) -compression format. - -Since this is a more complex configuration, we provide a Quilkin a yaml configuration file through a `ConfigMap` -stored on the cluster. - -Instead of connecting Xonotic directly, take the IP and port from the Agones hosted dedicated server, and replace the -`${GAMESERVER_IP}` and `${GAMESERVER_PORT}` values in a local copy of `client-compress.yaml`. Run this configuration -locally as: - -`quilkin -c ./client-compress.yaml run` - -From there connect to the local client proxy on "127.0.0.1:7000" via the "Multiplayer > Address" field in the -Xonotic client, and Quilkin will take care of compressing the data for you without having to change either the -client or the dedicated game server. - -## Metrics - -The Quilkin sidecars are also annotated with the -[appropriate Prometheus annotations](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus#scraping-pod-metrics-via-annotations) -to inform Prometheus on how to scrape the Quilkin proxies for metrics. +This is the code example for the "Quickstart: Quilkin with Agones and Xonotic (Sidecar)" Guide, linked via the homepage. diff --git a/examples/agones-xonotic-sidecar/sidecar-compress.yaml b/examples/agones-xonotic-sidecar/sidecar-compress.yaml index 3ec4073e12..7c7bdb6cae 100644 --- a/examples/agones-xonotic-sidecar/sidecar-compress.yaml +++ b/examples/agones-xonotic-sidecar/sidecar-compress.yaml @@ -56,9 +56,9 @@ spec: spec: containers: - name: xonotic - image: gcr.io/agones-images/xonotic-example:0.8 + image: us-docker.pkg.dev/agones-images/examples/xonotic-example:1.2 - name: quilkin - image: us-docker.pkg.dev/quilkin/release/quilkin:0.6.0 + image: us-docker.pkg.dev/quilkin/release/quilkin:0.7.0 args: - proxy - --port=26001 diff --git a/examples/agones-xonotic-sidecar/sidecar.yaml b/examples/agones-xonotic-sidecar/sidecar.yaml index b5c32d9fd3..f82e9ecc75 100644 --- a/examples/agones-xonotic-sidecar/sidecar.yaml +++ b/examples/agones-xonotic-sidecar/sidecar.yaml @@ -39,9 +39,9 @@ spec: spec: containers: - name: xonotic - image: gcr.io/agones-images/xonotic-example:0.8 + image: us-docker.pkg.dev/agones-images/examples/xonotic-example:1.2 - name: quilkin - image: us-docker.pkg.dev/quilkin/release/quilkin:0.6.0 + image: us-docker.pkg.dev/quilkin/release/quilkin:0.7.0 args: ["proxy", "--port", "26001", "--to", "127.0.0.1:26000"] livenessProbe: httpGet: diff --git a/examples/agones-xonotic-xds/README.md b/examples/agones-xonotic-xds/README.md index 42b1569a56..fa6318e285 100644 --- a/examples/agones-xonotic-xds/README.md +++ b/examples/agones-xonotic-xds/README.md @@ -1,111 +1,3 @@ -# Agones & Xonotic Example +# Agones & Xonotic via xDS Provider Example -An example of running [Xonotic](https://xonotic.org/) with Quilkin on an [Agones](https://agones.dev/) cluster, -utlising the Quilkin xDS Agones provider, with a TokenRouter to provide routing and access control to the -allocated `GameServer` instance. - -To interact with the demo, you will need to download the Xonotic client and an existing Agones Kubernetes cluster. - -## Installation on the Cluster - -To install Quilkin as an Agones integrated xDS control plane, we can create a deployment of Quilkin running in -`manage agones` mode, with the appropriate permissions. - -```shell -$ kubectl apply -f ./xds-control-plane.yaml -configmap/quilkin-xds-filter-config created -serviceaccount/quilkin-agones created -clusterrole.rbac.authorization.k8s.io/quilkin-agones created -rolebinding.rbac.authorization.k8s.io/quilkin-agones created -deployment.apps/quilkin-manage-agones created -service/quilkin-manage-agones created -$ kubectl get pods -NAME READY STATUS RESTARTS AGE -quilkin-manage-agones-68b47457d4-42fxl 1/1 Running 0 3s -``` - -This also creates an internal `Service` endpoint for our Quilkin proxy instances to connect to named -`quilkin-manage-agones`. - -To install the Quilkin Proxy pool which connects to the above xDS provider, we can split up a Deployment of Quilkin -instances that point to the aforementioned Service, like so: - -```shell -$ kubectl apply -f ./proxy-pool.yaml -deployment.apps/quilkin-proxies created -service/quilkin-proxies created -$ kubectl get pods -NAME READY STATUS RESTARTS AGE -quilkin-manage-agones-68b47457d4-42fxl 1/1 Running 0 20m -quilkin-proxies-7448987cfb-6kbsz 1/1 Running 0 3s -quilkin-proxies-7448987cfb-p6krc 1/1 Running 0 3s -quilkin-proxies-7448987cfb-s9gxz 1/1 Running 0 3s -``` - -We can now see that we have 3 proxies running alongside and connected to our xDS provider. - -## Create the `Fleet` - -Next, create the `Fleet` of Xonotic `GameServer` instances: - -```shell -$ kubectl apply -f ./fleet.yaml -fleet.agones.dev/xonotic created -$ kubectl get gameservers # run until they are Ready -NAME STATE ADDRESS PORT NODE AGE -xonotic-d7rfx-55j7q Ready 34.168.170.51 7226 gke-agones-default-534a3f8d-ifpc 34s -xonotic-d7rfx-nx7xr Ready 34.168.170.51 7984 gke-agones-default-534a3f8d-ifpc 34s -xonotic-d7rfx-sn5d6 Ready 34.168.170.51 7036 gke-agones-default-534a3f8d-ifpc 34s -``` - -To let the Quilkin xDS provider know what token will route to which `GameServer` we need to apply the -`quilkin.dev/tokens` annotation to an allocated `GameServer`, with the token content as its value - so let's create -an allocation, and apply the annotation all in one go! - -```shell -$ kubectl create -f ./gameserverallocation.yaml -gameserverallocation.allocation.agones.dev/xonotic-d7rfx-nx7xr created -$ kubectl get gs -NAME STATE ADDRESS PORT NODE AGE -xonotic-d7rfx-55j7q Allocated 34.168.170.51 7226 gke-agones-default-534a3f8d-ifpc 23m -xonotic-d7rfx-nx7xr Ready 34.168.170.51 7984 gke-agones-default-534a3f8d-ifpc 23m -xonotic-d7rfx-sn5d6 Ready 34.168.170.51 7036 gke-agones-default-534a3f8d-ifpc 23m -``` - -> Don't do this more than once, as then multiple allocated `GameServers` will have the same routing token! - -## Connecting Client Side - -Instead of connecting to Xonotic or an Agones `GameServer` directly, we'll want to grab the IP and exposed port of -the `Service` that fronts all our Quilkin proxies: - -```shell -$ kubectl get service quilkin-proxies -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -quilkin-proxies LoadBalancer 10.109.0.12 35.246.94.14 7000:30174/UDP 3h22m -``` - -We then take the EXTERNAL-IP and port from the `quilkin-proxies` service, and replace the`${LOADBALANCER_IP}` -with it in `client-token.yaml`. - -Run this configuration locally as: - -```shell -$ quilkin -c ./client-token.yaml proxy -{"timestamp":"2022-10-07T22:10:47.257635Z","level":"INFO","fields":{"message":"Starting Quilkin","version":"0.4.0-dev","commit":"c77260a2526542c564829a2c66935c60f00adcd2"},"target":"quilkin::cli"} -{"timestamp":"2022-10-07T22:10:47.258273Z","level":"INFO","fields":{"message":"Starting","port":7000,"proxy_id":"markmandel45"},"target":"quilkin::proxy"} -{"timestamp":"2022-10-07T22:10:47.258321Z","level":"INFO","fields":{"message":"Starting admin endpoint","address":"[::]:9092"},"target":"quilkin::admin"} -{"timestamp":"2022-10-07T22:10:47.258812Z","level":"INFO","fields":{"message":"Quilkin is ready"},"target":"quilkin::proxy"} -``` - -Now connect to the local client proxy on "127.0.0.1:7000" via the "Multiplayer > Address" field in the -Xonotic client, and Quilkin will take care of appending the routing token to all your UDP packets, which the Quilkin -proxies will route to the Allocated GameServer, and you can play a gamee! - -...And you didn't have to change the client or the dedicated game server 🤸 - -## Metrics - -The Quilkin instances are also annotated with the -[appropriate Prometheus annotations](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus#scraping-pod-metrics-via-annotations) -to inform Prometheus on how to scrape the Quilkin proxies for metrics. +This is the code example for the "Quickstart: Quilkin with Agones and Xonotic (xDS)" Guide, linked via the homepage. diff --git a/examples/agones-xonotic-xds/client-token.yaml b/examples/agones-xonotic-xds/client-token.yaml index 1b99fc536a..ee52927331 100644 --- a/examples/agones-xonotic-xds/client-token.yaml +++ b/examples/agones-xonotic-xds/client-token.yaml @@ -16,7 +16,7 @@ version: v1alpha1 filters: - - name: quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes + - name: quilkin.filters.concatenate.v1alpha1.Concatenate config: on_read: APPEND on_write: DO_NOTHING diff --git a/examples/agones-xonotic-xds/fleet.yaml b/examples/agones-xonotic-xds/fleet.yaml index 75a90db8ba..228854ccd3 100644 --- a/examples/agones-xonotic-xds/fleet.yaml +++ b/examples/agones-xonotic-xds/fleet.yaml @@ -33,4 +33,4 @@ spec: spec: containers: - name: xonotic - image: gcr.io/agones-images/xonotic-example:0.8 + image: us-docker.pkg.dev/agones-images/examples/xonotic-example:1.2 diff --git a/examples/agones-xonotic-xds/proxy-pool.yaml b/examples/agones-xonotic-xds/proxy-pool.yaml index e7ed982584..c95b060b9e 100644 --- a/examples/agones-xonotic-xds/proxy-pool.yaml +++ b/examples/agones-xonotic-xds/proxy-pool.yaml @@ -42,17 +42,26 @@ spec: spec: containers: - name: quilkin - image: us-docker.pkg.dev/quilkin/release/quilkin:0.6.0 + image: us-docker.pkg.dev/quilkin/release/quilkin:0.7.0 args: ["proxy", "--management-server", "http://quilkin-manage-agones:80"] env: - name: RUST_LOG value: info #,quilkin=trace livenessProbe: + periodSeconds: 5 failureThreshold: 3 httpGet: path: /live port: 8000 scheme: HTTP + readinessProbe: + periodSeconds: 1 + failureThreshold: 900 + successThreshold: 1 + httpGet: + path: /ready + port: 8000 + scheme: HTTP ports: - containerPort: 7777 protocol: UDP diff --git a/examples/agones-xonotic-xds/xds-control-plane.yaml b/examples/agones-xonotic-xds/xds-control-plane.yaml index 7d01591c1a..e4b0d8d973 100644 --- a/examples/agones-xonotic-xds/xds-control-plane.yaml +++ b/examples/agones-xonotic-xds/xds-control-plane.yaml @@ -113,13 +113,22 @@ spec: env: - name: RUST_LOG value: info - image: us-docker.pkg.dev/quilkin/release/quilkin:0.6.0 + image: us-docker.pkg.dev/quilkin/release/quilkin:0.7.0 livenessProbe: + periodSeconds: 5 failureThreshold: 3 httpGet: path: /live port: 8000 scheme: HTTP + readinessProbe: + periodSeconds: 1 + failureThreshold: 900 + successThreshold: 1 + httpGet: + path: /ready + port: 8000 + scheme: HTTP ports: - containerPort: 7800 protocol: TCP diff --git a/examples/quilkin-filter-example/README.md b/examples/quilkin-filter-example/README.md index 353724c9f7..b4a15cb622 100644 --- a/examples/quilkin-filter-example/README.md +++ b/examples/quilkin-filter-example/README.md @@ -1,4 +1,3 @@ # Quilkin filter example -This crate contains the code example on how to [add filters to Quilkin](https://github.com/googleforgames/quilkin/blob/main/docs/extensions/filters/writing_custom_filters.md) - +This is the code example for the "Writing Custom Filters" Guide, linked via the homepage. diff --git a/examples/quilkin-filter-example/src/main.rs b/examples/quilkin-filter-example/src/main.rs index 57e2c5ccad..1175c52d6c 100644 --- a/examples/quilkin-filter-example/src/main.rs +++ b/examples/quilkin-filter-example/src/main.rs @@ -113,6 +113,8 @@ async fn main() -> quilkin::Result<()> { ) }); - proxy.run(config.into(), shutdown_rx).await + let admin = quilkin::cli::Admin::Proxy(<_>::default()); + + proxy.run(config.into(), admin, shutdown_rx).await } // ANCHOR_END: run diff --git a/image/Dockerfile b/image/Dockerfile index c73cc4e673..298b7f66cd 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gcr.io/distroless/cc-debian11:nonroot as base +FROM gcr.io/distroless/cc-debian12:nonroot as base WORKDIR / COPY ./license.html . COPY ./dependencies-src.zip . -COPY ./target/image/quilkin . +COPY --chown=nonroot:nonroot ./target/image/quilkin . USER nonroot:nonroot ENTRYPOINT ["/quilkin"] diff --git a/proto/quilkin/filters/concatenate_bytes/v1alpha1/concatenate_bytes.proto b/proto/quilkin/filters/concatenate/v1alpha1/concatenate.proto similarity index 91% rename from proto/quilkin/filters/concatenate_bytes/v1alpha1/concatenate_bytes.proto rename to proto/quilkin/filters/concatenate/v1alpha1/concatenate.proto index b2c43f446e..e71589d464 100644 --- a/proto/quilkin/filters/concatenate_bytes/v1alpha1/concatenate_bytes.proto +++ b/proto/quilkin/filters/concatenate/v1alpha1/concatenate.proto @@ -16,9 +16,9 @@ syntax = "proto3"; -package quilkin.filters.concatenate_bytes.v1alpha1; +package quilkin.filters.concatenate.v1alpha1; -message ConcatenateBytes { +message Concatenate { enum Strategy { DoNothing = 0; Append = 1; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 19c4041fb7..28eebdf5c3 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -13,5 +13,5 @@ # limitations under the License. [toolchain] -channel = "1.72.0" +channel = "1.73.0" components = ["rustfmt", "clippy"] diff --git a/src/admin.rs b/src/admin.rs deleted file mode 100644 index c18afe4126..0000000000 --- a/src/admin.rs +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -mod health; - -use std::convert::Infallible; -use std::sync::Arc; - -use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Method, Request, Response, Server as HyperServer, StatusCode}; - -use self::health::Health; -use crate::config::Config; - -pub const PORT: u16 = 8000; - -/// Define which mode Quilkin is in. -#[derive(Copy, Clone, Debug)] -pub enum Mode { - Proxy, - Xds, -} - -pub fn server( - mode: Mode, - config: Arc, - address: Option, -) -> tokio::task::JoinHandle> { - let address = address.unwrap_or_else(|| (std::net::Ipv6Addr::UNSPECIFIED, PORT).into()); - let health = Health::new(); - tracing::info!(address = %address, "Starting admin endpoint"); - - let make_svc = make_service_fn(move |_conn| { - let config = config.clone(); - let health = health.clone(); - async move { - let config = config.clone(); - let health = health.clone(); - Ok::<_, Infallible>(service_fn(move |req| { - let config = config.clone(); - let health = health.clone(); - async move { Ok::<_, Infallible>(handle_request(req, mode, config, health)) } - })) - } - }); - - tokio::spawn(HyperServer::bind(&address).serve(make_svc)) -} - -fn handle_request( - request: Request, - mode: Mode, - config: Arc, - health: Health, -) -> Response { - match (request.method(), request.uri().path()) { - (&Method::GET, "/metrics") => collect_metrics(), - (&Method::GET, "/live" | "/livez") => health.check_healthy(), - (&Method::GET, "/ready" | "/readyz") => match mode { - Mode::Proxy => check_proxy_readiness(&config), - Mode::Xds => health.check_healthy(), - }, - (&Method::GET, "/config") => match serde_json::to_string(&config) { - Ok(body) => Response::builder() - .status(StatusCode::OK) - .header( - "Content-Type", - hyper::header::HeaderValue::from_static("application/json"), - ) - .body(Body::from(body)) - .unwrap(), - Err(err) => Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(Body::from(format!("failed to create config dump: {err}"))) - .unwrap(), - }, - (_, _) => { - let mut response = Response::new(Body::empty()); - *response.status_mut() = StatusCode::NOT_FOUND; - response - } - } -} - -fn check_proxy_readiness(config: &Config) -> Response { - if config.clusters.read().endpoints().count() > 0 { - return Response::new("ok".into()); - } - - let mut response = Response::new(Body::empty()); - *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - response -} - -fn collect_metrics() -> Response { - let mut response = Response::new(Body::empty()); - let mut buffer = vec![]; - let encoder = prometheus::TextEncoder::new(); - let body = - prometheus::Encoder::encode(&encoder, &crate::metrics::registry().gather(), &mut buffer) - .map_err(|error| tracing::warn!(%error, "Failed to encode metrics")) - .and_then(|_| { - String::from_utf8(buffer) - .map(Body::from) - .map_err(|error| tracing::warn!(%error, "Failed to convert metrics to utf8")) - }); - - match body { - Ok(body) => { - *response.body_mut() = body; - } - Err(_) => { - *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - } - }; - - response -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::endpoint::Endpoint; - - #[tokio::test] - async fn collect_metrics() { - let response = super::collect_metrics(); - assert_eq!(response.status(), hyper::StatusCode::OK); - } - - #[test] - fn check_proxy_readiness() { - let config = Config::default(); - assert_eq!(config.clusters.read().endpoints().count(), 0); - - let response = super::check_proxy_readiness(&config); - assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); - - config - .clusters - .write() - .insert_default([Endpoint::new((std::net::Ipv4Addr::LOCALHOST, 25999).into())].into()); - - let response = super::check_proxy_readiness(&config); - assert_eq!(response.status(), StatusCode::OK); - } -} diff --git a/src/cli.rs b/src/cli.rs index ebaba32e35..055e8bf628 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -14,6 +14,8 @@ * limitations under the License. */ +pub(crate) mod admin; + use std::{ path::{Path, PathBuf}, sync::Arc, @@ -23,12 +25,12 @@ use clap::builder::TypedValueParser; use clap::crate_version; use tokio::{signal, sync::watch}; -use crate::{admin::Mode, Config}; +use crate::Config; use strum_macros::{Display, EnumString}; pub use self::{ - agent::Agent, generate_config_schema::GenerateConfigSchema, manage::Manage, proxy::Proxy, - qcmp::Qcmp, relay::Relay, + admin::Admin, agent::Agent, generate_config_schema::GenerateConfigSchema, manage::Manage, + proxy::Proxy, qcmp::Qcmp, relay::Relay, }; macro_rules! define_port { @@ -106,10 +108,21 @@ pub enum Commands { } impl Commands { - pub fn admin_mode(&self) -> Option { + pub fn admin_mode(&self) -> Option { match self { - Self::Proxy(_) | Self::Agent(_) => Some(Mode::Proxy), - Self::Relay(_) | Self::Manage(_) => Some(Mode::Xds), + Self::Proxy(proxy) => Some(Admin::Proxy(proxy::RuntimeConfig { + idle_request_interval_secs: proxy.idle_request_interval_secs, + ..<_>::default() + })), + Self::Agent(agent) => Some(Admin::Agent(agent::RuntimeConfig { + idle_request_interval_secs: agent.idle_request_interval_secs, + ..<_>::default() + })), + Self::Relay(relay) => Some(Admin::Relay(relay::RuntimeConfig { + idle_request_interval_secs: relay.idle_request_interval_secs, + ..<_>::default() + })), + Self::Manage(_) => Some(Admin::Manage(<_>::default())), Self::GenerateConfigSchema(_) | Self::Qcmp(_) => None, } } @@ -148,24 +161,24 @@ impl Cli { "Starting Quilkin" ); - if let Commands::Qcmp(Qcmp::Ping(ping)) = self.command { - return ping.run().await; + // Non-long running commands (e.g. ones with no administration server) + // are executed here. + match self.command { + Commands::Qcmp(Qcmp::Ping(ping)) => return ping.run().await, + Commands::GenerateConfigSchema(generator) => { + return generator.generate_config_schema(); + } + _ => {} } tracing::debug!(cli = ?self, "config parameters"); let config = Arc::new(Self::read_config(self.config)?); - let _admin_task = self - .command - .admin_mode() - .filter(|_| !self.no_admin) - .map(|mode| { - tokio::spawn(crate::admin::server( - mode, - config.clone(), - self.admin_address, - )) - }); + let mode = self.command.admin_mode().unwrap(); + + if !self.no_admin { + mode.server(config.clone(), self.admin_address); + } let (shutdown_tx, shutdown_rx) = watch::channel::<()>(()); @@ -191,37 +204,45 @@ impl Cli { let fut = tryhard::retry_fn({ let shutdown_rx = shutdown_rx.clone(); + let mode = mode.clone(); move || match self.command.clone() { Commands::Agent(agent) => { let config = config.clone(); let shutdown_rx = shutdown_rx.clone(); - tokio::spawn( - async move { agent.run(config.clone(), shutdown_rx.clone()).await }, - ) + let mode = mode.clone(); + tokio::spawn(async move { + agent.run(config.clone(), mode, shutdown_rx.clone()).await + }) } Commands::Proxy(runner) => { let config = config.clone(); let shutdown_rx = shutdown_rx.clone(); - tokio::spawn( - async move { runner.run(config.clone(), shutdown_rx.clone()).await }, - ) + let mode = mode.clone(); + tokio::spawn(async move { + runner + .run(config.clone(), mode.clone(), shutdown_rx.clone()) + .await + }) } Commands::Manage(manager) => { let config = config.clone(); let shutdown_rx = shutdown_rx.clone(); + let mode = mode.clone(); tokio::spawn(async move { - manager.manage(config.clone(), shutdown_rx.clone()).await + manager + .manage(config.clone(), mode, shutdown_rx.clone()) + .await }) } - Commands::GenerateConfigSchema(generator) => { - tokio::spawn(std::future::ready(generator.generate_config_schema())) - } Commands::Relay(relay) => { let config = config.clone(); let shutdown_rx = shutdown_rx.clone(); - tokio::spawn(async move { relay.relay(config, shutdown_rx.clone()).await }) + let mode = mode.clone(); + tokio::spawn( + async move { relay.relay(config, mode, shutdown_rx.clone()).await }, + ) } - Commands::Qcmp(_) => unreachable!(), + Commands::GenerateConfigSchema(_) | Commands::Qcmp(_) => unreachable!(), } }) .retries(3) @@ -354,6 +375,7 @@ mod tests { region: None, sub_zone: None, zone: None, + idle_request_interval_secs: admin::IDLE_REQUEST_INTERVAL_SECS, qcmp_port: crate::test_utils::available_addr(&AddressType::Random) .await .port(), diff --git a/src/cli/admin.rs b/src/cli/admin.rs new file mode 100644 index 0000000000..1d743b93a4 --- /dev/null +++ b/src/cli/admin.rs @@ -0,0 +1,219 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +mod health; + +use std::convert::Infallible; +use std::sync::Arc; + +use hyper::service::{make_service_fn, service_fn}; +use hyper::{Body, Method, Request, Response, Server as HyperServer, StatusCode}; + +use self::health::Health; +use crate::config::Config; + +use super::{agent, manage, proxy, relay}; + +pub const PORT: u16 = 8000; + +pub(crate) const IDLE_REQUEST_INTERVAL_SECS: u64 = 30; + +/// The runtime mode of Quilkin, which contains various runtime configurations +/// specific to a mode. +#[derive(Clone, Debug)] +pub enum Admin { + Proxy(proxy::RuntimeConfig), + Relay(relay::RuntimeConfig), + Manage(manage::RuntimeConfig), + Agent(agent::RuntimeConfig), +} + +impl Admin { + #[track_caller] + pub fn unwrap_agent(&self) -> &agent::RuntimeConfig { + match self { + Self::Agent(config) => config, + _ => panic!("attempted to unwrap agent config when not in agent mode"), + } + } + + #[track_caller] + pub fn unwrap_proxy(&self) -> &proxy::RuntimeConfig { + match self { + Self::Proxy(config) => config, + _ => panic!("attempted to unwrap proxy config when not in proxy mode"), + } + } + + #[track_caller] + pub fn unwrap_relay(&self) -> &relay::RuntimeConfig { + match self { + Self::Relay(config) => config, + _ => panic!("attempted to unwrap relay config when not in relay mode"), + } + } + + #[track_caller] + pub fn unwrap_manage(&self) -> &manage::RuntimeConfig { + match self { + Self::Manage(config) => config, + _ => panic!("attempted to unwrap relay config when not in relay mode"), + } + } + + pub fn idle_request_interval_secs(&self) -> u64 { + match self { + Self::Proxy(config) => config.idle_request_interval_secs, + Self::Agent(config) => config.idle_request_interval_secs, + Self::Relay(config) => config.idle_request_interval_secs, + _ => IDLE_REQUEST_INTERVAL_SECS, + } + } + + pub fn server( + &self, + config: Arc, + address: Option, + ) -> tokio::task::JoinHandle> { + let address = address.unwrap_or_else(|| (std::net::Ipv6Addr::UNSPECIFIED, PORT).into()); + let health = Health::new(); + tracing::info!(address = %address, "Starting admin endpoint"); + + let mode = self.clone(); + let make_svc = make_service_fn(move |_conn| { + let config = config.clone(); + let health = health.clone(); + let mode = mode.clone(); + async move { + let config = config.clone(); + let health = health.clone(); + let mode = mode.clone(); + Ok::<_, Infallible>(service_fn(move |req| { + let config = config.clone(); + let health = health.clone(); + let mode = mode.clone(); + async move { Ok::<_, Infallible>(mode.handle_request(req, config, health)) } + })) + } + }); + + tokio::spawn(HyperServer::bind(&address).serve(make_svc)) + } + + fn is_ready(&self, config: &Config) -> bool { + match &self { + Self::Proxy(proxy) => proxy.is_ready(config), + Self::Agent(agent) => agent.is_ready(), + Self::Manage(manage) => manage.is_ready(), + Self::Relay(relay) => relay.is_ready(), + } + } + + fn handle_request( + &self, + request: Request, + config: Arc, + health: Health, + ) -> Response { + match (request.method(), request.uri().path()) { + (&Method::GET, "/metrics") => collect_metrics(), + (&Method::GET, "/live" | "/livez") => health.check_liveness(), + (&Method::GET, "/ready" | "/readyz") => check_readiness(|| self.is_ready(&config)), + (&Method::GET, "/config") => match serde_json::to_string(&config) { + Ok(body) => Response::builder() + .status(StatusCode::OK) + .header( + "Content-Type", + hyper::header::HeaderValue::from_static("application/json"), + ) + .body(Body::from(body)) + .unwrap(), + Err(err) => Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(format!("failed to create config dump: {err}"))) + .unwrap(), + }, + (_, _) => { + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::NOT_FOUND; + response + } + } + } +} + +fn check_readiness(check: impl Fn() -> bool) -> Response { + if (check)() { + return Response::new("ok".into()); + } + + let mut response = Response::new(Body::empty()); + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + response +} + +fn collect_metrics() -> Response { + let mut response = Response::new(Body::empty()); + let mut buffer = vec![]; + let encoder = prometheus::TextEncoder::new(); + let body = + prometheus::Encoder::encode(&encoder, &crate::metrics::registry().gather(), &mut buffer) + .map_err(|error| tracing::warn!(%error, "Failed to encode metrics")) + .and_then(|_| { + String::from_utf8(buffer) + .map(Body::from) + .map_err(|error| tracing::warn!(%error, "Failed to convert metrics to utf8")) + }); + + match body { + Ok(body) => { + *response.body_mut() = body; + } + Err(_) => { + *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + } + }; + + response +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::endpoint::Endpoint; + + #[tokio::test] + async fn collect_metrics() { + let response = super::collect_metrics(); + assert_eq!(response.status(), hyper::StatusCode::OK); + } + + #[test] + fn check_proxy_readiness() { + let config = crate::Config::default(); + assert_eq!(config.clusters.read().endpoints().count(), 0); + + let admin = Admin::Proxy(<_>::default()); + assert!(!admin.is_ready(&config)); + + config + .clusters + .write() + .insert_default([Endpoint::new((std::net::Ipv4Addr::LOCALHOST, 25999).into())].into()); + + assert!(admin.is_ready(&config)); + } +} diff --git a/src/admin/health.rs b/src/cli/admin/health.rs similarity index 93% rename from src/admin/health.rs rename to src/cli/admin/health.rs index 20282c5273..a685ff8490 100644 --- a/src/admin/health.rs +++ b/src/cli/admin/health.rs @@ -44,7 +44,7 @@ impl Health { } /// returns a HTTP 200 response if the proxy is healthy. - pub fn check_healthy(&self) -> Response { + pub fn check_liveness(&self) -> Response { if self.healthy.load(Relaxed) { return Response::new("ok".into()); }; @@ -65,14 +65,14 @@ mod tests { fn panic_hook() { let health = Health::new(); - let response = health.check_healthy(); + let response = health.check_liveness(); assert_eq!(response.status(), StatusCode::OK); let _ = std::panic::catch_unwind(|| { panic!("oh no!"); }); - let response = health.check_healthy(); + let response = health.check_liveness(); assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); } } diff --git a/src/cli/agent.rs b/src/cli/agent.rs index 9ef76506e1..c66f1fd0df 100644 --- a/src/cli/agent.rs +++ b/src/cli/agent.rs @@ -14,8 +14,12 @@ * limitations under the License. */ -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use super::Admin; use crate::config::Config; define_port!(7600); @@ -47,6 +51,10 @@ pub struct Agent { /// The configuration source for a management server. #[clap(subcommand)] pub provider: Option, + /// The interval in seconds at which the agent will wait for a discovery + /// request from a relay server before restarting the connection. + #[clap(long, env = "QUILKIN_IDLE_REQUEST_INTERVAL_SECS", default_value_t = super::admin::IDLE_REQUEST_INTERVAL_SECS)] + pub idle_request_interval_secs: u64, } impl Default for Agent { @@ -58,6 +66,7 @@ impl Default for Agent { zone: <_>::default(), sub_zone: <_>::default(), provider: <_>::default(), + idle_request_interval_secs: super::admin::IDLE_REQUEST_INTERVAL_SECS, } } } @@ -66,6 +75,7 @@ impl Agent { pub async fn run( &self, config: Arc, + mode: Admin, mut shutdown_rx: tokio::sync::watch::Receiver<()>, ) -> crate::Result<()> { let locality = (self.region.is_some() || self.zone.is_some() || self.sub_zone.is_some()) @@ -75,14 +85,21 @@ impl Agent { sub_zone: self.sub_zone.clone().unwrap_or_default(), }); + let runtime_config = mode.unwrap_agent(); + let _mds_task = if !self.relay.is_empty() { let _provider_task = match self.provider.as_ref() { - Some(provider) => Some(provider.spawn(config.clone(), locality.clone())), + Some(provider) => Some(provider.spawn( + config.clone(), + runtime_config.provider_is_healthy.clone(), + locality.clone(), + )), None => return Err(eyre::eyre!("no configuration provider given")), }; let task = crate::xds::client::MdsClient::connect( String::clone(&config.id.load()), + mode.clone(), self.relay.clone(), ); @@ -99,3 +116,17 @@ impl Agent { shutdown_rx.changed().await.map_err(From::from) } } + +#[derive(Clone, Debug, Default)] +pub struct RuntimeConfig { + pub idle_request_interval_secs: u64, + pub provider_is_healthy: Arc, + pub relay_is_healthy: Arc, +} + +impl RuntimeConfig { + pub fn is_ready(&self) -> bool { + self.provider_is_healthy.load(Ordering::SeqCst) + && self.relay_is_healthy.load(Ordering::SeqCst) + } +} diff --git a/src/cli/manage.rs b/src/cli/manage.rs index 22b226bd03..34e9f9703b 100644 --- a/src/cli/manage.rs +++ b/src/cli/manage.rs @@ -14,6 +14,13 @@ * limitations under the License. */ +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +use super::Admin; + use futures::TryFutureExt; define_port!(7800); @@ -49,6 +56,7 @@ impl Manage { pub async fn manage( &self, config: std::sync::Arc, + mode: Admin, mut shutdown_rx: tokio::sync::watch::Receiver<()>, ) -> crate::Result<()> { let locality = (self.region.is_some() || self.zone.is_some() || self.sub_zone.is_some()) @@ -64,12 +72,18 @@ impl Manage { .modify(|map| map.update_unlocated_endpoints(locality.clone())); } - let provider_task = self.provider.spawn(config.clone(), locality.clone()); + let runtime_config = mode.unwrap_manage(); + let provider_task = self.provider.spawn( + config.clone(), + runtime_config.provider_is_healthy.clone(), + locality.clone(), + ); let _relay_stream = if !self.relay.is_empty() { tracing::info!("connecting to relay server"); let client = crate::xds::client::MdsClient::connect( String::clone(&config.id.load()), + mode.clone(), self.relay.clone(), ) .await?; @@ -89,3 +103,14 @@ impl Manage { } } } + +#[derive(Clone, Debug, Default)] +pub struct RuntimeConfig { + pub provider_is_healthy: Arc, +} + +impl RuntimeConfig { + pub fn is_ready(&self) -> bool { + self.provider_is_healthy.load(Ordering::SeqCst) + } +} diff --git a/src/cli/proxy.rs b/src/cli/proxy.rs index 0b1e891a89..4dfe702e8f 100644 --- a/src/cli/proxy.rs +++ b/src/cli/proxy.rs @@ -14,10 +14,18 @@ * limitations under the License. */ -use std::{net::SocketAddr, sync::Arc, time::Duration}; +use std::{ + net::SocketAddr, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; use tonic::transport::Endpoint; +use super::Admin; use crate::{proxy::SessionMap, xds::ResourceType, Config, Result}; #[cfg(doc)] @@ -48,7 +56,7 @@ pub struct Proxy { pub to: Vec, /// The interval in seconds at which the relay will send a discovery request /// to an management server after receiving no updates. - #[clap(long, env = "QUILKIN_IDLE_REQUEST_INTERVAL_SECS", default_value_t = crate::xds::server::IDLE_REQUEST_INTERVAL_SECS)] + #[clap(long, env = "QUILKIN_IDLE_REQUEST_INTERVAL_SECS", default_value_t = super::admin::IDLE_REQUEST_INTERVAL_SECS)] pub idle_request_interval_secs: u64, } @@ -60,7 +68,7 @@ impl Default for Proxy { port: PORT, qcmp_port: QCMP_PORT, to: <_>::default(), - idle_request_interval_secs: crate::xds::server::IDLE_REQUEST_INTERVAL_SECS, + idle_request_interval_secs: super::admin::IDLE_REQUEST_INTERVAL_SECS, } } } @@ -70,6 +78,7 @@ impl Proxy { pub async fn run( &self, config: std::sync::Arc, + mode: Admin, mut shutdown_rx: tokio::sync::watch::Receiver<()>, ) -> crate::Result<()> { const SESSION_TIMEOUT_SECONDS: Duration = Duration::from_secs(60); @@ -114,11 +123,21 @@ impl Proxy { tracing::info!(port = self.port, proxy_id = &*id, "Starting"); let sessions = SessionMap::new(SESSION_TIMEOUT_SECONDS, SESSION_EXPIRY_POLL_INTERVAL); + let runtime_config = mode.unwrap_proxy(); let _xds_stream = if !self.management_server.is_empty() { - let client = - crate::xds::AdsClient::connect(String::clone(&id), self.management_server.clone()) - .await?; + { + let mut lock = runtime_config.xds_is_healthy.write(); + let check: Arc = <_>::default(); + *lock = Some(check.clone()); + } + + let client = crate::xds::AdsClient::connect( + String::clone(&id), + mode.clone(), + self.management_server.clone(), + ) + .await?; let mut stream = client.xds_client_stream(config.clone(), self.idle_request_interval_secs); @@ -399,3 +418,20 @@ mod tests { ); } } + +#[derive(Clone, Debug, Default)] +pub struct RuntimeConfig { + pub idle_request_interval_secs: u64, + // RwLock as this check is conditional on the proxy using xDS. + pub xds_is_healthy: Arc>>>, +} + +impl RuntimeConfig { + pub fn is_ready(&self, config: &Config) -> bool { + self.xds_is_healthy + .read() + .as_ref() + .map_or(true, |health| health.load(Ordering::SeqCst)) + && config.clusters.read().endpoints().count() != 0 + } +} diff --git a/src/cli/relay.rs b/src/cli/relay.rs index fc075653a4..e150b9bdb6 100644 --- a/src/cli/relay.rs +++ b/src/cli/relay.rs @@ -14,7 +14,10 @@ * limitations under the License. */ -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; use futures::StreamExt; @@ -36,7 +39,7 @@ pub struct Relay { pub xds_port: u16, /// The interval in seconds at which the relay will send a discovery request /// to an management server after receiving no updates. - #[clap(long, env = "QUILKIN_IDLE_REQUEST_INTERVAL_SECS", default_value_t = crate::xds::server::IDLE_REQUEST_INTERVAL_SECS)] + #[clap(long, env = "QUILKIN_IDLE_REQUEST_INTERVAL_SECS", default_value_t = super::admin::IDLE_REQUEST_INTERVAL_SECS)] pub idle_request_interval_secs: u64, #[clap(subcommand)] pub providers: Option, @@ -47,7 +50,7 @@ impl Default for Relay { Self { mds_port: PORT, xds_port: super::manage::PORT, - idle_request_interval_secs: crate::xds::server::IDLE_REQUEST_INTERVAL_SECS, + idle_request_interval_secs: super::admin::IDLE_REQUEST_INTERVAL_SECS, providers: None, } } @@ -57,6 +60,7 @@ impl Relay { pub async fn relay( &self, config: Arc, + mode: crate::cli::Admin, mut shutdown_rx: tokio::sync::watch::Receiver<()>, ) -> crate::Result<()> { let xds_server = crate::xds::server::spawn(self.xds_port, config.clone()); @@ -65,6 +69,7 @@ impl Relay { self.idle_request_interval_secs, config.clone(), )); + let runtime_config = mode.unwrap_relay(); let _provider_task = if let Some(Providers::Agones { config_namespace, .. @@ -72,45 +77,65 @@ impl Relay { { let config = config.clone(); let config_namespace = config_namespace.clone(); - Some(tokio::spawn(Providers::task(move || { - let config = config.clone(); - let config_namespace = config_namespace.clone(); - async move { - let client = tokio::time::timeout( - std::time::Duration::from_secs(5), - kube::Client::try_default(), - ) - .await??; + let provider_is_healthy = runtime_config.provider_is_healthy.clone(); + Some(tokio::spawn(Providers::task( + provider_is_healthy.clone(), + move || { + let config = config.clone(); + let config_namespace = config_namespace.clone(); + let provider_is_healthy = provider_is_healthy.clone(); + async move { + let client = tokio::time::timeout( + std::time::Duration::from_secs(5), + kube::Client::try_default(), + ) + .await??; - let configmap_reflector = - crate::config::providers::k8s::update_filters_from_configmap( - client.clone(), - config_namespace, - config.clone(), - ); + let configmap_reflector = + crate::config::providers::k8s::update_filters_from_configmap( + client.clone(), + config_namespace, + config.clone(), + ); - tokio::pin!(configmap_reflector); + tokio::pin!(configmap_reflector); - loop { - match configmap_reflector.next().await { - Some(Ok(_)) => (), - Some(Err(error)) => return Err(error), - None => break, + loop { + match configmap_reflector.next().await { + Some(Ok(_)) => { + provider_is_healthy.store(true, Ordering::SeqCst); + } + Some(Err(error)) => { + provider_is_healthy.store(false, Ordering::SeqCst); + return Err(error); + } + None => { + provider_is_healthy.store(false, Ordering::SeqCst); + break; + } + } } - } - tracing::info!("configmap stream ending"); - Ok(()) - } - }))) + tracing::info!("configmap stream ending"); + Ok(()) + } + }, + ))) } else if let Some(Providers::File { path }) = &self.providers { let config = config.clone(); let path = path.clone(); - Some(tokio::spawn(Providers::task(move || { - let config = config.clone(); - let path = path.clone(); - async move { crate::config::watch::fs(config, path, None).await } - }))) + let provider_is_healthy = runtime_config.provider_is_healthy.clone(); + Some(tokio::spawn(Providers::task( + provider_is_healthy.clone(), + move || { + let config = config.clone(); + let path = path.clone(); + let provider_is_healthy = provider_is_healthy.clone(); + async move { + crate::config::watch::fs(config, provider_is_healthy, path, None).await + } + }, + ))) } else { None }; @@ -126,3 +151,15 @@ impl Relay { } } } + +#[derive(Clone, Debug, Default)] +pub struct RuntimeConfig { + pub idle_request_interval_secs: u64, + pub provider_is_healthy: Arc, +} + +impl RuntimeConfig { + pub fn is_ready(&self) -> bool { + self.provider_is_healthy.load(Ordering::SeqCst) + } +} diff --git a/src/config/providers.rs b/src/config/providers.rs index 2bd331a3e1..ef543a71a1 100644 --- a/src/config/providers.rs +++ b/src/config/providers.rs @@ -1,3 +1,7 @@ +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; pub mod k8s; const RETRIES: u32 = 25; @@ -41,33 +45,47 @@ impl Providers { pub fn spawn( &self, config: std::sync::Arc, + health_check: Arc, locality: Option, ) -> tokio::task::JoinHandle> { match &self { Self::Agones { gameservers_namespace, config_namespace, - } => tokio::spawn(Self::task({ + } => tokio::spawn(Self::task(health_check.clone(), { let gameservers_namespace = gameservers_namespace.clone(); let config_namespace = config_namespace.clone(); + let health_check = health_check.clone(); move || { crate::config::watch::agones( gameservers_namespace.clone(), config_namespace.clone(), + health_check.clone(), locality.clone(), config.clone(), ) } })), - Self::File { path } => tokio::spawn(Self::task({ + Self::File { path } => tokio::spawn(Self::task(health_check.clone(), { let path = path.clone(); - move || crate::config::watch::fs(config.clone(), path.clone(), locality.clone()) + let health_check = health_check.clone(); + move || { + crate::config::watch::fs( + config.clone(), + health_check.clone(), + path.clone(), + locality.clone(), + ) + } })), } } #[tracing::instrument(level = "trace", skip_all)] - pub async fn task(task: impl FnMut() -> F) -> crate::Result<()> + pub async fn task( + health_check: Arc, + task: impl FnMut() -> F, + ) -> crate::Result<()> where F: std::future::Future>, { @@ -76,6 +94,7 @@ impl Providers { .exponential_backoff(BACKOFF_STEP) .max_delay(MAX_DELAY) .on_retry(|attempt, _, error: &eyre::Error| { + health_check.store(false, Ordering::SeqCst); let error = error.to_string(); async move { tracing::warn!(%attempt, %error, "provider task error, retrying"); diff --git a/src/config/watch/agones.rs b/src/config/watch/agones.rs index cf2e0dbfa9..5e7e1d9bfc 100644 --- a/src/config/watch/agones.rs +++ b/src/config/watch/agones.rs @@ -15,13 +15,17 @@ */ use futures::TryStreamExt; -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; use crate::{endpoint::Locality, Config}; pub async fn watch( gameservers_namespace: impl AsRef, config_namespace: impl AsRef, + health_check: Arc, locality: Option, config: Arc, ) -> crate::Result<()> { @@ -45,11 +49,15 @@ pub async fn watch( tokio::pin!(gameserver_reflector); loop { - let Some(_) = tokio::select! { - result = configmap_reflector.try_next() => result?, - result = gameserver_reflector.try_next() => result?, - } else { - break Ok(()); + let result = tokio::select! { + result = configmap_reflector.try_next() => result, + result = gameserver_reflector.try_next() => result, }; + + match result { + Ok(Some(_)) => health_check.store(true, Ordering::SeqCst), + Ok(None) => break Err(eyre::eyre!("kubernetes watch stream terminated")), + Err(error) => break Err(error), + } } } diff --git a/src/config/watch/fs.rs b/src/config/watch/fs.rs index 4d699cc1b1..bbf95ec3c0 100644 --- a/src/config/watch/fs.rs +++ b/src/config/watch/fs.rs @@ -14,7 +14,10 @@ * limitations under the License. */ -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; use notify::Watcher; use tracing::Instrument; @@ -23,6 +26,7 @@ use crate::Config; pub async fn watch( config: Arc, + health_check: Arc, path: impl Into, locality: Option, ) -> crate::Result<()> { @@ -42,6 +46,7 @@ pub async fn watch( let buf = tokio::fs::read(&path).await?; tracing::info!("applying initial configuration"); config.update_from_json(serde_yaml::from_slice(&buf)?, locality.clone())?; + health_check.store(true, Ordering::SeqCst); watcher.watch(&path, notify::RecursiveMode::Recursive)?; tracing::info!("watching file"); @@ -83,7 +88,7 @@ mod tests { tokio::fs::write(&file_path, serde_yaml::to_string(&source).unwrap()) .await .unwrap(); - let _handle = tokio::spawn(watch(dest.clone(), file_path.clone(), None)); + let _handle = tokio::spawn(watch(dest.clone(), <_>::default(), file_path.clone(), None)); tokio::time::sleep(std::time::Duration::from_millis(100)).await; source.clusters.modify(|clusters| { diff --git a/src/endpoint.rs b/src/endpoint.rs index 33d3038b50..462769ebc6 100644 --- a/src/endpoint.rs +++ b/src/endpoint.rs @@ -137,7 +137,7 @@ impl Ord for Endpoint { impl PartialOrd for Endpoint { fn partial_cmp(&self, other: &Self) -> Option { - self.address.partial_cmp(&other.address) + Some(self.cmp(other)) } } diff --git a/src/filters.rs b/src/filters.rs index ecc0928e1a..585c17b2ea 100644 --- a/src/filters.rs +++ b/src/filters.rs @@ -27,7 +27,7 @@ mod write; pub mod capture; pub mod compress; -pub mod concatenate_bytes; +pub mod concatenate; pub mod debug; pub mod drop; pub mod firewall; @@ -53,7 +53,7 @@ pub mod prelude { pub use self::{ capture::Capture, compress::Compress, - concatenate_bytes::ConcatenateBytes, + concatenate::Concatenate, debug::Debug, drop::Drop, error::{ConvertProtoConfigError, CreationError, FilterError}, diff --git a/src/filters/concatenate_bytes.rs b/src/filters/concatenate.rs similarity index 77% rename from src/filters/concatenate_bytes.rs rename to src/filters/concatenate.rs index 47d7fb003a..201d82b62d 100644 --- a/src/filters/concatenate_bytes.rs +++ b/src/filters/concatenate.rs @@ -14,28 +14,28 @@ * limitations under the License. */ -crate::include_proto!("quilkin.filters.concatenate_bytes.v1alpha1"); +crate::include_proto!("quilkin.filters.concatenate.v1alpha1"); mod config; use crate::filters::prelude::*; -use self::quilkin::filters::concatenate_bytes::v1alpha1 as proto; +use self::quilkin::filters::concatenate::v1alpha1 as proto; pub use config::{Config, Strategy}; -/// The `ConcatenateBytes` filter's job is to add a byte packet to either the +/// The `Concatenate` filter's job is to add a byte packet to either the /// beginning or end of each UDP packet that passes through. This is commonly /// used to provide an auth token to each packet, so they can be /// routed appropriately. -pub struct ConcatenateBytes { +pub struct Concatenate { on_read: Strategy, on_write: Strategy, bytes: Vec, } -impl ConcatenateBytes { +impl Concatenate { pub fn new(config: Config) -> Self { - ConcatenateBytes { + Concatenate { on_read: config.on_read, on_write: config.on_write, bytes: config.bytes, @@ -44,7 +44,7 @@ impl ConcatenateBytes { } #[async_trait::async_trait] -impl Filter for ConcatenateBytes { +impl Filter for Concatenate { async fn read(&self, ctx: &mut ReadContext) -> Result<(), FilterError> { match self.on_read { Strategy::Append => { @@ -74,12 +74,12 @@ impl Filter for ConcatenateBytes { } } -impl StaticFilter for ConcatenateBytes { - const NAME: &'static str = "quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes"; +impl StaticFilter for Concatenate { + const NAME: &'static str = "quilkin.filters.concatenate.v1alpha1.Concatenate"; type Configuration = Config; - type BinaryConfiguration = proto::ConcatenateBytes; + type BinaryConfiguration = proto::Concatenate; fn try_from_config(config: Option) -> Result { - Ok(ConcatenateBytes::new(Self::ensure_config_exists(config)?)) + Ok(Concatenate::new(Self::ensure_config_exists(config)?)) } } diff --git a/src/filters/concatenate_bytes/config.rs b/src/filters/concatenate/config.rs similarity index 76% rename from src/filters/concatenate_bytes/config.rs rename to src/filters/concatenate/config.rs index eb0e9dd426..326ccf42c9 100644 --- a/src/filters/concatenate_bytes/config.rs +++ b/src/filters/concatenate/config.rs @@ -31,7 +31,7 @@ pub enum Strategy { DoNothing, } -impl From for proto::concatenate_bytes::Strategy { +impl From for proto::concatenate::Strategy { fn from(strategy: Strategy) -> Self { match strategy { Strategy::Append => Self::Append, @@ -41,25 +41,25 @@ impl From for proto::concatenate_bytes::Strategy { } } -impl From for Strategy { - fn from(strategy: proto::concatenate_bytes::Strategy) -> Self { +impl From for Strategy { + fn from(strategy: proto::concatenate::Strategy) -> Self { match strategy { - proto::concatenate_bytes::Strategy::Append => Self::Append, - proto::concatenate_bytes::Strategy::Prepend => Self::Prepend, - proto::concatenate_bytes::Strategy::DoNothing => Self::DoNothing, + proto::concatenate::Strategy::Append => Self::Append, + proto::concatenate::Strategy::Prepend => Self::Prepend, + proto::concatenate::Strategy::DoNothing => Self::DoNothing, } } } -impl From for proto::concatenate_bytes::StrategyValue { +impl From for proto::concatenate::StrategyValue { fn from(strategy: Strategy) -> Self { Self { - value: proto::concatenate_bytes::Strategy::from(strategy) as i32, + value: proto::concatenate::Strategy::from(strategy) as i32, } } } -/// Config represents a `ConcatenateBytes` filter configuration. +/// Config represents a `Concatenate` filter configuration. #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, JsonSchema)] #[non_exhaustive] pub struct Config { @@ -77,7 +77,7 @@ pub struct Config { pub bytes: Vec, } -impl From for proto::ConcatenateBytes { +impl From for proto::Concatenate { fn from(config: Config) -> Self { Self { on_read: Some(config.on_read.into()), @@ -87,8 +87,8 @@ impl From for proto::ConcatenateBytes { } } -impl From for Config { - fn from(p: proto::ConcatenateBytes) -> Self { +impl From for Config { + fn from(p: proto::Concatenate) -> Self { let on_read = p .on_read .map(|p| p.value()) diff --git a/src/filters/set.rs b/src/filters/set.rs index 7b1af7ef35..b17e95ed36 100644 --- a/src/filters/set.rs +++ b/src/filters/set.rs @@ -29,7 +29,7 @@ pub type FilterMap = std::collections::HashMap<&'static str, Arc(ip) { - Ok(asn) => { - tracing::info!( - number = asn.r#as, - organization = asn.as_name, - country_code = asn.as_cc, - prefix = asn.prefix, - prefix_entity = asn.prefix_entity, - prefix_name = asn.prefix_name, - "maxmind information" - ); - - Some(asn) - } + Ok(asn) => Some(asn), Err(error) => { tracing::warn!(%ip, %error, "ip not found in maxmind database"); None diff --git a/src/proxy/sessions.rs b/src/proxy/sessions.rs index c7e71965c8..113b23d361 100644 --- a/src/proxy/sessions.rs +++ b/src/proxy/sessions.rs @@ -102,6 +102,18 @@ impl Session { tracing::debug!(source = %s.source, dest = ?s.dest, "Session created"); + if let Some(asn) = &s.asn_info { + tracing::debug!( + number = asn.r#as, + organization = asn.as_name, + country_code = asn.as_cc, + prefix = asn.prefix, + prefix_entity = asn.prefix_entity, + prefix_name = asn.prefix_name, + "maxmind information" + ); + } + self::metrics::total_sessions().inc(); s.active_session_metric().inc(); s.run(downstream_socket, shutdown_rx); diff --git a/src/test_utils.rs b/src/test_utils.rs index f70636a03b..a780ce4521 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -273,17 +273,14 @@ impl TestHelper { ) { let (shutdown_tx, shutdown_rx) = watch::channel::<()>(()); self.server_shutdown_tx.push(Some(shutdown_tx)); + let mode = crate::cli::Admin::Proxy(<_>::default()); if let Some(address) = with_admin { - tokio::spawn(crate::admin::server( - crate::admin::Mode::Proxy, - config.clone(), - address, - )); + mode.server(config.clone(), address); } tokio::spawn(async move { - server.run(config, shutdown_rx).await.unwrap(); + server.run(config, mode, shutdown_rx).await.unwrap(); }); } diff --git a/src/xds.rs b/src/xds.rs index 7c17261bcb..93a883df1e 100644 --- a/src/xds.rs +++ b/src/xds.rs @@ -198,7 +198,12 @@ mod tests { ..<_>::default() }; - tokio::spawn(async move { client_proxy.run(client_config, shutdown_rx).await }); + let proxy_admin = crate::cli::Admin::Proxy(<_>::default()); + tokio::spawn(async move { + client_proxy + .run(client_config, proxy_admin, shutdown_rx) + .await + }); tokio::time::sleep(std::time::Duration::from_millis(50)).await; tokio::time::sleep(std::time::Duration::from_millis(50)).await; @@ -296,13 +301,14 @@ mod tests { tokio::spawn(server::spawn(23456, config.clone())); let client = Client::connect( "test-client".into(), + crate::cli::Admin::Manage(<_>::default()), vec!["http://127.0.0.1:23456".try_into().unwrap()], ) .await .unwrap(); let mut stream = client.xds_client_stream( config.clone(), - crate::xds::server::IDLE_REQUEST_INTERVAL_SECS, + crate::cli::admin::IDLE_REQUEST_INTERVAL_SECS, ); tokio::time::sleep(std::time::Duration::from_millis(500)).await; @@ -320,14 +326,14 @@ mod tests { tokio::time::sleep(std::time::Duration::from_millis(500)).await; let filters = crate::filters::FilterChain::try_from(vec![ - ConcatenateBytes::as_filter_config(concatenate_bytes::Config { - on_read: concatenate_bytes::Strategy::Append, + Concatenate::as_filter_config(concatenate::Config { + on_read: concatenate::Strategy::Append, on_write: <_>::default(), bytes: b1.as_bytes().to_vec(), }) .unwrap(), - ConcatenateBytes::as_filter_config(concatenate_bytes::Config { - on_read: concatenate_bytes::Strategy::Append, + Concatenate::as_filter_config(concatenate::Config { + on_read: concatenate::Strategy::Append, on_write: <_>::default(), bytes: b2.as_bytes().to_vec(), }) diff --git a/src/xds/client.rs b/src/xds/client.rs index 7d189a9085..dac9b745e8 100644 --- a/src/xds/client.rs +++ b/src/xds/client.rs @@ -14,7 +14,7 @@ * limitations under the License. */ -use std::{collections::HashSet, sync::Arc, time::Duration}; +use std::{collections::HashSet, sync::atomic::Ordering, sync::Arc, time::Duration}; use futures::StreamExt; use rand::Rng; @@ -27,6 +27,7 @@ use tryhard::{ }; use crate::{ + cli::Admin, config::Config, xds::{ config::core::v3::Node, @@ -106,16 +107,22 @@ pub struct Client { client: C, identifier: Arc, management_servers: Vec, + mode: Admin, } impl Client { #[tracing::instrument(skip_all, level = "trace", fields(servers = ?management_servers))] - pub async fn connect(identifier: String, management_servers: Vec) -> Result { + pub async fn connect( + identifier: String, + mode: Admin, + management_servers: Vec, + ) -> Result { let client = Self::connect_with_backoff(&management_servers).await?; Ok(Self { client, identifier: Arc::from(identifier), management_servers, + mode, }) } @@ -240,6 +247,7 @@ impl AdsStream { client, identifier, management_servers, + mode, }: &AdsClient, config: Arc, idle_request_interval_secs: u64, @@ -247,6 +255,7 @@ impl AdsStream { let mut client = client.clone(); let identifier = identifier.clone(); let management_servers = management_servers.clone(); + let mode = mode.clone(); Self::connect( identifier.clone(), move |(mut requests, mut rx), subscribed_resources| async move { @@ -288,6 +297,7 @@ impl AdsStream { stream, move |resource| config.apply(resource), ); + let runtime_config = mode.unwrap_proxy(); loop { let next_response = tokio::time::timeout( @@ -297,6 +307,13 @@ impl AdsStream { match next_response.await { Ok(Some(Ok(ack))) => { + runtime_config + .xds_is_healthy + .read() + .as_deref() + .unwrap() + .store(true, Ordering::SeqCst); + tracing::trace!("received ack"); requests.send(ack)?; continue; @@ -323,6 +340,13 @@ impl AdsStream { } } + runtime_config + .xds_is_healthy + .read() + .as_deref() + .unwrap() + .store(false, Ordering::SeqCst); + tracing::info!("Lost connection to xDS, retrying"); client = AdsClient::connect_with_backoff(&management_servers).await?; rx = requests.subscribe(); @@ -358,12 +382,14 @@ impl MdsStream { client, identifier, management_servers, + mode, }: &MdsClient, config: Arc, ) -> Self { let mut client = client.clone(); let identifier = identifier.clone(); let management_servers = management_servers.clone(); + let mode = mode.clone(); Self::connect( identifier.clone(), move |(requests, mut rx), _| async move { @@ -390,15 +416,33 @@ impl MdsStream { let control_plane = super::server::ControlPlane::from_arc( config.clone(), - super::server::IDLE_REQUEST_INTERVAL_SECS, + mode.idle_request_interval_secs(), ); let mut stream = control_plane.stream_aggregated_resources(stream).await?; - while let Some(result) = stream.next().await { - let response = result?; - tracing::debug!(config=%serde_json::to_value(&config).unwrap(), "received discovery response"); - requests.send(response)?; + mode.unwrap_agent() + .relay_is_healthy + .store(true, Ordering::SeqCst); + + loop { + let timeout = tokio::time::timeout( + std::time::Duration::from_secs(mode.idle_request_interval_secs()), + stream.next(), + ); + + match timeout.await { + Ok(Some(result)) => { + let response = result?; + tracing::debug!(config=%serde_json::to_value(&config).unwrap(), "received discovery response"); + requests.send(response)?; + } + _ => break, + } } + mode.unwrap_agent() + .relay_is_healthy + .store(false, Ordering::SeqCst); + tracing::warn!("lost connection to relay server, retrying"); client = MdsClient::connect_with_backoff(&management_servers) .await diff --git a/src/xds/server.rs b/src/xds/server.rs index b7bdaaf07e..95d81c4899 100644 --- a/src/xds/server.rs +++ b/src/xds/server.rs @@ -38,8 +38,6 @@ use crate::{ }, }; -pub(crate) const IDLE_REQUEST_INTERVAL_SECS: u64 = 30; - #[tracing::instrument(skip_all)] pub fn spawn( port: u16, @@ -47,7 +45,7 @@ pub fn spawn( ) -> impl std::future::Future> { let server = AggregatedDiscoveryServiceServer::new(ControlPlane::from_arc( config, - IDLE_REQUEST_INTERVAL_SECS, + crate::cli::admin::IDLE_REQUEST_INTERVAL_SECS, )); let server = tonic::transport::Server::builder().add_service(server); tracing::info!("serving management server on port `{port}`"); @@ -424,7 +422,10 @@ mod tests { }; let config = Arc::new(Config::default()); - let client = ControlPlane::from_arc(config.clone(), IDLE_REQUEST_INTERVAL_SECS); + let client = ControlPlane::from_arc( + config.clone(), + crate::cli::admin::IDLE_REQUEST_INTERVAL_SECS, + ); let (tx, rx) = tokio::sync::mpsc::channel(256); let mut request = DiscoveryRequest { diff --git a/tests/concatenate_bytes.rs b/tests/concatenate.rs similarity index 93% rename from tests/concatenate_bytes.rs rename to tests/concatenate.rs index d10f0f1d3f..a74aca866b 100644 --- a/tests/concatenate_bytes.rs +++ b/tests/concatenate.rs @@ -21,12 +21,12 @@ use tokio::time::{timeout, Duration}; use quilkin::{ config::Filter, endpoint::Endpoint, - filters::{ConcatenateBytes, StaticFilter}, + filters::{Concatenate, StaticFilter}, test_utils::{AddressType, TestHelper}, }; #[tokio::test] -async fn concatenate_bytes() { +async fn concatenate() { let mut t = TestHelper::default(); let yaml = " on_read: APPEND @@ -45,7 +45,7 @@ bytes: YWJj #abc .modify(|clusters| clusters.insert_default([Endpoint::new(echo.clone())].into())); server_config.filters.store( quilkin::filters::FilterChain::try_from(vec![Filter { - name: ConcatenateBytes::factory().name().into(), + name: Concatenate::factory().name().into(), label: None, config: serde_yaml::from_str(yaml).unwrap(), }]) diff --git a/tests/filter_order.rs b/tests/filter_order.rs index 0267e49603..23e99a389d 100644 --- a/tests/filter_order.rs +++ b/tests/filter_order.rs @@ -21,7 +21,7 @@ use tokio::time::{timeout, Duration}; use quilkin::{ config::Filter, endpoint::Endpoint, - filters::{Compress, ConcatenateBytes, StaticFilter}, + filters::{Compress, Concatenate, StaticFilter}, test_utils::{AddressType, TestHelper}, }; @@ -65,12 +65,12 @@ on_write: DECOMPRESS server_config.filters.store( quilkin::filters::FilterChain::try_from(vec![ Filter { - name: ConcatenateBytes::factory().name().into(), + name: Concatenate::factory().name().into(), label: None, config: serde_yaml::from_str(yaml_concat_read).unwrap(), }, Filter { - name: ConcatenateBytes::factory().name().into(), + name: Concatenate::factory().name().into(), label: None, config: serde_yaml::from_str(yaml_concat_write).unwrap(), }, diff --git a/tests/match.rs b/tests/match.rs index ec354f728d..6462b9693c 100644 --- a/tests/match.rs +++ b/tests/match.rs @@ -40,18 +40,18 @@ suffix: on_read: metadataKey: quilkin.dev/capture fallthrough: - name: quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes + name: quilkin.filters.concatenate.v1alpha1.Concatenate config: on_read: APPEND bytes: ZGVm branches: - value: abc - name: quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes + name: quilkin.filters.concatenate.v1alpha1.Concatenate config: on_read: APPEND bytes: eHl6 # xyz - value: xyz - name: quilkin.filters.concatenate_bytes.v1alpha1.ConcatenateBytes + name: quilkin.filters.concatenate.v1alpha1.Concatenate config: on_read: APPEND bytes: YWJj # abc diff --git a/tests/qcmp.rs b/tests/qcmp.rs index 045cd044b3..82b2a49caa 100644 --- a/tests/qcmp.rs +++ b/tests/qcmp.rs @@ -50,9 +50,10 @@ async fn agent_ping() { }; let server_config = std::sync::Arc::new(quilkin::Config::default()); let (_tx, rx) = tokio::sync::watch::channel(()); + let admin = quilkin::cli::Admin::Agent(<_>::default()); tokio::spawn(async move { agent - .run(server_config, rx) + .run(server_config, admin, rx) .await .expect("Agent should run") });