From b050ef3e15d4c144f9ba7a8af7ef1e45575cb326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:44:49 +0100 Subject: [PATCH] Copy target scale down (#2054) * Config * More CRD docs * warning when target is not a deployment * Config docs * Changelog entry * Scale to 0 * mirrord-schema update * Removed 'Option' from CRD * Erroring out on scale down + target is not a deployment * Config fix * doc improvement * Format * Config finally fixed * Checking target type in 'LayerConfig::verify' * Doc updated * mirrord-schema updated * Format --- changelog.d/2053.added.md | 1 + mirrord-schema.json | 35 +++++++-- mirrord/cli/src/connection.rs | 4 +- mirrord/config/src/feature.rs | 12 ++-- mirrord/config/src/feature/copy_target.rs | 87 +++++++++++++++++++++++ mirrord/config/src/lib.rs | 18 ++++- mirrord/operator/src/client.rs | 51 ++++++++++--- mirrord/operator/src/crd.rs | 5 +- 8 files changed, 186 insertions(+), 27 deletions(-) create mode 100644 changelog.d/2053.added.md create mode 100644 mirrord/config/src/feature/copy_target.rs diff --git a/changelog.d/2053.added.md b/changelog.d/2053.added.md new file mode 100644 index 00000000000..f393af407a7 --- /dev/null +++ b/changelog.d/2053.added.md @@ -0,0 +1 @@ +Added option to scale down target deployment when using `copy target` feature. \ No newline at end of file diff --git a/mirrord-schema.json b/mirrord-schema.json index 996df2194ec..911b7fa527b 100644 --- a/mirrord-schema.json +++ b/mirrord-schema.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "LayerFileConfig", - "description": "mirrord allows for a high degree of customization when it comes to which features you want to enable, and how they should function.\n\nAll of the configuration fields have a default value, so a minimal configuration would be no configuration at all.\n\nThe configuration supports templating using the [Tera](https://keats.github.io/tera/docs/) template engine. Currently we don't provide additional values to the context, if you have anything you want us to provide please let us know.\n\nTo help you get started, here are examples of a basic configuration file, and a complete configuration file containing all fields.\n\n### Basic `config.json` {#root-basic}\n\n```json { \"target\": \"pod/bear-pod\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Basic `config.json` with templating {#root-basic-templating}\n\n```json { \"target\": \"{{ get_env(name=\"TARGET\", default=\"pod/fallback\") }}\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Complete `config.json` {#root-complete}\n\nDon't use this example as a starting point, it's just here to show you all the available options. ```json { \"accept_invalid_certificates\": false, \"skip_processes\": \"ide-debugger\", \"pause\": false, \"target\": { \"path\": \"pod/bear-pod\", \"namespace\": \"default\" }, \"connect_tcp\": null, \"agent\": { \"log_level\": \"info\", \"namespace\": \"default\", \"image\": \"ghcr.io/metalbear-co/mirrord:latest\", \"image_pull_policy\": \"IfNotPresent\", \"image_pull_secrets\": [ { \"secret-key\": \"secret\" } ], \"ttl\": 30, \"ephemeral\": false, \"communication_timeout\": 30, \"startup_timeout\": 360, \"network_interface\": \"eth0\", \"flush_connections\": true }, \"feature\": { \"env\": { \"include\": \"DATABASE_USER;PUBLIC_ENV\", \"exclude\": \"DATABASE_PASSWORD;SECRET_ENV\", \"override\": { \"DATABASE_CONNECTION\": \"db://localhost:7777/my-db\", \"LOCAL_BEAR\": \"panda\" } }, \"fs\": { \"mode\": \"write\", \"read_write\": \".+\\.json\" , \"read_only\": [ \".+\\.yaml\", \".+important-file\\.txt\" ], \"local\": [ \".+\\.js\", \".+\\.mjs\" ] }, \"network\": { \"incoming\": { \"mode\": \"steal\", \"http_header_filter\": { \"filter\": \"host: api\\..+\", \"ports\": [80, 8080] }, \"port_mapping\": [[ 7777, 8888 ]], \"ignore_localhost\": false, \"ignore_ports\": [9999, 10000] }, \"outgoing\": { \"tcp\": true, \"udp\": true, \"filter\": { \"local\": [\"tcp://1.1.1.0/24:1337\", \"1.1.5.0/24\", \"google.com\", \":53\"] }, \"ignore_localhost\": false, \"unix_streams\": \"bear.+\" }, \"dns\": false }, }, \"operator\": true, \"kubeconfig\": \"~/.kube/config\", \"sip_binaries\": \"bash\", \"telemetry\": true, \"kube_context\": \"my-cluster\" } ```\n\n# Options {#root-options}", + "description": "mirrord allows for a high degree of customization when it comes to which features you want to enable, and how they should function.\n\nAll of the configuration fields have a default value, so a minimal configuration would be no configuration at all.\n\nThe configuration supports templating using the [Tera](https://keats.github.io/tera/docs/) template engine. Currently we don't provide additional values to the context, if you have anything you want us to provide please let us know.\n\nTo help you get started, here are examples of a basic configuration file, and a complete configuration file containing all fields.\n\n### Basic `config.json` {#root-basic}\n\n```json { \"target\": \"pod/bear-pod\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Basic `config.json` with templating {#root-basic-templating}\n\n```json { \"target\": \"{{ get_env(name=\"TARGET\", default=\"pod/fallback\") }}\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Complete `config.json` {#root-complete}\n\nDon't use this example as a starting point, it's just here to show you all the available options. ```json { \"accept_invalid_certificates\": false, \"skip_processes\": \"ide-debugger\", \"pause\": false, \"target\": { \"path\": \"pod/bear-pod\", \"namespace\": \"default\" }, \"connect_tcp\": null, \"agent\": { \"log_level\": \"info\", \"namespace\": \"default\", \"image\": \"ghcr.io/metalbear-co/mirrord:latest\", \"image_pull_policy\": \"IfNotPresent\", \"image_pull_secrets\": [ { \"secret-key\": \"secret\" } ], \"ttl\": 30, \"ephemeral\": false, \"communication_timeout\": 30, \"startup_timeout\": 360, \"network_interface\": \"eth0\", \"flush_connections\": true }, \"feature\": { \"env\": { \"include\": \"DATABASE_USER;PUBLIC_ENV\", \"exclude\": \"DATABASE_PASSWORD;SECRET_ENV\", \"override\": { \"DATABASE_CONNECTION\": \"db://localhost:7777/my-db\", \"LOCAL_BEAR\": \"panda\" } }, \"fs\": { \"mode\": \"write\", \"read_write\": \".+\\.json\" , \"read_only\": [ \".+\\.yaml\", \".+important-file\\.txt\" ], \"local\": [ \".+\\.js\", \".+\\.mjs\" ] }, \"network\": { \"incoming\": { \"mode\": \"steal\", \"http_header_filter\": { \"filter\": \"host: api\\..+\", \"ports\": [80, 8080] }, \"port_mapping\": [[ 7777, 8888 ]], \"ignore_localhost\": false, \"ignore_ports\": [9999, 10000] }, \"outgoing\": { \"tcp\": true, \"udp\": true, \"filter\": { \"local\": [\"tcp://1.1.1.0/24:1337\", \"1.1.5.0/24\", \"google.com\", \":53\"] }, \"ignore_localhost\": false, \"unix_streams\": \"bear.+\" }, \"dns\": false, \"copy_target\": { \"scale_down\": false } }, }, \"operator\": true, \"kubeconfig\": \"~/.kube/config\", \"sip_binaries\": \"bash\", \"telemetry\": true, \"kube_context\": \"my-cluster\" } ```\n\n# Options {#root-options}", "type": "object", "properties": { "accept_invalid_certificates": { @@ -388,6 +388,27 @@ } ] }, + "CopyTargetFileConfig": { + "description": "Allows the user to target a pod created dynamically from the orignal [`target`](#target). The new pod inherits most of the original target's specification, e.g. labels.\n\n```json { \"feature\": { \"copy_target\": { \"scale_down\": true } } } ```\n\n```json { \"feature\": { \"copy_target\": true } } ```", + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "object", + "properties": { + "scale_down": { + "title": "feature.copy_target.scale_down {#feature-copy_target-scale_down}", + "description": "If this option is set and [`target`](#target) is a deployment, mirrord will scale it down to 0 for the time the copied pod is alive.", + "type": [ + "boolean", + "null" + ] + } + } + } + ] + }, "DeploymentTarget": { "description": " Mirror the deployment specified by [`DeploymentTarget::deployment`].", "type": "object", @@ -454,11 +475,15 @@ "type": "object", "properties": { "copy_target": { - "title": "feature.copy_target {#feature-copy-target}", + "title": "feature.copy_target {#feature-copy_target}", "description": "Creates a new copy of the target. mirrord will use this copy instead of the original target (e.g. intercept network traffic). This feature requires a [mirrord operator](https://mirrord.dev/docs/teams/introduction/).", - "type": [ - "boolean", - "null" + "anyOf": [ + { + "$ref": "#/definitions/CopyTargetFileConfig" + }, + { + "type": "null" + } ] }, "env": { diff --git a/mirrord/cli/src/connection.rs b/mirrord/cli/src/connection.rs index c88a86a9835..87e9a4a725a 100644 --- a/mirrord/cli/src/connection.rs +++ b/mirrord/cli/src/connection.rs @@ -95,8 +95,8 @@ where AgentConnection { sender: session.tx, receiver: session.rx }, )) } else { - if config.feature.copy_target { - return Err(CliError::FeatureRequiresOperatorError("copy pod".into())); + if config.feature.copy_target.enabled { + return Err(CliError::FeatureRequiresOperatorError("copy target".into())); } if matches!(config.target, mirrord_config::target::TargetConfig{ path: Some(mirrord_config::target::Target::Deployment{..}), ..}) { diff --git a/mirrord/config/src/feature.rs b/mirrord/config/src/feature.rs index 459cd84e9a1..8dccc8f5887 100644 --- a/mirrord/config/src/feature.rs +++ b/mirrord/config/src/feature.rs @@ -2,9 +2,9 @@ use mirrord_analytics::CollectAnalytics; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use self::{env::EnvConfig, fs::FsConfig, network::NetworkConfig}; -use crate::MirrordConfigSource; +use self::{copy_target::CopyTargetConfig, env::EnvConfig, fs::FsConfig, network::NetworkConfig}; +pub mod copy_target; pub mod env; pub mod fs; pub mod network; @@ -79,12 +79,12 @@ pub struct FeatureConfig { #[config(nested, toggleable)] pub network: NetworkConfig, - /// ## feature.copy_target {#feature-copy-target} + /// ## feature.copy_target {#feature-copy_target} /// /// Creates a new copy of the target. mirrord will use this copy instead of the original target /// (e.g. intercept network traffic). This feature requires a [mirrord operator](https://mirrord.dev/docs/teams/introduction/). - #[config(default = false, unstable)] - pub copy_target: bool, + #[config(nested, unstable)] + pub copy_target: CopyTargetConfig, } impl CollectAnalytics for &FeatureConfig { @@ -92,6 +92,6 @@ impl CollectAnalytics for &FeatureConfig { analytics.add("env", &self.env); analytics.add("fs", &self.fs); analytics.add("network", &self.network); - analytics.add("copy_target", self.copy_target); + analytics.add("copy_target", &self.copy_target); } } diff --git a/mirrord/config/src/feature/copy_target.rs b/mirrord/config/src/feature/copy_target.rs new file mode 100644 index 00000000000..c17232b3a85 --- /dev/null +++ b/mirrord/config/src/feature/copy_target.rs @@ -0,0 +1,87 @@ +//! Config for the `copy target` feature. [`CopyTargetFileConfig`] does follow the pattern of other +//! [`feature`](crate::feature) configs by not implementing +//! [`MirrordToggleableConfig`](crate::util::MirrordToggleableConfig). The reason for this is that +//! [`ToggleableConfig`](crate::util::ToggleableConfig) is enabled by default. This config should be +//! disabled unless explicitly enabled. + +use mirrord_analytics::CollectAnalytics; +use schemars::JsonSchema; +use serde::Deserialize; + +use crate::config::{ConfigContext, FromMirrordConfig, MirrordConfig, Result}; + +/// Allows the user to target a pod created dynamically from the orignal [`target`](#target). +/// The new pod inherits most of the original target's specification, e.g. labels. +/// +/// ```json +/// { +/// "feature": { +/// "copy_target": { +/// "scale_down": true +/// } +/// } +/// } +/// ``` +/// +/// ```json +/// { +/// "feature": { +/// "copy_target": true +/// } +/// } +/// ``` +#[derive(Clone, Debug, Deserialize, JsonSchema)] +#[cfg_attr(test, derive(PartialEq, Eq))] +#[serde(untagged)] +pub enum CopyTargetFileConfig { + Simple(bool), + Advanced { + /// ### feature.copy_target.scale_down {#feature-copy_target-scale_down} + /// + /// If this option is set and [`target`](#target) is a deployment, + /// mirrord will scale it down to 0 for the time the copied pod is alive. + scale_down: Option, + }, +} + +impl Default for CopyTargetFileConfig { + fn default() -> Self { + Self::Simple(false) + } +} + +impl MirrordConfig for CopyTargetFileConfig { + type Generated = CopyTargetConfig; + + fn generate_config(self, _context: &mut ConfigContext) -> Result { + let res = match self { + Self::Simple(enabled) => Self::Generated { + enabled, + scale_down: false, + }, + Self::Advanced { scale_down } => Self::Generated { + enabled: true, + scale_down: scale_down.unwrap_or_default(), + }, + }; + + Ok(res) + } +} + +impl FromMirrordConfig for CopyTargetConfig { + type Generator = CopyTargetFileConfig; +} + +#[derive(Clone, Debug)] +pub struct CopyTargetConfig { + pub enabled: bool, + pub scale_down: bool, +} + +impl CollectAnalytics for &CopyTargetConfig { + fn collect_analytics(&self, analytics: &mut mirrord_analytics::Analytics) { + analytics.add("enabled", self.enabled); + analytics.add("scale_down", self.scale_down); + } +} diff --git a/mirrord/config/src/lib.rs b/mirrord/config/src/lib.rs index ea0a1322898..8957a065e92 100644 --- a/mirrord/config/src/lib.rs +++ b/mirrord/config/src/lib.rs @@ -21,6 +21,7 @@ use config::{ConfigContext, ConfigError, MirrordConfig}; use mirrord_analytics::CollectAnalytics; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; +use target::Target; use tera::Tera; use tracing::warn; @@ -143,7 +144,10 @@ const PAUSE_WITHOUT_STEAL_WARNING: &str = /// "ignore_localhost": false, /// "unix_streams": "bear.+" /// }, -/// "dns": false +/// "dns": false, +/// "copy_target": { +/// "scale_down": false +/// } /// }, /// }, /// "operator": true, @@ -419,7 +423,7 @@ impl LayerConfig { } } - if self.feature.copy_target { + if self.feature.copy_target.enabled { if !self.operator { return Err(ConfigError::Conflict( "The copy target feature requires a mirrord operator, \ @@ -445,6 +449,16 @@ impl LayerConfig { .into(), ); } + + if self.feature.copy_target.scale_down + && !matches!(self.target.path, Some(Target::Deployment(..))) + { + return Err(ConfigError::Conflict( + "The scale down feature is compatible only with deployment targets, \ + please either disable this option or specify a deployment target." + .into(), + )); + } } Ok(()) diff --git a/mirrord/operator/src/client.rs b/mirrord/operator/src/client.rs index 777368d2348..e475c6f39ae 100644 --- a/mirrord/operator/src/client.rs +++ b/mirrord/operator/src/client.rs @@ -9,7 +9,9 @@ use mirrord_auth::{ certificate::Certificate, credential_store::CredentialStoreSync, error::AuthenticationError, }; use mirrord_config::{ - feature::network::incoming::ConcurrentSteal, target::TargetConfig, LayerConfig, + feature::network::incoming::ConcurrentSteal, + target::{Target, TargetConfig}, + LayerConfig, }; use mirrord_kube::{ api::kubernetes::{create_kube_api, get_k8s_resource_api}, @@ -33,8 +35,8 @@ static CONNECTION_CHANNEL_SIZE: usize = 1000; #[derive(Debug, Error)] pub enum OperatorApiError { - #[error("unable to create target for TargetConfig")] - InvalidTarget, + #[error("invalid target: {reason}")] + InvalidTarget { reason: String }, #[error(transparent)] HttpError(#[from] http::Error), #[error(transparent)] @@ -159,7 +161,7 @@ impl OperatorApi { /// Checks used config against operator specification. fn check_config(config: &LayerConfig, operator: &MirrordOperatorCrd) -> Result<()> { - if config.feature.copy_target { + if config.feature.copy_target.enabled { let feature_enabled = operator.spec.copy_target_enabled.unwrap_or(false); if !feature_enabled { @@ -168,6 +170,14 @@ impl OperatorApi { operator_version: operator.spec.operator_version.clone(), }); } + + if config.feature.copy_target.scale_down + && !matches!(config.target.path, Some(Target::Deployment(..))) + { + return Err(OperatorApiError::InvalidTarget { + reason: "scale down feature is enabled, but target is not a deployment".into(), + }); + } } Ok(()) @@ -238,14 +248,29 @@ impl OperatorApi { } version_progress.success(None); - let raw_target = operator_api - .fetch_target() - .await? - .ok_or(OperatorApiError::InvalidTarget)?; + let raw_target = + operator_api + .fetch_target() + .await? + .ok_or(OperatorApiError::InvalidTarget { + reason: "not found in the cluster".into(), + })?; - let target_to_connect = if config.feature.copy_target { + let target_to_connect = if config.feature.copy_target.enabled { let mut copy_progress = progress.subtask("copying target"); - let copied = operator_api.copy_target(&metadata, raw_target).await?; + + if config.feature.copy_target.scale_down { + let is_deployment = matches!(config.target.path, Some(Target::Deployment(..))); + if !is_deployment { + progress.warning( + "cannot scale down while copying target - target is not a deployment", + ) + } + } + + let copied = operator_api + .copy_target(&metadata, raw_target, config.feature.copy_target.scale_down) + .await?; copy_progress.success(None); OperatorSessionTarget::Copied(copied) @@ -465,18 +490,22 @@ impl OperatorApi { &self, session_metadata: &OperatorSessionMetadata, target: TargetCrd, + scale_down: bool, ) -> Result { let raw_target = target .spec .target .clone() - .ok_or(OperatorApiError::InvalidTarget)?; + .ok_or(OperatorApiError::InvalidTarget { + reason: "copy target feature is not compatible with targetless mode".into(), + })?; let requested = CopyTargetCrd::new( &target.name(), CopyTargetSpec { target: raw_target, idle_ttl: Some(Self::COPIED_POD_IDLE_TTL), + scale_down, }, ); diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 83c0fdb8d7c..ea8d8498096 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -135,9 +135,12 @@ pub enum OperatorFeatures { namespaced )] pub struct CopyTargetSpec { - /// Original target. + /// Original target. Only [`Target::Pod`] and [`Target::Deployment`] are accepted. pub target: Target, /// How long should the operator keep this pod alive after its creation. /// The pod is deleted when this timout has expired and there are no connected clients. pub idle_ttl: Option, + /// Should the operator scale down target deployment to 0 while this pod is alive. + /// Ignored if [`Target`] is not [`Target::Deployment`]. + pub scale_down: bool, }