Skip to content

Commit

Permalink
Copy target scale down (metalbear-co#2054)
Browse files Browse the repository at this point in the history
* Config

* More CRD docs

* warning when target is not a deployment

* Config docs

* Changelog entry

* Scale to 0

* mirrord-schema update

* Removed 'Option' from CRD

* Erroring out on scale down + target is not a deployment

* Config fix

* doc improvement

* Format

* Config finally fixed

* Checking target type in 'LayerConfig::verify'

* Doc updated

* mirrord-schema updated

* Format
  • Loading branch information
Razz4780 authored Nov 6, 2023
1 parent 658de32 commit b050ef3
Show file tree
Hide file tree
Showing 8 changed files with 186 additions and 27 deletions.
1 change: 1 addition & 0 deletions changelog.d/2053.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added option to scale down target deployment when using `copy target` feature.
35 changes: 30 additions & 5 deletions mirrord-schema.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "LayerFileConfig",
"description": "mirrord allows for a high degree of customization when it comes to which features you want to enable, and how they should function.\n\nAll of the configuration fields have a default value, so a minimal configuration would be no configuration at all.\n\nThe configuration supports templating using the [Tera](https://keats.github.io/tera/docs/) template engine. Currently we don't provide additional values to the context, if you have anything you want us to provide please let us know.\n\nTo help you get started, here are examples of a basic configuration file, and a complete configuration file containing all fields.\n\n### Basic `config.json` {#root-basic}\n\n```json { \"target\": \"pod/bear-pod\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Basic `config.json` with templating {#root-basic-templating}\n\n```json { \"target\": \"{{ get_env(name=\"TARGET\", default=\"pod/fallback\") }}\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Complete `config.json` {#root-complete}\n\nDon't use this example as a starting point, it's just here to show you all the available options. ```json { \"accept_invalid_certificates\": false, \"skip_processes\": \"ide-debugger\", \"pause\": false, \"target\": { \"path\": \"pod/bear-pod\", \"namespace\": \"default\" }, \"connect_tcp\": null, \"agent\": { \"log_level\": \"info\", \"namespace\": \"default\", \"image\": \"ghcr.io/metalbear-co/mirrord:latest\", \"image_pull_policy\": \"IfNotPresent\", \"image_pull_secrets\": [ { \"secret-key\": \"secret\" } ], \"ttl\": 30, \"ephemeral\": false, \"communication_timeout\": 30, \"startup_timeout\": 360, \"network_interface\": \"eth0\", \"flush_connections\": true }, \"feature\": { \"env\": { \"include\": \"DATABASE_USER;PUBLIC_ENV\", \"exclude\": \"DATABASE_PASSWORD;SECRET_ENV\", \"override\": { \"DATABASE_CONNECTION\": \"db://localhost:7777/my-db\", \"LOCAL_BEAR\": \"panda\" } }, \"fs\": { \"mode\": \"write\", \"read_write\": \".+\\.json\" , \"read_only\": [ \".+\\.yaml\", \".+important-file\\.txt\" ], \"local\": [ \".+\\.js\", \".+\\.mjs\" ] }, \"network\": { \"incoming\": { \"mode\": \"steal\", \"http_header_filter\": { \"filter\": \"host: api\\..+\", \"ports\": [80, 8080] }, \"port_mapping\": [[ 7777, 8888 ]], \"ignore_localhost\": false, \"ignore_ports\": [9999, 10000] }, \"outgoing\": { \"tcp\": true, \"udp\": true, \"filter\": { \"local\": [\"tcp://1.1.1.0/24:1337\", \"1.1.5.0/24\", \"google.com\", \":53\"] }, \"ignore_localhost\": false, \"unix_streams\": \"bear.+\" }, \"dns\": false }, }, \"operator\": true, \"kubeconfig\": \"~/.kube/config\", \"sip_binaries\": \"bash\", \"telemetry\": true, \"kube_context\": \"my-cluster\" } ```\n\n# Options {#root-options}",
"description": "mirrord allows for a high degree of customization when it comes to which features you want to enable, and how they should function.\n\nAll of the configuration fields have a default value, so a minimal configuration would be no configuration at all.\n\nThe configuration supports templating using the [Tera](https://keats.github.io/tera/docs/) template engine. Currently we don't provide additional values to the context, if you have anything you want us to provide please let us know.\n\nTo help you get started, here are examples of a basic configuration file, and a complete configuration file containing all fields.\n\n### Basic `config.json` {#root-basic}\n\n```json { \"target\": \"pod/bear-pod\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Basic `config.json` with templating {#root-basic-templating}\n\n```json { \"target\": \"{{ get_env(name=\"TARGET\", default=\"pod/fallback\") }}\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Complete `config.json` {#root-complete}\n\nDon't use this example as a starting point, it's just here to show you all the available options. ```json { \"accept_invalid_certificates\": false, \"skip_processes\": \"ide-debugger\", \"pause\": false, \"target\": { \"path\": \"pod/bear-pod\", \"namespace\": \"default\" }, \"connect_tcp\": null, \"agent\": { \"log_level\": \"info\", \"namespace\": \"default\", \"image\": \"ghcr.io/metalbear-co/mirrord:latest\", \"image_pull_policy\": \"IfNotPresent\", \"image_pull_secrets\": [ { \"secret-key\": \"secret\" } ], \"ttl\": 30, \"ephemeral\": false, \"communication_timeout\": 30, \"startup_timeout\": 360, \"network_interface\": \"eth0\", \"flush_connections\": true }, \"feature\": { \"env\": { \"include\": \"DATABASE_USER;PUBLIC_ENV\", \"exclude\": \"DATABASE_PASSWORD;SECRET_ENV\", \"override\": { \"DATABASE_CONNECTION\": \"db://localhost:7777/my-db\", \"LOCAL_BEAR\": \"panda\" } }, \"fs\": { \"mode\": \"write\", \"read_write\": \".+\\.json\" , \"read_only\": [ \".+\\.yaml\", \".+important-file\\.txt\" ], \"local\": [ \".+\\.js\", \".+\\.mjs\" ] }, \"network\": { \"incoming\": { \"mode\": \"steal\", \"http_header_filter\": { \"filter\": \"host: api\\..+\", \"ports\": [80, 8080] }, \"port_mapping\": [[ 7777, 8888 ]], \"ignore_localhost\": false, \"ignore_ports\": [9999, 10000] }, \"outgoing\": { \"tcp\": true, \"udp\": true, \"filter\": { \"local\": [\"tcp://1.1.1.0/24:1337\", \"1.1.5.0/24\", \"google.com\", \":53\"] }, \"ignore_localhost\": false, \"unix_streams\": \"bear.+\" }, \"dns\": false, \"copy_target\": { \"scale_down\": false } }, }, \"operator\": true, \"kubeconfig\": \"~/.kube/config\", \"sip_binaries\": \"bash\", \"telemetry\": true, \"kube_context\": \"my-cluster\" } ```\n\n# Options {#root-options}",
"type": "object",
"properties": {
"accept_invalid_certificates": {
Expand Down Expand Up @@ -388,6 +388,27 @@
}
]
},
"CopyTargetFileConfig": {
"description": "Allows the user to target a pod created dynamically from the orignal [`target`](#target). The new pod inherits most of the original target's specification, e.g. labels.\n\n```json { \"feature\": { \"copy_target\": { \"scale_down\": true } } } ```\n\n```json { \"feature\": { \"copy_target\": true } } ```",
"anyOf": [
{
"type": "boolean"
},
{
"type": "object",
"properties": {
"scale_down": {
"title": "feature.copy_target.scale_down {#feature-copy_target-scale_down}",
"description": "If this option is set and [`target`](#target) is a deployment, mirrord will scale it down to 0 for the time the copied pod is alive.",
"type": [
"boolean",
"null"
]
}
}
}
]
},
"DeploymentTarget": {
"description": "<!--${internal}--> Mirror the deployment specified by [`DeploymentTarget::deployment`].",
"type": "object",
Expand Down Expand Up @@ -454,11 +475,15 @@
"type": "object",
"properties": {
"copy_target": {
"title": "feature.copy_target {#feature-copy-target}",
"title": "feature.copy_target {#feature-copy_target}",
"description": "Creates a new copy of the target. mirrord will use this copy instead of the original target (e.g. intercept network traffic). This feature requires a [mirrord operator](https://mirrord.dev/docs/teams/introduction/).",
"type": [
"boolean",
"null"
"anyOf": [
{
"$ref": "#/definitions/CopyTargetFileConfig"
},
{
"type": "null"
}
]
},
"env": {
Expand Down
4 changes: 2 additions & 2 deletions mirrord/cli/src/connection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ where
AgentConnection { sender: session.tx, receiver: session.rx },
))
} else {
if config.feature.copy_target {
return Err(CliError::FeatureRequiresOperatorError("copy pod".into()));
if config.feature.copy_target.enabled {
return Err(CliError::FeatureRequiresOperatorError("copy target".into()));
}

if matches!(config.target, mirrord_config::target::TargetConfig{ path: Some(mirrord_config::target::Target::Deployment{..}), ..}) {
Expand Down
12 changes: 6 additions & 6 deletions mirrord/config/src/feature.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ use mirrord_analytics::CollectAnalytics;
use mirrord_config_derive::MirrordConfig;
use schemars::JsonSchema;

use self::{env::EnvConfig, fs::FsConfig, network::NetworkConfig};
use crate::MirrordConfigSource;
use self::{copy_target::CopyTargetConfig, env::EnvConfig, fs::FsConfig, network::NetworkConfig};

pub mod copy_target;
pub mod env;
pub mod fs;
pub mod network;
Expand Down Expand Up @@ -79,19 +79,19 @@ pub struct FeatureConfig {
#[config(nested, toggleable)]
pub network: NetworkConfig,

/// ## feature.copy_target {#feature-copy-target}
/// ## feature.copy_target {#feature-copy_target}
///
/// Creates a new copy of the target. mirrord will use this copy instead of the original target
/// (e.g. intercept network traffic). This feature requires a [mirrord operator](https://mirrord.dev/docs/teams/introduction/).
#[config(default = false, unstable)]
pub copy_target: bool,
#[config(nested, unstable)]
pub copy_target: CopyTargetConfig,
}

impl CollectAnalytics for &FeatureConfig {
fn collect_analytics(&self, analytics: &mut mirrord_analytics::Analytics) {
analytics.add("env", &self.env);
analytics.add("fs", &self.fs);
analytics.add("network", &self.network);
analytics.add("copy_target", self.copy_target);
analytics.add("copy_target", &self.copy_target);
}
}
87 changes: 87 additions & 0 deletions mirrord/config/src/feature/copy_target.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
//! Config for the `copy target` feature. [`CopyTargetFileConfig`] does follow the pattern of other
//! [`feature`](crate::feature) configs by not implementing
//! [`MirrordToggleableConfig`](crate::util::MirrordToggleableConfig). The reason for this is that
//! [`ToggleableConfig`](crate::util::ToggleableConfig) is enabled by default. This config should be
//! disabled unless explicitly enabled.
use mirrord_analytics::CollectAnalytics;
use schemars::JsonSchema;
use serde::Deserialize;

use crate::config::{ConfigContext, FromMirrordConfig, MirrordConfig, Result};

/// Allows the user to target a pod created dynamically from the orignal [`target`](#target).
/// The new pod inherits most of the original target's specification, e.g. labels.
///
/// ```json
/// {
/// "feature": {
/// "copy_target": {
/// "scale_down": true
/// }
/// }
/// }
/// ```
///
/// ```json
/// {
/// "feature": {
/// "copy_target": true
/// }
/// }
/// ```
#[derive(Clone, Debug, Deserialize, JsonSchema)]
#[cfg_attr(test, derive(PartialEq, Eq))]
#[serde(untagged)]
pub enum CopyTargetFileConfig {
Simple(bool),
Advanced {
/// ### feature.copy_target.scale_down {#feature-copy_target-scale_down}
///
/// If this option is set and [`target`](#target) is a deployment,
/// mirrord will scale it down to 0 for the time the copied pod is alive.
scale_down: Option<bool>,
},
}

impl Default for CopyTargetFileConfig {
fn default() -> Self {
Self::Simple(false)
}
}

impl MirrordConfig for CopyTargetFileConfig {
type Generated = CopyTargetConfig;

fn generate_config(self, _context: &mut ConfigContext) -> Result<Self::Generated> {
let res = match self {
Self::Simple(enabled) => Self::Generated {
enabled,
scale_down: false,
},
Self::Advanced { scale_down } => Self::Generated {
enabled: true,
scale_down: scale_down.unwrap_or_default(),
},
};

Ok(res)
}
}

impl FromMirrordConfig for CopyTargetConfig {
type Generator = CopyTargetFileConfig;
}

#[derive(Clone, Debug)]
pub struct CopyTargetConfig {
pub enabled: bool,
pub scale_down: bool,
}

impl CollectAnalytics for &CopyTargetConfig {
fn collect_analytics(&self, analytics: &mut mirrord_analytics::Analytics) {
analytics.add("enabled", self.enabled);
analytics.add("scale_down", self.scale_down);
}
}
18 changes: 16 additions & 2 deletions mirrord/config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ use config::{ConfigContext, ConfigError, MirrordConfig};
use mirrord_analytics::CollectAnalytics;
use mirrord_config_derive::MirrordConfig;
use schemars::JsonSchema;
use target::Target;
use tera::Tera;
use tracing::warn;

Expand Down Expand Up @@ -143,7 +144,10 @@ const PAUSE_WITHOUT_STEAL_WARNING: &str =
/// "ignore_localhost": false,
/// "unix_streams": "bear.+"
/// },
/// "dns": false
/// "dns": false,
/// "copy_target": {
/// "scale_down": false
/// }
/// },
/// },
/// "operator": true,
Expand Down Expand Up @@ -419,7 +423,7 @@ impl LayerConfig {
}
}

if self.feature.copy_target {
if self.feature.copy_target.enabled {
if !self.operator {
return Err(ConfigError::Conflict(
"The copy target feature requires a mirrord operator, \
Expand All @@ -445,6 +449,16 @@ impl LayerConfig {
.into(),
);
}

if self.feature.copy_target.scale_down
&& !matches!(self.target.path, Some(Target::Deployment(..)))
{
return Err(ConfigError::Conflict(
"The scale down feature is compatible only with deployment targets, \
please either disable this option or specify a deployment target."
.into(),
));
}
}

Ok(())
Expand Down
51 changes: 40 additions & 11 deletions mirrord/operator/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ use mirrord_auth::{
certificate::Certificate, credential_store::CredentialStoreSync, error::AuthenticationError,
};
use mirrord_config::{
feature::network::incoming::ConcurrentSteal, target::TargetConfig, LayerConfig,
feature::network::incoming::ConcurrentSteal,
target::{Target, TargetConfig},
LayerConfig,
};
use mirrord_kube::{
api::kubernetes::{create_kube_api, get_k8s_resource_api},
Expand All @@ -33,8 +35,8 @@ static CONNECTION_CHANNEL_SIZE: usize = 1000;

#[derive(Debug, Error)]
pub enum OperatorApiError {
#[error("unable to create target for TargetConfig")]
InvalidTarget,
#[error("invalid target: {reason}")]
InvalidTarget { reason: String },
#[error(transparent)]
HttpError(#[from] http::Error),
#[error(transparent)]
Expand Down Expand Up @@ -159,7 +161,7 @@ impl OperatorApi {

/// Checks used config against operator specification.
fn check_config(config: &LayerConfig, operator: &MirrordOperatorCrd) -> Result<()> {
if config.feature.copy_target {
if config.feature.copy_target.enabled {
let feature_enabled = operator.spec.copy_target_enabled.unwrap_or(false);

if !feature_enabled {
Expand All @@ -168,6 +170,14 @@ impl OperatorApi {
operator_version: operator.spec.operator_version.clone(),
});
}

if config.feature.copy_target.scale_down
&& !matches!(config.target.path, Some(Target::Deployment(..)))
{
return Err(OperatorApiError::InvalidTarget {
reason: "scale down feature is enabled, but target is not a deployment".into(),
});
}
}

Ok(())
Expand Down Expand Up @@ -238,14 +248,29 @@ impl OperatorApi {
}
version_progress.success(None);

let raw_target = operator_api
.fetch_target()
.await?
.ok_or(OperatorApiError::InvalidTarget)?;
let raw_target =
operator_api
.fetch_target()
.await?
.ok_or(OperatorApiError::InvalidTarget {
reason: "not found in the cluster".into(),
})?;

let target_to_connect = if config.feature.copy_target {
let target_to_connect = if config.feature.copy_target.enabled {
let mut copy_progress = progress.subtask("copying target");
let copied = operator_api.copy_target(&metadata, raw_target).await?;

if config.feature.copy_target.scale_down {
let is_deployment = matches!(config.target.path, Some(Target::Deployment(..)));
if !is_deployment {
progress.warning(
"cannot scale down while copying target - target is not a deployment",
)
}
}

let copied = operator_api
.copy_target(&metadata, raw_target, config.feature.copy_target.scale_down)
.await?;
copy_progress.success(None);

OperatorSessionTarget::Copied(copied)
Expand Down Expand Up @@ -465,18 +490,22 @@ impl OperatorApi {
&self,
session_metadata: &OperatorSessionMetadata,
target: TargetCrd,
scale_down: bool,
) -> Result<CopyTargetCrd> {
let raw_target = target
.spec
.target
.clone()
.ok_or(OperatorApiError::InvalidTarget)?;
.ok_or(OperatorApiError::InvalidTarget {
reason: "copy target feature is not compatible with targetless mode".into(),
})?;

let requested = CopyTargetCrd::new(
&target.name(),
CopyTargetSpec {
target: raw_target,
idle_ttl: Some(Self::COPIED_POD_IDLE_TTL),
scale_down,
},
);

Expand Down
5 changes: 4 additions & 1 deletion mirrord/operator/src/crd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,12 @@ pub enum OperatorFeatures {
namespaced
)]
pub struct CopyTargetSpec {
/// Original target.
/// Original target. Only [`Target::Pod`] and [`Target::Deployment`] are accepted.
pub target: Target,
/// How long should the operator keep this pod alive after its creation.
/// The pod is deleted when this timout has expired and there are no connected clients.
pub idle_ttl: Option<u32>,
/// Should the operator scale down target deployment to 0 while this pod is alive.
/// Ignored if [`Target`] is not [`Target::Deployment`].
pub scale_down: bool,
}

0 comments on commit b050ef3

Please sign in to comment.