Skip to content
This repository has been archived by the owner on Oct 16, 2024. It is now read-only.

add and emit pool owner metadata for alerting #327

Merged
merged 11 commits into from
Oct 13, 2023
Merged

add and emit pool owner metadata for alerting #327

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
6934faa
add and emit pool owner metadata for alerting
gmdfalk Jun 27, 2023
d5a6c23
newline
gmdfalk Jun 29, 2023
f32e387
tests failing locally
gmdfalk Jun 29, 2023
327c8fa
fix formatting
gmdfalk Jul 5, 2023
aad905e
Merge branch 'master' into u/gfalk/add_pool_owner
gmdfalk Aug 21, 2023
553ee0d
add team label to all autoscaler metrics
gmdfalk Aug 21, 2023
53fcf7c
update tests
gmdfalk Aug 21, 2023
b912f4b
formatting
gmdfalk Aug 21, 2023
69d9308
dummy commit
gmdfalk Oct 11, 2023
ffc4439
precommit
gmdfalk Oct 11, 2023
987f308
use team instead of pool_owner for emitted metrics
gmdfalk Oct 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions acceptance/srv-configs/clusterman-clusters/local-dev/default.kubernetes
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,4 @@ autoscaling:
instance_loss_threshold: 3

alert_on_max_capacity: false
pool_owner: compute_infra
1 change: 1 addition & 0 deletions acceptance/srv-configs/clusterman-clusters/local-dev/default.mesos
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,4 @@ autoscale_signal:
minute_range: 10

alert_on_max_capacity: false
pool_owner: compute_infra
2 changes: 1 addition & 1 deletion clusterman/autoscaler/autoscaler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ def run(self, dry_run: bool = False, timestamp: Optional[arrow.Arrow] = None) ->
self.target_capacity_gauge.set(new_target_capacity, {"dry_run": dry_run})
self.max_capacity_gauge.set(
self.pool_manager.max_capacity,
{"dry_run": dry_run, "alert_on_max_capacity": self.pool_manager.alert_on_max_capacity},
{"dry_run": dry_run, "alert_on_max_capacity": self.pool_manager.alert_on_max_capacity, "team": self.pool_manager.pool_owner},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we s/team/pool_owner to keep things consistent between config and metric labeling?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I knowingly set it to team because that's what our alertmanager automatically resolves for ticketing/alerting.
If we don't set it to team here, we'll have to relabel in the prometheus query for the max_pool_capacity alerting.

Still want to keep it at pool_owner? @nemacysts @jfongatyelp

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh, that's fine then - might be good to add that in a comment here tho since i'm not sure how many people are aware of that (or maybe i'm just the only one that didn't know this :p)

)
self.setpoint_gauge.set(self.autoscaling_config.setpoint, {"dry_run": dry_run})
self._emit_requested_resource_metrics(resource_request, dry_run=dry_run)
Expand Down
1 change: 1 addition & 0 deletions clusterman/autoscaler/pool_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ def __init__(
"autoscaling.killable_nodes_prioritizing_v2", default=False
)
self.alert_on_max_capacity = self.pool_config.read_bool("alert_on_max_capacity", default=True)
self.pool_owner = self.pool_config.read_string("pool_owner", default="compute_infra")
monitoring_info = {"cluster": cluster, "pool": pool}
self.killable_nodes_counter = get_monitoring_client().create_counter(SFX_KILLABLE_NODES_COUNT, monitoring_info)

Expand Down
1 change: 1 addition & 0 deletions clusterman/simulator/simulated_pool_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def __init__(
MAX_MIN_NODE_SCALEIN_UPTIME_SECONDS,
)
self.alert_on_max_capacity = self.pool_config.read_bool("alert_on_max_capacity", default=True)
self.pool_owner = self.pool_config.read_string("pool_owner", default="compute_infra")
self.killable_nodes_prioritizing_v2 = self.pool_config.read_bool(
"autoscaling.killable_nodes_prioritizing_v2", default=False
)
Expand Down
3 changes: 2 additions & 1 deletion examples/schemas/pool.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@
"additionalProperties": false
},
"sensu_config": {"$ref": "definitions.json#sensu_config"},
"alert_on_max_capacity": {"type": "boolean"}
"alert_on_max_capacity": {"type": "boolean"},
"pool_owner": {"type": "string"}
},
"additionalProperties": false
}
2 changes: 2 additions & 0 deletions itests/environment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ def setup_configurations(context):
],
},
"alert_on_max_capacity": True,
"pool_owner": "compute_infra",
}
kube_pool_config = {
"resource_groups": [
Expand All @@ -144,6 +145,7 @@ def setup_configurations(context):
"period_minutes": 7,
},
"alert_on_max_capacity": True,
"pool_owner": "compute_infra",
}
with staticconf.testing.MockConfiguration(
boto_config, namespace=CREDENTIALS_NAMESPACE
Expand Down
4 changes: 3 additions & 1 deletion tests/autoscaler/autoscaler_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ def pool_configs():
"max_weight_to_remove": 10,
},
"alert_on_max_capacity": True,
"pool_owner": "compute_infra",
},
namespace=POOL_NAMESPACE.format(pool="bar", scheduler="mesos"),
):
Expand Down Expand Up @@ -91,6 +92,7 @@ def mock_autoscaler():
"alert_on_max_capacity",
namespace=POOL_NAMESPACE.format(pool="bar", scheduler="mesos"),
)
mock_autoscaler.pool_manager.pool_owner = "compute_infra"
mock_autoscaler.pool_manager.non_orphan_fulfilled_capacity = 0

mock_autoscaler.target_capacity_gauge = mock.Mock(spec=GaugeProtocol)
Expand Down Expand Up @@ -160,7 +162,7 @@ def test_autoscaler_run(dry_run, mock_autoscaler, run_timestamp):

assert mock_autoscaler.target_capacity_gauge.set.call_args == mock.call(100, {"dry_run": dry_run})
assert mock_autoscaler.max_capacity_gauge.set.call_args == mock.call(
mock_autoscaler.pool_manager.max_capacity, {"dry_run": dry_run, "alert_on_max_capacity": True}
mock_autoscaler.pool_manager.max_capacity, {"dry_run": dry_run, "alert_on_max_capacity": True, "pool_owner": "compute_infra"}
)
assert mock_autoscaler.setpoint_gauge.set.call_args == mock.call(0.7, {"dry_run": dry_run})
assert mock_autoscaler._compute_target_capacity.call_args == mock.call(resource_request)
Expand Down
2 changes: 2 additions & 0 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ def clusterman_pool_config():
],
},
"alert_on_max_capacity": True,
"pool_owner": "compute_infra",
}
with staticconf.testing.MockConfiguration(config, namespace="bar.mesos_config"):
yield
Expand Down Expand Up @@ -202,6 +203,7 @@ def clusterman_k8s_pool_config():
"disable_autoscaling": False,
},
"alert_on_max_capacity": False,
"pool_owner": "foo",
}
with staticconf.testing.MockConfiguration(config, namespace="bar.kubernetes_config"):
yield
Expand Down