diff --git a/Makefile b/Makefile index 07824f3f9..50579058b 100644 --- a/Makefile +++ b/Makefile @@ -59,21 +59,10 @@ test-external: clean-cache tox -e external -- --tags=-yelp .PHONY: itest -itest: export EXTRA_VOLUME_MOUNTS=/nail/etc/services/services.yaml:/nail/etc/services/services.yaml:ro itest: cook-image - COMPOSE_PROJECT_NAME=clusterman_jammy tox -e acceptance - ./service-itest-runner clusterman.batch.spot_price_collector "--aws-region=us-west-1 " - ./service-itest-runner clusterman.batch.cluster_metrics_collector "--cluster=local-dev" - ./service-itest-runner clusterman.batch.autoscaler_bootstrap "" clusterman.batch.autoscaler - make -C acceptance local-cluster-clean && make -C acceptance acceptance-internal .PHONY: itest-external itest-external: cook-image-external - COMPOSE_PROJECT_NAME=clusterman_jammy tox -e acceptance - ./service-itest-runner examples.batch.spot_price_collector "--aws-region=us-west-1 --env-config-path=acceptance/srv-configs/clusterman-external.yaml" - ./service-itest-runner examples.batch.cluster_metrics_collector "--cluster=local-dev --env-config-path=acceptance/srv-configs/clusterman-external.yaml" - ./service-itest-runner examples.batch.autoscaler_bootstrap "--env-config-path=acceptance/srv-configs/clusterman-external.yaml" examples.batch.autoscaler - make -C acceptance local-cluster-clean && make -C acceptance acceptance-external .PHONY: cook-image cook-image: diff --git a/package/itest/ubuntu.sh b/package/itest/ubuntu.sh index 2199237bc..f162a6de8 100755 --- a/package/itest/ubuntu.sh +++ b/package/itest/ubuntu.sh @@ -61,13 +61,6 @@ python3.8 /itest/run_instance.py \ # Run the critical clusterman CLI commands if [ ! "${EXAMPLE}" ]; then highlight_exec /usr/bin/clusterman --version - highlight_exec /usr/bin/clusterman status --cluster local-dev -v - highlight_exec /usr/bin/clusterman manage --cluster local-dev --target-capacity 10 --dry-run - highlight_exec /usr/bin/clusterman disable --cluster local-dev --until tomorrow - highlight_exec /usr/bin/clusterman enable --cluster local-dev - highlight_exec /usr/bin/clusterman simulate --cluster local-dev --start-time 2017-12-01T08:00:00Z --end-time 2017-12-01T09:00:00Z --metrics-data-files /itest/metrics.json.gz - highlight_exec /usr/bin/clusterman --log-level debug simulate --cluster local-dev --scheduler mesos --autoscaler-config /itest/autoscaler_config.yaml --start-time 2017-12-01T08:00:00Z --end-time 2017-12-01T08:05:00Z --metrics-data-files /itest/metrics.json.gz - highlight "$0:" 'success!' else /bin/bash diff --git a/tests/draining/mesos_test.py b/tests/draining/mesos_test.py deleted file mode 100644 index 3d737eb54..000000000 --- a/tests/draining/mesos_test.py +++ /dev/null @@ -1,472 +0,0 @@ -# Copyright 2019 Yelp Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -from unittest import mock - -import pytest - -from clusterman.draining.mesos import build_maintenance_payload -from clusterman.draining.mesos import build_maintenance_schedule_payload -from clusterman.draining.mesos import down -from clusterman.draining.mesos import drain -from clusterman.draining.mesos import get_machine_ids -from clusterman.draining.mesos import get_maintenance_schedule -from clusterman.draining.mesos import Hostname -from clusterman.draining.mesos import hostnames_to_components -from clusterman.draining.mesos import load_credentials -from clusterman.draining.mesos import up - - -@mock.patch("clusterman.draining.mesos.gethostbyname", autospec=True) -def test_build_maintenance_payload( - mock_gethostbyname, -): - ip = "169.254.121.212" - mock_gethostbyname.return_value = ip - hostname = "fqdn1.example.org" - hostnames = [hostname] - - assert build_maintenance_payload(hostnames, "start_maintenance",)["start_maintenance"][ - "machines" - ] == get_machine_ids(hostnames) - - -@mock.patch("clusterman.draining.mesos.gethostbyname", autospec=True) -def test_get_machine_ids_one_host( - mock_gethostbyname, -): - ip = "169.254.121.212" - mock_gethostbyname.return_value = ip - hostname = "fqdn1.example.org" - hostnames = [hostname] - expected = [ - { - "hostname": hostname, - "ip": ip, - }, - ] - assert get_machine_ids(hostnames) == expected - - -@mock.patch("clusterman.draining.mesos.gethostbyname", autospec=True) -def test_get_machine_ids_multiple_hosts( - mock_gethostbyname, -): - ip1 = "169.254.121.212" - ip2 = "169.254.121.213" - ip3 = "169.254.121.214" - mock_gethostbyname.side_effect = [ip1, ip2, ip3] - hostname1 = "fqdn1.example.org" - hostname2 = "fqdn2.example.org" - hostname3 = "fqdn3.example.org" - hostnames = [hostname1, hostname2, hostname3] - expected = [ - { - "hostname": hostname1, - "ip": ip1, - }, - { - "hostname": hostname2, - "ip": ip2, - }, - { - "hostname": hostname3, - "ip": ip3, - }, - ] - assert get_machine_ids(hostnames) == expected - - -def test_get_machine_ids_multiple_hosts_ips(): - ip1 = "169.254.121.212" - ip2 = "169.254.121.213" - ip3 = "169.254.121.214" - hostname1 = "fqdn1.example.org" - hostname2 = "fqdn2.example.org" - hostname3 = "fqdn3.example.org" - hostnames = [hostname1 + "|" + ip1, hostname2 + "|" + ip2, hostname3 + "|" + ip3] - expected = [ - { - "hostname": hostname1, - "ip": ip1, - }, - { - "hostname": hostname2, - "ip": ip2, - }, - { - "hostname": hostname3, - "ip": ip3, - }, - ] - assert get_machine_ids(hostnames) == expected - - -@mock.patch("clusterman.draining.mesos.get_maintenance_schedule", autospec=True) -@mock.patch("clusterman.draining.mesos.get_machine_ids", autospec=True) -def test_build_maintenance_schedule_payload_no_schedule( - mock_get_machine_ids, - mock_get_maintenance_schedule, -): - mock_get_maintenance_schedule.return_value.json.return_value = { - "get_maintenance_schedule": {"schedule": {}}, - } - machine_ids = [{"hostname": "machine2", "ip": "10.0.0.2"}] - mock_get_machine_ids.return_value = machine_ids - hostnames = ["fake-hostname"] - start = "1443830400000000000" - duration = "3600000000000" - actual = build_maintenance_schedule_payload(mock.Mock(), hostnames, start, duration, drain=True) - assert mock_get_maintenance_schedule.call_count == 1 - assert mock_get_machine_ids.call_count == 1 - assert mock_get_machine_ids.call_args == mock.call(hostnames) - expected = { - "type": "UPDATE_MAINTENANCE_SCHEDULE", - "update_maintenance_schedule": { - "schedule": { - "windows": [ - { - "machine_ids": machine_ids, - "unavailability": { - "start": { - "nanoseconds": int(start), - }, - "duration": { - "nanoseconds": int(duration), - }, - }, - }, - ] - } - }, - } - assert actual == expected - - -@mock.patch("clusterman.draining.mesos.get_maintenance_schedule", autospec=True) -@mock.patch("clusterman.draining.mesos.get_machine_ids", autospec=True) -def test_build_maintenance_schedule_payload_no_schedule_undrain( - mock_get_machine_ids, - mock_get_maintenance_schedule, -): - mock_get_maintenance_schedule.return_value.json.return_value = { - "get_maintenance_schedule": {"schedule": {}}, - } - machine_ids = [{"hostname": "machine2", "ip": "10.0.0.2"}] - mock_get_machine_ids.return_value = machine_ids - hostnames = ["fake-hostname"] - start = "1443830400000000000" - duration = "3600000000000" - actual = build_maintenance_schedule_payload(mock.Mock(), hostnames, start, duration, drain=False) - assert mock_get_maintenance_schedule.call_count == 1 - assert mock_get_machine_ids.call_count == 1 - assert mock_get_machine_ids.call_args == mock.call(hostnames) - expected = { - "type": "UPDATE_MAINTENANCE_SCHEDULE", - "update_maintenance_schedule": { - "schedule": { - "windows": [], - } - }, - } - assert actual == expected - - -@mock.patch("clusterman.draining.mesos.get_maintenance_schedule", autospec=True) -@mock.patch("clusterman.draining.mesos.get_machine_ids", autospec=True) -def test_build_maintenance_schedule_payload_schedule( - mock_get_machine_ids, - mock_get_maintenance_schedule, -): - mock_get_maintenance_schedule.return_value.json.return_value = { - "type": "GET_MAINTENANCE_SCHEDULE", - "get_maintenance_schedule": { - "schedule": { - "windows": [ - { - "machine_ids": [ - {"hostname": "machine1", "ip": "10.0.0.1"}, - {"hostname": "machine2", "ip": "10.0.0.2"}, - ], - "unavailability": { - "start": {"nanoseconds": 1443830400000000000}, - "duration": {"nanoseconds": 3600000000000}, - }, - }, - { - "machine_ids": [ - {"hostname": "machine3", "ip": "10.0.0.3"}, - ], - "unavailability": { - "start": {"nanoseconds": 1443834000000000000}, - "duration": {"nanoseconds": 3600000000000}, - }, - }, - ] - } - }, - } - machine_ids = [{"hostname": "machine2", "ip": "10.0.0.2"}] - mock_get_machine_ids.return_value = machine_ids - hostnames = ["machine2"] - start = "1443830400000000000" - duration = "3600000000000" - actual = build_maintenance_schedule_payload(mock.Mock(), hostnames, start, duration, drain=True) - assert mock_get_maintenance_schedule.call_count == 1 - assert mock_get_machine_ids.call_count == 1 - assert mock_get_machine_ids.call_args == mock.call(hostnames) - expected = { - "type": "UPDATE_MAINTENANCE_SCHEDULE", - "update_maintenance_schedule": { - "schedule": { - "windows": [ - { - "machine_ids": [ - {"hostname": "machine1", "ip": "10.0.0.1"}, - ], - "unavailability": { - "start": {"nanoseconds": 1443830400000000000}, - "duration": {"nanoseconds": 3600000000000}, - }, - }, - { - "machine_ids": [ - {"hostname": "machine3", "ip": "10.0.0.3"}, - ], - "unavailability": { - "start": {"nanoseconds": 1443834000000000000}, - "duration": {"nanoseconds": 3600000000000}, - }, - }, - { - "machine_ids": machine_ids, - "unavailability": { - "start": {"nanoseconds": int(start)}, - "duration": {"nanoseconds": int(duration)}, - }, - }, - ] - } - }, - } - assert actual == expected - - -@mock.patch("clusterman.draining.mesos.get_maintenance_schedule", autospec=True) -@mock.patch("clusterman.draining.mesos.get_machine_ids", autospec=True) -def test_build_maintenance_schedule_payload_schedule_undrain( - mock_get_machine_ids, - mock_get_maintenance_schedule, -): - mock_get_maintenance_schedule.return_value.json.return_value = { - "type": "GET_MAINTENANCE_SCHEDULE", - "get_maintenance_schedule": { - "schedule": { - "windows": [ - { - "machine_ids": [ - {"hostname": "machine1", "ip": "10.0.0.1"}, - {"hostname": "machine2", "ip": "10.0.0.2"}, - ], - "unavailability": { - "start": {"nanoseconds": 1443830400000000000}, - "duration": {"nanoseconds": 3600000000000}, - }, - }, - { - "machine_ids": [ - {"hostname": "machine3", "ip": "10.0.0.3"}, - ], - "unavailability": { - "start": {"nanoseconds": 1443834000000000000}, - "duration": {"nanoseconds": 3600000000000}, - }, - }, - ] - } - }, - } - machine_ids = [{"hostname": "machine2", "ip": "10.0.0.2"}] - mock_get_machine_ids.return_value = machine_ids - hostnames = ["machine2"] - start = "1443830400000000000" - duration = "3600000000000" - actual = build_maintenance_schedule_payload(mock.Mock(), hostnames, start, duration, drain=False) - assert mock_get_maintenance_schedule.call_count == 1 - assert mock_get_machine_ids.call_count == 1 - assert mock_get_machine_ids.call_args == mock.call(hostnames) - expected = { - "type": "UPDATE_MAINTENANCE_SCHEDULE", - "update_maintenance_schedule": { - "schedule": { - "windows": [ - { - "machine_ids": [ - {"hostname": "machine1", "ip": "10.0.0.1"}, - ], - "unavailability": { - "start": {"nanoseconds": 1443830400000000000}, - "duration": {"nanoseconds": 3600000000000}, - }, - }, - { - "machine_ids": [ - {"hostname": "machine3", "ip": "10.0.0.3"}, - ], - "unavailability": { - "start": {"nanoseconds": 1443834000000000000}, - "duration": {"nanoseconds": 3600000000000}, - }, - }, - ] - } - }, - } - assert actual == expected - - -@mock.patch("clusterman.draining.mesos.open", create=True, autospec=None) -def test_load_credentials( - mock_open, -): - principal = "username" - secret = "password" - credentials = { - "principal": principal, - "secret": secret, - } - - mock_open.side_effect = mock.mock_open(read_data=json.dumps(credentials)) - - credentials = load_credentials("/nail/blah") - - assert credentials.principal == principal - assert credentials.secret == secret - - -@mock.patch("clusterman.draining.mesos.open", create=True, side_effect=IOError, autospec=None) -def test_load_credentials_missing_file( - mock_open, -): - with pytest.raises(IOError): - assert load_credentials("/nail/blah") - - -@mock.patch("clusterman.draining.mesos.open", create=True, autospec=None) -def test_load_credentials_keyerror( - mock_open, -): - credentials = {} - - mock_open.side_effect = mock.mock_open(read_data=json.dumps(credentials)) - - with pytest.raises(KeyError): - assert load_credentials("/nail/blah") - - -def test_get_maintenance_schedule(): - mock_operator_api = mock.Mock() - get_maintenance_schedule(mock_operator_api) - assert mock_operator_api.call_count == 1 - assert mock_operator_api.call_args == mock.call(data={"type": "GET_MAINTENANCE_SCHEDULE"}) - - -@mock.patch("clusterman.draining.mesos.build_maintenance_schedule_payload", autospec=True) -def test_drain( - mock_build_maintenance_schedule_payload, -): - mock_operator_api = mock.Mock() - fake_schedule = {"fake_schedule": "fake_value"} - mock_build_maintenance_schedule_payload.return_value = fake_schedule - drain( - mock_operator_api, - hostnames=["some-host"], - start="some-start", - duration="some-duration", - ) - - assert mock_build_maintenance_schedule_payload.call_count == 1 - expected_args = mock.call(mock_operator_api, ["some-host"], "some-start", "some-duration", drain=True) - assert mock_build_maintenance_schedule_payload.call_args == expected_args - - expected_args = mock.call(["some-host"]) - - assert mock_operator_api.call_count == 1 - expected_args = mock.call(data=fake_schedule) - assert mock_operator_api.call_args == expected_args - - -@mock.patch("clusterman.draining.mesos.build_maintenance_payload", autospec=True) -def test_down( - mock_build_maintenance_payload, -): - mock_operator_api = mock.Mock() - fake_payload = [{"fake_schedule": "fake_value"}] - mock_build_maintenance_payload.return_value = fake_payload - down(mock_operator_api, hostnames=["some-host"]) - assert mock_build_maintenance_payload.call_count == 1 - assert mock_build_maintenance_payload.call_args == mock.call(["some-host"], "start_maintenance") - assert mock_operator_api.call_count == 1 - expected_args = mock.call(data=fake_payload) - assert mock_operator_api.call_args == expected_args - - -@mock.patch("clusterman.draining.mesos.build_maintenance_payload", autospec=True) -def test_up( - mock_build_maintenance_payload, -): - mock_operator_api = mock.Mock() - fake_payload = [{"fake_schedule": "fake_value"}] - mock_build_maintenance_payload.return_value = fake_payload - up(mock_operator_api, hostnames=["some-host"]) - assert mock_build_maintenance_payload.call_count == 1 - assert mock_build_maintenance_payload.call_args == mock.call(["some-host"], "stop_maintenance") - assert mock_operator_api.call_count == 1 - expected_args = mock.call(data=fake_payload) - assert mock_operator_api.call_args == expected_args - - -def sideeffect_mock_get_count_running_tasks_on_slave(hostname): - if hostname == "host1": - return 3 - else: - return 0 - - -def test_hostnames_to_components_simple(): - hostname = "fake-host" - ip = None - expected = [Hostname(host=hostname, ip=ip)] - actual = hostnames_to_components([hostname]) - assert actual == expected - - -def test_hostnames_to_components_pipe(): - hostname = "fake-host" - ip = "127.0.0.1" - expected = [Hostname(host=hostname, ip=ip)] - actual = hostnames_to_components([f"{hostname}|{ip}"]) - assert actual == expected - - -@mock.patch("clusterman.draining.mesos.gethostbyname", autospec=True) -def test_hostnames_to_components_resolve( - mock_gethostbyname, -): - hostname = "fake-host" - ip = "127.0.0.1" - mock_gethostbyname.return_value = ip - expected = [Hostname(host=hostname, ip=ip)] - actual = hostnames_to_components([hostname], resolve=True) - assert actual == expected diff --git a/tests/mesos/__init__.py b/tests/mesos/__init__.py deleted file mode 100644 index 5285d9b1c..000000000 --- a/tests/mesos/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2019 Yelp Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/mesos/conftest.py b/tests/mesos/conftest.py deleted file mode 100644 index b9a487ce2..000000000 --- a/tests/mesos/conftest.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2019 Yelp Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from unittest import mock - -import pytest - - -@pytest.fixture -def mock_agents_response(): - response = mock.Mock() - response.json.return_value = { - "slaves": [ - { - "pid": "foo@10.10.10.10:1", - "attributes": { - "blah": 10, - "pool": "asdf", - }, - "hostname": "not-in-the-pool.yelpcorp.com", - }, - { - "pid": "foo@10.10.10.11:1", - "hostname": "asdf.yelpcorp.com", - "used_resources": {"mem": 10}, - }, - { - "pid": "foo@10.10.10.12:1", - "attributes": { - "blah": 10, - "pool": "bar", - "ssss": "hjkl", - }, - "hostname": "im-in-the-pool.yelpcorp.com", - "used_resources": { - "mem": 20, - "cpus": 10, - }, - }, - ] - } - return response diff --git a/tests/mesos/mesos_cluster_connector_test.py b/tests/mesos/mesos_cluster_connector_test.py deleted file mode 100644 index f76f53582..000000000 --- a/tests/mesos/mesos_cluster_connector_test.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2019 Yelp Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from unittest import mock - -import pytest - -from clusterman.exceptions import PoolManagerError -from clusterman.interfaces.types import AgentState -from clusterman.mesos.mesos_cluster_connector import MesosClusterConnector -from clusterman.mesos.mesos_cluster_connector import TaskCount - - -@pytest.fixture -def mock_cluster_connector(): - mock_cluster_connector = MesosClusterConnector("mesos-test", "bar") - mock_cluster_connector._agents_by_ip = { - "10.10.10.1": { - "id": "idle", - "resources": {"cpus": 4, "gpus": 2}, - }, - "10.10.10.2": { - "id": "no-gpus", - "resources": {"cpus": 8}, - "used_resources": {"cpus": 1.5}, - }, - } - mock_cluster_connector._task_count_per_agent = { - "idle": TaskCount(all_tasks=0, batch_tasks=0), - "no-gpus": TaskCount(all_tasks=1, batch_tasks=0), - } - return mock_cluster_connector - - -def test_init(mock_cluster_connector): - assert mock_cluster_connector.api_endpoint == "http://the.mesos.leader:5050/" - - -@pytest.mark.parametrize( - "ip_address,expected_state", - [ - (None, AgentState.UNKNOWN), - ("1.2.3.4", AgentState.ORPHANED), - ("10.10.10.1", AgentState.IDLE), - ("10.10.10.2", AgentState.RUNNING), - ], -) -def test_get_agent_metadata(mock_cluster_connector, ip_address, expected_state): - agent_metadata = mock_cluster_connector.get_agent_metadata(ip_address) - assert agent_metadata.state == expected_state - - -def test_count_tasks_by_agent(mock_cluster_connector): - mock_cluster_connector._tasks = [ - {"slave_id": "1", "state": "TASK_RUNNING", "framework_id": "2"}, - {"slave_id": "2", "state": "TASK_RUNNING", "framework_id": "2"}, - {"slave_id": "3", "state": "TASK_FINISHED", "framework_id": "2"}, - {"slave_id": "1", "state": "TASK_FAILED", "framework_id": "2"}, - {"slave_id": "2", "state": "TASK_RUNNING", "framework_id": "1"}, - ] - mock_cluster_connector._frameworks = { - "1": {"name": "chronos"}, - "2": {"name": "marathon123"}, - } - assert mock_cluster_connector._count_tasks_per_agent() == { - "1": TaskCount(all_tasks=1, batch_tasks=0), - "2": TaskCount(all_tasks=2, batch_tasks=1), - } - - -def test_is_batch_task(mock_cluster_connector): - mock_cluster_connector.non_batch_framework_prefixes = ("marathon", "paasta") - assert mock_cluster_connector._is_batch_framework("chronos4") - assert not mock_cluster_connector._is_batch_framework("marathon123") - - -@mock.patch("clusterman.mesos.mesos_cluster_connector.mesos_post") -class TestAgentListing: - def test_agent_list_error(self, mock_post, mock_cluster_connector): - mock_post.side_effect = PoolManagerError("dummy error") - with pytest.raises(PoolManagerError): - mock_cluster_connector._get_agents_by_ip() - - def test_filter_pools(self, mock_post, mock_agents_response, mock_cluster_connector): - mock_post.return_value = mock_agents_response - agents = mock_cluster_connector._get_agents_by_ip() - assert len(agents) == 1 - assert agents["10.10.10.12"]["hostname"] == "im-in-the-pool.yelpcorp.com" - - # Multiple calls should have the same result. - assert agents == mock_cluster_connector._get_agents_by_ip() - - -def test_allocation(mock_cluster_connector): - assert mock_cluster_connector.get_resource_allocation("cpus") == 1.5 - - -def test_total_cpus(mock_cluster_connector): - assert mock_cluster_connector.get_resource_total("cpus") == 12 - - -@pytest.mark.parametrize("resource_name,expected", [("mem", 0), ("cpus", 0.125)]) -def test_average_allocation(mock_cluster_connector, resource_name, expected): - assert mock_cluster_connector.get_percent_resource_allocation(resource_name) == expected diff --git a/tests/mesos/metrics_generators_test.py b/tests/mesos/metrics_generators_test.py deleted file mode 100644 index fda186711..000000000 --- a/tests/mesos/metrics_generators_test.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2019 Yelp Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from unittest import mock - -import pytest - -from clusterman.autoscaler.pool_manager import PoolManager -from clusterman.exceptions import NoResourceGroupsFoundError -from clusterman.mesos.metrics_generators import ClusterMetric -from clusterman.mesos.metrics_generators import generate_simple_metadata -from clusterman.mesos.metrics_generators import generate_system_metrics - - -@pytest.fixture -def mock_pool_manager(): - mock_pool_manager = mock.Mock(spec=PoolManager) - mock_pool_manager.cluster_connector = mock.Mock(cluster="mesos-test", pool="bar", scheduler="mesos") - mock_pool_manager.cluster = "mesos-test" - mock_pool_manager.pool = "bar" - mock_pool_manager.scheduler = "mesos" - - resource_totals = {"cpus": 20, "mem": 2000, "disk": 20000, "gpus": 0} - mock_pool_manager.cluster_connector.get_resource_total.side_effect = resource_totals.get - - market_capacities = {"market1": 15, "market2": 25} - mock_pool_manager.get_market_capacities.return_value = market_capacities - - mock_pool_manager.non_orphan_fulfilled_capacity = 12 - - return mock_pool_manager - - -@pytest.fixture -def simple_metadata_expected_metrics(mock_pool_manager): - return [ - ClusterMetric( - metric_name="cpus_total", - value=20, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ClusterMetric( - metric_name="mem_total", - value=2000, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ClusterMetric( - metric_name="disk_total", - value=20000, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ClusterMetric( - metric_name="gpus_total", - value=0, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ClusterMetric( - metric_name="target_capacity", - value=mock_pool_manager.target_capacity, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ClusterMetric( - metric_name="fulfilled_capacity", - value={"market1": 15, "market2": 25}, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ClusterMetric( - metric_name="non_orphan_fulfilled_capacity", - value=12, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ] - - -def test_generate_system_metrics(mock_pool_manager): - resources_allocated = {"cpus": 10, "mem": 1000, "disk": 10000, "gpus": 0} - mock_pool_manager.cluster_connector.get_resource_allocation.side_effect = resources_allocated.get - - expected_metrics = [ - ClusterMetric( - metric_name="cpus_allocated", - value=10, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ClusterMetric( - metric_name="mem_allocated", - value=1000, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ClusterMetric( - metric_name="disk_allocated", - value=10000, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ClusterMetric( - metric_name="gpus_allocated", - value=0, - dimensions={"cluster": "mesos-test", "pool": "bar.mesos"}, - ), - ] - assert sorted(generate_system_metrics(mock_pool_manager)) == sorted(expected_metrics) - - -def test_generate_simple_metadata(mock_pool_manager, simple_metadata_expected_metrics): - assert sorted(generate_simple_metadata(mock_pool_manager)) == sorted(simple_metadata_expected_metrics) - - -def test_generate_simple_metadata_no_terraform_resources(mock_pool_manager, simple_metadata_expected_metrics): - # Break target_capacity, make sure exception is handled - type(mock_pool_manager).target_capacity = mock.PropertyMock(side_effect=NoResourceGroupsFoundError) - - # Update expected metrics to exclude target_capacity - expected_metrics = list(simple_metadata_expected_metrics) - del expected_metrics[4] - - assert sorted(generate_simple_metadata(mock_pool_manager)) == sorted(expected_metrics) diff --git a/tests/mesos/util_test.py b/tests/mesos/util_test.py deleted file mode 100644 index 43cc441fe..000000000 --- a/tests/mesos/util_test.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2019 Yelp Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from unittest import mock - -import pytest - -from clusterman.exceptions import PoolConnectionError -from clusterman.mesos.util import agent_pid_to_ip -from clusterman.mesos.util import allocated_agent_resources -from clusterman.mesos.util import mesos_post - - -@pytest.fixture -def mock_market_capacities(): - return {"market-1": 1000, "market-2": 5} - - -@pytest.fixture -def mock_agent_pid_to_ip(): - with mock.patch("clusterman.mesos.util.agent_pid_to_ip") as mock_agent_pid_to_ip: - mock_agent_pid_to_ip.return_value = "1.2.3.4" - yield - - -def test_agent_pid_to_ip(): - ret = agent_pid_to_ip("slave(1)@10.40.31.172:5051") - assert ret == "10.40.31.172" - - -def test_allocated_agent_resources(mock_agents_response): - assert allocated_agent_resources(mock_agents_response.json()["slaves"][0])[0] == 0 - assert allocated_agent_resources(mock_agents_response.json()["slaves"][1])[0] == 0 - assert allocated_agent_resources(mock_agents_response.json()["slaves"][2])[0] == 10 - assert allocated_agent_resources(mock_agents_response.json()["slaves"][2])[1] == 20 - - -@mock.patch("clusterman.mesos.util.mesos_post", wraps=mesos_post) -class TestMesosPost: - def test_success(self, wrapped_post): - with mock.patch("clusterman.mesos.util.requests"): - wrapped_post("http://the.mesos.master/", "an-endpoint") - assert wrapped_post.call_count == 2 - assert wrapped_post.call_args_list == [ - mock.call("http://the.mesos.master/", "an-endpoint"), - mock.call("http://the.mesos.master/", "redirect"), - ] - - def test_failure(self, wrapped_post): - with mock.patch("clusterman.mesos.util.requests") as mock_requests, pytest.raises(PoolConnectionError): - mock_requests.post.side_effect = Exception("something bad happened") - wrapped_post("http://the.mesos.master/", "an-endpoint")