From f85faa7e1b327fdcaf43ebb677bb5dc6a4d5f41d Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Wed, 30 Sep 2020 15:46:28 +0800 Subject: [PATCH 01/32] update ElastiCache default limits --- awslimitchecker/services/elasticache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/awslimitchecker/services/elasticache.py b/awslimitchecker/services/elasticache.py index b4312bad..28ac3b5f 100644 --- a/awslimitchecker/services/elasticache.py +++ b/awslimitchecker/services/elasticache.py @@ -168,7 +168,7 @@ def get_limits(self): limits['Nodes'] = AwsLimit( 'Nodes', self, - 100, + 300, self.warning_threshold, self.critical_threshold, limit_type='AWS::ElastiCache::CacheNode', @@ -177,7 +177,7 @@ def get_limits(self): limits['Nodes per Cluster'] = AwsLimit( 'Nodes per Cluster', self, - 20, + 40, self.warning_threshold, self.critical_threshold, limit_type='AWS::ElastiCache::CacheNode', @@ -186,7 +186,7 @@ def get_limits(self): limits['Subnet Groups'] = AwsLimit( 'Subnet Groups', self, - 50, + 150, self.warning_threshold, self.critical_threshold, limit_type='AWS::ElastiCache::SubnetGroup', @@ -204,7 +204,7 @@ def get_limits(self): limits['Parameter Groups'] = AwsLimit( 'Parameter Groups', self, - 20, + 150, self.warning_threshold, self.critical_threshold, limit_type='AWS::ElastiCache::ParameterGroup', From 9fe498e2387486e9dac20a60751801811754b19b Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Wed, 30 Sep 2020 21:35:30 +0800 Subject: [PATCH 02/32] add support for missing RDS limits (fixes #489) --- awslimitchecker/services/rds.py | 40 +++++++++++++++++++ .../tests/services/result_fixtures.py | 20 ++++++++++ awslimitchecker/tests/services/test_rds.py | 22 +++++++++- 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/awslimitchecker/services/rds.py b/awslimitchecker/services/rds.py index 2c0080a2..64e69877 100644 --- a/awslimitchecker/services/rds.py +++ b/awslimitchecker/services/rds.py @@ -69,6 +69,10 @@ class _RDSService(_AwsService): 'ReadReplicasPerMaster': 'Read replicas per master', 'DBClusters': 'DB Clusters', 'DBClusterParameterGroups': 'DB Cluster Parameter Groups', + 'DBInstanceRoles': 'DB Instance Roles', + 'DBClusterRoles': 'DB Cluster Roles', + 'CustomEndpointsPerDBCluster': 'Custom Endpoints Per DB Cluster', + 'ManualClusterSnapshots': 'Manual Cluster Snapshots', } def find_usage(self): @@ -284,6 +288,42 @@ def get_limits(self): ta_limit_name='Cluster parameter groups', quotas_name='DB cluster parameter groups' ) + limits['Manual Cluster Snapshots'] = AwsLimit( + 'Manual Cluster Snapshots', + self, + 100, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::RDS::DBCluster', + quotas_name='Manual Cluster Snapshots' + ) + limits['Custom Endpoints Per DB Cluster'] = AwsLimit( + 'Custom Endpoints Per DB Cluster', + self, + 5, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::RDS::DBCluster', + quotas_name='Custom Endpoints Per DB Cluster' + ) + limits['DB Instance Roles'] = AwsLimit( + 'DB Instance Roles', + self, + 5, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::RDS::DBInstance', + quotas_name='DB Instance Roles' + ) + limits['DB Cluster Roles'] = AwsLimit( + 'DB Cluster Roles', + self, + 5, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::RDS::DBCluster', + quotas_name='DB Cluster Roles' + ) self.limits = limits return limits diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 6ece80d4..8c9d3ca2 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -1113,6 +1113,26 @@ class RDS(object): 'AccountQuotaName': 'DBClusterParameterGroups', 'Used': 6 }, + { + 'Max': 11, + 'AccountQuotaName': 'DBInstanceRoles', + 'Used': 1 + }, + { + 'Max': 12, + 'AccountQuotaName': 'DBClusterRoles', + 'Used': 2 + }, + { + 'Max': 13, + 'AccountQuotaName': 'CustomEndpointsPerDBCluster', + 'Used': 3 + }, + { + 'Max': 101, + 'AccountQuotaName': 'ManualClusterSnapshots', + 'Used': 5 + }, { 'Max': 98, 'AccountQuotaName': 'Foo', diff --git a/awslimitchecker/tests/services/test_rds.py b/awslimitchecker/tests/services/test_rds.py index c543e936..763b5e1d 100644 --- a/awslimitchecker/tests/services/test_rds.py +++ b/awslimitchecker/tests/services/test_rds.py @@ -86,7 +86,11 @@ def test_get_limits(self): 'Event Subscriptions', 'Read replicas per master', 'DB Clusters', - 'DB Cluster Parameter Groups' + 'DB Cluster Parameter Groups', + 'DB Cluster Roles', + 'DB Instance Roles', + 'Custom Endpoints Per DB Cluster', + 'Manual Cluster Snapshots' ]) for name, limit in res.items(): assert limit.service == cls @@ -340,3 +344,19 @@ def test_update_limits_from_api(self): lim = cls.limits['DB Cluster Parameter Groups'] assert lim.api_limit == 51 assert lim.get_current_usage()[0].get_value() == 6 + + lim = cls.limits['Manual Cluster Snapshots'] + assert lim.api_limit == 101 + assert lim.get_current_usage()[0].get_value() == 5 + + lim = cls.limits['DB Instance Roles'] + assert lim.api_limit == 11 + assert lim.get_current_usage()[0].get_value() == 1 + + lim = cls.limits['DB Cluster Roles'] + assert lim.api_limit == 12 + assert lim.get_current_usage()[0].get_value() == 2 + + lim = cls.limits['Custom Endpoints Per DB Cluster'] + assert lim.api_limit == 13 + assert lim.get_current_usage()[0].get_value() == 3 From 21f965d48106e2b335fd3404592bd168e8bdf387 Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Wed, 30 Sep 2020 23:16:39 +0800 Subject: [PATCH 03/32] #472 - Initial support of EKS service --- awslimitchecker/services/__init__.py | 1 + awslimitchecker/services/eks.py | 115 +++++++++++++++ .../tests/services/result_fixtures.py | 10 ++ awslimitchecker/tests/services/test_eks.py | 132 ++++++++++++++++++ 4 files changed, 258 insertions(+) create mode 100644 awslimitchecker/services/eks.py create mode 100644 awslimitchecker/tests/services/test_eks.py diff --git a/awslimitchecker/services/__init__.py b/awslimitchecker/services/__init__.py index 4b6c5d4b..e84cfe90 100644 --- a/awslimitchecker/services/__init__.py +++ b/awslimitchecker/services/__init__.py @@ -51,6 +51,7 @@ from awslimitchecker.services.elasticache import _ElastiCacheService from awslimitchecker.services.elasticbeanstalk import _ElasticBeanstalkService from awslimitchecker.services.elb import _ElbService +from awslimitchecker.services.eks import _EksService from awslimitchecker.services.firehose import _FirehoseService from awslimitchecker.services.iam import _IamService from awslimitchecker.services.lambdafunc import _LambdaService diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py new file mode 100644 index 00000000..b9f1d5ef --- /dev/null +++ b/awslimitchecker/services/eks.py @@ -0,0 +1,115 @@ +""" +awslimitchecker/services/eks.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import abc # noqa +import logging + +from .base import _AwsService +from ..limit import AwsLimit +from ..utils import paginate_dict + +logger = logging.getLogger(__name__) + + +class _EksService(_AwsService): + + service_name = 'EKS' + api_name = 'eks' # AWS API name to connect to (boto3.client) + quotas_service_code = 'eks' + + def find_usage(self): + """ + Determine the current usage for each limit of this service, + and update corresponding Limit via + :py:meth:`~.AwsLimit._add_current_usage`. + """ + logger.debug("Checking usage for service %s", self.service_name) + self.connect() + for lim in self.limits.values(): + lim._reset_usage() + self._find_clusters() + self._have_usage = True + logger.debug("Done checking usage.") + + def _find_clusters(self): + usage = paginate_dict( + self.conn.list_clusters, + alc_marker_path=['NextToken'], + alc_data_path=['ResourceListName'], + alc_marker_param='NextToken' + ) + self.limits['Clusters']._add_current_usage( + len(usage['clusters']), + resource_id=self._boto3_connection_kwargs['region_name'], + aws_type='AWS::EKS::Cluster') + + def get_limits(self): + """ + Return all known limits for this service, as a dict of their names + to :py:class:`~.AwsLimit` objects. + + :returns: dict of limit names to :py:class:`~.AwsLimit` objects + :rtype: dict + """ + if self.limits != {}: + return self.limits + limits = {} + limits['Clusters'] = AwsLimit( + 'Clusters', + self, + 100, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Cluster', + ) + self.limits = limits + return limits + + def required_iam_permissions(self): + """ + Return a list of IAM Actions required for this Service to function + properly. All Actions will be shown with an Effect of "Allow" + and a Resource of "*". + + :returns: list of IAM Action strings + :rtype: list + """ + return [ + "eks:ListClusters", + ] diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 6ece80d4..0bcec900 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -4564,3 +4564,13 @@ class CloudTrail(object): } ] } + + +class EKS(object): + + test_find_clusters = { + 'clusters': [ + 'devel', + 'prod', + ] + } diff --git a/awslimitchecker/tests/services/test_eks.py b/awslimitchecker/tests/services/test_eks.py new file mode 100644 index 00000000..4da62397 --- /dev/null +++ b/awslimitchecker/tests/services/test_eks.py @@ -0,0 +1,132 @@ +""" +awslimitchecker/tests/services/test_eks.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import sys +from awslimitchecker.tests.services import result_fixtures +from awslimitchecker.services.eks import _EksService + +# https://code.google.com/p/mock/issues/detail?id=249 +# py>=3.4 should use unittest.mock not the mock package on pypi +if ( + sys.version_info[0] < 3 or + sys.version_info[0] == 3 and sys.version_info[1] < 4 +): + from mock import patch, call, Mock, DEFAULT +else: + from unittest.mock import patch, call, Mock, DEFAULT + + +pbm = 'awslimitchecker.services.eks' # module patch base +pb = '%s._EksService' % pbm # class patch pase + + +class Test_EksService(object): + + def test_init(self): + """test __init__()""" + cls = _EksService(21, 43, {}, None) + assert cls.service_name == 'EKS' + assert cls.api_name == 'eks' + assert cls.conn is None + assert cls.warning_threshold == 21 + assert cls.critical_threshold == 43 + + def test_get_limits(self): + cls = _EksService(21, 43, {}, None) + cls.limits = {} + res = cls.get_limits() + assert sorted(res.keys()) == sorted([ + 'Clusters', + ]) + for name, limit in res.items(): + assert limit.service == cls + assert limit.def_warning_threshold == 21 + assert limit.def_critical_threshold == 43 + + def test_get_limits_again(self): + """test that existing limits dict is returned on subsequent calls""" + mock_limits = Mock() + cls = _EksService(21, 43, {}, None) + cls.limits = mock_limits + res = cls.get_limits() + assert res == mock_limits + + def test_find_usage(self): + """test find usage method calls other methods""" + mock_conn = Mock() + with patch('%s.connect' % pb) as mock_connect: + with patch.multiple( + pb, + _find_clusters=DEFAULT, + ) as mocks: + cls = _EksService(21, 43, {}, None) + cls.conn = mock_conn + assert cls._have_usage is False + cls.find_usage() + assert mock_connect.mock_calls == [call()] + assert cls._have_usage is True + assert mock_conn.mock_calls == [] + for x in [ + '_find_clusters', + ]: + assert mocks[x].mock_calls == [call()] + + def test_find_clusters(self): + response = result_fixtures.EKS.test_find_clusters + limit_key = 'Clusters' + + mock_conn = Mock() + mock_conn.list_clusters.return_value = response + + cls = _EksService(21, 43, {'region_name': 'us-west-2'}, None) + cls.conn = mock_conn + cls._find_clusters() + + assert mock_conn.mock_calls == [ + call.list_clusters() + ] + assert len(cls.limits[limit_key].get_current_usage()) == 1 + assert cls.limits[limit_key].get_current_usage()[ + 0].get_value() == 2 + + def test_required_iam_permissions(self): + cls = _EksService(21, 43, {}, None) + assert cls.required_iam_permissions() == [ + "eks:ListClusters" + ] From 07b3dd683d8a2c60e988260444766ef002522224 Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Thu, 1 Oct 2020 23:59:36 +0800 Subject: [PATCH 04/32] add EKS Control Plane Security Groups limit and usage support --- awslimitchecker/services/eks.py | 33 +++++++++++++-- .../tests/services/result_fixtures.py | 27 ++++++++++++- awslimitchecker/tests/services/test_eks.py | 40 +++++++++++++------ 3 files changed, 82 insertions(+), 18 deletions(-) diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py index b9f1d5ef..995a932e 100644 --- a/awslimitchecker/services/eks.py +++ b/awslimitchecker/services/eks.py @@ -63,19 +63,35 @@ def find_usage(self): self.connect() for lim in self.limits.values(): lim._reset_usage() - self._find_clusters() + self._find_clusters_usage() self._have_usage = True logger.debug("Done checking usage.") - def _find_clusters(self): - usage = paginate_dict( + def _find_clusters_usage(self): + clusters_info = paginate_dict( self.conn.list_clusters, alc_marker_path=['NextToken'], alc_data_path=['ResourceListName'], alc_marker_param='NextToken' ) + + cluster_list = clusters_info['clusters'] + + for cluster in cluster_list: + describe_cluster_response = self.conn.describe_cluster( + name=cluster + ) + security_group_id_list = describe_cluster_response['cluster'][ + 'resourcesVpcConfig']['securityGroupIds'] + self.limits[ + 'Control plane security groups per cluster']._add_current_usage( + len(security_group_id_list), + resource_id=cluster, + aws_type='AWS::EKS::Cluster' + ) + self.limits['Clusters']._add_current_usage( - len(usage['clusters']), + len(cluster_list), resource_id=self._boto3_connection_kwargs['region_name'], aws_type='AWS::EKS::Cluster') @@ -98,6 +114,14 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EKS::Cluster', ) + limits['Control plane security groups per cluster'] = AwsLimit( + 'Control plane security groups per cluster', + self, + 4, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Cluster', + ) self.limits = limits return limits @@ -112,4 +136,5 @@ def required_iam_permissions(self): """ return [ "eks:ListClusters", + "eks:DescribeCluster" ] diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 0bcec900..80444a05 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -4568,9 +4568,34 @@ class CloudTrail(object): class EKS(object): - test_find_clusters = { + test_find_clusters_usage_list = { 'clusters': [ 'devel', 'prod', ] } + + test_find_clusters_usage_describe = [ + { + 'cluster': { + 'name': 'devel', + 'resourcesVpcConfig': { + 'securityGroupIds': [ + 'abc-1234', + + ], + }, + } + }, + { + 'cluster': { + 'name': 'prod', + 'resourcesVpcConfig': { + 'securityGroupIds': [ + 'foo-1234', + 'bar-1234' + ], + }, + } + } + ] diff --git a/awslimitchecker/tests/services/test_eks.py b/awslimitchecker/tests/services/test_eks.py index 4da62397..c413d4a9 100644 --- a/awslimitchecker/tests/services/test_eks.py +++ b/awslimitchecker/tests/services/test_eks.py @@ -47,9 +47,9 @@ sys.version_info[0] < 3 or sys.version_info[0] == 3 and sys.version_info[1] < 4 ): - from mock import patch, call, Mock, DEFAULT + from mock import patch, call, Mock, DEFAULT, ANY else: - from unittest.mock import patch, call, Mock, DEFAULT + from unittest.mock import patch, call, Mock, DEFAULT, ANY pbm = 'awslimitchecker.services.eks' # module patch base @@ -73,6 +73,7 @@ def test_get_limits(self): res = cls.get_limits() assert sorted(res.keys()) == sorted([ 'Clusters', + 'Control plane security groups per cluster', ]) for name, limit in res.items(): assert limit.service == cls @@ -93,7 +94,7 @@ def test_find_usage(self): with patch('%s.connect' % pb) as mock_connect: with patch.multiple( pb, - _find_clusters=DEFAULT, + _find_clusters_usage=DEFAULT, ) as mocks: cls = _EksService(21, 43, {}, None) cls.conn = mock_conn @@ -103,30 +104,43 @@ def test_find_usage(self): assert cls._have_usage is True assert mock_conn.mock_calls == [] for x in [ - '_find_clusters', + '_find_clusters_usage', ]: assert mocks[x].mock_calls == [call()] - def test_find_clusters(self): - response = result_fixtures.EKS.test_find_clusters - limit_key = 'Clusters' + def test_find_clusters_usage(self): + list_clusters = result_fixtures.EKS.test_find_clusters_usage_list + describe_cluster = result_fixtures.EKS.test_find_clusters_usage_describe + clusters_limit_key = 'Clusters' + security_group_limit_key = 'Control plane security groups per cluster' mock_conn = Mock() - mock_conn.list_clusters.return_value = response + mock_conn.list_clusters.return_value = list_clusters + mock_conn.describe_cluster.side_effect = describe_cluster cls = _EksService(21, 43, {'region_name': 'us-west-2'}, None) cls.conn = mock_conn - cls._find_clusters() + cls._find_clusters_usage() assert mock_conn.mock_calls == [ - call.list_clusters() + call.list_clusters(), + call.describe_cluster(name=ANY), + call.describe_cluster(name=ANY) ] - assert len(cls.limits[limit_key].get_current_usage()) == 1 - assert cls.limits[limit_key].get_current_usage()[ + assert len(cls.limits[clusters_limit_key].get_current_usage()) == 1 + assert cls.limits[clusters_limit_key].get_current_usage()[ 0].get_value() == 2 + assert len(cls.limits[ + security_group_limit_key].get_current_usage()) == 2 + assert cls.limits[security_group_limit_key].get_current_usage()[ + 0].get_value() == 1 + assert cls.limits[security_group_limit_key].get_current_usage()[ + 1].get_value() == 2 + def test_required_iam_permissions(self): cls = _EksService(21, 43, {}, None) assert cls.required_iam_permissions() == [ - "eks:ListClusters" + "eks:ListClusters", + "eks:DescribeCluster" ] From 34b4b0a3eeb82d8d1d3eacea889a5c8ff3eafe87 Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Fri, 2 Oct 2020 00:44:18 +0800 Subject: [PATCH 05/32] add EKS Managed node groups per cluster limit and usage support --- awslimitchecker/services/eks.py | 30 ++++++++++++++++--- .../tests/services/result_fixtures.py | 14 +++++++++ awslimitchecker/tests/services/test_eks.py | 19 ++++++++++-- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py index 995a932e..4584057e 100644 --- a/awslimitchecker/services/eks.py +++ b/awslimitchecker/services/eks.py @@ -70,9 +70,9 @@ def find_usage(self): def _find_clusters_usage(self): clusters_info = paginate_dict( self.conn.list_clusters, - alc_marker_path=['NextToken'], - alc_data_path=['ResourceListName'], - alc_marker_param='NextToken' + alc_marker_path=['nextToken'], + alc_data_path=['clusters'], + alc_marker_param='nextToken' ) cluster_list = clusters_info['clusters'] @@ -90,6 +90,19 @@ def _find_clusters_usage(self): aws_type='AWS::EKS::Cluster' ) + list_nodegroup_response = paginate_dict( + self.conn.list_nodegroups, + clusterName=cluster, + alc_marker_path=['nextToken'], + alc_data_path=['nodegroups'], + alc_marker_param='nextToken' + ) + nodegroup_list = list_nodegroup_response['nodegroups'] + self.limits['Managed node groups per cluster']._add_current_usage( + len(nodegroup_list), + resource_id=cluster, + aws_type='AWS::EKS::Cluster') + self.limits['Clusters']._add_current_usage( len(cluster_list), resource_id=self._boto3_connection_kwargs['region_name'], @@ -122,6 +135,14 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EKS::Cluster', ) + limits['Managed node groups per cluster'] = AwsLimit( + 'Managed node groups per cluster', + self, + 30, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Nodegroup', + ) self.limits = limits return limits @@ -136,5 +157,6 @@ def required_iam_permissions(self): """ return [ "eks:ListClusters", - "eks:DescribeCluster" + "eks:DescribeCluster", + "eks:ListNodegroups" ] diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 80444a05..c84fcfc7 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -4599,3 +4599,17 @@ class EKS(object): } } ] + + test_find_clusters_usage_nodegrps = [ + { + 'nodegroups': [ + 'managed-ng-1', + 'managed-ng-2' + ] + }, + { + 'nodegroups': [ + 'managed-ng-3', + ] + } + ] diff --git a/awslimitchecker/tests/services/test_eks.py b/awslimitchecker/tests/services/test_eks.py index c413d4a9..b3075f78 100644 --- a/awslimitchecker/tests/services/test_eks.py +++ b/awslimitchecker/tests/services/test_eks.py @@ -74,6 +74,7 @@ def test_get_limits(self): assert sorted(res.keys()) == sorted([ 'Clusters', 'Control plane security groups per cluster', + 'Managed node groups per cluster', ]) for name, limit in res.items(): assert limit.service == cls @@ -111,12 +112,16 @@ def test_find_usage(self): def test_find_clusters_usage(self): list_clusters = result_fixtures.EKS.test_find_clusters_usage_list describe_cluster = result_fixtures.EKS.test_find_clusters_usage_describe + list_nodegroups = result_fixtures.EKS.test_find_clusters_usage_nodegrps + clusters_limit_key = 'Clusters' security_group_limit_key = 'Control plane security groups per cluster' + nodegroup_limit_key = 'Managed node groups per cluster' mock_conn = Mock() mock_conn.list_clusters.return_value = list_clusters mock_conn.describe_cluster.side_effect = describe_cluster + mock_conn.list_nodegroups.side_effect = list_nodegroups cls = _EksService(21, 43, {'region_name': 'us-west-2'}, None) cls.conn = mock_conn @@ -125,7 +130,9 @@ def test_find_clusters_usage(self): assert mock_conn.mock_calls == [ call.list_clusters(), call.describe_cluster(name=ANY), - call.describe_cluster(name=ANY) + call.list_nodegroups(clusterName=ANY), + call.describe_cluster(name=ANY), + call.list_nodegroups(clusterName=ANY), ] assert len(cls.limits[clusters_limit_key].get_current_usage()) == 1 assert cls.limits[clusters_limit_key].get_current_usage()[ @@ -138,9 +145,17 @@ def test_find_clusters_usage(self): assert cls.limits[security_group_limit_key].get_current_usage()[ 1].get_value() == 2 + assert len(cls.limits[ + nodegroup_limit_key].get_current_usage()) == 2 + assert cls.limits[nodegroup_limit_key].get_current_usage()[ + 0].get_value() == 2 + assert cls.limits[nodegroup_limit_key].get_current_usage()[ + 1].get_value() == 1 + def test_required_iam_permissions(self): cls = _EksService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "eks:ListClusters", - "eks:DescribeCluster" + "eks:DescribeCluster", + "eks:ListNodegroups" ] From d5dc7f1f7afc85f30b086512d8b88013b8f2a012 Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Fri, 2 Oct 2020 15:31:40 +0800 Subject: [PATCH 06/32] add EKS 'Public endpoint access CIDR ranges per cluster' limit and usage support --- awslimitchecker/services/eks.py | 18 +++++++++++++++++- .../tests/services/result_fixtures.py | 8 ++++++++ awslimitchecker/tests/services/test_eks.py | 9 +++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py index 4584057e..8a5108e5 100644 --- a/awslimitchecker/services/eks.py +++ b/awslimitchecker/services/eks.py @@ -89,7 +89,15 @@ def _find_clusters_usage(self): resource_id=cluster, aws_type='AWS::EKS::Cluster' ) - + publicAccessCidrs_list = describe_cluster_response['cluster'][ + 'resourcesVpcConfig']['publicAccessCidrs'] + self.limits[ + 'Public endpoint access CIDR ranges per cluster' + ]._add_current_usage( + len(publicAccessCidrs_list), + resource_id=cluster, + aws_type='AWS::EKS::Cluster' + ) list_nodegroup_response = paginate_dict( self.conn.list_nodegroups, clusterName=cluster, @@ -143,6 +151,14 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EKS::Nodegroup', ) + limits['Public endpoint access CIDR ranges per cluster'] = AwsLimit( + 'Public endpoint access CIDR ranges per cluster', + self, + 40, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Cluster', + ) self.limits = limits return limits diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index c84fcfc7..6c84eb8c 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -4584,6 +4584,11 @@ class EKS(object): 'abc-1234', ], + 'publicAccessCidrs': [ + '1.1.1.1/32', + '2.2.2.0/24', + '203.0.113.5/32' + ] }, } }, @@ -4595,6 +4600,9 @@ class EKS(object): 'foo-1234', 'bar-1234' ], + 'publicAccessCidrs': [ + '1.1.1.1/32', + ] }, } } diff --git a/awslimitchecker/tests/services/test_eks.py b/awslimitchecker/tests/services/test_eks.py index b3075f78..f4adf7a4 100644 --- a/awslimitchecker/tests/services/test_eks.py +++ b/awslimitchecker/tests/services/test_eks.py @@ -75,6 +75,7 @@ def test_get_limits(self): 'Clusters', 'Control plane security groups per cluster', 'Managed node groups per cluster', + 'Public endpoint access CIDR ranges per cluster', ]) for name, limit in res.items(): assert limit.service == cls @@ -117,6 +118,7 @@ def test_find_clusters_usage(self): clusters_limit_key = 'Clusters' security_group_limit_key = 'Control plane security groups per cluster' nodegroup_limit_key = 'Managed node groups per cluster' + public_cidr_limit_key = 'Public endpoint access CIDR ranges per cluster' mock_conn = Mock() mock_conn.list_clusters.return_value = list_clusters @@ -152,6 +154,13 @@ def test_find_clusters_usage(self): assert cls.limits[nodegroup_limit_key].get_current_usage()[ 1].get_value() == 1 + assert len(cls.limits[ + public_cidr_limit_key].get_current_usage()) == 2 + assert cls.limits[public_cidr_limit_key].get_current_usage()[ + 0].get_value() == 3 + assert cls.limits[public_cidr_limit_key].get_current_usage()[ + 1].get_value() == 1 + def test_required_iam_permissions(self): cls = _EksService(21, 43, {}, None) assert cls.required_iam_permissions() == [ From 1a636c2233c704669c2f4d5b7996ed5438922f68 Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Fri, 2 Oct 2020 15:34:29 +0800 Subject: [PATCH 07/32] fix variable name to follow snake_case style --- awslimitchecker/services/eks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py index 8a5108e5..a47d9f62 100644 --- a/awslimitchecker/services/eks.py +++ b/awslimitchecker/services/eks.py @@ -89,12 +89,12 @@ def _find_clusters_usage(self): resource_id=cluster, aws_type='AWS::EKS::Cluster' ) - publicAccessCidrs_list = describe_cluster_response['cluster'][ + public_access_cidrs_list = describe_cluster_response['cluster'][ 'resourcesVpcConfig']['publicAccessCidrs'] self.limits[ 'Public endpoint access CIDR ranges per cluster' ]._add_current_usage( - len(publicAccessCidrs_list), + len(public_access_cidrs_list), resource_id=cluster, aws_type='AWS::EKS::Cluster' ) From 11e293566e5a2cce249eb6e0d42d98a8bc4fcd13 Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Fri, 2 Oct 2020 15:57:39 +0800 Subject: [PATCH 08/32] add EKS 'Fargate profiles per cluster' limit and usage support --- awslimitchecker/services/eks.py | 27 ++++++++++++++++++- .../tests/services/result_fixtures.py | 14 ++++++++++ awslimitchecker/tests/services/test_eks.py | 16 ++++++++++- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py index a47d9f62..ba92c8fc 100644 --- a/awslimitchecker/services/eks.py +++ b/awslimitchecker/services/eks.py @@ -98,6 +98,7 @@ def _find_clusters_usage(self): resource_id=cluster, aws_type='AWS::EKS::Cluster' ) + list_nodegroup_response = paginate_dict( self.conn.list_nodegroups, clusterName=cluster, @@ -111,6 +112,21 @@ def _find_clusters_usage(self): resource_id=cluster, aws_type='AWS::EKS::Cluster') + list_fargate_profiles_response = paginate_dict( + self.conn.list_fargate_profiles, + clusterName=cluster, + alc_marker_path=['nextToken'], + alc_data_path=['fargateProfileNames'], + alc_marker_param='nextToken' + ) + fargate_profiles_list = list_fargate_profiles_response[ + 'fargateProfileNames' + ] + self.limits['Fargate profiles per cluster']._add_current_usage( + len(fargate_profiles_list), + resource_id=cluster, + aws_type='AWS::EKS::FargateProfile') + self.limits['Clusters']._add_current_usage( len(cluster_list), resource_id=self._boto3_connection_kwargs['region_name'], @@ -159,6 +175,14 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EKS::Cluster', ) + limits['Fargate profiles per cluster'] = AwsLimit( + 'Fargate profiles per cluster', + self, + 10, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::FargateProfile', + ) self.limits = limits return limits @@ -174,5 +198,6 @@ def required_iam_permissions(self): return [ "eks:ListClusters", "eks:DescribeCluster", - "eks:ListNodegroups" + "eks:ListNodegroups", + "eks:ListFargateProfiles" ] diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 6c84eb8c..4e4b9791 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -4621,3 +4621,17 @@ class EKS(object): ] } ] + + test_find_clusters_usage_fargates = [ + { + 'fargateProfileNames': [ + 'foo', + ] + }, + { + 'fargateProfileNames': [ + 'bar', + 'baz', + ] + } + ] diff --git a/awslimitchecker/tests/services/test_eks.py b/awslimitchecker/tests/services/test_eks.py index f4adf7a4..0599c6a2 100644 --- a/awslimitchecker/tests/services/test_eks.py +++ b/awslimitchecker/tests/services/test_eks.py @@ -76,6 +76,7 @@ def test_get_limits(self): 'Control plane security groups per cluster', 'Managed node groups per cluster', 'Public endpoint access CIDR ranges per cluster', + 'Fargate profiles per cluster', ]) for name, limit in res.items(): assert limit.service == cls @@ -114,16 +115,19 @@ def test_find_clusters_usage(self): list_clusters = result_fixtures.EKS.test_find_clusters_usage_list describe_cluster = result_fixtures.EKS.test_find_clusters_usage_describe list_nodegroups = result_fixtures.EKS.test_find_clusters_usage_nodegrps + list_fargates = result_fixtures.EKS.test_find_clusters_usage_fargates clusters_limit_key = 'Clusters' security_group_limit_key = 'Control plane security groups per cluster' nodegroup_limit_key = 'Managed node groups per cluster' public_cidr_limit_key = 'Public endpoint access CIDR ranges per cluster' + fargate_profiles_limit_key = 'Fargate profiles per cluster' mock_conn = Mock() mock_conn.list_clusters.return_value = list_clusters mock_conn.describe_cluster.side_effect = describe_cluster mock_conn.list_nodegroups.side_effect = list_nodegroups + mock_conn.list_fargate_profiles.side_effect = list_fargates cls = _EksService(21, 43, {'region_name': 'us-west-2'}, None) cls.conn = mock_conn @@ -133,8 +137,10 @@ def test_find_clusters_usage(self): call.list_clusters(), call.describe_cluster(name=ANY), call.list_nodegroups(clusterName=ANY), + call.list_fargate_profiles(clusterName=ANY), call.describe_cluster(name=ANY), call.list_nodegroups(clusterName=ANY), + call.list_fargate_profiles(clusterName=ANY) ] assert len(cls.limits[clusters_limit_key].get_current_usage()) == 1 assert cls.limits[clusters_limit_key].get_current_usage()[ @@ -161,10 +167,18 @@ def test_find_clusters_usage(self): assert cls.limits[public_cidr_limit_key].get_current_usage()[ 1].get_value() == 1 + assert len(cls.limits[ + fargate_profiles_limit_key].get_current_usage()) == 2 + assert cls.limits[fargate_profiles_limit_key].get_current_usage()[ + 0].get_value() == 1 + assert cls.limits[fargate_profiles_limit_key].get_current_usage()[ + 1].get_value() == 2 + def test_required_iam_permissions(self): cls = _EksService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "eks:ListClusters", "eks:DescribeCluster", - "eks:ListNodegroups" + "eks:ListNodegroups", + "eks:ListFargateProfiles" ] From 9a8b668a74d39f2c458bd008e1cc2bde2f9862c2 Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Fri, 2 Oct 2020 23:28:54 +0800 Subject: [PATCH 09/32] add EKS 'Selectors per Fargate profile' limit and usage support --- awslimitchecker/services/eks.py | 22 ++++++- .../tests/services/result_fixtures.py | 64 +++++++++++++++++++ awslimitchecker/tests/services/test_eks.py | 30 ++++++++- 3 files changed, 113 insertions(+), 3 deletions(-) diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py index ba92c8fc..67214545 100644 --- a/awslimitchecker/services/eks.py +++ b/awslimitchecker/services/eks.py @@ -127,6 +127,17 @@ def _find_clusters_usage(self): resource_id=cluster, aws_type='AWS::EKS::FargateProfile') + for fargate_profile_name in fargate_profiles_list: + fargate_info = self.conn.describe_fargate_profile( + clusterName=cluster, + fargateProfileName=fargate_profile_name + ) + profile_selectors = fargate_info['fargateProfile']['selectors'] + self.limits['Selectors per Fargate profile']._add_current_usage( + len(profile_selectors), + resource_id=f"{cluster}.{fargate_profile_name}", + aws_type='AWS::EKS::FargateProfile') + self.limits['Clusters']._add_current_usage( len(cluster_list), resource_id=self._boto3_connection_kwargs['region_name'], @@ -183,6 +194,14 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EKS::FargateProfile', ) + limits['Selectors per Fargate profile'] = AwsLimit( + 'Selectors per Fargate profile', + self, + 5, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::FargateProfile', + ) self.limits = limits return limits @@ -199,5 +218,6 @@ def required_iam_permissions(self): "eks:ListClusters", "eks:DescribeCluster", "eks:ListNodegroups", - "eks:ListFargateProfiles" + "eks:ListFargateProfiles", + "eks:DescribeFargateProfile", ] diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 4e4b9791..6bfb9712 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -4635,3 +4635,67 @@ class EKS(object): ] } ] + + test_find_clusters_usage_fargate_prof = [ + { + 'fargateProfile': { + 'fargateProfileName': 'foo', + 'selectors': [ + { + 'namespace': "test_namespace1", + 'labels': { + 'key1': 'value1', + } + }, + ], + } + }, + { + 'fargateProfile': { + 'fargateProfileName': 'bar', + 'selectors': [ + { + 'namespace': "test_namespace1", + 'labels': { + 'key1': 'value1', + } + }, + { + 'namespace': "test_namespace2", + 'labels': { + 'key1': 'value1', + 'key2': 'value2', + } + }, + ], + } + }, + { + 'fargateProfile': { + 'fargateProfileName': 'baz', + 'selectors': [ + { + 'namespace': "test_namespace1", + 'labels': { + 'key1': 'value1', + } + }, + { + 'namespace': "test_namespace2", + 'labels': { + 'key1': 'value1', + 'key2': 'value2', + } + }, + { + 'namespace': "test_namespace3", + 'labels': { + 'key1': 'value1', + 'key2': 'value2', + 'key3': 'value3', + } + }, + ], + } + } + ] diff --git a/awslimitchecker/tests/services/test_eks.py b/awslimitchecker/tests/services/test_eks.py index 0599c6a2..52fa195e 100644 --- a/awslimitchecker/tests/services/test_eks.py +++ b/awslimitchecker/tests/services/test_eks.py @@ -77,6 +77,7 @@ def test_get_limits(self): 'Managed node groups per cluster', 'Public endpoint access CIDR ranges per cluster', 'Fargate profiles per cluster', + 'Selectors per Fargate profile', ]) for name, limit in res.items(): assert limit.service == cls @@ -116,18 +117,21 @@ def test_find_clusters_usage(self): describe_cluster = result_fixtures.EKS.test_find_clusters_usage_describe list_nodegroups = result_fixtures.EKS.test_find_clusters_usage_nodegrps list_fargates = result_fixtures.EKS.test_find_clusters_usage_fargates + dsc_fargate = result_fixtures.EKS.test_find_clusters_usage_fargate_prof clusters_limit_key = 'Clusters' security_group_limit_key = 'Control plane security groups per cluster' nodegroup_limit_key = 'Managed node groups per cluster' public_cidr_limit_key = 'Public endpoint access CIDR ranges per cluster' fargate_profiles_limit_key = 'Fargate profiles per cluster' + selectors_limit_key = 'Selectors per Fargate profile' mock_conn = Mock() mock_conn.list_clusters.return_value = list_clusters mock_conn.describe_cluster.side_effect = describe_cluster mock_conn.list_nodegroups.side_effect = list_nodegroups mock_conn.list_fargate_profiles.side_effect = list_fargates + mock_conn.describe_fargate_profile.side_effect = dsc_fargate cls = _EksService(21, 43, {'region_name': 'us-west-2'}, None) cls.conn = mock_conn @@ -138,9 +142,21 @@ def test_find_clusters_usage(self): call.describe_cluster(name=ANY), call.list_nodegroups(clusterName=ANY), call.list_fargate_profiles(clusterName=ANY), + call.describe_fargate_profile( + clusterName=ANY, + fargateProfileName=ANY + ), call.describe_cluster(name=ANY), call.list_nodegroups(clusterName=ANY), - call.list_fargate_profiles(clusterName=ANY) + call.list_fargate_profiles(clusterName=ANY), + call.describe_fargate_profile( + clusterName=ANY, + fargateProfileName=ANY + ), + call.describe_fargate_profile( + clusterName=ANY, + fargateProfileName=ANY + ), ] assert len(cls.limits[clusters_limit_key].get_current_usage()) == 1 assert cls.limits[clusters_limit_key].get_current_usage()[ @@ -174,11 +190,21 @@ def test_find_clusters_usage(self): assert cls.limits[fargate_profiles_limit_key].get_current_usage()[ 1].get_value() == 2 + assert len(cls.limits[ + selectors_limit_key].get_current_usage()) == 3 + assert cls.limits[selectors_limit_key].get_current_usage()[ + 0].get_value() == 1 + assert cls.limits[selectors_limit_key].get_current_usage()[ + 1].get_value() == 2 + assert cls.limits[selectors_limit_key].get_current_usage()[ + 2].get_value() == 3 + def test_required_iam_permissions(self): cls = _EksService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "eks:ListClusters", "eks:DescribeCluster", "eks:ListNodegroups", - "eks:ListFargateProfiles" + "eks:ListFargateProfiles", + "eks:DescribeFargateProfile" ] From cdde694aabee25598226253d12efcb0de0cb7d89 Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Fri, 2 Oct 2020 23:52:04 +0800 Subject: [PATCH 10/32] add EKS 'Label pairs per Fargate profile selector' limit and usage support --- awslimitchecker/services/eks.py | 19 +++++++++++++++++++ awslimitchecker/tests/services/test_eks.py | 17 +++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py index 67214545..c1ca9860 100644 --- a/awslimitchecker/services/eks.py +++ b/awslimitchecker/services/eks.py @@ -138,6 +138,17 @@ def _find_clusters_usage(self): resource_id=f"{cluster}.{fargate_profile_name}", aws_type='AWS::EKS::FargateProfile') + for selector in profile_selectors: + label_pairs = selector['labels'] + self.limits[ + 'Label pairs per Fargate profile selector' + ]._add_current_usage( + len(label_pairs), + resource_id=( + f"{cluster}.{fargate_profile_name}.{selector}" + ), + aws_type='AWS::EKS::FargateProfile') + self.limits['Clusters']._add_current_usage( len(cluster_list), resource_id=self._boto3_connection_kwargs['region_name'], @@ -202,6 +213,14 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EKS::FargateProfile', ) + limits['Label pairs per Fargate profile selector'] = AwsLimit( + 'Label pairs per Fargate profile selector', + self, + 5, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::FargateProfile', + ) self.limits = limits return limits diff --git a/awslimitchecker/tests/services/test_eks.py b/awslimitchecker/tests/services/test_eks.py index 52fa195e..f56e4a7c 100644 --- a/awslimitchecker/tests/services/test_eks.py +++ b/awslimitchecker/tests/services/test_eks.py @@ -78,6 +78,7 @@ def test_get_limits(self): 'Public endpoint access CIDR ranges per cluster', 'Fargate profiles per cluster', 'Selectors per Fargate profile', + 'Label pairs per Fargate profile selector', ]) for name, limit in res.items(): assert limit.service == cls @@ -125,6 +126,7 @@ def test_find_clusters_usage(self): public_cidr_limit_key = 'Public endpoint access CIDR ranges per cluster' fargate_profiles_limit_key = 'Fargate profiles per cluster' selectors_limit_key = 'Selectors per Fargate profile' + label_pairs_limit_key = 'Label pairs per Fargate profile selector' mock_conn = Mock() mock_conn.list_clusters.return_value = list_clusters @@ -199,6 +201,21 @@ def test_find_clusters_usage(self): assert cls.limits[selectors_limit_key].get_current_usage()[ 2].get_value() == 3 + assert len(cls.limits[ + label_pairs_limit_key].get_current_usage()) == 6 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 0].get_value() == 1 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 1].get_value() == 1 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 2].get_value() == 2 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 3].get_value() == 1 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 4].get_value() == 2 + assert cls.limits[label_pairs_limit_key].get_current_usage()[ + 5].get_value() == 3 + def test_required_iam_permissions(self): cls = _EksService(21, 43, {}, None) assert cls.required_iam_permissions() == [ From c645bd3545a215eca7cd097a0e6ebfe6d75ee458 Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Fri, 2 Oct 2020 23:55:50 +0800 Subject: [PATCH 11/32] add EKS 'Nodes per managed node group' limit support (no ability to measure usage) --- awslimitchecker/services/eks.py | 8 ++++++++ awslimitchecker/tests/services/test_eks.py | 1 + 2 files changed, 9 insertions(+) diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py index c1ca9860..bfa384c0 100644 --- a/awslimitchecker/services/eks.py +++ b/awslimitchecker/services/eks.py @@ -189,6 +189,14 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EKS::Nodegroup', ) + limits['Nodes per managed node group'] = AwsLimit( + 'Nodes per managed node group', + self, + 100, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::EKS::Cluster', + ) limits['Public endpoint access CIDR ranges per cluster'] = AwsLimit( 'Public endpoint access CIDR ranges per cluster', self, diff --git a/awslimitchecker/tests/services/test_eks.py b/awslimitchecker/tests/services/test_eks.py index f56e4a7c..34c518de 100644 --- a/awslimitchecker/tests/services/test_eks.py +++ b/awslimitchecker/tests/services/test_eks.py @@ -75,6 +75,7 @@ def test_get_limits(self): 'Clusters', 'Control plane security groups per cluster', 'Managed node groups per cluster', + 'Nodes per managed node group', 'Public endpoint access CIDR ranges per cluster', 'Fargate profiles per cluster', 'Selectors per Fargate profile', From 9fc375dd44853a358b0f91172de18b7b4a25dcea Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Sat, 3 Oct 2020 17:35:35 +0800 Subject: [PATCH 12/32] replace use of fstring as project still supports python 3.5 --- awslimitchecker/services/eks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/awslimitchecker/services/eks.py b/awslimitchecker/services/eks.py index bfa384c0..5e28c267 100644 --- a/awslimitchecker/services/eks.py +++ b/awslimitchecker/services/eks.py @@ -135,7 +135,7 @@ def _find_clusters_usage(self): profile_selectors = fargate_info['fargateProfile']['selectors'] self.limits['Selectors per Fargate profile']._add_current_usage( len(profile_selectors), - resource_id=f"{cluster}.{fargate_profile_name}", + resource_id="{}.{}".format(cluster, fargate_profile_name), aws_type='AWS::EKS::FargateProfile') for selector in profile_selectors: @@ -145,7 +145,11 @@ def _find_clusters_usage(self): ]._add_current_usage( len(label_pairs), resource_id=( - f"{cluster}.{fargate_profile_name}.{selector}" + "{}.{}.{}".format( + cluster, + fargate_profile_name, + selector + ) ), aws_type='AWS::EKS::FargateProfile') From 90019df9357035274641ec48cdf3c4f79b2d27eb Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Sat, 3 Oct 2020 22:36:44 +0800 Subject: [PATCH 13/32] issue #495 - Initial support for Kinesis service --- awslimitchecker/services/__init__.py | 1 + awslimitchecker/services/kinesis.py | 128 ++++++++++++ .../tests/services/result_fixtures.py | 7 + .../tests/services/test_kinesis.py | 196 ++++++++++++++++++ 4 files changed, 332 insertions(+) create mode 100644 awslimitchecker/services/kinesis.py create mode 100644 awslimitchecker/tests/services/test_kinesis.py diff --git a/awslimitchecker/services/__init__.py b/awslimitchecker/services/__init__.py index 4b6c5d4b..771e7273 100644 --- a/awslimitchecker/services/__init__.py +++ b/awslimitchecker/services/__init__.py @@ -53,6 +53,7 @@ from awslimitchecker.services.elb import _ElbService from awslimitchecker.services.firehose import _FirehoseService from awslimitchecker.services.iam import _IamService +from awslimitchecker.services.kinesis import _KinesisService from awslimitchecker.services.lambdafunc import _LambdaService from awslimitchecker.services.rds import _RDSService from awslimitchecker.services.redshift import _RedshiftService diff --git a/awslimitchecker/services/kinesis.py b/awslimitchecker/services/kinesis.py new file mode 100644 index 00000000..e3c38258 --- /dev/null +++ b/awslimitchecker/services/kinesis.py @@ -0,0 +1,128 @@ +""" +awslimitchecker/services/kinesis.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import abc # noqa +import logging + +from .base import _AwsService +from ..limit import AwsLimit + +logger = logging.getLogger(__name__) + + +class _KinesisService(_AwsService): + + service_name = 'Kinesis' + api_name = 'kinesis' + quotas_service_code = 'kinesis' + + def find_usage(self): + """ + Determine the current usage for each limit of this service, + and update corresponding Limit via + :py:meth:`~.AwsLimit._add_current_usage`. + """ + logger.debug("Checking usage for service %s", self.service_name) + self.connect() + for lim in self.limits.values(): + lim._reset_usage() + self._find_shards() + self._have_usage = True + logger.debug("Done checking usage.") + + def _find_shards(self): + describe_limits_response = self.conn.describe_limits() + self.limits['Shards per Region']._add_current_usage( + describe_limits_response['OpenShardCount'], + resource_id=self._boto3_connection_kwargs['region_name'], + aws_type='AWS::Kinesis::Stream' + ) + + def get_limits(self): + """ + Return all known limits for this service, as a dict of their names + to :py:class:`~.AwsLimit` objects. + + :returns: dict of limit names to :py:class:`~.AwsLimit` objects + :rtype: dict + """ + if self.limits != {}: + return self.limits + + self.connect() + region_name = self.conn._client_config.region_name + regions_500_shards = ['us-east-1', 'us-west-2', 'eu-west-1'] + + limits = {} + + limits['Shards per Region'] = AwsLimit( + 'Shards per Region', + self, + 500 if region_name in regions_500_shards else 200, + self.warning_threshold, + self.critical_threshold, + limit_type='AWS::Kinesis::Stream', + ) + self.limits = limits + return limits + + def _update_limits_from_api(self): + """ + Call the service's API action to retrieve limit/quota information, and + update AwsLimit objects in ``self.limits`` with this information. + """ + logger.debug("Updating limits for Kinesis from the AWS API") + self.connect() + describe_limits_response = self.conn.describe_limits() + self.limits['Shards per Region']._set_api_limit( + describe_limits_response['ShardLimit'] + ) + + def required_iam_permissions(self): + """ + Return a list of IAM Actions required for this Service to function + properly. All Actions will be shown with an Effect of "Allow" + and a Resource of "*". + + :returns: list of IAM Action strings + :rtype: list + """ + return [ + 'kinesis:DescribeLimits', + ] diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 6ece80d4..5b3006db 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -4564,3 +4564,10 @@ class CloudTrail(object): } ] } + + +class Kinesis(object): + mock_describe_limits = { + 'ShardLimit': 700, + 'OpenShardCount': 555 + } diff --git a/awslimitchecker/tests/services/test_kinesis.py b/awslimitchecker/tests/services/test_kinesis.py new file mode 100644 index 00000000..81cf58d0 --- /dev/null +++ b/awslimitchecker/tests/services/test_kinesis.py @@ -0,0 +1,196 @@ +""" +awslimitchecker/tests/services/test_kinesis.py + +The latest version of this package is available at: + + +################################################################################ +Copyright 2015-2018 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +################################################################################ +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +################################################################################ + +AUTHORS: +Jason Antman +################################################################################ +""" + +import sys +from awslimitchecker.tests.services import result_fixtures +from awslimitchecker.limit import AwsLimit +from awslimitchecker.services.kinesis import _KinesisService + +# https://code.google.com/p/mock/issues/detail?id=249 +# py>=3.4 should use unittest.mock not the mock package on pypi +if ( + sys.version_info[0] < 3 or + sys.version_info[0] == 3 and sys.version_info[1] < 4 +): + from mock import patch, call, Mock, DEFAULT +else: + from unittest.mock import patch, call, Mock, DEFAULT + + +pbm = 'awslimitchecker.services.kinesis' # module patch base +pb = '%s._KinesisService' % pbm # class patch pase + + +class Test_KinesisService(object): + + def test_init(self): + """test __init__()""" + with patch('%s.get_limits' % pb): + cls = _KinesisService(21, 43, {}, None) + assert cls.service_name == 'Kinesis' + assert cls.api_name == 'kinesis' + assert cls.conn is None + assert cls.warning_threshold == 21 + assert cls.critical_threshold == 43 + + def test_get_limits(self): + mock_conn = Mock() + m_client = Mock() + type(m_client).region_name = 'ap-southeast-2' + type(mock_conn)._client_config = m_client + + def se_conn(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as mock_connect: + mock_connect.side_effect = se_conn + cls = _KinesisService(21, 43, {}, None) + + cls.limits = {} + res = cls.get_limits() + assert sorted(res.keys()) == sorted([ + 'Shards per Region', + ]) + for name, limit in res.items(): + assert limit.service == cls + assert limit.def_warning_threshold == 21 + assert limit.def_critical_threshold == 43 + + limits = cls.limits + assert len(limits) == 1 + assert limits['Shards per Region'].default_limit == 200 + + def test_get_limits_us_east_1(self): + mock_conn = Mock() + m_client = Mock() + type(m_client).region_name = 'us-east-1' + type(mock_conn)._client_config = m_client + + def se_conn(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as mock_connect: + mock_connect.side_effect = se_conn + cls = _KinesisService(21, 43, {}, None) + + limits = cls.limits + for x in limits: + assert isinstance(limits[x], AwsLimit) + assert x == limits[x].name + assert limits[x].service == cls + + assert len(limits) == 1 + assert limits['Shards per Region'].default_limit == 500 + + def test_get_limits_again(self): + """test that existing limits dict is returned on subsequent calls""" + mock_limits = Mock() + cls = _KinesisService(21, 43, {}, None) + cls.limits = mock_limits + res = cls.get_limits() + assert res == mock_limits + + def test_find_usage(self): + mock_conn = Mock() + + def se_conn(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as mock_connect: + mock_connect.side_effect = se_conn + with patch.multiple( + pb, + _find_shards=DEFAULT, + ) as mocks: + cls = _KinesisService(21, 43, {}, None) + cls.conn = mock_conn + assert cls._have_usage is False + cls.find_usage() + assert mock_connect.mock_calls == [call(cls), call(cls)] + assert cls._have_usage is True + assert mock_conn.mock_calls == [] + for x in [ + '_find_shards', + ]: + assert mocks[x].mock_calls == [call()] + + def test_find_shards(self): + response = result_fixtures.Kinesis.mock_describe_limits + limit_key = 'Shards per Region' + + mock_conn = Mock() + mock_conn.describe_limits.return_value = response + + cls = _KinesisService(21, 43, {'region_name': 'us-west-2'}, None) + cls.conn = mock_conn + cls._find_shards() + + assert mock_conn.mock_calls == [ + call.describe_limits() + ] + assert len(cls.limits[limit_key].get_current_usage()) == 1 + assert cls.limits[limit_key].get_current_usage()[ + 0].get_value() == 555 + + def test_update_limits_from_api(self): + response = result_fixtures.Kinesis.mock_describe_limits + mock_conn = Mock() + mock_conn.describe_limits.return_value = response + + def se_conn(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as mock_connect: + mock_connect.side_effect = se_conn + cls = _KinesisService(21, 43, {'region_name': 'us-west-2'}, None) + assert len(cls.limits) == 1 + cls.conn = mock_conn + cls._update_limits_from_api() + + assert mock_connect.mock_calls == [call(cls), call(cls)] + assert mock_conn.mock_calls == [call.describe_limits()] + assert len(cls.limits) == 1 + lim = cls.limits['Shards per Region'].get_limit() + assert lim == 700 + + def test_required_iam_permissions(self): + cls = _KinesisService(21, 43, {}, None) + assert cls.required_iam_permissions() == [ + 'kinesis:DescribeLimits', + ] From 4ccedfc0e9a2df9c798e87607e37d78765e11f42 Mon Sep 17 00:00:00 2001 From: Reshad Patuck Date: Sun, 25 Oct 2020 16:55:03 +0530 Subject: [PATCH 14/32] feat(vpc.py): add quota_name for Entries per route table. --- awslimitchecker/services/vpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awslimitchecker/services/vpc.py b/awslimitchecker/services/vpc.py index b3b0c181..9554436c 100644 --- a/awslimitchecker/services/vpc.py +++ b/awslimitchecker/services/vpc.py @@ -309,6 +309,7 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EC2::Route', limit_subtype='AWS::EC2::RouteTable', + quota_name='Routes per route table' ) limits['Internet gateways'] = AwsLimit( From 1aefc0a06b6ae3bb4e49066fb5bd80ae0938c8cf Mon Sep 17 00:00:00 2001 From: Reshad Patuck Date: Sun, 25 Oct 2020 17:16:42 +0530 Subject: [PATCH 15/32] fix(vpc.py): quotas_name not quota_name! --- awslimitchecker/services/vpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awslimitchecker/services/vpc.py b/awslimitchecker/services/vpc.py index 9554436c..848de973 100644 --- a/awslimitchecker/services/vpc.py +++ b/awslimitchecker/services/vpc.py @@ -309,7 +309,7 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::EC2::Route', limit_subtype='AWS::EC2::RouteTable', - quota_name='Routes per route table' + quotas_name='Routes per route table' ) limits['Internet gateways'] = AwsLimit( From 7d84e810234965a99eda34d04d0b49d863a4f623 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Tue, 1 Dec 2020 09:50:31 -0500 Subject: [PATCH 16/32] Fixes #466 - Disable TA by default except in China and GovCloud --- CHANGES.rst | 16 +++ awslimitchecker/tests/test_trustedadvisor.py | 129 ++++++++++++++++--- awslimitchecker/trustedadvisor.py | 30 +++++ docs/source/cli_usage.rst | 6 + docs/source/cli_usage.rst.template | 6 + docs/source/development.rst | 3 + docs/source/getting_started.rst | 3 + docs/source/internals.rst | 3 + docs/source/python_usage.rst | 9 ++ 9 files changed, 184 insertions(+), 21 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 95746971..99a9c0c5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,22 @@ Changelog ========= +.. _changelog.10_0_0: + +10.0.0 (2020-12-07) +------------------- + +**Important:** This release makes significant changes to how Trusted Advisor is used. + +* `Issue #466 `__ - **Significant** changes to Trusted Advisor support. + + * In June 2019, AWS `announced `__ the new Service Quotas service (great name) that allows us to retrieve limit/quota information from a unified API. In addition, many individual services now provide limit information via their own APIs. At this point (late 2020) all of the limit/quota information that was previously available via Trusted Advisor is now available via a combination of the individual service APIs and Service Quotas. + * In February 2020, the layout of Trusted Advisor checks was changed, and the "Performance / Service Limits" check that we previously used to obtain limit information was moved to its own category in Trusted Advisor. While I can't confirm this, as far as I can tell, this change was only made in the standard AWS regions/partitions (i.e. not GovCloud or China). + * awslimitchecker still has not been updated for this new Trusted Advisor layout. + * This release **disables Trusted Advisor by default outside China and GovCloud**, as it provides no additional information outside of these regions/partitions. + * If you are running in China or GovCloud and have issues with awslimitchecker retrieving information from Trusted Advisor, please `open an issue `__. + * My current intent is to leave Trusted Advisor support in this state until Service Quotas is available in China and GovCloud, at which point I plan on completely removing all Trusted Advisor support. + .. _changelog.9_0_0: 9.0.0 (2020-09-22) diff --git a/awslimitchecker/tests/test_trustedadvisor.py b/awslimitchecker/tests/test_trustedadvisor.py index 6bb5e562..7d2f4277 100644 --- a/awslimitchecker/tests/test_trustedadvisor.py +++ b/awslimitchecker/tests/test_trustedadvisor.py @@ -53,9 +53,9 @@ sys.version_info[0] < 3 or sys.version_info[0] == 3 and sys.version_info[1] < 4 ): - from mock import patch, call, Mock + from mock import patch, call, Mock, DEFAULT else: - from unittest.mock import patch, call, Mock + from unittest.mock import patch, call, Mock, DEFAULT pbm = 'awslimitchecker.trustedadvisor' @@ -163,35 +163,122 @@ def setup(self): def test_simple(self): mock_results = Mock() - with patch('%s.connect' % pb, autospec=True) as mock_connect: - with patch('%s._poll' % pb, autospec=True) as mock_poll: - with patch('%s._update_services' % pb, - autospec=True) as mock_update_services: - mock_poll.return_value = mock_results - self.cls.update_limits() - assert mock_connect.mock_calls == [call(self.cls)] - assert mock_poll.mock_calls == [call(self.cls)] - assert mock_update_services.mock_calls == [ + with patch.multiple( + pb, + connect=DEFAULT, + _poll=DEFAULT, + _update_services=DEFAULT, + _dont_use_ta=DEFAULT, + autospec=True + ) as mocks: + mocks['_poll'].return_value = mock_results + mocks['_dont_use_ta'].return_value = False + self.cls.update_limits() + assert mocks['connect'].mock_calls == [call(self.cls)] + assert mocks['_poll'].mock_calls == [call(self.cls)] + assert mocks['_update_services'].mock_calls == [ call(self.cls, mock_results) ] def test_again(self): mock_results = Mock() self.cls.limits_updated = True - with patch('%s.connect' % pb, autospec=True) as mock_connect: - with patch('%s._poll' % pb, autospec=True) as mock_poll: - with patch('%s._update_services' % pb, - autospec=True) as mock_update_services: - with patch('%s.logger' % pbm) as mock_logger: - mock_poll.return_value = mock_results - self.cls.update_limits() - assert mock_connect.mock_calls == [] - assert mock_poll.mock_calls == [] - assert mock_update_services.mock_calls == [] + with patch.multiple( + pb, + connect=DEFAULT, + _poll=DEFAULT, + _update_services=DEFAULT, + _dont_use_ta=DEFAULT, + autospec=True + ) as mocks: + mocks['_poll'].return_value = mock_results + mocks['_dont_use_ta'].return_value = False + with patch('%s.logger' % pbm) as mock_logger: + self.cls.update_limits() + assert mocks['connect'].mock_calls == [] + assert mocks['_poll'].mock_calls == [] + assert mocks['_update_services'].mock_calls == [] assert mock_logger.mock_calls == [ call.debug('Already polled TA; skipping update') ] + def test_dont_use(self): + mock_results = Mock() + with patch.multiple( + pb, + connect=DEFAULT, + _poll=DEFAULT, + _update_services=DEFAULT, + _dont_use_ta=DEFAULT, + autospec=True + ) as mocks: + mocks['_poll'].return_value = mock_results + mocks['_dont_use_ta'].return_value = True + self.cls.update_limits() + assert mocks['connect'].mock_calls == [call(self.cls)] + assert mocks['_poll'].mock_calls == [] + assert mocks['_update_services'].mock_calls == [] + + +class TestDontUseTa(): + + def setup(self): + self.mock_conn = Mock() + self.mock_client_config = Mock() + type(self.mock_client_config).region_name = 'us-east-1' + type(self.mock_conn)._client_config = self.mock_client_config + self.cls = TrustedAdvisor({}, {}) + self.cls.conn = self.mock_conn + + self.mock_svc1 = Mock(spec_set=_AwsService) + self.mock_svc2 = Mock(spec_set=_AwsService) + self.services = { + 'SvcFoo': self.mock_svc1, + 'SvcBar': self.mock_svc2, + } + + @patch.dict( + 'os.environ', + {'AWS_REGION': 'us-east-1'}, + clear=True + ) + def test_simple(self): + assert self.cls._dont_use_ta() is True + + @patch.dict( + 'os.environ', + {'AWS_REGION': 'us-east-1', 'FORCE_USE_TA': 'true'}, + clear=True + ) + def test_env_var(self): + assert self.cls._dont_use_ta() is False + + @patch.dict( + 'os.environ', + {'AWS_REGION': 'us-east-1', 'FORCE_USE_TA': 'nottrue'}, + clear=True + ) + def test_env_var_false(self): + assert self.cls._dont_use_ta() is True + + @patch.dict( + 'os.environ', + {'AWS_REGION': 'cn-foo-1'}, + clear=True + ) + def test_china(self): + type(self.mock_client_config).region_name = 'cn-foo-1' + assert self.cls._dont_use_ta() is False + + @patch.dict( + 'os.environ', + {'AWS_REGION': 'us-gov-east-1'}, + clear=True + ) + def test_gov_cloud(self): + type(self.mock_client_config).region_name = 'us-gov-east-1' + assert self.cls._dont_use_ta() is False + class TestGetLimitCheckId(object): diff --git a/awslimitchecker/trustedadvisor.py b/awslimitchecker/trustedadvisor.py index 1e2bf0c0..800f52d6 100644 --- a/awslimitchecker/trustedadvisor.py +++ b/awslimitchecker/trustedadvisor.py @@ -37,6 +37,7 @@ ################################################################################ """ +import os from botocore.exceptions import ClientError from dateutil import parser import logging @@ -142,10 +143,39 @@ def update_limits(self): logger.debug('Already polled TA; skipping update') return self.connect() + if self._dont_use_ta(): + logger.info( + 'Not using Trusted Advisor in regions outside of China or ' + 'GovCloud; export FORCE_USE_TA=true to override.' + ) + return ta_results = self._poll() self._update_services(ta_results) self.limits_updated = True + def _dont_use_ta(self): + """ + If we are connecting to a region outside of China or GovCloud, and do + not have the ``FORCE_USE_TA`` environment variable set to ``true``, + don't use Trusted Advisor at all. + + :return: whether whether to skip using TA + :rtype: bool + """ + if os.environ.get('FORCE_USE_TA', '') == 'true': + logger.debug( + 'Found env var FORCE_USE_TA set to "true"; forcing Trusted ' + 'Advisor polling.' + ) + return False + region_name = self.conn._client_config.region_name + if region_name.startswith('cn-') or region_name.startswith('us-gov-'): + logger.debug( + 'Using Trusted Advisor in region: %s', region_name + ) + return False + return True + def _poll(self): """ Poll Trusted Advisor (Support) API for limit checks. diff --git a/docs/source/cli_usage.rst b/docs/source/cli_usage.rst index 0baf9349..01618e1b 100644 --- a/docs/source/cli_usage.rst +++ b/docs/source/cli_usage.rst @@ -267,6 +267,9 @@ from the Service Quotas service. Disabling Trusted Advisor Checks ++++++++++++++++++++++++++++++++ +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + Using the ``--skip-ta`` option will disable attempting to query limit information from Trusted Advisor for all commands. @@ -684,6 +687,9 @@ between your account and the 123456789012 destination account; see the Partitions and Trusted Advisor Regions ++++++++++++++++++++++++++++++++++++++ +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + awslimitchecker currently supports operating against non-standard `partitions `_, such as GovCloud and AWS China (Beijing). Partition names, as seen in the ``partition`` field of ARNs, can be specified with the ``--role-partition`` option to awslimitchecker, like ``--role-partition=aws-cn`` for the China (Beijing) partition. Similarly, the region name to use for the ``support`` API for Trusted Advisor can be specified with the ``--ta-api-region`` option, like ``--ta-api-region=us-gov-west-1``. .. _cli_usage.throttling: diff --git a/docs/source/cli_usage.rst.template b/docs/source/cli_usage.rst.template index c6b57166..0e03434e 100644 --- a/docs/source/cli_usage.rst.template +++ b/docs/source/cli_usage.rst.template @@ -69,6 +69,9 @@ from the Service Quotas service. Disabling Trusted Advisor Checks ++++++++++++++++++++++++++++++++ +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + Using the ``--skip-ta`` option will disable attempting to query limit information from Trusted Advisor for all commands. @@ -325,6 +328,9 @@ between your account and the 123456789012 destination account; see the Partitions and Trusted Advisor Regions ++++++++++++++++++++++++++++++++++++++ +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + awslimitchecker currently supports operating against non-standard `partitions `_, such as GovCloud and AWS China (Beijing). Partition names, as seen in the ``partition`` field of ARNs, can be specified with the ``--role-partition`` option to awslimitchecker, like ``--role-partition=aws-cn`` for the China (Beijing) partition. Similarly, the region name to use for the ``support`` API for Trusted Advisor can be specified with the ``--ta-api-region`` option, like ``--ta-api-region=us-gov-west-1``. .. _cli_usage.throttling: diff --git a/docs/source/development.rst b/docs/source/development.rst index c432ab6e..813ff0dc 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -206,6 +206,9 @@ include 'NextToken' or another pagination marker, should be called through Trusted Advisor Checks ---------------------- +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + So long as the ``Service`` and ``Limit`` name strings returned by the Trusted Advisor (Support) API exactly match how they are set on the corresponding :py:class:`~._AwsService` and :py:class:`~.AwsLimit` objects, no code changes are needed to support new limit checks from TA. diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index ae277ff5..2e3edb9f 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -153,6 +153,9 @@ to check limits in multiple regions, simply run the script multiple times, once Trusted Advisor --------------- +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + awslimitchecker supports retrieving your current service limits via the `Trusted Advisor `_ `"Service Limits" performance check `_ diff --git a/docs/source/internals.rst b/docs/source/internals.rst index 21b909bc..ec05023c 100644 --- a/docs/source/internals.rst +++ b/docs/source/internals.rst @@ -40,6 +40,9 @@ to the services and limits without any connection to AWS. This is utilized by th Trusted Advisor --------------- +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + When :py:class:`~awslimitchecker.checker.AwsLimitChecker` is initialized, it also initializes an instance of :py:class:`~awslimitchecker.trustedadvisor.TrustedAdvisor`. In :py:meth:`~.AwsLimitChecker.get_limits`, :py:meth:`~.AwsLimitChecker.find_usage` and :py:meth:`~.AwsLimitChecker.check_thresholds`, when called with diff --git a/docs/source/python_usage.rst b/docs/source/python_usage.rst index 48e112fb..67bb5410 100644 --- a/docs/source/python_usage.rst +++ b/docs/source/python_usage.rst @@ -56,6 +56,9 @@ parameter to the class constructor: Refreshing Trusted Advisor Check Results ++++++++++++++++++++++++++++++++++++++++ +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + Trusted Advisor check refresh behavior is controlled by the ``ta_refresh_mode`` and ``ta_refresh_timeout`` parameters on the :py:class:`~awslimitchecker.checker.AwsLimitChecker` constructor, which are passed through to the :py:class:`~awslimitchecker.trustedadvisor.TrustedAdvisor` @@ -235,6 +238,9 @@ crossed the critical threshold: Disabling Trusted Advisor ++++++++++++++++++++++++++ +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + To disable querying Trusted Advisor for limit information, call :py:meth:`~.AwsLimitChecker.get_limits` or :py:meth:`~.AwsLimitChecker.check_thresholds` with ``use_ta=False``: @@ -259,6 +265,9 @@ in to the :py:class:`~.AwsLimitChecker` class constructor: Partitions and Trusted Advisor Regions ++++++++++++++++++++++++++++++++++++++ +.. attention:: + Trusted Advisor support in awslimitchecker is deprecated outside of the China and GovCloud regions, and now defaults to disabled/skipped in standard AWS, as the information available from TA can now be retrieved faster and more accurately via other means. See :ref:`changelog.10_0_0` for further information. + awslimitchecker currently supports operating against non-standard `partitions `_, such as GovCloud and AWS China (Beijing). Partition names, as seen in the ``partition`` field of ARNs, can be specified with the ``role_partition`` keyword argument to the :py:class:`~.AwsLimitChecker` class. Similarly, the region name to use for the ``support`` API for Trusted Advisor can be specified with the ``ta_api_region`` keyword argument to the :py:class:`~.AwsLimitChecker` class. Skipping Specific Services From fdfe764d35ebffc6feecfd816a952579be956965 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Tue, 1 Dec 2020 12:00:57 -0500 Subject: [PATCH 17/32] Update for move of CI builds from travis-ci.org to travis-ci.com --- CHANGES.rst | 1 + README.rst | 8 ++++---- dev/release.py | 4 ++-- dev/release_utils.py | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 99a9c0c5..ec5be0f7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,7 @@ Changelog * This release **disables Trusted Advisor by default outside China and GovCloud**, as it provides no additional information outside of these regions/partitions. * If you are running in China or GovCloud and have issues with awslimitchecker retrieving information from Trusted Advisor, please `open an issue `__. * My current intent is to leave Trusted Advisor support in this state until Service Quotas is available in China and GovCloud, at which point I plan on completely removing all Trusted Advisor support. +* Migrate CI builds from travis-ci.org to travis-ci.com. .. _changelog.9_0_0: diff --git a/README.rst b/README.rst index 3ad7cda7..fd81df96 100644 --- a/README.rst +++ b/README.rst @@ -27,8 +27,8 @@ awslimitchecker Master: -.. image:: https://secure.travis-ci.org/jantman/awslimitchecker.png?branch=master - :target: http://travis-ci.org/jantman/awslimitchecker +.. image:: https://api.travis-ci.com/jantman/awslimitchecker.svg?branch=master + :target: http://travis-ci.com/jantman/awslimitchecker :alt: travis-ci for master branch .. image:: https://codecov.io/github/jantman/awslimitchecker/coverage.svg?branch=master @@ -41,8 +41,8 @@ Master: Develop: -.. image:: https://secure.travis-ci.org/jantman/awslimitchecker.png?branch=develop - :target: http://travis-ci.org/jantman/awslimitchecker +.. image:: https://api.travis-ci.com/jantman/awslimitchecker.svg?branch=develop + :target: http://travis-ci.com/jantman/awslimitchecker :alt: travis-ci for develop branch .. image:: https://codecov.io/github/jantman/awslimitchecker/coverage.svg?branch=develop diff --git a/dev/release.py b/dev/release.py index 3e3daa90..44ddfcd9 100755 --- a/dev/release.py +++ b/dev/release.py @@ -163,7 +163,7 @@ def run(self): 'and then re-run.' % (build.number, build.id)) logger.info('OK, Travis build passed.') logger.info( - 'Build URL: ', build.id ) if not prompt_user( @@ -316,7 +316,7 @@ def wait_for_travis(self, pr_num): headsha = pr.head.sha while True: for s in self._gh._repo.statuses(sha=headsha): - if s.context == 'continuous-integration/travis-ci/pr': + if s.context == 'Travis CI - Pull Request': logger.info( 'Last TravisCI PR status: %s <%s> (at %s)', s.state, s.target_url, s.updated_at diff --git a/dev/release_utils.py b/dev/release_utils.py index 6674075c..48af9ca3 100644 --- a/dev/release_utils.py +++ b/dev/release_utils.py @@ -46,7 +46,7 @@ try: from travispy import TravisPy - from travispy.travispy import PUBLIC + from travispy.travispy import PRIVATE except ImportError: raise SystemExit( "ERROR: TravisPy not installed. Please run 'pip install TravisPy'" @@ -98,7 +98,7 @@ def __init__(self): 'Please export your GitHub PAT as the "GITHUB_TOKEN" env var' ) logger.debug('Connecting to TravisCI API...') - self._travis = TravisPy.github_auth(token) + self._travis = TravisPy.github_auth(token, uri=PRIVATE) def commit_latest_build_status(self, commit): """ From 2380a1f1cf0212652dec2c23ff778b267537d907 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Tue, 1 Dec 2020 12:29:34 -0500 Subject: [PATCH 18/32] more travis-ci.com updates --- .gitignore | 1 + dev/release.py | 8 ++++---- dev/requirements_dev.txt | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index ebc3f5e2..78e3f07d 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,4 @@ results/ .release_position.json dev/terraform.tfstate man/ +share/ diff --git a/dev/release.py b/dev/release.py index 44ddfcd9..6d64c932 100755 --- a/dev/release.py +++ b/dev/release.py @@ -315,13 +315,13 @@ def wait_for_travis(self, pr_num): pr = self._gh._repo.pull_request(pr_num) headsha = pr.head.sha while True: - for s in self._gh._repo.statuses(sha=headsha): - if s.context == 'Travis CI - Pull Request': + for s in self._gh._repo.commit(headsha).check_runs(): + if s.name == 'Travis CI - Pull Request': logger.info( 'Last TravisCI PR status: %s <%s> (at %s)', - s.state, s.target_url, s.updated_at + s.conclusion, s.html_url, s.completed_at ) - if s.state == 'success': + if s.conclusion == 'success': return True logger.debug('No successful TravisCI PR build; trying again in 15s') sleep(15) diff --git a/dev/requirements_dev.txt b/dev/requirements_dev.txt index 3b4b02e1..05ad1f96 100644 --- a/dev/requirements_dev.txt +++ b/dev/requirements_dev.txt @@ -2,4 +2,4 @@ tox==2.7.0 twine webhook2lambda2sqs wheel -github3.py==1.0.0a4 \ No newline at end of file +github3.py==1.3.0 \ No newline at end of file From b7fdcb4ecfd5ba7a8c958abdf8bbf7a30126d3ee Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Wed, 2 Dec 2020 08:38:20 -0500 Subject: [PATCH 19/32] Fixes #503 - handle (A|E)LB per Region quota with units of either Count or None --- CHANGES.rst | 1 + awslimitchecker/quotas.py | 5 ++-- awslimitchecker/services/elb.py | 27 ++++++++++++++++++++-- awslimitchecker/tests/services/test_elb.py | 21 ++++++++++++++++- 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ec5be0f7..fed6c0f9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -17,6 +17,7 @@ Changelog * If you are running in China or GovCloud and have issues with awslimitchecker retrieving information from Trusted Advisor, please `open an issue `__. * My current intent is to leave Trusted Advisor support in this state until Service Quotas is available in China and GovCloud, at which point I plan on completely removing all Trusted Advisor support. * Migrate CI builds from travis-ci.org to travis-ci.com. +* `Issue #503 `__ - Fix ``Units set to "None"`` error when retrieving load balancer data from Service Quotas. We now allow the (A|E)LB per Region quota with a unit of either "Count" (prior to November 2020) or "None" (November 2020 on). .. _changelog.9_0_0: diff --git a/awslimitchecker/quotas.py b/awslimitchecker/quotas.py index 5b350b46..38b5bb85 100644 --- a/awslimitchecker/quotas.py +++ b/awslimitchecker/quotas.py @@ -145,10 +145,11 @@ def get_quota_value( return converter(val, svc[quota_name.lower()]['Unit'], units) logger.error( 'ERROR: Service Quota service_code=%s QuotaName="%s" has ' - 'Units set to "%s"; awslimitchecker does not know how to ' + 'Units set to "%s", but expected units to be "%s"; ' + 'awslimitchecker does not know how to ' 'handle this. This quota will be ignored. Please open a bug ' 'report.', service_code, quota_name, - svc[quota_name.lower()]['Unit'] + svc[quota_name.lower()]['Unit'], units ) return None return val diff --git a/awslimitchecker/services/elb.py b/awslimitchecker/services/elb.py index 635e9430..f2eb9d36 100644 --- a/awslimitchecker/services/elb.py +++ b/awslimitchecker/services/elb.py @@ -52,6 +52,27 @@ ELBV2_MAX_RETRY_ATTEMPTS = 12 +def allow_count_or_none_units(value, in_unit, out_unit): + """ + This is a unit converter for Service Quotas; see + :py:meth:`.ServiceQuotasClient.get_quota_value` for details. + + This is a work-around for + https://github.com/jantman/awslimitchecker/issues/503 where, sometime + between 2020-11-02 and 2020-11-09, the quota unit for Application Load + Balancers per Region and Classic Load Balancers per Region changed, without + announcement or warning, from "Count" to "None". This converter allows both + options and treats them identically. + """ + if in_unit not in ['None', 'Count'] or out_unit != 'Count': + logger.error( + 'ERROR: cannot convert Service Quotas (A|E)LB limit value from ' + 'units of "%s" to units of "%s"', in_unit, out_unit + ) + return None + return value + + class _ElbService(_AwsService): """ Note that ELB (ELBv1) and ALB (ELBv2) are combined in the same service. @@ -273,7 +294,8 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::ElasticLoadBalancing::LoadBalancer', quotas_name='Classic Load Balancers per Region', - quotas_unit='Count' + quotas_unit='Count', + quotas_unit_converter=allow_count_or_none_units ) limits['Listeners per load balancer'] = AwsLimit( 'Listeners per load balancer', @@ -302,7 +324,8 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::ElasticLoadBalancingV2::LoadBalancer', quotas_name='Application Load Balancers per Region', - quotas_unit='Count' + quotas_unit='Count', + quotas_unit_converter=allow_count_or_none_units ) limits['Target groups'] = AwsLimit( 'Target groups', diff --git a/awslimitchecker/tests/services/test_elb.py b/awslimitchecker/tests/services/test_elb.py index a18ad7c4..95f54a4f 100644 --- a/awslimitchecker/tests/services/test_elb.py +++ b/awslimitchecker/tests/services/test_elb.py @@ -39,7 +39,7 @@ import sys from awslimitchecker.tests.services import result_fixtures -from awslimitchecker.services.elb import _ElbService +from awslimitchecker.services.elb import _ElbService, allow_count_or_none_units # https://code.google.com/p/mock/issues/detail?id=249 # py>=3.4 should use unittest.mock not the mock package on pypi @@ -56,6 +56,18 @@ pb = '%s._ElbService' % pbm # patch base path +class TestAllowCountOrNoneUnits: + + def test_none(self): + assert allow_count_or_none_units(10, 'None', 'Count') == 10 + + def test_count(self): + assert allow_count_or_none_units(10, 'Count', 'Count') == 10 + + def test_other(self): + assert allow_count_or_none_units(10, 'Other', 'Count') is None + + class Test_ElbService(object): def test_init(self): @@ -86,6 +98,13 @@ def test_get_limits(self): assert limit.service == cls assert limit.def_warning_threshold == 21 assert limit.def_critical_threshold == 43 + if name in [ + 'Application load balancers', + 'Classic load balancers', + ]: + assert limit.quotas_unit_converter == allow_count_or_none_units + else: + assert limit.quotas_unit_converter is None def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" From 0bb9d3e53967997361cf1dc8b913f0e2ffff6518 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Wed, 2 Dec 2020 08:50:43 -0500 Subject: [PATCH 20/32] Fix integration tests for #505 --- awslimitchecker/tests/test_integration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awslimitchecker/tests/test_integration.py b/awslimitchecker/tests/test_integration.py index be42da6b..43e6894c 100644 --- a/awslimitchecker/tests/test_integration.py +++ b/awslimitchecker/tests/test_integration.py @@ -181,8 +181,8 @@ def test_verify_limits(self, checker_args, creds_type, service_name, use_ta, assert len(records) == 0, "awslimitchecker emitted unexpected log " \ "messages at WARN or higher: \n%s" % "\n".join(records) polls = logs.num_ta_polls - assert polls == 1, "awslimitchecker should have polled Trusted " \ - "Advisor once, but polled %s times" % polls + assert polls == 0, "awslimitchecker should have polled Trusted " \ + "Advisor ZERO times, but polled %s times" % polls @pytest.mark.integration @skip_if_pr From 8bbab924c75bed45fd41cc799eda8b33bd045865 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Wed, 2 Dec 2020 09:11:48 -0500 Subject: [PATCH 21/32] Changelog for #490 --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index fed6c0f9..eb7bd49d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,6 +18,7 @@ Changelog * My current intent is to leave Trusted Advisor support in this state until Service Quotas is available in China and GovCloud, at which point I plan on completely removing all Trusted Advisor support. * Migrate CI builds from travis-ci.org to travis-ci.com. * `Issue #503 `__ - Fix ``Units set to "None"`` error when retrieving load balancer data from Service Quotas. We now allow the (A|E)LB per Region quota with a unit of either "Count" (prior to November 2020) or "None" (November 2020 on). +* `PR #490 `__ - Add missing RDS limits: ``Manual Cluster Snapshots``, ``Custom Endpoints Per DB Cluster``, ``DB Instance Roles``, and ``DB Cluster Roles``. Thanks to `sebasrp `__ for this contribution! .. _changelog.9_0_0: From 49e2bfb4053f449157907ad6ede9c96c4bd677ad Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Wed, 2 Dec 2020 09:34:08 -0500 Subject: [PATCH 22/32] Changelog for #472 / #494 --- CHANGES.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index eb7bd49d..bfc4ef20 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,7 +6,11 @@ Changelog 10.0.0 (2020-12-07) ------------------- -**Important:** This release makes significant changes to how Trusted Advisor is used. +**Important:** This release makes significant changes to how Trusted Advisor is used; see below. + +**Important:** This release requires the following new IAM permissions: ``eks:ListClusters``, ``eks:DescribeCluster``, ``eks:ListNodegroups``, ``eks:ListFargateProfiles``, ``eks:DescribeFargateProfile``. + +**Important:** This release introduces a number of new limits, as well as new services. Please see below for details. * `Issue #466 `__ - **Significant** changes to Trusted Advisor support. @@ -18,7 +22,8 @@ Changelog * My current intent is to leave Trusted Advisor support in this state until Service Quotas is available in China and GovCloud, at which point I plan on completely removing all Trusted Advisor support. * Migrate CI builds from travis-ci.org to travis-ci.com. * `Issue #503 `__ - Fix ``Units set to "None"`` error when retrieving load balancer data from Service Quotas. We now allow the (A|E)LB per Region quota with a unit of either "Count" (prior to November 2020) or "None" (November 2020 on). -* `PR #490 `__ - Add missing RDS limits: ``Manual Cluster Snapshots``, ``Custom Endpoints Per DB Cluster``, ``DB Instance Roles``, and ``DB Cluster Roles``. Thanks to `sebasrp `__ for this contribution! +* `Issue #489 `__ / `PR #490 `__ - Add missing RDS limits: ``Manual Cluster Snapshots``, ``Custom Endpoints Per DB Cluster``, ``DB Instance Roles``, and ``DB Cluster Roles``. Thanks to `sebasrp `__ for this contribution! +* `Issue #472 `__ / `PR #494 `__ - Add support for the ``EKS`` service, and 8 new limits for it. All limit information, as well as current usage information for this service, is retrieved from Service Quotas. **As such, this service will not currently have any utilization data in China or GovCloud.** Thanks to `sebasrp `__ for this contribution! .. _changelog.9_0_0: From bf94f3c0134360d3bc8bc25abc5852dd972f1700 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Wed, 2 Dec 2020 09:53:36 -0500 Subject: [PATCH 23/32] Changelog for #495 / #496 --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index bfc4ef20..a2388073 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -23,7 +23,8 @@ Changelog * Migrate CI builds from travis-ci.org to travis-ci.com. * `Issue #503 `__ - Fix ``Units set to "None"`` error when retrieving load balancer data from Service Quotas. We now allow the (A|E)LB per Region quota with a unit of either "Count" (prior to November 2020) or "None" (November 2020 on). * `Issue #489 `__ / `PR #490 `__ - Add missing RDS limits: ``Manual Cluster Snapshots``, ``Custom Endpoints Per DB Cluster``, ``DB Instance Roles``, and ``DB Cluster Roles``. Thanks to `sebasrp `__ for this contribution! -* `Issue #472 `__ / `PR #494 `__ - Add support for the ``EKS`` service, and 8 new limits for it. All limit information, as well as current usage information for this service, is retrieved from Service Quotas. **As such, this service will not currently have any utilization data in China or GovCloud.** Thanks to `sebasrp `__ for this contribution! +* `Issue #472 `__ / `PR #494 `__ - Add support for the ``EKS`` service, and 8 new limits for it. Thanks to `sebasrp `__ for this contribution! +* `Issue #495 `__ / `PR #496 `__ - Add support for the ``Kinesis`` service, and one new limit for it. Thanks to `sebasrp `__ for this contribution! .. _changelog.9_0_0: From 4a23d3fc5cf0d9751bc159595b426388175fea8b Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Wed, 2 Dec 2020 10:09:08 -0500 Subject: [PATCH 24/32] Changelog for #499 --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a2388073..80cc00b6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,7 +8,7 @@ Changelog **Important:** This release makes significant changes to how Trusted Advisor is used; see below. -**Important:** This release requires the following new IAM permissions: ``eks:ListClusters``, ``eks:DescribeCluster``, ``eks:ListNodegroups``, ``eks:ListFargateProfiles``, ``eks:DescribeFargateProfile``. +**Important:** This release requires the following new IAM permissions: ``eks:ListClusters``, ``eks:DescribeCluster``, ``eks:ListNodegroups``, ``eks:ListFargateProfiles``, ``eks:DescribeFargateProfile``, ``kinesis:DescribeLimits``. **Important:** This release introduces a number of new limits, as well as new services. Please see below for details. @@ -25,6 +25,7 @@ Changelog * `Issue #489 `__ / `PR #490 `__ - Add missing RDS limits: ``Manual Cluster Snapshots``, ``Custom Endpoints Per DB Cluster``, ``DB Instance Roles``, and ``DB Cluster Roles``. Thanks to `sebasrp `__ for this contribution! * `Issue #472 `__ / `PR #494 `__ - Add support for the ``EKS`` service, and 8 new limits for it. Thanks to `sebasrp `__ for this contribution! * `Issue #495 `__ / `PR #496 `__ - Add support for the ``Kinesis`` service, and one new limit for it. Thanks to `sebasrp `__ for this contribution! +* `PR #499 `__ - Set quota_name for VPC "Entries per route table" limit, so that the current limit will be automatically retrieved from Service Quotas. Thanks to `patuck `__ for this contribution! .. _changelog.9_0_0: From e87235a986ebfdb7d910c5b367ea53d13a820763 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Wed, 2 Dec 2020 11:52:46 -0500 Subject: [PATCH 25/32] Issue #498 - support Service Quota for EC2/Rules per VPC security group limit --- awslimitchecker/services/ec2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awslimitchecker/services/ec2.py b/awslimitchecker/services/ec2.py index 6d68a746..03f8599a 100644 --- a/awslimitchecker/services/ec2.py +++ b/awslimitchecker/services/ec2.py @@ -744,6 +744,8 @@ def _get_limits_networking(self): self.critical_threshold, limit_type='AWS::EC2::SecurityGroup', limit_subtype='AWS::EC2::VPC', + quotas_name='Inbound or outbound rules per security group', + quotas_service_code='vpc' ) limits['VPC Elastic IP addresses (EIPs)'] = AwsLimit( 'VPC Elastic IP addresses (EIPs)', From 9aae8670edb64f63900d1851e0ea8845d82ae294 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Wed, 2 Dec 2020 12:00:46 -0500 Subject: [PATCH 26/32] Fix #498 - switch EC2/Security groups per VPC limit for new EC2/VPC security groups per Region limit --- CHANGES.rst | 16 +++++++++++++--- awslimitchecker/services/ec2.py | 22 +++++++++++----------- awslimitchecker/tests/services/test_ec2.py | 19 +++++++++---------- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 80cc00b6..11c42492 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,11 +6,16 @@ Changelog 10.0.0 (2020-12-07) ------------------- -**Important:** This release makes significant changes to how Trusted Advisor is used; see below. +IMPORTANT - Breaking Changes +++++++++++++++++++++++++++++ -**Important:** This release requires the following new IAM permissions: ``eks:ListClusters``, ``eks:DescribeCluster``, ``eks:ListNodegroups``, ``eks:ListFargateProfiles``, ``eks:DescribeFargateProfile``, ``kinesis:DescribeLimits``. +* This release makes significant changes to how Trusted Advisor is used; see below. +* This release requires the following new IAM permissions: ``eks:ListClusters``, ``eks:DescribeCluster``, ``eks:ListNodegroups``, ``eks:ListFargateProfiles``, ``eks:DescribeFargateProfile``, ``kinesis:DescribeLimits``. +* This release introduces a number of new limits, as well as new services. Please see below for details. +* This release **removes** the ``EC2/Security groups per VPC`` limit, which no longer exists, and adds the new ``EC2/VPC security groups per Region`` limit. -**Important:** This release introduces a number of new limits, as well as new services. Please see below for details. +All Changes ++++++++++++ * `Issue #466 `__ - **Significant** changes to Trusted Advisor support. @@ -26,6 +31,11 @@ Changelog * `Issue #472 `__ / `PR #494 `__ - Add support for the ``EKS`` service, and 8 new limits for it. Thanks to `sebasrp `__ for this contribution! * `Issue #495 `__ / `PR #496 `__ - Add support for the ``Kinesis`` service, and one new limit for it. Thanks to `sebasrp `__ for this contribution! * `PR #499 `__ - Set quota_name for VPC "Entries per route table" limit, so that the current limit will be automatically retrieved from Service Quotas. Thanks to `patuck `__ for this contribution! +* `Issue #498 `__ - Fix multiple issues relating to VPC limits: + + * Update the EC2 / ``Rules per VPC security group`` limit to support retrieving the current limit value from Service Quotas. + * Remove the ``EC2/Security groups per VPC`` limit, which no longer exists. + * Add the new ``EC2/VPC security groups per Region`` limit. .. _changelog.9_0_0: diff --git a/awslimitchecker/services/ec2.py b/awslimitchecker/services/ec2.py index 03f8599a..fbf2216b 100644 --- a/awslimitchecker/services/ec2.py +++ b/awslimitchecker/services/ec2.py @@ -640,12 +640,12 @@ def _get_limits_spot(self): def _find_usage_networking_sgs(self): """calculate usage for VPC-related things""" logger.debug("Getting usage for EC2 VPC resources") - sgs_per_vpc = defaultdict(int) + sg_count = 0 rules_per_sg = defaultdict(int) for sg in self.resource_conn.security_groups.all(): if sg.vpc_id is None: continue - sgs_per_vpc[sg.vpc_id] += 1 + sg_count += 1 """ see: https://github.com/jantman/awslimitchecker/issues/431 @@ -676,12 +676,10 @@ def _find_usage_networking_sgs(self): ) rules_per_sg[sg.id] = max(counts) # set usage - for vpc_id, count in sgs_per_vpc.items(): - self.limits['Security groups per VPC']._add_current_usage( - count, - aws_type='AWS::EC2::VPC', - resource_id=vpc_id, - ) + self.limits['VPC security groups per Region']._add_current_usage( + sg_count, + aws_type='AWS::EC2::SecurityGroup', + ) for sg_id, count in rules_per_sg.items(): self.limits['Rules per VPC security group']._add_current_usage( count, @@ -727,14 +725,16 @@ def _get_limits_networking(self): :rtype: dict """ limits = {} - limits['Security groups per VPC'] = AwsLimit( - 'Security groups per VPC', + limits['VPC security groups per Region'] = AwsLimit( + 'VPC security groups per Region', self, - 500, + 2500, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::SecurityGroup', limit_subtype='AWS::EC2::VPC', + quotas_name='VPC security groups per Region', + quotas_service_code='vpc' ) limits['Rules per VPC security group'] = AwsLimit( 'Rules per VPC security group', diff --git a/awslimitchecker/tests/services/test_ec2.py b/awslimitchecker/tests/services/test_ec2.py index 533d8756..a79e1517 100644 --- a/awslimitchecker/tests/services/test_ec2.py +++ b/awslimitchecker/tests/services/test_ec2.py @@ -777,18 +777,13 @@ def test_simple(self): assert mock_logger.mock_calls == [ call.debug("Getting usage for EC2 VPC resources"), ] - limit = cls.limits['Security groups per VPC'] + limit = cls.limits['VPC security groups per Region'] # relies on AwsLimitUsage sorting by numeric usage value sorted_usage = sorted(limit.get_current_usage()) - assert len(sorted_usage) == 2 + assert len(sorted_usage) == 1 assert sorted_usage[0].limit == limit - assert sorted_usage[0].get_value() == 1 - assert sorted_usage[0].resource_id == 'vpc-bbb' - assert sorted_usage[0].aws_type == 'AWS::EC2::VPC' - assert sorted_usage[1].limit == limit - assert sorted_usage[1].get_value() == 2 - assert sorted_usage[1].resource_id == 'vpc-aaa' - assert sorted_usage[1].aws_type == 'AWS::EC2::VPC' + assert sorted_usage[0].get_value() == 3 + assert sorted_usage[0].aws_type == 'AWS::EC2::SecurityGroup' limit = cls.limits['Rules per VPC security group'] sorted_usage = sorted(limit.get_current_usage()) @@ -886,7 +881,7 @@ def test_simple(self): cls = _Ec2Service(21, 43, {}, None) limits = cls._get_limits_networking() expected = [ - 'Security groups per VPC', + 'VPC security groups per Region', 'Rules per VPC security group', 'VPC Elastic IP addresses (EIPs)', 'Elastic IP addresses (EIPs)', @@ -913,6 +908,10 @@ def test_simple(self): assert limits[ 'Elastic IP addresses (EIPs)' ].quotas_unit == 'None' + vpcsg = limits['VPC security groups per Region'] + assert vpcsg.quota_name == 'VPC security groups per Region' + assert vpcsg.quotas_service_code == 'vpc' + assert vpcsg.default_limit == 2500 class TestGetLimitsSpot(object): From 23928b57655a52ba8a78076615006eaa3e592180 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Wed, 2 Dec 2020 15:42:25 -0500 Subject: [PATCH 27/32] Fix #501 - update for new calculation of VPC/Network interfaces per Region limit --- CHANGES.rst | 2 + awslimitchecker/services/vpc.py | 26 +--------- .../tests/services/result_fixtures.py | 43 +-------------- awslimitchecker/tests/services/test_vpc.py | 52 +------------------ 4 files changed, 5 insertions(+), 118 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 11c42492..126f5ace 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -37,6 +37,8 @@ All Changes * Remove the ``EC2/Security groups per VPC`` limit, which no longer exists. * Add the new ``EC2/VPC security groups per Region`` limit. +* `Issue #501 `__ - Update ``VPC/Network interfaces per Region`` limit for new calculation method. + .. _changelog.9_0_0: 9.0.0 (2020-09-22) diff --git a/awslimitchecker/services/vpc.py b/awslimitchecker/services/vpc.py index 848de973..83feb37b 100644 --- a/awslimitchecker/services/vpc.py +++ b/awslimitchecker/services/vpc.py @@ -48,8 +48,6 @@ logger = logging.getLogger(__name__) -DEFAULT_ENI_LIMIT = 350 - class _VpcService(_AwsService): @@ -344,7 +342,7 @@ def get_limits(self): limits['Network interfaces per Region'] = AwsLimit( 'Network interfaces per Region', self, - DEFAULT_ENI_LIMIT, + 5000, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::NetworkInterface' @@ -352,28 +350,6 @@ def get_limits(self): self.limits = limits return limits - def _update_limits_from_api(self): - """ - Query EC2's DescribeAccountAttributes API action and - update the network interface limit, as needed. Updates ``self.limits``. - - More info on the network interface limit, from the docs: - 'This limit is the greater of either the default limit (350) or your - On-Demand Instance limit multiplied by 5. - The default limit for On-Demand Instances is 20.' - """ - self.connect() - self.connect_resource() - logger.info("Querying EC2 DescribeAccountAttributes for limits") - attribs = self.conn.describe_account_attributes() - for attrib in attribs['AccountAttributes']: - if attrib['AttributeName'] == 'max-instances': - val = attrib['AttributeValues'][0]['AttributeValue'] - if int(val) * 5 > DEFAULT_ENI_LIMIT: - limit_name = 'Network interfaces per Region' - self.limits[limit_name]._set_api_limit(int(val) * 5) - logger.debug("Done setting limits from API") - def required_iam_permissions(self): """ Return a list of IAM Actions required for this Service to function diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index b5c8f982..32dcc3f8 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -221,6 +221,7 @@ class EBS(object): class VPC(object): + test_find_usage_vpcs = { 'Vpcs': [ { @@ -620,48 +621,6 @@ class VPC(object): } } - test_update_limits_from_api_high_max_instances = { - 'ResponseMetadata': { - 'HTTPStatusCode': 200, - 'RequestId': '16b85906-ab0d-4134-b8bb-df3e6120c6c7' - }, - 'AccountAttributes': [ - { - 'AttributeName': 'max-instances', - 'AttributeValues': [ - { - 'AttributeValue': '400' - } - ] - } - ] - } - - test_update_limits_from_api_low_max_instances = { - 'ResponseMetadata': { - 'HTTPStatusCode': 200, - 'RequestId': '16b85906-ab0d-4134-b8bb-df3e6120c6c7' - }, - 'AccountAttributes': [ - { - 'AttributeName': 'max-instances', - 'AttributeValues': [ - { - 'AttributeValue': '50' - } - ] - }, - { - 'AttributeName': 'something-else', - 'AttributeValues': [ - { - 'AttributeValue': '1' - } - ] - } - ] - } - class RDS(object): test_find_usage_instances = [] diff --git a/awslimitchecker/tests/services/test_vpc.py b/awslimitchecker/tests/services/test_vpc.py index 0744d5e2..e968935a 100644 --- a/awslimitchecker/tests/services/test_vpc.py +++ b/awslimitchecker/tests/services/test_vpc.py @@ -39,7 +39,7 @@ import sys from awslimitchecker.tests.services import result_fixtures -from awslimitchecker.services.vpc import _VpcService, DEFAULT_ENI_LIMIT +from awslimitchecker.services.vpc import _VpcService from botocore.exceptions import ClientError @@ -382,56 +382,6 @@ def test_find_usage_network_interfaces(self): }]), ] - def test_update_limits_from_api_high_max_instances(self): - fixtures = result_fixtures.VPC() - response = fixtures.test_update_limits_from_api_high_max_instances - - mock_conn = Mock() - mock_client_conn = Mock() - mock_client_conn.describe_account_attributes.return_value = response - - cls = _VpcService(21, 43, {}, None) - cls.resource_conn = mock_conn - cls.conn = mock_client_conn - with patch('awslimitchecker.services.vpc.logger') as mock_logger: - cls._update_limits_from_api() - assert mock_conn.mock_calls == [] - assert mock_client_conn.mock_calls == [ - call.describe_account_attributes() - ] - assert mock_logger.mock_calls == [ - call.info("Querying EC2 DescribeAccountAttributes for limits"), - call.debug('Done setting limits from API') - ] - assert cls.limits['Network interfaces per Region'].api_limit == 2000 - assert cls.limits['Network interfaces per Region'].get_limit() == 2000 - - def test_update_limits_from_api_low_max_instances(self): - fixtures = result_fixtures.VPC() - response = fixtures.test_update_limits_from_api_low_max_instances - - mock_conn = Mock() - mock_client_conn = Mock() - mock_client_conn.describe_account_attributes.return_value = response - - cls = _VpcService(21, 43, {}, None) - cls.resource_conn = mock_conn - cls.conn = mock_client_conn - with patch('awslimitchecker.services.vpc.logger') as mock_logger: - cls._update_limits_from_api() - assert mock_conn.mock_calls == [] - assert mock_client_conn.mock_calls == [ - call.describe_account_attributes() - ] - assert mock_logger.mock_calls == [ - call.info("Querying EC2 DescribeAccountAttributes for limits"), - call.debug('Done setting limits from API') - ] - - limit_name = 'Network interfaces per Region' - assert cls.limits[limit_name].api_limit is None - assert cls.limits[limit_name].get_limit() == DEFAULT_ENI_LIMIT - def test_required_iam_permissions(self): cls = _VpcService(21, 43, {}, None) assert cls.required_iam_permissions() == [ From bc217b655e7e756afad7e7eeba1d9f3b5f57492c Mon Sep 17 00:00:00 2001 From: Sebastian Rodriguez <331558+sebasrp@users.noreply.github.com> Date: Thu, 3 Dec 2020 11:16:40 +0800 Subject: [PATCH 28/32] update node per cluster limit to match documentation --- awslimitchecker/services/elasticache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awslimitchecker/services/elasticache.py b/awslimitchecker/services/elasticache.py index 28ac3b5f..09e1f54c 100644 --- a/awslimitchecker/services/elasticache.py +++ b/awslimitchecker/services/elasticache.py @@ -177,7 +177,7 @@ def get_limits(self): limits['Nodes per Cluster'] = AwsLimit( 'Nodes per Cluster', self, - 40, + 20, self.warning_threshold, self.critical_threshold, limit_type='AWS::ElastiCache::CacheNode', From d116207e07bd7318c5b2e57c287048f8a4a3c2a4 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Mon, 7 Dec 2020 14:06:24 -0500 Subject: [PATCH 29/32] CHANGELOG for #488 / #491 --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 126f5ace..bd724e69 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -38,6 +38,7 @@ All Changes * Add the new ``EC2/VPC security groups per Region`` limit. * `Issue #501 `__ - Update ``VPC/Network interfaces per Region`` limit for new calculation method. +* `Issue #488 `__ / `PR #491 `__ - Update new ElastiCache default limits. Thanks to `sebasrp `__ for this contribution! .. _changelog.9_0_0: From d0593d8c4b0de4b423f6222e1606dea6774de646 Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Mon, 7 Dec 2020 14:46:20 -0500 Subject: [PATCH 30/32] bump version to 10.0.0 --- awslimitchecker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awslimitchecker/version.py b/awslimitchecker/version.py index 35342f25..2e910cb6 100644 --- a/awslimitchecker/version.py +++ b/awslimitchecker/version.py @@ -47,7 +47,7 @@ except ImportError: logger.error("Unable to import versionfinder", exc_info=True) -_VERSION_TUP = (9, 0, 0) +_VERSION_TUP = (10, 0, 0) _VERSION = '.'.join([str(x) for x in _VERSION_TUP]) _PROJECT_URL = 'https://github.com/jantman/awslimitchecker' From 704b79c33d7dc0c28bfa456f0c1bffdb5f83d9cb Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Mon, 7 Dec 2020 14:49:36 -0500 Subject: [PATCH 31/32] fix bug in release.py --- dev/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release.py b/dev/release.py index 6d64c932..c5248254 100755 --- a/dev/release.py +++ b/dev/release.py @@ -106,7 +106,7 @@ def run(self): for line in fh.readlines(): line = line.strip() if ( - line == '------------------' and + line == '-' * len(expected) and last_line == expected ): have_ver = True From 53c11b67f6dab6ca69d6dd967e93f1d94f7bc64e Mon Sep 17 00:00:00 2001 From: Jason Antman Date: Mon, 7 Dec 2020 15:10:35 -0500 Subject: [PATCH 32/32] rebuild docs --- docs/build_generated_docs.py | 9 +- docs/source/awslimitchecker.services.eks.rst | 8 ++ .../awslimitchecker.services.kinesis.rst | 8 ++ docs/source/awslimitchecker.services.rst | 2 + docs/source/cli_usage.rst | 12 +- docs/source/cli_usage.rst.template | 34 ++++- docs/source/iam_policy.rst | 6 + docs/source/limits.rst | 117 +++++++++++------- 8 files changed, 136 insertions(+), 60 deletions(-) create mode 100644 docs/source/awslimitchecker.services.eks.rst create mode 100644 docs/source/awslimitchecker.services.kinesis.rst diff --git a/docs/build_generated_docs.py b/docs/build_generated_docs.py index c4f03947..69d84dfa 100644 --- a/docs/build_generated_docs.py +++ b/docs/build_generated_docs.py @@ -167,7 +167,7 @@ def limits_for_ec2(): regions still use the old per-instance-type limits. Please see the sections for either :ref:`limits.ec2-standard` or :ref:`limits.ec2-nonvcpu` for details. - + """) limit_info += '.. _limits.ec2-standard:\n\n' limit_info += "EC2 - Standard Regions\n" @@ -180,10 +180,10 @@ def limits_for_ec2(): (On-Demand, Reserved, etc.). The value that awslimitchecker reports for Running On-Demand Instances current usage will *not* match the number of instances you see in the Console or API. - + **Important:** The limits for **Running On-Demand Instances** are now measured in vCPU count per instance family, not instance count per instance - type. + type. """) + "\n" limit_info += "\n" limit_info += format_limits_for_service( @@ -299,9 +299,6 @@ def build_runner_examples(): '--limit="AutoScaling/Launch configurations"=456', '-l', ], - 'check_thresholds': ['awslimitchecker', '--no-color'], - 'check_thresholds_custom': ['awslimitchecker', '-W', '97', - '--critical=98', '--no-color'], 'iam_policy': ['awslimitchecker', '--iam-policy'], 'list_metrics': ['awslimitchecker', '--list-metrics-providers'], 'list_alerts': ['awslimitchecker', '--list-alert-providers'], diff --git a/docs/source/awslimitchecker.services.eks.rst b/docs/source/awslimitchecker.services.eks.rst new file mode 100644 index 00000000..83796793 --- /dev/null +++ b/docs/source/awslimitchecker.services.eks.rst @@ -0,0 +1,8 @@ +awslimitchecker.services.eks module +=================================== + +.. automodule:: awslimitchecker.services.eks + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/source/awslimitchecker.services.kinesis.rst b/docs/source/awslimitchecker.services.kinesis.rst new file mode 100644 index 00000000..50f96665 --- /dev/null +++ b/docs/source/awslimitchecker.services.kinesis.rst @@ -0,0 +1,8 @@ +awslimitchecker.services.kinesis module +======================================= + +.. automodule:: awslimitchecker.services.kinesis + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/source/awslimitchecker.services.rst b/docs/source/awslimitchecker.services.rst index 10ee0387..e849f0a5 100644 --- a/docs/source/awslimitchecker.services.rst +++ b/docs/source/awslimitchecker.services.rst @@ -24,11 +24,13 @@ Submodules awslimitchecker.services.ec2 awslimitchecker.services.ecs awslimitchecker.services.efs + awslimitchecker.services.eks awslimitchecker.services.elasticache awslimitchecker.services.elasticbeanstalk awslimitchecker.services.elb awslimitchecker.services.firehose awslimitchecker.services.iam + awslimitchecker.services.kinesis awslimitchecker.services.lambdafunc awslimitchecker.services.rds awslimitchecker.services.redshift diff --git a/docs/source/cli_usage.rst b/docs/source/cli_usage.rst index 01618e1b..7a16f21f 100644 --- a/docs/source/cli_usage.rst +++ b/docs/source/cli_usage.rst @@ -461,8 +461,6 @@ threshold only, and another has crossed the critical threshold): VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=5, us-east-1c= (...) VPC/Virtual private gateways (limit 5) WARNING: 4 - - .. _cli_usage.threshold_overrides: Set Custom Thresholds @@ -482,8 +480,6 @@ To set the warning threshold of 50% and a critical threshold of 75% when checkin S3/Buckets (limit 100) CRITICAL: 946 VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=5, us-east-1c= (...) - - You can also set custom thresholds on a per-limit basis using the ``--threshold-override-json`` CLI option, which accepts the path to a JSON file (local or an s3:// URL) matching the format described in @@ -530,17 +526,15 @@ Using a command like: .. code-block:: console (venv)$ awslimitchecker -W 97 --critical=98 --no-color --threshold-override-json=s3://bucketname/path/overrides.json - DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: some_app_name_here=5 + DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: some_app_name (...) DynamoDB/Tables Per Region (limit 256) CRITICAL: 554 - EC2/Rules per VPC security group (limit 50) CRITICAL: sg-aaaaaaaa=49, sg-bbbbb (...) - EC2/Security groups per VPC (limit 500) CRITICAL: vpc-cccccccc=726, vpc-c (...) + EC2/Rules per VPC security group (limit 50) CRITICAL: sg-cccccccc=49, sg-eeeee (...) + EC2/Security groups per VPC (limit 500) CRITICAL: vpc-dddddddd=726, vpc-c (...) (...) RDS/VPC Security Groups (limit 5) CRITICAL: 5 S3/Buckets (limit 100) CRITICAL: 946 VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=5, us-east-1c= (...) - - .. _cli_usage.metrics: Enable Metrics Provider diff --git a/docs/source/cli_usage.rst.template b/docs/source/cli_usage.rst.template index 0e03434e..38a0a5bf 100644 --- a/docs/source/cli_usage.rst.template +++ b/docs/source/cli_usage.rst.template @@ -187,7 +187,18 @@ usage value, or both; this functionality is not currently present in the command To check all limits against their thresholds (in this example, one limit has crossed the warning threshold only, and another has crossed the critical threshold): -{check_thresholds} +.. code-block:: console + + (venv)$ awslimitchecker --no-color + CloudFormation/Stacks (limit 4000) WARNING: 3396 + DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: some_app_name (...) + DynamoDB/Tables Per Region (limit 256) CRITICAL: 554 + EBS/Active snapshots (limit 40000.0) WARNING: 33387 + EC2/Rules per VPC security group (limit 50) CRITICAL: sg-aaaaaaaa=50, sg-bbbb (...) + (...) + VPC/Entries per route table (limit 50) WARNING: rtb-aaaaaaaa=43, rtb-bbbb (...) + VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=5, us-east-1c= (...) + VPC/Virtual private gateways (limit 5) WARNING: 4 .. _cli_usage.threshold_overrides: @@ -196,7 +207,17 @@ Set Custom Thresholds To set the warning threshold of 50% and a critical threshold of 75% when checking limits: -{check_thresholds_custom} +.. code-block:: console + + (venv)$ awslimitchecker -W 97 --critical=98 --no-color + DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: some_app_name (...) + DynamoDB/Tables Per Region (limit 256) CRITICAL: 554 + EC2/Rules per VPC security group (limit 50) CRITICAL: sg-cccccccc=49, sg-eeeee (...) + EC2/Security groups per VPC (limit 500) CRITICAL: vpc-dddddddd=726, vpc-c (...) + (...) + RDS/VPC Security Groups (limit 5) CRITICAL: 5 + S3/Buckets (limit 100) CRITICAL: 946 + VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=5, us-east-1c= (...) You can also set custom thresholds on a per-limit basis using the ``--threshold-override-json`` CLI option, which accepts the path to a JSON file @@ -212,7 +233,14 @@ Using a command like: .. code-block:: console (venv)$ awslimitchecker -W 97 --critical=98 --no-color --threshold-override-json=s3://bucketname/path/overrides.json -{check_thresholds_custom-output-only} + DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: some_app_name (...) + DynamoDB/Tables Per Region (limit 256) CRITICAL: 554 + EC2/Rules per VPC security group (limit 50) CRITICAL: sg-cccccccc=49, sg-eeeee (...) + EC2/Security groups per VPC (limit 500) CRITICAL: vpc-dddddddd=726, vpc-c (...) + (...) + RDS/VPC Security Groups (limit 5) CRITICAL: 5 + S3/Buckets (limit 100) CRITICAL: 946 + VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=5, us-east-1c= (...) .. _cli_usage.metrics: diff --git a/docs/source/iam_policy.rst b/docs/source/iam_policy.rst index daa85e87..0878346c 100644 --- a/docs/source/iam_policy.rst +++ b/docs/source/iam_policy.rst @@ -71,6 +71,11 @@ services that do not affect the results of this program. "ecs:DescribeServices", "ecs:ListClusters", "ecs:ListServices", + "eks:DescribeCluster", + "eks:DescribeFargateProfile", + "eks:ListClusters", + "eks:ListFargateProfiles", + "eks:ListNodegroups", "elasticache:DescribeCacheClusters", "elasticache:DescribeCacheParameterGroups", "elasticache:DescribeCacheSecurityGroups", @@ -86,6 +91,7 @@ services that do not affect the results of this program. "elasticloadbalancing:DescribeTargetGroups", "firehose:ListDeliveryStreams", "iam:GetAccountSummary", + "kinesis:DescribeLimits", "lambda:GetAccountSettings", "rds:DescribeAccountAttributes", "rds:DescribeDBInstances", diff --git a/docs/source/limits.rst b/docs/source/limits.rst index be14de68..be1a0302 100644 --- a/docs/source/limits.rst +++ b/docs/source/limits.rst @@ -167,26 +167,26 @@ number of instances you see in the Console or API. **Important:** The limits for **Running On-Demand Instances** are now measured in vCPU count per instance family, not instance count per instance -type. +type. ==================================================================== =============== ======== ======= ==== Limit Trusted Advisor Quotas API Default ==================================================================== =============== ======== ======= ==== -Elastic IP addresses (EIPs) |check| |check| |check| 5 +Elastic IP addresses (EIPs) |check| 5 Max active spot fleets per region 1000 Max launch specifications per spot fleet 50 Max spot instance requests per region 20 Max target capacity for all spot fleets in region 5000 Max target capacity per spot fleet 3000 -Rules per VPC security group 60 +Rules per VPC security group |check| 60 Running On-Demand All F instances |check| 128 Running On-Demand All G instances |check| 128 Running On-Demand All P instances |check| 128 Running On-Demand All Standard (A, C, D, H, I, M, R, T, Z) instances |check| 1152 Running On-Demand All X instances |check| 128 -Security groups per VPC 500 -VPC Elastic IP addresses (EIPs) |check| |check| |check| 5 +VPC Elastic IP addresses (EIPs) |check| 5 +VPC security groups per Region |check| 2500 VPC security groups per elastic network interface |check| 5 ==================================================================== =============== ======== ======= ==== @@ -516,6 +516,24 @@ Limit Trusted Advisor Quotas API Default File systems |check| 1000 ============ =============== ======== ======= ==== +.. _limits.EKS: + +EKS +---- + +============================================== =============== ======== ======= === +Limit Trusted Advisor Quotas API Default +============================================== =============== ======== ======= === +Clusters |check| 100 +Control plane security groups per cluster |check| 4 +Fargate profiles per cluster |check| 10 +Label pairs per Fargate profile selector |check| 5 +Managed node groups per cluster |check| 30 +Nodes per managed node group |check| 100 +Public endpoint access CIDR ranges per cluster |check| 40 +Selectors per Fargate profile |check| 5 +============================================== =============== ======== ======= === + .. _limits.ELB: ELB @@ -544,11 +562,11 @@ ElastiCache ======================== =============== ======== ======= === Limit Trusted Advisor Quotas API Default ======================== =============== ======== ======= === -Nodes 100 +Nodes 300 Nodes per Cluster 20 -Parameter Groups 20 +Parameter Groups 150 Security Groups 50 -Subnet Groups 50 +Subnet Groups 150 Subnets per subnet group 20 ======================== =============== ======== ======= === @@ -593,6 +611,17 @@ Server certificates |check| |check| |check| 20 Users |check| |check| |check| 5000 ====================== =============== ======== ======= ===== +.. _limits.Kinesis: + +Kinesis +-------- + +================= =============== ======== ======= === +Limit Trusted Advisor Quotas API Default +================= =============== ======== ======= === +Shards per Region |check| |check| 500 +================= =============== ======== ======= === + .. _limits.Lambda: Lambda @@ -614,25 +643,29 @@ Unreserved Concurrent Executions |check| 1000 RDS ---- -============================ =============== ======== ======= ====== -Limit Trusted Advisor Quotas API Default -============================ =============== ======== ======= ====== -DB Cluster Parameter Groups |check| |check| |check| 50 -DB Clusters |check| |check| |check| 40 -DB instances |check| |check| |check| 40 -DB parameter groups |check| |check| |check| 50 -DB security groups |check| |check| |check| 25 -DB snapshots per user |check| |check| 100 -Event Subscriptions |check| |check| |check| 20 -Max auths per security group |check| |check| |check| 20 -Option Groups |check| |check| 20 -Read replicas per master |check| |check| |check| 5 -Reserved Instances |check| |check| 40 -Storage quota (GB) |check| |check| |check| 100000 -Subnet Groups |check| |check| |check| 50 -Subnets per Subnet Group |check| |check| |check| 20 -VPC Security Groups 5 -============================ =============== ======== ======= ====== +=============================== =============== ======== ======= ====== +Limit Trusted Advisor Quotas API Default +=============================== =============== ======== ======= ====== +Custom Endpoints Per DB Cluster |check| 5 +DB Cluster Parameter Groups |check| |check| 50 +DB Cluster Roles |check| 5 +DB Clusters |check| |check| 40 +DB Instance Roles |check| 5 +DB instances |check| |check| 40 +DB parameter groups |check| |check| 50 +DB security groups |check| |check| 25 +DB snapshots per user |check| 100 +Event Subscriptions |check| |check| 20 +Manual Cluster Snapshots |check| 100 +Max auths per security group |check| |check| 20 +Option Groups |check| |check| 20 +Read replicas per master |check| |check| 5 +Reserved Instances |check| |check| 40 +Storage quota (GB) |check| |check| 100000 +Subnet Groups |check| |check| 50 +Subnets per Subnet Group |check| |check| 20 +VPC Security Groups 5 +=============================== =============== ======== ======= ====== .. _limits.Redshift: @@ -661,7 +694,7 @@ zone. As such, each zone may have a different limit value. Limit Trusted Advisor Quotas API Default ================================ =============== ======== ======= ===== Record sets per hosted zone |check| 10000 -VPC associations per hosted zone |check| 100 +VPC associations per hosted zone 100 ================================ =============== ======== ======= ===== .. _limits.S3: @@ -683,7 +716,7 @@ SES =================== =============== ======== ======= === Limit Trusted Advisor Quotas API Default =================== =============== ======== ======= === -Daily sending quota |check| |check| 200 +Daily sending quota |check| 200 =================== =============== ======== ======= === .. _limits.VPC: @@ -691,20 +724,20 @@ Daily sending quota |check| |check| 200 VPC ---- -============================= =============== ======== ======= === +============================= =============== ======== ======= ==== Limit Trusted Advisor Quotas API Default -============================= =============== ======== ======= === -Entries per route table 50 -Internet gateways |check| |check| 5 -NAT Gateways per AZ 5 -Network ACLs per VPC 200 -Network interfaces per Region |check| 350 -Route tables per VPC 200 -Rules per network ACL 20 -Subnets per VPC 200 -VPCs |check| |check| 5 -Virtual private gateways 5 -============================= =============== ======== ======= === +============================= =============== ======== ======= ==== +Entries per route table |check| 50 +Internet gateways |check| 5 +NAT Gateways per AZ |check| 5 +Network ACLs per VPC |check| 200 +Network interfaces per Region |check| 5000 +Route tables per VPC |check| 200 +Rules per network ACL |check| 20 +Subnets per VPC |check| 200 +VPCs |check| 5 +Virtual private gateways 5 +============================= =============== ======== ======= ====