diff --git a/pkg/awsclient/awsclient.go b/pkg/awsclient/awsclient.go index afee465..44d3d41 100644 --- a/pkg/awsclient/awsclient.go +++ b/pkg/awsclient/awsclient.go @@ -31,6 +31,14 @@ type Client interface { DescribePendingMaintenanceActionsAll(ctx context.Context) ([]*rds.ResourcePendingMaintenanceActions, error) DescribeDBInstancesAll(ctx context.Context) ([]*rds.DBInstance, error) + // VPC + DescribeRouteTablesWithContext(ctx context.Context, input *ec2.DescribeRouteTablesInput, opts ...request.Option) (*ec2.DescribeRouteTablesOutput, error) + DescribeVpcsWithContext(ctx context.Context, input *ec2.DescribeVpcsInput, opts ...request.Option) (*ec2.DescribeVpcsOutput, error) + DescribeVpcsPagesWithContext(ctx context.Context, input *ec2.DescribeVpcsInput, fn func(*ec2.DescribeVpcsOutput, bool) bool, opts ...request.Option) error + DescribeSubnetsPagesWithContext(ctx context.Context, input *ec2.DescribeSubnetsInput, fn func(*ec2.DescribeSubnetsOutput, bool) bool, opts ...request.Option) error + DescribeVpcEndpointsPagesWithContext(ctx context.Context, input *ec2.DescribeVpcEndpointsInput, fn func(*ec2.DescribeVpcEndpointsOutput, bool) bool, opts ...request.Option) error + DescribeRouteTablesPagesWithContext(ctx context.Context, input *ec2.DescribeRouteTablesInput, fn func(*ec2.DescribeRouteTablesOutput, bool) bool, opts ...request.Option) error + // Service Quota GetServiceQuotaWithContext(ctx aws.Context, input *servicequotas.GetServiceQuotaInput, opts ...request.Option) (*servicequotas.GetServiceQuotaOutput, error) } @@ -115,6 +123,47 @@ func (c *awsClient) DescribeDBInstancesAll(ctx context.Context) ([]*rds.DBInstan return instances, nil } +// VPC Functions +func (c *awsClient) DescribeRouteTablesWithContext(ctx context.Context, input *ec2.DescribeRouteTablesInput, opts ...request.Option) (*ec2.DescribeRouteTablesOutput, error) { + return c.ec2Client.DescribeRouteTablesWithContext(ctx, input, opts...) +} + +func (c *awsClient) DescribeVpcsWithContext(ctx context.Context, input *ec2.DescribeVpcsInput, opts ...request.Option) (*ec2.DescribeVpcsOutput, error) { + return c.ec2Client.DescribeVpcsWithContext(ctx, input, opts...) +} + +func (c *awsClient) DescribeVpcsPagesWithContext(ctx context.Context, input *ec2.DescribeVpcsInput, fn func(*ec2.DescribeVpcsOutput, bool) bool, opts ...request.Option) error { + err := c.ec2Client.DescribeVpcsPagesWithContext(ctx, input, fn, opts...) + if err != nil { + AwsExporterMetrics.IncrementErrors() + } + return err +} + +func (c *awsClient) DescribeSubnetsPagesWithContext(ctx context.Context, input *ec2.DescribeSubnetsInput, fn func(*ec2.DescribeSubnetsOutput, bool) bool, opts ...request.Option) error { + err := c.ec2Client.DescribeSubnetsPagesWithContext(ctx, input, fn, opts...) + if err != nil { + AwsExporterMetrics.IncrementErrors() + } + return err +} + +func (c *awsClient) DescribeVpcEndpointsPagesWithContext(ctx context.Context, input *ec2.DescribeVpcEndpointsInput, fn func(*ec2.DescribeVpcEndpointsOutput, bool) bool, opts ...request.Option) error { + err := c.ec2Client.DescribeVpcEndpointsPagesWithContext(ctx, input, fn, opts...) + if err != nil { + AwsExporterMetrics.IncrementErrors() + } + return err +} + +func (c *awsClient) DescribeRouteTablesPagesWithContext(ctx context.Context, input *ec2.DescribeRouteTablesInput, fn func(*ec2.DescribeRouteTablesOutput, bool) bool, opts ...request.Option) error { + err := c.ec2Client.DescribeRouteTablesPagesWithContext(ctx, input, fn, opts...) + if err != nil { + AwsExporterMetrics.IncrementErrors() + } + return err +} + func NewClientFromSession(sess *session.Session) Client { return &awsClient{ ec2Client: ec2.New(sess), diff --git a/pkg/awsclient/mock/zz_generated.mock_client.go b/pkg/awsclient/mock/zz_generated.mock_client.go index eca1ca8..1aee42f 100644 --- a/pkg/awsclient/mock/zz_generated.mock_client.go +++ b/pkg/awsclient/mock/zz_generated.mock_client.go @@ -141,6 +141,64 @@ func (mr *MockClientMockRecorder) DescribePendingMaintenanceActionsPagesWithCont return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribePendingMaintenanceActionsPagesWithContext", reflect.TypeOf((*MockClient)(nil).DescribePendingMaintenanceActionsPagesWithContext), varargs...) } +// DescribeRouteTablesPagesWithContext mocks base method. +func (m *MockClient) DescribeRouteTablesPagesWithContext(ctx context.Context, input *ec2.DescribeRouteTablesInput, fn func(*ec2.DescribeRouteTablesOutput, bool) bool, opts ...request.Option) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, input, fn} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeRouteTablesPagesWithContext", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DescribeRouteTablesPagesWithContext indicates an expected call of DescribeRouteTablesPagesWithContext. +func (mr *MockClientMockRecorder) DescribeRouteTablesPagesWithContext(ctx, input, fn interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, input, fn}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeRouteTablesPagesWithContext", reflect.TypeOf((*MockClient)(nil).DescribeRouteTablesPagesWithContext), varargs...) +} + +// DescribeRouteTablesWithContext mocks base method. +func (m *MockClient) DescribeRouteTablesWithContext(ctx context.Context, input *ec2.DescribeRouteTablesInput, opts ...request.Option) (*ec2.DescribeRouteTablesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, input} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeRouteTablesWithContext", varargs...) + ret0, _ := ret[0].(*ec2.DescribeRouteTablesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeRouteTablesWithContext indicates an expected call of DescribeRouteTablesWithContext. +func (mr *MockClientMockRecorder) DescribeRouteTablesWithContext(ctx, input interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, input}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeRouteTablesWithContext", reflect.TypeOf((*MockClient)(nil).DescribeRouteTablesWithContext), varargs...) +} + +// DescribeSubnetsPagesWithContext mocks base method. +func (m *MockClient) DescribeSubnetsPagesWithContext(ctx context.Context, input *ec2.DescribeSubnetsInput, fn func(*ec2.DescribeSubnetsOutput, bool) bool, opts ...request.Option) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, input, fn} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeSubnetsPagesWithContext", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DescribeSubnetsPagesWithContext indicates an expected call of DescribeSubnetsPagesWithContext. +func (mr *MockClientMockRecorder) DescribeSubnetsPagesWithContext(ctx, input, fn interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, input, fn}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeSubnetsPagesWithContext", reflect.TypeOf((*MockClient)(nil).DescribeSubnetsPagesWithContext), varargs...) +} + // DescribeTransitGatewaysWithContext mocks base method. func (m *MockClient) DescribeTransitGatewaysWithContext(ctx aws.Context, input *ec2.DescribeTransitGatewaysInput, opts ...request.Option) (*ec2.DescribeTransitGatewaysOutput, error) { m.ctrl.T.Helper() @@ -161,6 +219,64 @@ func (mr *MockClientMockRecorder) DescribeTransitGatewaysWithContext(ctx, input return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeTransitGatewaysWithContext", reflect.TypeOf((*MockClient)(nil).DescribeTransitGatewaysWithContext), varargs...) } +// DescribeVpcEndpointsPagesWithContext mocks base method. +func (m *MockClient) DescribeVpcEndpointsPagesWithContext(ctx context.Context, input *ec2.DescribeVpcEndpointsInput, fn func(*ec2.DescribeVpcEndpointsOutput, bool) bool, opts ...request.Option) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, input, fn} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeVpcEndpointsPagesWithContext", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DescribeVpcEndpointsPagesWithContext indicates an expected call of DescribeVpcEndpointsPagesWithContext. +func (mr *MockClientMockRecorder) DescribeVpcEndpointsPagesWithContext(ctx, input, fn interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, input, fn}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeVpcEndpointsPagesWithContext", reflect.TypeOf((*MockClient)(nil).DescribeVpcEndpointsPagesWithContext), varargs...) +} + +// DescribeVpcsPagesWithContext mocks base method. +func (m *MockClient) DescribeVpcsPagesWithContext(ctx context.Context, input *ec2.DescribeVpcsInput, fn func(*ec2.DescribeVpcsOutput, bool) bool, opts ...request.Option) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, input, fn} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeVpcsPagesWithContext", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// DescribeVpcsPagesWithContext indicates an expected call of DescribeVpcsPagesWithContext. +func (mr *MockClientMockRecorder) DescribeVpcsPagesWithContext(ctx, input, fn interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, input, fn}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeVpcsPagesWithContext", reflect.TypeOf((*MockClient)(nil).DescribeVpcsPagesWithContext), varargs...) +} + +// DescribeVpcsWithContext mocks base method. +func (m *MockClient) DescribeVpcsWithContext(ctx context.Context, input *ec2.DescribeVpcsInput, opts ...request.Option) (*ec2.DescribeVpcsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, input} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeVpcsWithContext", varargs...) + ret0, _ := ret[0].(*ec2.DescribeVpcsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeVpcsWithContext indicates an expected call of DescribeVpcsWithContext. +func (mr *MockClientMockRecorder) DescribeVpcsWithContext(ctx, input interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, input}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeVpcsWithContext", reflect.TypeOf((*MockClient)(nil).DescribeVpcsWithContext), varargs...) +} + // GetServiceQuotaWithContext mocks base method. func (m *MockClient) GetServiceQuotaWithContext(ctx aws.Context, input *servicequotas.GetServiceQuotaInput, opts ...request.Option) (*servicequotas.GetServiceQuotaOutput, error) { m.ctrl.T.Helper() diff --git a/pkg/vpc.go b/pkg/vpc.go index f26aade..12f1d21 100644 --- a/pkg/vpc.go +++ b/pkg/vpc.go @@ -83,37 +83,44 @@ func NewVPCExporter(sess []*session.Session, logger log.Logger, config VPCConfig func (e *VPCExporter) CollectInRegion(session *session.Session, region *string, wg *sync.WaitGroup) { defer wg.Done() - ec2Svc := ec2.New(session) - quotaSvc := servicequotas.New(session) - - e.collectVpcsPerRegionQuota(quotaSvc, *region) - e.collectVpcsPerRegionUsage(ec2Svc, *region) - e.collectRoutesTablesPerVpcQuota(quotaSvc, *region) - e.collectInterfaceVpcEndpointsPerVpcQuota(quotaSvc, *region) - e.collectSubnetsPerVpcQuota(quotaSvc, *region) - e.collectIPv4BlocksPerVpcQuota(quotaSvc, *region) + awsClient := awsclient.NewClientFromSession(session) + + e.collectVpcsPerRegionQuota(awsClient, *region) + e.collectVpcsPerRegionUsage(awsClient, *region) + e.collectRoutesTablesPerVpcQuota(awsClient, *region) + e.collectInterfaceVpcEndpointsPerVpcQuota(awsClient, *region) + e.collectSubnetsPerVpcQuota(awsClient, *region) + e.collectIPv4BlocksPerVpcQuota(awsClient, *region) vpcCtx, vpcCancel := context.WithTimeout(context.Background(), e.timeout) defer vpcCancel() - allVpcs, err := ec2Svc.DescribeVpcsWithContext(vpcCtx, &ec2.DescribeVpcsInput{}) + var allVpcs []*ec2.Vpc + err := awsClient.DescribeVpcsPagesWithContext(vpcCtx, &ec2.DescribeVpcsInput{}, func(out *ec2.DescribeVpcsOutput, lastPage bool) bool { + allVpcs = append(allVpcs, out.Vpcs...) + return !lastPage + }) if err != nil { level.Error(e.logger).Log("msg", "Call to DescribeVpcs failed", "region", region, "err", err) } else { - for i, _ := range allVpcs.Vpcs { - e.collectSubnetsPerVpcUsage(allVpcs.Vpcs[i], ec2Svc, *region) - e.collectInterfaceVpcEndpointsPerVpcUsage(allVpcs.Vpcs[i], ec2Svc, *region) - e.collectRoutesTablesPerVpcUsage(allVpcs.Vpcs[i], ec2Svc, *region) - e.collectIPv4BlocksPerVpcUsage(allVpcs.Vpcs[i], ec2Svc, *region) + for i := range allVpcs { + e.collectSubnetsPerVpcUsage(allVpcs[i], awsClient, *region) + e.collectInterfaceVpcEndpointsPerVpcUsage(allVpcs[i], awsClient, *region) + e.collectRoutesTablesPerVpcUsage(allVpcs[i], awsClient, *region) + e.collectIPv4BlocksPerVpcUsage(allVpcs[i], awsClient, *region) } } - e.collectRoutesPerRouteTableQuota(quotaSvc, *region) + e.collectRoutesPerRouteTableQuota(awsClient, *region) routesCtx, routesCancel := context.WithTimeout(context.Background(), e.timeout) defer routesCancel() - allRouteTables, err := ec2Svc.DescribeRouteTablesWithContext(routesCtx, &ec2.DescribeRouteTablesInput{}) + var allRouteTables []*ec2.RouteTable + err = awsClient.DescribeRouteTablesPagesWithContext(routesCtx, &ec2.DescribeRouteTablesInput{}, func(out *ec2.DescribeRouteTablesOutput, lastPage bool) bool { + allRouteTables = append(allRouteTables, out.RouteTables...) + return !lastPage + }) if err != nil { level.Error(e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", region, "err", err) } else { - for i, _ := range allRouteTables.RouteTables { - e.collectRoutesPerRouteTableUsage(allRouteTables.RouteTables[i], ec2Svc, *region) + for i := range allRouteTables { + e.collectRoutesPerRouteTableUsage(allRouteTables[i], awsClient, *region) } } } @@ -123,7 +130,7 @@ func (e *VPCExporter) CollectLoop() { wg := &sync.WaitGroup{} wg.Add(len(e.sessions)) - for i, _ := range e.sessions { + for i := range e.sessions { session := e.sessions[i] region := session.Config.Region go e.CollectInRegion(session, region, wg) @@ -142,7 +149,7 @@ func (e *VPCExporter) Collect(ch chan<- prometheus.Metric) { } } -func (e *VPCExporter) GetQuotaValue(client *servicequotas.ServiceQuotas, serviceCode string, quotaCode string) (float64, error) { +func (e *VPCExporter) GetQuotaValue(client awsclient.Client, serviceCode string, quotaCode string) (float64, error) { ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout) defer cancelFunc() sqOutput, err := client.GetServiceQuotaWithContext(ctx, &servicequotas.GetServiceQuotaInput{ @@ -157,7 +164,7 @@ func (e *VPCExporter) GetQuotaValue(client *servicequotas.ServiceQuotas, service return *sqOutput.Quota.Value, nil } -func (e *VPCExporter) collectVpcsPerRegionQuota(client *servicequotas.ServiceQuotas, region string) { +func (e *VPCExporter) collectVpcsPerRegionQuota(client awsclient.Client, region string) { quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_VPCS_PER_REGION) if err != nil { level.Error(e.logger).Log("msg", "Call to VpcsPerRegion ServiceQuota failed", "region", region, "err", err) @@ -167,20 +174,23 @@ func (e *VPCExporter) collectVpcsPerRegionQuota(client *servicequotas.ServiceQuo e.cache.AddMetric(prometheus.MustNewConstMetric(e.VpcsPerRegionQuota, prometheus.GaugeValue, quota, region)) } -func (e *VPCExporter) collectVpcsPerRegionUsage(ec2Svc *ec2.EC2, region string) { +func (e *VPCExporter) collectVpcsPerRegionUsage(client awsclient.Client, region string) { ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout) defer cancelFunc() - describeVpcsOutput, err := ec2Svc.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{}) + numVpcs := 0 + err := client.DescribeVpcsPagesWithContext(ctx, &ec2.DescribeVpcsInput{}, func(page *ec2.DescribeVpcsOutput, lastPage bool) bool { + numVpcs += len(page.Vpcs) + return !lastPage + }) if err != nil { level.Error(e.logger).Log("msg", "Call to DescribeVpcs failed", "region", region, "err", err) awsclient.AwsExporterMetrics.IncrementErrors() return } - usage := len(describeVpcsOutput.Vpcs) - e.cache.AddMetric(prometheus.MustNewConstMetric(e.VpcsPerRegionUsage, prometheus.GaugeValue, float64(usage), region)) + e.cache.AddMetric(prometheus.MustNewConstMetric(e.VpcsPerRegionUsage, prometheus.GaugeValue, float64(numVpcs), region)) } -func (e *VPCExporter) collectSubnetsPerVpcQuota(client *servicequotas.ServiceQuotas, region string) { +func (e *VPCExporter) collectSubnetsPerVpcQuota(client awsclient.Client, region string) { quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_SUBNETS_PER_VPC) if err != nil { level.Error(e.logger).Log("msg", "Call to SubnetsPerVpc ServiceQuota failed", "region", region, "err", err) @@ -190,25 +200,27 @@ func (e *VPCExporter) collectSubnetsPerVpcQuota(client *servicequotas.ServiceQuo e.cache.AddMetric(prometheus.MustNewConstMetric(e.SubnetsPerVpcQuota, prometheus.GaugeValue, quota, region)) } -func (e *VPCExporter) collectSubnetsPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.EC2, region string) { +func (e *VPCExporter) collectSubnetsPerVpcUsage(vpc *ec2.Vpc, client awsclient.Client, region string) { ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout) defer cancelFunc() - describeSubnetsOutput, err := ec2Svc.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{ - Filters: []*ec2.Filter{&ec2.Filter{ + numSubnets := 0 + err := client.DescribeSubnetsPagesWithContext(ctx, &ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{{ Name: aws.String("vpc-id"), Values: []*string{vpc.VpcId}, - }}, + }}}, func(page *ec2.DescribeSubnetsOutput, lastPage bool) bool { + numSubnets += len(page.Subnets) + return !lastPage }) if err != nil { level.Error(e.logger).Log("msg", "Call to DescribeSubnets failed", "region", region, "err", err) awsclient.AwsExporterMetrics.IncrementErrors() return } - usage := len(describeSubnetsOutput.Subnets) - e.cache.AddMetric(prometheus.MustNewConstMetric(e.SubnetsPerVpcUsage, prometheus.GaugeValue, float64(usage), region, *vpc.VpcId)) + e.cache.AddMetric(prometheus.MustNewConstMetric(e.SubnetsPerVpcUsage, prometheus.GaugeValue, float64(numSubnets), region, *vpc.VpcId)) } -func (e *VPCExporter) collectRoutesPerRouteTableQuota(client *servicequotas.ServiceQuotas, region string) { +func (e *VPCExporter) collectRoutesPerRouteTableQuota(client awsclient.Client, region string) { quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_ROUTES_PER_ROUTE_TABLE) if err != nil { level.Error(e.logger).Log("msg", "Call to RoutesPerRouteTable ServiceQuota failed", "region", region, "err", err) @@ -218,12 +230,16 @@ func (e *VPCExporter) collectRoutesPerRouteTableQuota(client *servicequotas.Serv e.cache.AddMetric(prometheus.MustNewConstMetric(e.RoutesPerRouteTableQuota, prometheus.GaugeValue, quota, region)) } -func (e *VPCExporter) collectRoutesPerRouteTableUsage(rtb *ec2.RouteTable, ec2Svc *ec2.EC2, region string) { +func (e *VPCExporter) collectRoutesPerRouteTableUsage(rtb *ec2.RouteTable, client awsclient.Client, region string) { ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout) defer cancelFunc() - descRouteTableOutput, err := ec2Svc.DescribeRouteTablesWithContext(ctx, &ec2.DescribeRouteTablesInput{ + descRouteTableOutput, err := client.DescribeRouteTablesWithContext(ctx, &ec2.DescribeRouteTablesInput{ RouteTableIds: []*string{rtb.RouteTableId}, }) + if len(descRouteTableOutput.RouteTables) != 1 { + level.Error(e.logger).Log("msg", "Unexpected number of routetables (!= 1) returned from DescribeRouteTables") + return + } if err != nil { level.Error(e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", region, "err", err) awsclient.AwsExporterMetrics.IncrementErrors() @@ -233,7 +249,7 @@ func (e *VPCExporter) collectRoutesPerRouteTableUsage(rtb *ec2.RouteTable, ec2Sv e.cache.AddMetric(prometheus.MustNewConstMetric(e.RoutesPerRouteTableUsage, prometheus.GaugeValue, float64(quota), region, *rtb.VpcId, *rtb.RouteTableId)) } -func (e *VPCExporter) collectInterfaceVpcEndpointsPerVpcQuota(client *servicequotas.ServiceQuotas, region string) { +func (e *VPCExporter) collectInterfaceVpcEndpointsPerVpcQuota(client awsclient.Client, region string) { quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_INTERFACE_VPC_ENDPOINTS_PER_VPC) if err != nil { level.Error(e.logger).Log("msg", "Call to InterfaceVpcEndpointsPerVpc ServiceQuota failed", "region", region, "err", err) @@ -243,25 +259,27 @@ func (e *VPCExporter) collectInterfaceVpcEndpointsPerVpcQuota(client *servicequo e.cache.AddMetric(prometheus.MustNewConstMetric(e.InterfaceVpcEndpointsPerVpcQuota, prometheus.GaugeValue, quota, region)) } -func (e *VPCExporter) collectInterfaceVpcEndpointsPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.EC2, region string) { +func (e *VPCExporter) collectInterfaceVpcEndpointsPerVpcUsage(vpc *ec2.Vpc, client awsclient.Client, region string) { ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout) defer cancelFunc() - descVpcEndpoints, err := ec2Svc.DescribeVpcEndpointsWithContext(ctx, &ec2.DescribeVpcEndpointsInput{ - Filters: []*ec2.Filter{{ - Name: aws.String("vpc-id"), - Values: []*string{vpc.VpcId}, - }}, + + numEndpoints := 0 + descEndpointsInput := &ec2.DescribeVpcEndpointsInput{ + Filters: []*ec2.Filter{{Name: aws.String("vpc-id"), Values: []*string{vpc.VpcId}}}, + } + err := client.DescribeVpcEndpointsPagesWithContext(ctx, descEndpointsInput, func(page *ec2.DescribeVpcEndpointsOutput, lastPage bool) bool { + numEndpoints += len(page.VpcEndpoints) + return !lastPage }) if err != nil { level.Error(e.logger).Log("msg", "Call to DescribeVpcEndpoints failed", "region", region, "err", err) awsclient.AwsExporterMetrics.IncrementErrors() return } - quota := len(descVpcEndpoints.VpcEndpoints) - e.cache.AddMetric(prometheus.MustNewConstMetric(e.InterfaceVpcEndpointsPerVpcUsage, prometheus.GaugeValue, float64(quota), region, *vpc.VpcId)) + e.cache.AddMetric(prometheus.MustNewConstMetric(e.InterfaceVpcEndpointsPerVpcUsage, prometheus.GaugeValue, float64(numEndpoints), region, *vpc.VpcId)) } -func (e *VPCExporter) collectRoutesTablesPerVpcQuota(client *servicequotas.ServiceQuotas, region string) { +func (e *VPCExporter) collectRoutesTablesPerVpcQuota(client awsclient.Client, region string) { quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_ROUTE_TABLES_PER_VPC) if err != nil { level.Error(e.logger).Log("msg", "Call to RoutesTablesPerVpc ServiceQuota failed", "region", region, "err", err) @@ -271,25 +289,28 @@ func (e *VPCExporter) collectRoutesTablesPerVpcQuota(client *servicequotas.Servi e.cache.AddMetric(prometheus.MustNewConstMetric(e.RouteTablesPerVpcQuota, prometheus.GaugeValue, quota, region)) } -func (e *VPCExporter) collectRoutesTablesPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.EC2, region string) { +func (e *VPCExporter) collectRoutesTablesPerVpcUsage(vpc *ec2.Vpc, client awsclient.Client, region string) { ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout) defer cancelFunc() - descRouteTables, err := ec2Svc.DescribeRouteTablesWithContext(ctx, &ec2.DescribeRouteTablesInput{ + var numRouteTables int + input := &ec2.DescribeRouteTablesInput{ Filters: []*ec2.Filter{{ Name: aws.String("vpc-id"), Values: []*string{vpc.VpcId}, - }}, + }}} + err := client.DescribeRouteTablesPagesWithContext(ctx, input, func(page *ec2.DescribeRouteTablesOutput, lastPage bool) bool { + numRouteTables += len(page.RouteTables) + return !lastPage }) if err != nil { level.Error(e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", region, "err", err) awsclient.AwsExporterMetrics.IncrementErrors() return } - quota := len(descRouteTables.RouteTables) - e.cache.AddMetric(prometheus.MustNewConstMetric(e.RouteTablesPerVpcUsage, prometheus.GaugeValue, float64(quota), region, *vpc.VpcId)) + e.cache.AddMetric(prometheus.MustNewConstMetric(e.RouteTablesPerVpcUsage, prometheus.GaugeValue, float64(numRouteTables), region, *vpc.VpcId)) } -func (e *VPCExporter) collectIPv4BlocksPerVpcQuota(client *servicequotas.ServiceQuotas, region string) { +func (e *VPCExporter) collectIPv4BlocksPerVpcQuota(client awsclient.Client, region string) { quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_IPV4_BLOCKS_PER_VPC) if err != nil { level.Error(e.logger).Log("msg", "Call to IPv4BlocksPerVpc ServiceQuota failed", "region", region, "err", err) @@ -299,10 +320,10 @@ func (e *VPCExporter) collectIPv4BlocksPerVpcQuota(client *servicequotas.Service e.cache.AddMetric(prometheus.MustNewConstMetric(e.IPv4BlocksPerVpcQuota, prometheus.GaugeValue, quota, region)) } -func (e *VPCExporter) collectIPv4BlocksPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.EC2, region string) { +func (e *VPCExporter) collectIPv4BlocksPerVpcUsage(vpc *ec2.Vpc, client awsclient.Client, region string) { ctx, cancelFunc := context.WithTimeout(context.Background(), e.timeout) defer cancelFunc() - descVpcs, err := ec2Svc.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{ + descVpcs, err := client.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{ VpcIds: []*string{vpc.VpcId}, }) if err != nil { diff --git a/pkg/vpc_test.go b/pkg/vpc_test.go new file mode 100644 index 0000000..48d9899 --- /dev/null +++ b/pkg/vpc_test.go @@ -0,0 +1,88 @@ +package pkg + +import ( + "testing" + "time" + + "github.com/app-sre/aws-resource-exporter/pkg/awsclient/mock" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/go-kit/kit/log" + "github.com/golang/mock/gomock" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" +) + +var timeout time.Duration = 10 * time.Second + +func Test_collectIPv4BlocksPerVpcUsage(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mock := mock.NewMockClient(ctrl) + vpc := ec2.Vpc{ + VpcId: aws.String("1"), + CidrBlockAssociationSet: []*ec2.VpcCidrBlockAssociation{ + {}, + }, + } + + mock.EXPECT().DescribeVpcsWithContext(gomock.Any(), &ec2.DescribeVpcsInput{ + VpcIds: []*string{vpc.VpcId}, + }).Return(&ec2.DescribeVpcsOutput{ + Vpcs: []*ec2.Vpc{&vpc}, + }, nil) + + e := NewVPCExporter([]*session.Session{}, log.NewNopLogger(), VPCConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + Interval: &timeout, + Timeout: &timeout, + CacheTTL: &timeout, + }, + Regions: []string{"any"}, + }, "1") + e.collectIPv4BlocksPerVpcUsage(&vpc, mock, "any") + metrics := e.cache.GetAllMetrics() + assert.Len(t, metrics, 1) + + var dto dto.Metric + metrics[0].Write(&dto) + assert.Equal(t, float64(1), *dto.Gauge.Value) +} + +func Test_collectRoutesPerTablePerVpcUsage(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mock := mock.NewMockClient(ctrl) + rtb := ec2.RouteTable{ + VpcId: aws.String("1"), + RouteTableId: aws.String("1"), + Routes: []*ec2.Route{ + {}, + }, + } + + mock.EXPECT().DescribeRouteTablesWithContext(gomock.Any(), &ec2.DescribeRouteTablesInput{ + RouteTableIds: []*string{rtb.RouteTableId}, + }).Return(&ec2.DescribeRouteTablesOutput{ + RouteTables: []*ec2.RouteTable{&rtb}, + }, nil) + + e := NewVPCExporter([]*session.Session{}, log.NewNopLogger(), VPCConfig{ + BaseConfig: BaseConfig{ + Enabled: true, + Interval: &timeout, + Timeout: &timeout, + CacheTTL: &timeout, + }, + Regions: []string{"any"}, + }, "1") + e.collectRoutesPerRouteTableUsage(&rtb, mock, "any") + metrics := e.cache.GetAllMetrics() + assert.Len(t, metrics, 1) + + var dto dto.Metric + metrics[0].Write(&dto) + assert.Equal(t, float64(1), *dto.Gauge.Value) +}