added listing role-tags as a second step

go.mod

@@ -4,7 +4,7 @@ go 1.14
 
 require (
 	github.com/AlecAivazis/survey/v2 v2.0.8
-	github.com/aws/aws-sdk-go v1.33.7
+	github.com/aws/aws-sdk-go v1.33.9
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/chanzuckerberg/go-misc v0.0.0-20200716151046-c2bf84000e9f
@@ -28,6 +28,7 @@ require (
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/stretchr/testify v1.6.1
+	golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 // indirect
 	golang.org/x/text v0.3.3 // indirect
 	gopkg.in/ini.v1 v1.57.0
 )

go.sum

@@ -15,8 +15,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-lambda-go v1.17.0/go.mod h1:FEwgPLE6+8wcGBTe5cJN3JWurd1Ztm9zN4jsXsjzKKw=
 github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.33.7 h1:vOozL5hmWHHriRviVTQnUwz8l05RS0rehmEFymI+/x8=
-github.com/aws/aws-sdk-go v1.33.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.33.9 h1:nkC8YxL1nxwshIoO3UM2486Ph+zs7IZWjhRHjmXeCPw=
+github.com/aws/aws-sdk-go v1.33.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
@@ -489,6 +489,8 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 h1:gVCS+QOncANNPlmlO1AhlU3oxs4V9z+gTtPwIk3p2N8=
+golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

pkg/aws_config_server/list_roles.go

@@ -137,7 +137,7 @@ func listRoles(ctx context.Context, svc iamiface.IAMAPI, configParams *AWSConfig
 		return output, errors.Wrap(err, "Error listing IAM roles")
 	}
 
-	return filterRoles(ctx, svc, output, configParams)
+	return output, err
 }
 
 type Action []string

pkg/aws_config_server/parallelization_functions.go

@@ -8,7 +8,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/service/iam"
-	"github.com/aws/aws-sdk-go/service/iam/iamiface"
 	"github.com/chanzuckerberg/aws-oidc/pkg/okta"
 	"github.com/hashicorp/go-multierror"
 	"github.com/honeycombio/beeline-go"
@@ -81,14 +80,20 @@ func (a *ClientIDToAWSRoles) populateMapping(
 		return errors.Wrap(err, "Unable to parallelize mapping generation process")
 	}
 
+	// For each of those roles in the mappingsList, filter them out using filterRoles
 	for _, mapping := range mappingsList {
 		for clientID, configList := range mapping {
-			for _, config := range configList {
+			filteredConfigs, err := a.awsTagFilter(ctx, configList, configParams.RolesConcurrency)
+			if err != nil {
+				return errors.Wrapf(err, "Unable to filter these configs: %v", filteredConfigs)
+			}
+
+			for _, config := range filteredConfigs {
 				if _, ok := a.clientRoleMapping[clientID]; !ok {
-					a.clientRoleMapping[clientID] = []ConfigProfile{config}
+					a.clientRoleMapping[clientID] = []ConfigProfile{*config}
 					continue
 				}
-				a.clientRoleMapping[clientID] = append(a.clientRoleMapping[clientID], config)
+				a.clientRoleMapping[clientID] = append(a.clientRoleMapping[clientID], *config)
 			}
 		}
 	}
@@ -97,65 +102,71 @@ func (a *ClientIDToAWSRoles) populateMapping(
 }
 
 // We can skip over roles with specific tags
-func filterRoles(
+func (a *ClientIDToAWSRoles) awsTagFilter(
 	ctx context.Context,
-	svc iamiface.IAMAPI,
-	roles []*iam.Role,
-	configParams *AWSConfigGenerationParams) ([]*iam.Role, error) {
+	configList []ConfigProfile,
+	rolesConcurrency int) ([]*ConfigProfile, error) {
 
-	ctx, span := beeline.StartSpan(ctx, "filtering AWS roles")
+	ctx, span := beeline.StartSpan(ctx, "filtering AWS Configs")
 	defer span.Send()
 
-	if configParams.RolesConcurrency == 0 {
-		return nil, errors.Errorf("Set configParams.RolesConcurrency to a value > 0")
+	// Despite filtering AWS Config objects, we implemented concurrency for _role-based_actions_
+	if rolesConcurrency == 0 {
+		return nil, errors.Errorf("Set rolesConcurrency to a value > 0")
 	}
 
-	filterRolesFunc := func(ctx context.Context, role *iam.Role) (*iam.Role, error) {
-		tags, err := listRoleTags(ctx, svc, role.RoleName)
+	filterConfigsFunc := func(ctx context.Context, config ConfigProfile) (*ConfigProfile, error) {
+		workerAWSConfig := &aws.Config{
+			Credentials:                   stscreds.NewCredentials(a.awsSession, config.RoleARN.String()),
+			CredentialsChainVerboseErrors: aws.Bool(true),
+			Retryer:                       a.awsSession.Config.Retryer,
+		}
+		svc := a.awsClient.WithIAM(workerAWSConfig).IAM.Svc
+
+		tags, err := listRoleTags(ctx, svc, &config.RoleName)
 		if err != nil {
-			return nil, errors.Wrapf(err, "error listing tags for %s", *role.RoleName)
+			return nil, errors.Wrapf(err, "error listing tags for %s", config.RoleName)
 		}
 
 		if shouldSkipTags(tags) {
 			return nil, nil
 		}
 
 		// After all of this... return the role if it fulfills all requirements
-		return role, nil
+		return &config, nil
 	}
 
-	return parallelizeFilterRoles(ctx, configParams.RolesConcurrency, roles, filterRolesFunc)
+	return parallelizeFilterConfigs(ctx, rolesConcurrency, configList, filterConfigsFunc)
 }
 
-func parallelizeFilterRoles(ctx context.Context,
+func parallelizeFilterConfigs(ctx context.Context,
 	concurrencyLimit int,
-	queue []*iam.Role,
-	action func(context.Context, *iam.Role) (*iam.Role, error),
-) ([]*iam.Role, error) {
-	logrus.Debug("in parallelize function")
+	queue []ConfigProfile,
+	action func(context.Context, ConfigProfile) (*ConfigProfile, error)) ([]*ConfigProfile, error) {
+	logrus.Debug("start of parallelize function")
 
 	// wg to track goroutines
 	wg := sync.WaitGroup{}
 	// to aggregate errors, make it buffered so we don't block on errors
 	errs := make(chan error, len(queue))
 
 	// how much concurrent work we're allowed to do
-	scheduledQueue := make(chan *iam.Role, concurrencyLimit)
+	scheduledQueue := make(chan *ConfigProfile, concurrencyLimit)
 
 	// outputRoles to aggregate all our roles
-	outputChannel := make(chan *iam.Role, len(queue))
+	outputChannel := make(chan *ConfigProfile, len(queue))
 	logrus.Debug("made all the channels")
 
 	// the goroutine that will process one element at a time
-	processor := func(scheduledQueue <-chan *iam.Role, outputList chan<- *iam.Role) {
+	processor := func(scheduledQueue <-chan *ConfigProfile, outputList chan<- *ConfigProfile) {
 		defer wg.Done()
 
 		for element := range scheduledQueue {
 			if element == nil {
 				continue
 			}
 
-			output, err := action(ctx, element)
+			output, err := action(ctx, *element)
 			if err != nil {
 				errs <- err
 				continue
@@ -175,7 +186,7 @@ func parallelizeFilterRoles(ctx context.Context,
 
 	// // schedule all the work
 	for _, element := range queue {
-		scheduledQueue <- element
+		scheduledQueue <- &element
 	}
 	close(scheduledQueue) // signal processors they can stop
 
@@ -193,7 +204,7 @@ func parallelizeFilterRoles(ctx context.Context,
 	// small lesson: don't set a length for the outputList!
 	// Or else we'll get nil pointers in the output,
 	// 	which will cause segmentation violations
-	outputList := []*iam.Role{}
+	outputList := []*ConfigProfile{}
 	for element := range outputChannel {
 		outputList = append(outputList, element)
 	}