diff --git a/go.mod b/go.mod index 60949fc9..dea8ebbc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.14 require ( github.com/AlecAivazis/survey/v2 v2.0.8 - github.com/aws/aws-sdk-go v1.33.7 + github.com/aws/aws-sdk-go v1.33.9 github.com/blang/semver v3.5.1+incompatible github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect github.com/chanzuckerberg/go-misc v0.0.0-20200716151046-c2bf84000e9f @@ -28,6 +28,7 @@ require ( github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spf13/cobra v1.0.0 github.com/stretchr/testify v1.6.1 + golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 // indirect golang.org/x/text v0.3.3 // indirect gopkg.in/ini.v1 v1.57.0 ) diff --git a/go.sum b/go.sum index f49860be..56951f55 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-lambda-go v1.17.0/go.mod h1:FEwgPLE6+8wcGBTe5cJN3JWurd1Ztm9zN4jsXsjzKKw= github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.33.7 h1:vOozL5hmWHHriRviVTQnUwz8l05RS0rehmEFymI+/x8= -github.com/aws/aws-sdk-go v1.33.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.33.9 h1:nkC8YxL1nxwshIoO3UM2486Ph+zs7IZWjhRHjmXeCPw= +github.com/aws/aws-sdk-go v1.33.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -489,6 +489,8 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 h1:gVCS+QOncANNPlmlO1AhlU3oxs4V9z+gTtPwIk3p2N8= +golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/pkg/aws_config_server/list_roles.go b/pkg/aws_config_server/list_roles.go index 36b45fb7..99884a14 100644 --- a/pkg/aws_config_server/list_roles.go +++ b/pkg/aws_config_server/list_roles.go @@ -137,7 +137,7 @@ func listRoles(ctx context.Context, svc iamiface.IAMAPI, configParams *AWSConfig return output, errors.Wrap(err, "Error listing IAM roles") } - return filterRoles(ctx, svc, output, configParams) + return output, err } type Action []string diff --git a/pkg/aws_config_server/parallelization_functions.go b/pkg/aws_config_server/parallelization_functions.go index cd2b510a..57bc1ac2 100644 --- a/pkg/aws_config_server/parallelization_functions.go +++ b/pkg/aws_config_server/parallelization_functions.go @@ -8,7 +8,6 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/service/iam" - "github.com/aws/aws-sdk-go/service/iam/iamiface" "github.com/chanzuckerberg/aws-oidc/pkg/okta" "github.com/hashicorp/go-multierror" "github.com/honeycombio/beeline-go" @@ -81,14 +80,20 @@ func (a *ClientIDToAWSRoles) populateMapping( return errors.Wrap(err, "Unable to parallelize mapping generation process") } + // For each of those roles in the mappingsList, filter them out using filterRoles for _, mapping := range mappingsList { for clientID, configList := range mapping { - for _, config := range configList { + filteredConfigs, err := a.awsTagFilter(ctx, configList, configParams.RolesConcurrency) + if err != nil { + return errors.Wrapf(err, "Unable to filter these configs: %v", filteredConfigs) + } + + for _, config := range filteredConfigs { if _, ok := a.clientRoleMapping[clientID]; !ok { - a.clientRoleMapping[clientID] = []ConfigProfile{config} + a.clientRoleMapping[clientID] = []ConfigProfile{*config} continue } - a.clientRoleMapping[clientID] = append(a.clientRoleMapping[clientID], config) + a.clientRoleMapping[clientID] = append(a.clientRoleMapping[clientID], *config) } } } @@ -97,23 +102,30 @@ func (a *ClientIDToAWSRoles) populateMapping( } // We can skip over roles with specific tags -func filterRoles( +func (a *ClientIDToAWSRoles) awsTagFilter( ctx context.Context, - svc iamiface.IAMAPI, - roles []*iam.Role, - configParams *AWSConfigGenerationParams) ([]*iam.Role, error) { + configList []ConfigProfile, + rolesConcurrency int) ([]*ConfigProfile, error) { - ctx, span := beeline.StartSpan(ctx, "filtering AWS roles") + ctx, span := beeline.StartSpan(ctx, "filtering AWS Configs") defer span.Send() - if configParams.RolesConcurrency == 0 { - return nil, errors.Errorf("Set configParams.RolesConcurrency to a value > 0") + // Despite filtering AWS Config objects, we implemented concurrency for _role-based_actions_ + if rolesConcurrency == 0 { + return nil, errors.Errorf("Set rolesConcurrency to a value > 0") } - filterRolesFunc := func(ctx context.Context, role *iam.Role) (*iam.Role, error) { - tags, err := listRoleTags(ctx, svc, role.RoleName) + filterConfigsFunc := func(ctx context.Context, config ConfigProfile) (*ConfigProfile, error) { + workerAWSConfig := &aws.Config{ + Credentials: stscreds.NewCredentials(a.awsSession, config.RoleARN.String()), + CredentialsChainVerboseErrors: aws.Bool(true), + Retryer: a.awsSession.Config.Retryer, + } + svc := a.awsClient.WithIAM(workerAWSConfig).IAM.Svc + + tags, err := listRoleTags(ctx, svc, &config.RoleName) if err != nil { - return nil, errors.Wrapf(err, "error listing tags for %s", *role.RoleName) + return nil, errors.Wrapf(err, "error listing tags for %s", config.RoleName) } if shouldSkipTags(tags) { @@ -121,18 +133,17 @@ func filterRoles( } // After all of this... return the role if it fulfills all requirements - return role, nil + return &config, nil } - return parallelizeFilterRoles(ctx, configParams.RolesConcurrency, roles, filterRolesFunc) + return parallelizeFilterConfigs(ctx, rolesConcurrency, configList, filterConfigsFunc) } -func parallelizeFilterRoles(ctx context.Context, +func parallelizeFilterConfigs(ctx context.Context, concurrencyLimit int, - queue []*iam.Role, - action func(context.Context, *iam.Role) (*iam.Role, error), -) ([]*iam.Role, error) { - logrus.Debug("in parallelize function") + queue []ConfigProfile, + action func(context.Context, ConfigProfile) (*ConfigProfile, error)) ([]*ConfigProfile, error) { + logrus.Debug("start of parallelize function") // wg to track goroutines wg := sync.WaitGroup{} @@ -140,14 +151,14 @@ func parallelizeFilterRoles(ctx context.Context, errs := make(chan error, len(queue)) // how much concurrent work we're allowed to do - scheduledQueue := make(chan *iam.Role, concurrencyLimit) + scheduledQueue := make(chan *ConfigProfile, concurrencyLimit) // outputRoles to aggregate all our roles - outputChannel := make(chan *iam.Role, len(queue)) + outputChannel := make(chan *ConfigProfile, len(queue)) logrus.Debug("made all the channels") // the goroutine that will process one element at a time - processor := func(scheduledQueue <-chan *iam.Role, outputList chan<- *iam.Role) { + processor := func(scheduledQueue <-chan *ConfigProfile, outputList chan<- *ConfigProfile) { defer wg.Done() for element := range scheduledQueue { @@ -155,7 +166,7 @@ func parallelizeFilterRoles(ctx context.Context, continue } - output, err := action(ctx, element) + output, err := action(ctx, *element) if err != nil { errs <- err continue @@ -175,7 +186,7 @@ func parallelizeFilterRoles(ctx context.Context, // // schedule all the work for _, element := range queue { - scheduledQueue <- element + scheduledQueue <- &element } close(scheduledQueue) // signal processors they can stop @@ -193,7 +204,7 @@ func parallelizeFilterRoles(ctx context.Context, // small lesson: don't set a length for the outputList! // Or else we'll get nil pointers in the output, // which will cause segmentation violations - outputList := []*iam.Role{} + outputList := []*ConfigProfile{} for element := range outputChannel { outputList = append(outputList, element) }