From deca23ab6b8a4deab4caaf2fcec588375c319c68 Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Sun, 16 Jun 2024 17:16:48 +0200 Subject: [PATCH 01/10] Refactor to handle changes implemented by https://github.com/nais/api/pull/43 --- internal/reconcilers/manager.go | 65 ++++++++++++++---------------- internal/reconcilers/queue.go | 13 +++--- internal/reconcilers/queue_test.go | 2 +- 3 files changed, 38 insertions(+), 42 deletions(-) diff --git a/internal/reconcilers/manager.go b/internal/reconcilers/manager.go index aff6119..7995bdb 100644 --- a/internal/reconcilers/manager.go +++ b/internal/reconcilers/manager.go @@ -23,11 +23,10 @@ import ( type ctxKey int const ( - ctxCorrelationID ctxKey = iota + reconcilerTimeout = time.Minute * 15 + ctxCorrelationID ctxKey = iota ) -const reconcilerTimeout = time.Minute * 15 - type Manager struct { apiclient *apiclient.APIClient reconcilers []Reconciler @@ -35,7 +34,7 @@ type Manager struct { reconcilersToEnable []string log logrus.FieldLogger pubsubSubscription *pubsub.Subscription - syncQueueChan <-chan Input + syncQueueChan <-chan ReconcileRequest syncQueue Queue inFlight InFlight @@ -126,7 +125,7 @@ func (m *Manager) ListenForEvents(ctx context.Context) { switch event { case protoapi.EventTypes_EVENT_TEAM_DELETED, protoapi.EventTypes_EVENT_TEAM_UPDATED: - input := Input{ + input := ReconcileRequest{ CorrelationID: correlationID.String(), } @@ -136,7 +135,6 @@ func (m *Manager) ListenForEvents(ctx context.Context) { } if event == protoapi.EventTypes_EVENT_TEAM_DELETED { obj = &protoapi.EventTeamDeleted{} - input.Delete = true } else { obj = &protoapi.EventTeamUpdated{} } @@ -218,37 +216,35 @@ func (m *Manager) Close() { // syncTeam will mark the team as "in flight" and start the reconciliation process. If the team is already in flight, it // will be added to the back of the queue. Based on the input, the team will either be deleted or reconciled. -func (m *Manager) syncTeam(ctx context.Context, input Input) { - log := m.log.WithField("team", input.TeamSlug) +func (m *Manager) syncTeam(ctx context.Context, req ReconcileRequest) { + log := m.log.WithField("team", req.TeamSlug) - if !m.inFlight.Set(input.TeamSlug) { + if !m.inFlight.Set(req.TeamSlug) { log.Info("already in flight - adding to back of queue") time.Sleep(10 * time.Second) - if err := m.syncQueue.Add(input); err != nil { + if err := m.syncQueue.Add(req); err != nil { log.WithError(err).Error("failed while re-queueing team that is in flight") } return } - defer m.inFlight.Remove(input.TeamSlug) + defer m.inFlight.Remove(req.TeamSlug) - resp, err := m.apiclient.Teams().Get(ctx, &protoapi.GetTeamRequest{Slug: input.TeamSlug}) + resp, err := m.apiclient.Teams().Get(ctx, &protoapi.GetTeamRequest{Slug: req.TeamSlug}) if err != nil { if status.Code(err) == codes.NotFound { log.Info("team not found, team will not be requeued") return } - log.WithError(err).Error("error while getting team") - if err := m.syncQueue.Add(input); err != nil { - log.WithError(err).Error("failed while re-queueing team that is in flight") + log.WithError(err).Error("error while getting team, requeuing") + if err := m.syncQueue.Add(req); err != nil { + log.WithError(err).Error("failed while requeuing team") } return } - team := resp.Team - ctx, cancel := context.WithTimeout(ctx, reconcilerTimeout) defer cancel() @@ -258,10 +254,11 @@ func (m *Manager) syncTeam(ctx context.Context, input Input) { return } - if input.Delete { - m.deleteTeam(ctx, reconcilers, team, input) - } else { - m.reconcileTeam(ctx, reconcilers, team, input) + team := resp.Team + if team.CanBeDeleted { + m.deleteTeam(ctx, reconcilers, team, req) + } else if !team.IsDeleted() { + m.reconcileTeam(ctx, reconcilers, team, req) } } @@ -286,17 +283,17 @@ func (m *Manager) enabledReconcilers(ctx context.Context) ([]Reconciler, error) // deleteTeam will pass the team through to all enabled reconcilers, effectively deleting the team from all configured // external systems. -func (m *Manager) deleteTeam(ctx context.Context, reconcilers []Reconciler, naisTeam *protoapi.Team, input Input) { +func (m *Manager) deleteTeam(ctx context.Context, reconcilers []Reconciler, naisTeam *protoapi.Team, req ReconcileRequest) { teamStart := time.Now() - log := m.log.WithField("team", input.TeamSlug) + log := m.log.WithField("team", req.TeamSlug) - if input.CorrelationID != "" { - log = log.WithField("correlation_id", input.CorrelationID) - ctx = context.WithValue(ctx, ctxCorrelationID, input.CorrelationID) + if req.CorrelationID != "" { + log = log.WithField("correlation_id", req.CorrelationID) + ctx = context.WithValue(ctx, ctxCorrelationID, req.CorrelationID) } - if input.TraceID != "" { - log = log.WithField("trace_id", input.TraceID) + if req.TraceID != "" { + log = log.WithField("trace_id", req.TraceID) } log.WithField("time", teamStart).Debugf("start team deletion process") @@ -318,10 +315,10 @@ func (m *Manager) deleteTeam(ctx context.Context, reconcilers []Reconciler, nais log.WithError(err).Errorf("error during team deletion") req := &protoapi.SetReconcilerErrorForTeamRequest{ - CorrelationId: input.CorrelationID, + CorrelationId: req.CorrelationID, ReconcilerName: r.Name(), ErrorMessage: err.Error(), - TeamSlug: input.TeamSlug, + TeamSlug: req.TeamSlug, } if _, err := m.apiclient.Reconcilers().SetReconcilerErrorForTeam(ctx, req); err != nil { log.WithError(err).Errorf("error while adding deletion error") @@ -329,7 +326,7 @@ func (m *Manager) deleteTeam(ctx context.Context, reconcilers []Reconciler, nais } else { req := &protoapi.RemoveReconcilerErrorForTeamRequest{ ReconcilerName: r.Name(), - TeamSlug: input.TeamSlug, + TeamSlug: req.TeamSlug, } if _, err := m.apiclient.Reconcilers().RemoveReconcilerErrorForTeam(ctx, req); err != nil { log.WithError(err).Errorf("error while removing deletion error") @@ -354,7 +351,7 @@ func (m *Manager) deleteTeam(ctx context.Context, reconcilers []Reconciler, nais if successfulDelete { req := &protoapi.DeleteTeamRequest{ - Slug: input.TeamSlug, + Slug: req.TeamSlug, } if _, err := m.apiclient.Teams().Delete(ctx, req); err != nil { log.WithError(err).Errorf("error while deleting team") @@ -377,7 +374,7 @@ func (m *Manager) deleteTeam(ctx context.Context, reconcilers []Reconciler, nais // reconcileTeam will pass the team through to all enabled reconcilers, effectively synchronizing the team to all // configured external systems. -func (m *Manager) reconcileTeam(ctx context.Context, reconcilers []Reconciler, naisTeam *protoapi.Team, input Input) { +func (m *Manager) reconcileTeam(ctx context.Context, reconcilers []Reconciler, naisTeam *protoapi.Team, input ReconcileRequest) { teamStart := time.Now() log := m.log.WithField("team", input.TeamSlug) @@ -473,7 +470,7 @@ func (m *Manager) scheduleAllTeams(ctx context.Context, correlationID uuid.UUID) m.log.WithField("num_teams", len(teams)).Debugf("fetched teams from API") for _, team := range teams { - err := m.syncQueue.Add(Input{ + err := m.syncQueue.Add(ReconcileRequest{ CorrelationID: correlationID.String(), TeamSlug: team.Slug, }) diff --git a/internal/reconcilers/queue.go b/internal/reconcilers/queue.go index 5772c61..122f005 100644 --- a/internal/reconcilers/queue.go +++ b/internal/reconcilers/queue.go @@ -7,33 +7,32 @@ import ( const reconcilerQueueSize = 4096 -type Input struct { +type ReconcileRequest struct { CorrelationID string TraceID string TeamSlug string - Delete bool } type Queue interface { - Add(Input) error + Add(ReconcileRequest) error Close() } type queue struct { - queue chan Input + queue chan ReconcileRequest closed bool lock sync.Mutex } -func NewQueue() (Queue, <-chan Input) { - ch := make(chan Input, reconcilerQueueSize) +func NewQueue() (Queue, <-chan ReconcileRequest) { + ch := make(chan ReconcileRequest, reconcilerQueueSize) return &queue{ queue: ch, closed: false, }, ch } -func (q *queue) Add(input Input) error { +func (q *queue) Add(input ReconcileRequest) error { q.lock.Lock() defer q.lock.Unlock() diff --git a/internal/reconcilers/queue_test.go b/internal/reconcilers/queue_test.go index dba0401..ad2e79b 100644 --- a/internal/reconcilers/queue_test.go +++ b/internal/reconcilers/queue_test.go @@ -9,7 +9,7 @@ import ( ) func Test_Queue(t *testing.T) { - input := reconcilers.Input{ + input := reconcilers.ReconcileRequest{ TeamSlug: "test-team", CorrelationID: uuid.New().String(), } From 395f63abd389d7e0d85d5b213c1870cf47e3e585 Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Sun, 16 Jun 2024 17:20:23 +0200 Subject: [PATCH 02/10] Rename var --- internal/reconcilers/queue.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/reconcilers/queue.go b/internal/reconcilers/queue.go index 122f005..b6441f0 100644 --- a/internal/reconcilers/queue.go +++ b/internal/reconcilers/queue.go @@ -32,7 +32,7 @@ func NewQueue() (Queue, <-chan ReconcileRequest) { }, ch } -func (q *queue) Add(input ReconcileRequest) error { +func (q *queue) Add(req ReconcileRequest) error { q.lock.Lock() defer q.lock.Unlock() @@ -40,7 +40,7 @@ func (q *queue) Add(input ReconcileRequest) error { return fmt.Errorf("team reconciler channel is closed") } - q.queue <- input + q.queue <- req return nil } From 16d278fe05bdd6aaa97b5cc8fa354f8b3fb35eba Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Mon, 17 Jun 2024 10:23:17 +0200 Subject: [PATCH 03/10] No need to add already deleted teams to the queue --- internal/reconcilers/manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/reconcilers/manager.go b/internal/reconcilers/manager.go index 7995bdb..4b8a90a 100644 --- a/internal/reconcilers/manager.go +++ b/internal/reconcilers/manager.go @@ -470,6 +470,10 @@ func (m *Manager) scheduleAllTeams(ctx context.Context, correlationID uuid.UUID) m.log.WithField("num_teams", len(teams)).Debugf("fetched teams from API") for _, team := range teams { + if team.IsDeleted() { + continue + } + err := m.syncQueue.Add(ReconcileRequest{ CorrelationID: correlationID.String(), TeamSlug: team.Slug, From 059aaae05a769c89f93852bfdc528bf6fc05fd4b Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Thu, 20 Jun 2024 22:48:34 +0200 Subject: [PATCH 04/10] Use separate commands for fetching teams for reconcile / delete --- internal/reconcilers/manager.go | 57 ++++++++++++++++++++++++--------- internal/reconcilers/queue.go | 1 + 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/internal/reconcilers/manager.go b/internal/reconcilers/manager.go index 4b8a90a..d1923fe 100644 --- a/internal/reconcilers/manager.go +++ b/internal/reconcilers/manager.go @@ -125,15 +125,18 @@ func (m *Manager) ListenForEvents(ctx context.Context) { switch event { case protoapi.EventTypes_EVENT_TEAM_DELETED, protoapi.EventTypes_EVENT_TEAM_UPDATED: - input := ReconcileRequest{ - CorrelationID: correlationID.String(), - } var obj interface { proto.Message GetSlug() string } + + input := ReconcileRequest{ + CorrelationID: correlationID.String(), + } + if event == protoapi.EventTypes_EVENT_TEAM_DELETED { + input.Delete = true obj = &protoapi.EventTeamDeleted{} } else { obj = &protoapi.EventTeamUpdated{} @@ -145,8 +148,7 @@ func (m *Manager) ListenForEvents(ctx context.Context) { } input.TeamSlug = obj.GetSlug() - err := m.syncQueue.Add(input) - if err != nil { + if err := m.syncQueue.Add(input); err != nil { msg.Nack() m.log.WithError(err).Error("error while adding team to queue") return @@ -255,9 +257,9 @@ func (m *Manager) syncTeam(ctx context.Context, req ReconcileRequest) { } team := resp.Team - if team.CanBeDeleted { + if req.Delete { m.deleteTeam(ctx, reconcilers, team, req) - } else if !team.IsDeleted() { + } else { m.reconcileTeam(ctx, reconcilers, team, req) } } @@ -462,21 +464,35 @@ func (m *Manager) scheduleAllTeams(ctx context.Context, correlationID uuid.UUID) return nil } - teams, err := getTeams(ctx, m.apiclient.Teams()) + teams, err := getTeamsToBeReconciled(ctx, m.apiclient.Teams()) if err != nil { return err } - m.log.WithField("num_teams", len(teams)).Debugf("fetched teams from API") + m.log.WithField("num_teams", len(teams)).Debugf("fetched teams to be reconciled from API") for _, team := range teams { - if team.IsDeleted() { - continue + err := m.syncQueue.Add(ReconcileRequest{ + CorrelationID: correlationID.String(), + TeamSlug: team.Slug, + }) + if err != nil { + m.log.WithField("team", team.Slug).WithError(err).Errorf("error while adding team to queue") } + } + teams, err = getTeamsToBeDeleted(ctx, m.apiclient.Teams()) + if err != nil { + return err + } + + m.log.WithField("num_teams", len(teams)).Debugf("fetched teams to be deleted from API") + + for _, team := range teams { err := m.syncQueue.Add(ReconcileRequest{ CorrelationID: correlationID.String(), TeamSlug: team.Slug, + Delete: true, }) if err != nil { m.log.WithField("team", team.Slug).WithError(err).Errorf("error while adding team to queue") @@ -486,10 +502,21 @@ func (m *Manager) scheduleAllTeams(ctx context.Context, correlationID uuid.UUID) return nil } -// getTeams retrieves all teams from the NAIS API -func getTeams(ctx context.Context, client protoapi.TeamsClient) ([]*protoapi.Team, error) { - it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.ListTeamsResponse, error) { - return client.List(ctx, &protoapi.ListTeamsRequest{Limit: limit, Offset: offset}) +func getTeamsToBeReconciled(ctx context.Context, client protoapi.TeamsClient) ([]*protoapi.Team, error) { + it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.TeamsToBeReconciledResponse, error) { + return client.ToBeReconciled(ctx, &protoapi.TeamsToBeReconciledRequest{Limit: limit, Offset: offset}) + }) + + teams := make([]*protoapi.Team, 0) + for it.Next() { + teams = append(teams, it.Value()) + } + return teams, it.Err() +} + +func getTeamsToBeDeleted(ctx context.Context, client protoapi.TeamsClient) ([]*protoapi.Team, error) { + it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.TeamsToBeDeletedResponse, error) { + return client.ToBeDeleted(ctx, &protoapi.TeamsToBeDeletedRequest{Limit: limit, Offset: offset}) }) teams := make([]*protoapi.Team, 0) diff --git a/internal/reconcilers/queue.go b/internal/reconcilers/queue.go index b6441f0..28cb1c9 100644 --- a/internal/reconcilers/queue.go +++ b/internal/reconcilers/queue.go @@ -11,6 +11,7 @@ type ReconcileRequest struct { CorrelationID string TraceID string TeamSlug string + Delete bool } type Queue interface { From 830a61ab13631923adea7fd07f40985575bdc99e Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Wed, 26 Jun 2024 14:24:47 +0200 Subject: [PATCH 05/10] Schedule active and deletable teams --- internal/reconcilers/manager.go | 77 ++++++++++++++++----------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/internal/reconcilers/manager.go b/internal/reconcilers/manager.go index d1923fe..44eec17 100644 --- a/internal/reconcilers/manager.go +++ b/internal/reconcilers/manager.go @@ -3,6 +3,7 @@ package reconcilers import ( "context" "errors" + "fmt" "slices" "time" @@ -464,66 +465,64 @@ func (m *Manager) scheduleAllTeams(ctx context.Context, correlationID uuid.UUID) return nil } - teams, err := getTeamsToBeReconciled(ctx, m.apiclient.Teams()) - if err != nil { - return err + if err := m.scheduleActiveTeams(ctx, correlationID); err != nil { + m.log.WithError(err).Error("error while scheduling active teams for reconciliation") } - m.log.WithField("num_teams", len(teams)).Debugf("fetched teams to be reconciled from API") - - for _, team := range teams { - err := m.syncQueue.Add(ReconcileRequest{ - CorrelationID: correlationID.String(), - TeamSlug: team.Slug, - }) - if err != nil { - m.log.WithField("team", team.Slug).WithError(err).Errorf("error while adding team to queue") - } + if err := m.scheduleDeletableTeams(ctx, correlationID); err != nil { + m.log.WithError(err).Error("error while scheduling teams for deletion") } - teams, err = getTeamsToBeDeleted(ctx, m.apiclient.Teams()) - if err != nil { - return err - } + return nil +} - m.log.WithField("num_teams", len(teams)).Debugf("fetched teams to be deleted from API") +func (m *Manager) scheduleActiveTeams(ctx context.Context, correlationID uuid.UUID) error { + it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.ListActiveTeamsResponse, error) { + return m.apiclient.Teams().ListActive(ctx, &protoapi.ListActiveTeamsRequest{Limit: limit, Offset: offset}) + }) - for _, team := range teams { + num := 0 + for it.Next() { err := m.syncQueue.Add(ReconcileRequest{ CorrelationID: correlationID.String(), - TeamSlug: team.Slug, - Delete: true, + TeamSlug: it.Value().Slug, }) if err != nil { - m.log.WithField("team", team.Slug).WithError(err).Errorf("error while adding team to queue") + return fmt.Errorf("error while adding team to queue: %w", err) } + num++ + } + if err := it.Err(); err != nil { + return fmt.Errorf("error while fetching active teams for reconciliation: %w", err) } + m.log.WithField("num_teams", num).Debugf("added teams to reconcile queue") return nil } -func getTeamsToBeReconciled(ctx context.Context, client protoapi.TeamsClient) ([]*protoapi.Team, error) { - it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.TeamsToBeReconciledResponse, error) { - return client.ToBeReconciled(ctx, &protoapi.TeamsToBeReconciledRequest{Limit: limit, Offset: offset}) +func (m *Manager) scheduleDeletableTeams(ctx context.Context, correlationID uuid.UUID) error { + it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.ListDeletableTeamsResponse, error) { + return m.apiclient.Teams().ListDeletable(ctx, &protoapi.ListDeletableTeamsRequest{Limit: limit, Offset: offset}) }) - teams := make([]*protoapi.Team, 0) + num := 0 for it.Next() { - teams = append(teams, it.Value()) + err := m.syncQueue.Add(ReconcileRequest{ + CorrelationID: correlationID.String(), + TeamSlug: it.Value().Slug, + Delete: true, + }) + if err != nil { + return fmt.Errorf("error while adding team to queue: %w", err) + } + num++ } - return teams, it.Err() -} - -func getTeamsToBeDeleted(ctx context.Context, client protoapi.TeamsClient) ([]*protoapi.Team, error) { - it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.TeamsToBeDeletedResponse, error) { - return client.ToBeDeleted(ctx, &protoapi.TeamsToBeDeletedRequest{Limit: limit, Offset: offset}) - }) - - teams := make([]*protoapi.Team, 0) - for it.Next() { - teams = append(teams, it.Value()) + if err := it.Err(); err != nil { + return fmt.Errorf("error while fetching teams for deletion: %w", err) } - return teams, it.Err() + + m.log.WithField("num_teams", num).Debugf("added teams to delete queue") + return nil } // getReconcilers retrieves all reconcilers from the NAIS API From 1072e272eac608b409b12ac943be88e3ca744f73 Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Mon, 1 Jul 2024 12:02:40 +0200 Subject: [PATCH 06/10] Bump dependencies --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 332b297..5323682 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grafana/grafana-openapi-client-go v0.0.0-20240430202104-3ad0f7e4ee52 github.com/joho/godotenv v1.5.1 - github.com/nais/api v0.0.0-20240614093353-fe1f73775e6a + github.com/nais/api v0.0.0-20240701095637-9cac722281ee github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11 github.com/prometheus/client_golang v1.19.1 github.com/sethvargo/go-envconfig v1.0.3 diff --git a/go.sum b/go.sum index 445dfd2..a7ef29a 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,10 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/nais/api v0.0.0-20240614093353-fe1f73775e6a h1:tL5JOFS+gUWimotAAcH6zygtb3Ef1C6hrq2WvGk3l4c= github.com/nais/api v0.0.0-20240614093353-fe1f73775e6a/go.mod h1:X0Jp7pnYDRxKWfi46CJxyPc0f09uAaSh3d4idCTsLtY= +github.com/nais/api v0.0.0-20240628141327-c193beb23a8c h1:KVAkgA4Q/UPmK9uNBw52t26R/KRqsHC+Tdh1cTYe6uw= +github.com/nais/api v0.0.0-20240628141327-c193beb23a8c/go.mod h1:uvppNeqRtp+LElLlFS0Qh+O00OJ08ac+o/UaVMj/9cc= +github.com/nais/api v0.0.0-20240701095637-9cac722281ee h1:l3fjq9Rp/LmzkfdUtVB/PDeIyJP22trXojVaFoGlNxY= +github.com/nais/api v0.0.0-20240701095637-9cac722281ee/go.mod h1:uvppNeqRtp+LElLlFS0Qh+O00OJ08ac+o/UaVMj/9cc= github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11 h1:2Cy7cjTM7PyiaUi3+aLsRSg6T0R6AqlduPNmT9LBMSY= github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11/go.mod h1:BdPp1+tyE1hSMZigr7aGmhcTqaksf9Pxxbo4ZAi4On8= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= From 2feb3d86d454a75f6e2d7528363e694bbb6b3b16 Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Tue, 2 Jul 2024 10:07:34 +0200 Subject: [PATCH 07/10] Bump dependencies --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5323682..af8a1b0 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grafana/grafana-openapi-client-go v0.0.0-20240430202104-3ad0f7e4ee52 github.com/joho/godotenv v1.5.1 - github.com/nais/api v0.0.0-20240701095637-9cac722281ee + github.com/nais/api v0.0.0-20240702080604-ae6c0f51e69b github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11 github.com/prometheus/client_golang v1.19.1 github.com/sethvargo/go-envconfig v1.0.3 diff --git a/go.sum b/go.sum index a7ef29a..f512492 100644 --- a/go.sum +++ b/go.sum @@ -210,6 +210,10 @@ github.com/nais/api v0.0.0-20240628141327-c193beb23a8c h1:KVAkgA4Q/UPmK9uNBw52t2 github.com/nais/api v0.0.0-20240628141327-c193beb23a8c/go.mod h1:uvppNeqRtp+LElLlFS0Qh+O00OJ08ac+o/UaVMj/9cc= github.com/nais/api v0.0.0-20240701095637-9cac722281ee h1:l3fjq9Rp/LmzkfdUtVB/PDeIyJP22trXojVaFoGlNxY= github.com/nais/api v0.0.0-20240701095637-9cac722281ee/go.mod h1:uvppNeqRtp+LElLlFS0Qh+O00OJ08ac+o/UaVMj/9cc= +github.com/nais/api v0.0.0-20240701131232-caa8442f03ef h1:GTyWZvbW80QORreDPRP5bB0dUx57d46QqyCxoewiHHM= +github.com/nais/api v0.0.0-20240701131232-caa8442f03ef/go.mod h1:uvppNeqRtp+LElLlFS0Qh+O00OJ08ac+o/UaVMj/9cc= +github.com/nais/api v0.0.0-20240702080604-ae6c0f51e69b h1:eLl227vMRDmjlQ+vhY2qg9Ku6YAt6a5qzHp64ZryLoY= +github.com/nais/api v0.0.0-20240702080604-ae6c0f51e69b/go.mod h1:BBRdhHlB5TK0cAShs650TvA7PVh9LUU9V1qgxinZ4EU= github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11 h1:2Cy7cjTM7PyiaUi3+aLsRSg6T0R6AqlduPNmT9LBMSY= github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11/go.mod h1:BdPp1+tyE1hSMZigr7aGmhcTqaksf9Pxxbo4ZAi4On8= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= From 274a5b8b93e51d8177d86254a3df1f6cc8a69439 Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Tue, 2 Jul 2024 10:07:57 +0200 Subject: [PATCH 08/10] go mod tidy --- go.sum | 8 -------- 1 file changed, 8 deletions(-) diff --git a/go.sum b/go.sum index f512492..b7ac3b9 100644 --- a/go.sum +++ b/go.sum @@ -204,14 +204,6 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/nais/api v0.0.0-20240614093353-fe1f73775e6a h1:tL5JOFS+gUWimotAAcH6zygtb3Ef1C6hrq2WvGk3l4c= -github.com/nais/api v0.0.0-20240614093353-fe1f73775e6a/go.mod h1:X0Jp7pnYDRxKWfi46CJxyPc0f09uAaSh3d4idCTsLtY= -github.com/nais/api v0.0.0-20240628141327-c193beb23a8c h1:KVAkgA4Q/UPmK9uNBw52t26R/KRqsHC+Tdh1cTYe6uw= -github.com/nais/api v0.0.0-20240628141327-c193beb23a8c/go.mod h1:uvppNeqRtp+LElLlFS0Qh+O00OJ08ac+o/UaVMj/9cc= -github.com/nais/api v0.0.0-20240701095637-9cac722281ee h1:l3fjq9Rp/LmzkfdUtVB/PDeIyJP22trXojVaFoGlNxY= -github.com/nais/api v0.0.0-20240701095637-9cac722281ee/go.mod h1:uvppNeqRtp+LElLlFS0Qh+O00OJ08ac+o/UaVMj/9cc= -github.com/nais/api v0.0.0-20240701131232-caa8442f03ef h1:GTyWZvbW80QORreDPRP5bB0dUx57d46QqyCxoewiHHM= -github.com/nais/api v0.0.0-20240701131232-caa8442f03ef/go.mod h1:uvppNeqRtp+LElLlFS0Qh+O00OJ08ac+o/UaVMj/9cc= github.com/nais/api v0.0.0-20240702080604-ae6c0f51e69b h1:eLl227vMRDmjlQ+vhY2qg9Ku6YAt6a5qzHp64ZryLoY= github.com/nais/api v0.0.0-20240702080604-ae6c0f51e69b/go.mod h1:BBRdhHlB5TK0cAShs650TvA7PVh9LUU9V1qgxinZ4EU= github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11 h1:2Cy7cjTM7PyiaUi3+aLsRSg6T0R6AqlduPNmT9LBMSY= From dbd95159c176498315c2141eaef66131ad7ee99d Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Tue, 2 Jul 2024 10:52:01 +0200 Subject: [PATCH 09/10] Remove delete flag from message and check the newly added team field instead --- go.mod | 2 +- go.sum | 4 ++-- internal/reconcilers/manager.go | 42 +++++---------------------------- internal/reconcilers/queue.go | 1 - 4 files changed, 9 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index af8a1b0..58b3bb4 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grafana/grafana-openapi-client-go v0.0.0-20240430202104-3ad0f7e4ee52 github.com/joho/godotenv v1.5.1 - github.com/nais/api v0.0.0-20240702080604-ae6c0f51e69b + github.com/nais/api v0.0.0-20240702084911-c0f9a3595889 github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11 github.com/prometheus/client_golang v1.19.1 github.com/sethvargo/go-envconfig v1.0.3 diff --git a/go.sum b/go.sum index b7ac3b9..e609d30 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,8 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/nais/api v0.0.0-20240702080604-ae6c0f51e69b h1:eLl227vMRDmjlQ+vhY2qg9Ku6YAt6a5qzHp64ZryLoY= -github.com/nais/api v0.0.0-20240702080604-ae6c0f51e69b/go.mod h1:BBRdhHlB5TK0cAShs650TvA7PVh9LUU9V1qgxinZ4EU= +github.com/nais/api v0.0.0-20240702084911-c0f9a3595889 h1:CDBinYp0rIeVc7VfVqBmiDAJr7+5l7GVchGvkgJg3xw= +github.com/nais/api v0.0.0-20240702084911-c0f9a3595889/go.mod h1:BBRdhHlB5TK0cAShs650TvA7PVh9LUU9V1qgxinZ4EU= github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11 h1:2Cy7cjTM7PyiaUi3+aLsRSg6T0R6AqlduPNmT9LBMSY= github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11/go.mod h1:BdPp1+tyE1hSMZigr7aGmhcTqaksf9Pxxbo4ZAi4On8= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= diff --git a/internal/reconcilers/manager.go b/internal/reconcilers/manager.go index 44eec17..af1d0b3 100644 --- a/internal/reconcilers/manager.go +++ b/internal/reconcilers/manager.go @@ -137,7 +137,6 @@ func (m *Manager) ListenForEvents(ctx context.Context) { } if event == protoapi.EventTypes_EVENT_TEAM_DELETED { - input.Delete = true obj = &protoapi.EventTeamDeleted{} } else { obj = &protoapi.EventTeamUpdated{} @@ -258,7 +257,7 @@ func (m *Manager) syncTeam(ctx context.Context, req ReconcileRequest) { } team := resp.Team - if req.Delete { + if team.DeleteKeyConfirmedAt != nil { m.deleteTeam(ctx, reconcilers, team, req) } else { m.reconcileTeam(ctx, reconcilers, team, req) @@ -465,20 +464,16 @@ func (m *Manager) scheduleAllTeams(ctx context.Context, correlationID uuid.UUID) return nil } - if err := m.scheduleActiveTeams(ctx, correlationID); err != nil { - m.log.WithError(err).Error("error while scheduling active teams for reconciliation") - } - - if err := m.scheduleDeletableTeams(ctx, correlationID); err != nil { - m.log.WithError(err).Error("error while scheduling teams for deletion") + if err := m.scheduleTeams(ctx, correlationID); err != nil { + m.log.WithError(err).Error("error while scheduling teams for reconciliation") } return nil } -func (m *Manager) scheduleActiveTeams(ctx context.Context, correlationID uuid.UUID) error { - it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.ListActiveTeamsResponse, error) { - return m.apiclient.Teams().ListActive(ctx, &protoapi.ListActiveTeamsRequest{Limit: limit, Offset: offset}) +func (m *Manager) scheduleTeams(ctx context.Context, correlationID uuid.UUID) error { + it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.ListTeamsResponse, error) { + return m.apiclient.Teams().List(ctx, &protoapi.ListTeamsRequest{Limit: limit, Offset: offset}) }) num := 0 @@ -500,31 +495,6 @@ func (m *Manager) scheduleActiveTeams(ctx context.Context, correlationID uuid.UU return nil } -func (m *Manager) scheduleDeletableTeams(ctx context.Context, correlationID uuid.UUID) error { - it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.ListDeletableTeamsResponse, error) { - return m.apiclient.Teams().ListDeletable(ctx, &protoapi.ListDeletableTeamsRequest{Limit: limit, Offset: offset}) - }) - - num := 0 - for it.Next() { - err := m.syncQueue.Add(ReconcileRequest{ - CorrelationID: correlationID.String(), - TeamSlug: it.Value().Slug, - Delete: true, - }) - if err != nil { - return fmt.Errorf("error while adding team to queue: %w", err) - } - num++ - } - if err := it.Err(); err != nil { - return fmt.Errorf("error while fetching teams for deletion: %w", err) - } - - m.log.WithField("num_teams", num).Debugf("added teams to delete queue") - return nil -} - // getReconcilers retrieves all reconcilers from the NAIS API func getReconcilers(ctx context.Context, client protoapi.ReconcilersClient) ([]*protoapi.Reconciler, error) { it := iterator.New(ctx, 100, func(limit, offset int64) (*protoapi.ListReconcilersResponse, error) { diff --git a/internal/reconcilers/queue.go b/internal/reconcilers/queue.go index 28cb1c9..b6441f0 100644 --- a/internal/reconcilers/queue.go +++ b/internal/reconcilers/queue.go @@ -11,7 +11,6 @@ type ReconcileRequest struct { CorrelationID string TraceID string TeamSlug string - Delete bool } type Queue interface { From 953f32236c82f4e85b9bea587ae38e014b76a846 Mon Sep 17 00:00:00 2001 From: Christer Edvartsen Date: Tue, 2 Jul 2024 11:15:04 +0200 Subject: [PATCH 10/10] Bump version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 58b3bb4..c55c8fe 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grafana/grafana-openapi-client-go v0.0.0-20240430202104-3ad0f7e4ee52 github.com/joho/godotenv v1.5.1 - github.com/nais/api v0.0.0-20240702084911-c0f9a3595889 + github.com/nais/api v0.0.0-20240702091139-fe19bb3460df github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11 github.com/prometheus/client_golang v1.19.1 github.com/sethvargo/go-envconfig v1.0.3 diff --git a/go.sum b/go.sum index e609d30..b6e904f 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,8 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/nais/api v0.0.0-20240702084911-c0f9a3595889 h1:CDBinYp0rIeVc7VfVqBmiDAJr7+5l7GVchGvkgJg3xw= -github.com/nais/api v0.0.0-20240702084911-c0f9a3595889/go.mod h1:BBRdhHlB5TK0cAShs650TvA7PVh9LUU9V1qgxinZ4EU= +github.com/nais/api v0.0.0-20240702091139-fe19bb3460df h1:SUv7kJfxqMgK3/O4jE4XeCDimsxtQ1m3f92T0uVnE+s= +github.com/nais/api v0.0.0-20240702091139-fe19bb3460df/go.mod h1:BBRdhHlB5TK0cAShs650TvA7PVh9LUU9V1qgxinZ4EU= github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11 h1:2Cy7cjTM7PyiaUi3+aLsRSg6T0R6AqlduPNmT9LBMSY= github.com/nais/dependencytrack v0.0.0-20240610080458-c76185696c11/go.mod h1:BdPp1+tyE1hSMZigr7aGmhcTqaksf9Pxxbo4ZAi4On8= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=