From 09b8f86484fd8df97039f2406b5150430d36ef72 Mon Sep 17 00:00:00 2001 From: hugoShaka Date: Tue, 3 Dec 2024 17:52:28 -0500 Subject: [PATCH] autoupdate: reconcile rollout status and add strategy interface --- .../teleport/autoupdate/v1/autoupdate.pb.go | 80 +++-- .../teleport/autoupdate/v1/autoupdate.proto | 6 + api/types/maintenance.go | 8 +- api/types/maintenance_test.go | 2 +- lib/autoupdate/rollout/controller.go | 4 + lib/autoupdate/rollout/reconciler.go | 182 +++++++++-- lib/autoupdate/rollout/reconciler_test.go | 297 +++++++++++++++++- lib/autoupdate/rollout/strategy.go | 72 +++++ lib/autoupdate/rollout/strategy_test.go | 287 +++++++++++++++++ 9 files changed, 886 insertions(+), 52 deletions(-) create mode 100644 lib/autoupdate/rollout/strategy.go create mode 100644 lib/autoupdate/rollout/strategy_test.go diff --git a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go index eab6a3f5830fc..f0a5af8c20e08 100644 --- a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go +++ b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go @@ -982,6 +982,12 @@ type AutoUpdateAgentRolloutStatusGroup struct { LastUpdateTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=last_update_time,json=lastUpdateTime,proto3" json:"last_update_time,omitempty"` // last_update_reason is the trigger for the last update LastUpdateReason string `protobuf:"bytes,5,opt,name=last_update_reason,json=lastUpdateReason,proto3" json:"last_update_reason,omitempty"` + // config_days when the update can run. Supported values are "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" and "*" + ConfigDays []string `protobuf:"bytes,6,rep,name=config_days,json=configDays,proto3" json:"config_days,omitempty"` + // config_start_hour to initiate update + ConfigStartHour int32 `protobuf:"varint,7,opt,name=config_start_hour,json=configStartHour,proto3" json:"config_start_hour,omitempty"` + // config_wait_days after last group succeeds before this group can run. This can only be used when the strategy is "halt-on-failure". + ConfigWaitDays int64 `protobuf:"varint,9,opt,name=config_wait_days,json=configWaitDays,proto3" json:"config_wait_days,omitempty"` } func (x *AutoUpdateAgentRolloutStatusGroup) Reset() { @@ -1049,6 +1055,27 @@ func (x *AutoUpdateAgentRolloutStatusGroup) GetLastUpdateReason() string { return "" } +func (x *AutoUpdateAgentRolloutStatusGroup) GetConfigDays() []string { + if x != nil { + return x.ConfigDays + } + return nil +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetConfigStartHour() int32 { + if x != nil { + return x.ConfigStartHour + } + return 0 +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetConfigWaitDays() int64 { + if x != nil { + return x.ConfigWaitDays + } + return 0 +} + var File_teleport_autoupdate_v1_autoupdate_proto protoreflect.FileDescriptor var file_teleport_autoupdate_v1_autoupdate_proto_rawDesc = []byte{ @@ -1201,7 +1228,7 @@ var file_teleport_autoupdate_v1_autoupdate_proto_rawDesc = []byte{ 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x06, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x22, 0xaf, 0x02, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x06, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x22, 0xa6, 0x03, 0x0a, 0x21, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, @@ -1220,29 +1247,36 @@ var file_teleport_autoupdate_v1_autoupdate_proto_rawDesc = []byte{ 0x70, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, - 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x2a, - 0xf7, 0x01, 0x0a, 0x19, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, - 0x65, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2d, 0x0a, - 0x29, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, - 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, - 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x2b, 0x0a, 0x27, - 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, - 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, - 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x28, 0x0a, 0x24, 0x41, 0x55, 0x54, - 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, - 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, - 0x45, 0x10, 0x02, 0x12, 0x26, 0x0a, 0x22, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, + 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, + 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x64, 0x61, 0x79, 0x73, 0x18, 0x06, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x44, 0x61, 0x79, 0x73, + 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x72, 0x74, 0x48, 0x6f, 0x75, 0x72, 0x12, 0x28, 0x0a, 0x10, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x77, 0x61, 0x69, 0x74, 0x5f, 0x64, 0x61, 0x79, 0x73, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x57, 0x61, + 0x69, 0x74, 0x44, 0x61, 0x79, 0x73, 0x2a, 0xf7, 0x01, 0x0a, 0x19, 0x41, 0x75, 0x74, 0x6f, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, + 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x00, 0x12, 0x2b, 0x0a, 0x27, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x12, 0x2c, 0x0a, 0x28, 0x41, - 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, - 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x4f, 0x4c, - 0x4c, 0x45, 0x44, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, 0x42, 0x56, 0x5a, 0x54, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, - 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, - 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, 0x01, + 0x12, 0x28, 0x0a, 0x24, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, + 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, + 0x45, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x26, 0x0a, 0x22, 0x41, 0x55, + 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, + 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x4f, 0x4e, 0x45, + 0x10, 0x03, 0x12, 0x2c, 0x0a, 0x28, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, + 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x45, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x45, 0x44, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, + 0x42, 0x56, 0x5a, 0x54, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, + 0x72, 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, + 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x75, + 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/proto/teleport/autoupdate/v1/autoupdate.proto b/api/proto/teleport/autoupdate/v1/autoupdate.proto index 5c7527d0177cf..6bcb62a6497a8 100644 --- a/api/proto/teleport/autoupdate/v1/autoupdate.proto +++ b/api/proto/teleport/autoupdate/v1/autoupdate.proto @@ -178,6 +178,12 @@ message AutoUpdateAgentRolloutStatusGroup { google.protobuf.Timestamp last_update_time = 4; // last_update_reason is the trigger for the last update string last_update_reason = 5; + // config_days when the update can run. Supported values are "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" and "*" + repeated string config_days = 6; + // config_start_hour to initiate update + int32 config_start_hour = 7; + // config_wait_days after last group succeeds before this group can run. This can only be used when the strategy is "halt-on-failure". + int64 config_wait_days = 9; } // AutoUpdateAgentGroupState represents the agent group state. This state controls whether the agents from this group diff --git a/api/types/maintenance.go b/api/types/maintenance.go index 9cab6a9ad4765..65d2f7271c6fc 100644 --- a/api/types/maintenance.go +++ b/api/types/maintenance.go @@ -45,10 +45,10 @@ var validWeekdays = [7]time.Weekday{ time.Saturday, } -// parseWeekday attempts to interpret a string as a time.Weekday. In the interest of flexibility, +// ParseWeekday attempts to interpret a string as a time.Weekday. In the interest of flexibility, // parsing is case-insensitive and supports the common three-letter shorthand accepted by many // common scheduling utilites (e.g. contab, systemd timers). -func parseWeekday(s string) (day time.Weekday, ok bool) { +func ParseWeekday(s string) (day time.Weekday, ok bool) { for _, w := range validWeekdays { if strings.EqualFold(w.String(), s) || strings.EqualFold(w.String()[:3], s) { return w, true @@ -75,7 +75,7 @@ func (w *AgentUpgradeWindow) generator(from time.Time) func() (start time.Time, var weekdays []time.Weekday for _, d := range w.Weekdays { - if p, ok := parseWeekday(d); ok { + if p, ok := ParseWeekday(d); ok { weekdays = append(weekdays, p) } } @@ -203,7 +203,7 @@ func (m *ClusterMaintenanceConfigV1) CheckAndSetDefaults() error { } for _, day := range m.Spec.AgentUpgrades.Weekdays { - if _, ok := parseWeekday(day); !ok { + if _, ok := ParseWeekday(day); !ok { return trace.BadParameter("invalid weekday in agent upgrade window: %q", day) } } diff --git a/api/types/maintenance_test.go b/api/types/maintenance_test.go index 203006a8dee37..40296dbd60f9a 100644 --- a/api/types/maintenance_test.go +++ b/api/types/maintenance_test.go @@ -205,7 +205,7 @@ func TestWeekdayParser(t *testing.T) { } for _, tt := range tts { - day, ok := parseWeekday(tt.input) + day, ok := ParseWeekday(tt.input) if tt.fail { require.False(t, ok) continue diff --git a/lib/autoupdate/rollout/controller.go b/lib/autoupdate/rollout/controller.go index 53a3741f8050a..9dc986f559c60 100644 --- a/lib/autoupdate/rollout/controller.go +++ b/lib/autoupdate/rollout/controller.go @@ -56,12 +56,16 @@ func NewController(client Client, log *slog.Logger, clock clockwork.Clock) (*Con if clock == nil { return nil, trace.BadParameter("missing clock") } + return &Controller{ clock: clock, log: log, reconciler: reconciler{ clt: client, log: log, + rolloutStrategies: []rolloutStrategy{ + // TODO(hugoShaka): add the strategies here as we implement them + }, }, }, nil } diff --git a/lib/autoupdate/rollout/reconciler.go b/lib/autoupdate/rollout/reconciler.go index 2fc04634c72f9..dd9d9a040d4cc 100644 --- a/lib/autoupdate/rollout/reconciler.go +++ b/lib/autoupdate/rollout/reconciler.go @@ -25,9 +25,13 @@ import ( "time" "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" update "github.com/gravitational/teleport/api/types/autoupdate" + "github.com/gravitational/teleport/api/utils" ) const ( @@ -35,6 +39,17 @@ const ( defaultConfigMode = update.AgentsUpdateModeEnabled defaultStrategy = update.AgentsStrategyHaltOnError maxConflictRetry = 3 + + defaultGroupName = "default" + defaultStartHour = 12 + + // Common reasons + updateReasonCreated = "created" + updateReasonReconcilerError = "reconciler_error" +) + +var ( + defaultUpdateDays = []string{"Mon", "Tue", "Wed", "Thu"} ) // reconciler reconciles the AutoUpdateAgentRollout singleton based on the content of the AutoUpdateVersion and @@ -42,8 +57,11 @@ const ( // - we reconcile 2 resources with one // - both input and output are singletons, we don't need the multi resource logic nor stream/paginated APIs type reconciler struct { - clt Client - log *slog.Logger + clt Client + log *slog.Logger + clock clockwork.Clock + + rolloutStrategies []rolloutStrategy // mutex ensures we only run one reconciliation at a time mutex sync.Mutex @@ -131,10 +149,26 @@ func (r *reconciler) tryReconcile(ctx context.Context) error { if err != nil { return trace.Wrap(err, "mutating rollout") } + newStatus, err := r.computeStatus(ctx, existingRollout, newSpec, config.GetSpec().GetAgents().GetSchedules()) + if err != nil { + return trace.Wrap(err, "computing rollout status") + } + + // there was an existing rollout, we must figure if something changed + specChanged := !proto.Equal(existingRollout.GetSpec(), newSpec) + statusChanged := !proto.Equal(existingRollout.GetStatus(), newStatus) + rolloutChanged := specChanged || statusChanged + + // if nothing changed, no need to update the resource + if !rolloutChanged { + r.log.DebugContext(ctx, "rollout unchanged") + return nil + } - // if there are no existing rollout, we create a new one + // if there are no existing rollout, we create a new one and set the status if !rolloutExists { rollout, err := update.NewAutoUpdateAgentRollout(newSpec) + rollout.Status = newStatus if err != nil { return trace.Wrap(err, "validating new rollout") } @@ -142,27 +176,10 @@ func (r *reconciler) tryReconcile(ctx context.Context) error { return trace.Wrap(err, "creating rollout") } - // there was an existing rollout, we must figure if something changed - specChanged := existingRollout.GetSpec().GetStartVersion() != newSpec.GetStartVersion() || - existingRollout.GetSpec().GetTargetVersion() != newSpec.GetTargetVersion() || - existingRollout.GetSpec().GetAutoupdateMode() != newSpec.GetAutoupdateMode() || - existingRollout.GetSpec().GetStrategy() != newSpec.GetStrategy() || - existingRollout.GetSpec().GetSchedule() != newSpec.GetSchedule() - - // TODO: reconcile the status here when we'll add group support. - // Even if the spec does not change, we might still have to update the status: - // - sync groups with the ones from the user config - // - progress the rollout across groups - - // if nothing changed, no need to update the resource - if !specChanged { - r.log.DebugContext(ctx, "rollout unchanged") - return nil - } - - // something changed, we replace the old spec with the new one, validate and update the resource - // we don't create a new resource to keep the revision ID and + // If there was a previous rollout, we update its spec and status and do an update. + // We don't create a new resource to keep the metadata containing the revision ID. existingRollout.Spec = newSpec + existingRollout.Status = newStatus err = update.ValidateAutoUpdateAgentRollout(existingRollout) if err != nil { return trace.Wrap(err, "validating mutated rollout") @@ -233,3 +250,122 @@ func getMode(configMode, versionMode string) (string, error) { } return codeToAgentMode[versionCode], nil } + +// computeStatus computes the new rollout status based on the existing rollout, +// new rollout spec, and autoupdate_config. existingRollout might be nil if this +// is a new rollout. +// Even if the returned new status might be derived from the existing rollout +// status, it is a new deep-cloned structure. +func (r *reconciler) computeStatus( + ctx context.Context, + existingRollout *autoupdate.AutoUpdateAgentRollout, + newSpec *autoupdate.AutoUpdateAgentRolloutSpec, + configSchedules *autoupdate.AgentAutoUpdateSchedules, +) (*autoupdate.AutoUpdateAgentRolloutStatus, error) { + + var status *autoupdate.AutoUpdateAgentRolloutStatus + + // First, we check if a major spec change happened and we should reset the rollout status + shouldResetRollout := existingRollout.GetSpec().GetStartVersion() != newSpec.GetStartVersion() || + existingRollout.GetSpec().GetTargetVersion() != newSpec.GetTargetVersion() || + existingRollout.GetSpec().GetSchedule() != newSpec.GetSchedule() || + existingRollout.GetSpec().GetStrategy() != newSpec.GetStrategy() + + // We create a new status if the rollout should be reset or the previous status was nil + if shouldResetRollout || existingRollout.GetStatus() == nil { + status = new(autoupdate.AutoUpdateAgentRolloutStatus) + } else { + status = utils.CloneProtoMsg(existingRollout.GetStatus()) + } + + // Then, we check if the selected schedule uses groups + switch newSpec.GetSchedule() { + case update.AgentsScheduleImmediate: + // There are no groups with the immediate schedule, we must clean them + status.Groups = nil + return status, nil + case update.AgentsScheduleRegular: + // Regular schedule has groups, we will compute them after + default: + return nil, trace.BadParameter("unsupported agent schedule type %q", newSpec.GetSchedule()) + } + + // capture the current time to put it in the status update timestamps and to + // compute the group state changes + now := r.clock.Now() + + // If this is a new rollout or the rollout has been reset, we create groups from the config + groups := status.GetGroups() + var err error + if len(groups) == 0 { + groups, err = makeGroupsStatus(configSchedules, now) + if err != nil { + return nil, trace.Wrap(err, "creating groups status") + } + } + + err = r.progressRollout(ctx, newSpec.GetStrategy(), groups) + // Failing to progress the update is not a hard failure. + // We expected to update the status even if something went wrong to surface the failed reconciliation and potential errors to the user. + if err != nil { + r.log.ErrorContext(ctx, "Errors encountered during rollout progress. Some groups might not get updated properly.", + "error", err) + } + + status.Groups = groups + return status, nil +} + +// progressRollout picks the right rollout strategy and updates groups to progress the rollout. +// groups are updated in place. +// If an error is returned, the groups should still be upserted, depending on the strategy, +// failing to update a group might not be fatal (other groups can still progress independently). +func (r *reconciler) progressRollout(ctx context.Context, strategyName string, groups []*autoupdate.AutoUpdateAgentRolloutStatusGroup) error { + for _, strategy := range r.rolloutStrategies { + if strategy.name() == strategyName { + return strategy.progressRollout(ctx, groups) + } + } + return trace.NotImplemented("rollout strategy %q not implemented", strategyName) +} + +// makeGroupStatus creates the autoupdate_agent_rollout.status.groups based on the autoupdate_config. +// This should be called if the status groups have not been initialized or must be reset. +func makeGroupsStatus(schedules *autoupdate.AgentAutoUpdateSchedules, now time.Time) ([]*autoupdate.AutoUpdateAgentRolloutStatusGroup, error) { + configGroups := schedules.GetRegular() + if len(configGroups) == 0 { + defaultGroup, err := defaultConfigGroup() + if err != nil { + return nil, trace.Wrap(err, "retrieving default group") + } + configGroups = []*autoupdate.AgentAutoUpdateGroup{defaultGroup} + } + + groups := make([]*autoupdate.AutoUpdateAgentRolloutStatusGroup, len(configGroups)) + for i, group := range configGroups { + groups[i] = &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: group.Name, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: group.Days, + ConfigStartHour: group.StartHour, + ConfigWaitDays: group.WaitDays, + } + } + return groups, nil +} + +// defaultConfigGroup returns the default group in case of missing autoupdate_config resource. +// This is a function and not a variable because we will need to add more logic there in the future +// lookup maintenance information from RFD 109's cluster_maintenance_config. +func defaultConfigGroup() (*autoupdate.AgentAutoUpdateGroup, error) { + // TODO: get group from CMC if possible + return &autoupdate.AgentAutoUpdateGroup{ + Name: defaultGroupName, + Days: defaultUpdateDays, + StartHour: defaultStartHour, + WaitDays: 0, + }, nil +} diff --git a/lib/autoupdate/rollout/reconciler_test.go b/lib/autoupdate/rollout/reconciler_test.go index 4d24563f7b32f..388c7bceb8492 100644 --- a/lib/autoupdate/rollout/reconciler_test.go +++ b/lib/autoupdate/rollout/reconciler_test.go @@ -20,13 +20,17 @@ package rollout import ( "context" + "sync" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/google/uuid" "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" update "github.com/gravitational/teleport/api/types/autoupdate" @@ -39,7 +43,7 @@ import ( // The comparison does not take into account the proto internal state. func rolloutEquals(expected *autoupdate.AutoUpdateAgentRollout) require.ValueAssertionFunc { return func(t require.TestingT, i interface{}, _ ...interface{}) { - require.IsType(t, &autoupdate.AutoUpdateAgentRollout{}, i) + require.IsType(t, &autoupdate.AutoUpdateAgentRollout{}, i, "resource should be an autoupdate_agent_rollout") actual := i.(*autoupdate.AutoUpdateAgentRollout) require.Empty(t, cmp.Diff(expected, actual, protocmp.Transform())) } @@ -181,6 +185,7 @@ func TestTryReconcile(t *testing.T) { Strategy: update.AgentsStrategyHaltOnError, }) require.NoError(t, err) + upToDateRollout.Status = &autoupdate.AutoUpdateAgentRolloutStatus{} outOfDateRollout, err := update.NewAutoUpdateAgentRollout(&autoupdate.AutoUpdateAgentRolloutSpec{ StartVersion: "1.2.2", @@ -190,6 +195,7 @@ func TestTryReconcile(t *testing.T) { Strategy: update.AgentsStrategyHaltOnError, }) require.NoError(t, err) + outOfDateRollout.Status = &autoupdate.AutoUpdateAgentRolloutStatus{} tests := []struct { name string @@ -354,6 +360,7 @@ func TestReconciler_Reconcile(t *testing.T) { Strategy: update.AgentsStrategyHaltOnError, }) require.NoError(t, err) + upToDateRollout.Status = &autoupdate.AutoUpdateAgentRolloutStatus{} outOfDateRollout, err := update.NewAutoUpdateAgentRollout(&autoupdate.AutoUpdateAgentRolloutSpec{ StartVersion: "1.2.2", @@ -363,6 +370,7 @@ func TestReconciler_Reconcile(t *testing.T) { Strategy: update.AgentsStrategyHaltOnError, }) require.NoError(t, err) + outOfDateRollout.Status = &autoupdate.AutoUpdateAgentRolloutStatus{} // Those tests are not written in table format because the fixture setup it too complex and this would harm // readability. @@ -565,3 +573,290 @@ func TestReconciler_Reconcile(t *testing.T) { client.checkIfEmpty(t) }) } + +func Test_makeGroupsStatus(t *testing.T) { + now := time.Now() + + tests := []struct { + name string + schedules *autoupdate.AgentAutoUpdateSchedules + expected []*autoupdate.AutoUpdateAgentRolloutStatusGroup + }{ + { + name: "nil schedules", + schedules: nil, + expected: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: defaultGroupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: defaultUpdateDays, + ConfigStartHour: defaultStartHour, + ConfigWaitDays: 0, + }, + }, + }, + { + name: "no groups in schedule", + schedules: &autoupdate.AgentAutoUpdateSchedules{Regular: make([]*autoupdate.AgentAutoUpdateGroup, 0)}, + expected: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: defaultGroupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: defaultUpdateDays, + ConfigStartHour: defaultStartHour, + ConfigWaitDays: 0, + }, + }, + }, + { + name: "one group in schedule", + schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + { + Name: "group1", + Days: everyWeekday, + StartHour: matchingStartHour, + WaitDays: 0, + }, + }, + }, + expected: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "group1", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: everyWeekday, + ConfigStartHour: matchingStartHour, + ConfigWaitDays: 0, + }, + }, + }, + { + name: "multiple groups in schedule", + schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + { + Name: "group1", + Days: everyWeekday, + StartHour: matchingStartHour, + WaitDays: 0, + }, + { + Name: "group2", + Days: everyWeekdayButSunday, + StartHour: nonMatchingStartHour, + WaitDays: 1, + }, + }, + }, + expected: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "group1", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: everyWeekday, + ConfigStartHour: matchingStartHour, + ConfigWaitDays: 0, + }, + { + Name: "group2", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: everyWeekdayButSunday, + ConfigStartHour: nonMatchingStartHour, + ConfigWaitDays: 1, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := makeGroupsStatus(tt.schedules, now) + require.NoError(t, err) + require.Equal(t, tt.expected, result) + }) + } +} + +const fakeRolloutStrategyName = "fake" + +type fakeRolloutStrategy struct { + strategyName string + // calls counts how many times the fake rollout strategy was called. + // This is not thread safe. + calls int +} + +func (f *fakeRolloutStrategy) name() string { + return f.strategyName +} + +func (f *fakeRolloutStrategy) progressRollout(ctx context.Context, groups []*autoupdate.AutoUpdateAgentRolloutStatusGroup) error { + f.calls++ + return nil +} + +func Test_reconciler_computeStatus(t *testing.T) { + log := utils.NewSlogLoggerForTests() + clock := clockwork.NewFakeClock() + ctx := context.Background() + + oldStatus := &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "old group", + }, + }, + } + oldSpec := &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName, + } + schedules := &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + { + Name: "new group", + Days: everyWeekday, + }, + }, + } + newGroups, err := makeGroupsStatus(schedules, clock.Now()) + require.NoError(t, err) + newStatus := &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: newGroups, + } + + tests := []struct { + name string + existingRollout *autoupdate.AutoUpdateAgentRollout + newSpec *autoupdate.AutoUpdateAgentRolloutSpec + expectedStatus *autoupdate.AutoUpdateAgentRolloutStatus + expectedStrategyCalls int + }{ + { + name: "status is reset if start version changes", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.2", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName, + }, + // status should have been reset and is now the new status + expectedStatus: newStatus, + expectedStrategyCalls: 1, + }, + { + name: "status is reset if target version changes", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.5", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName, + }, + // status should have been reset and is now the new status + expectedStatus: newStatus, + expectedStrategyCalls: 1, + }, + { + name: "status is reset if strategy changes", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName + "2", + }, + // status should have been reset and is now the new status + expectedStatus: newStatus, + expectedStrategyCalls: 1, + }, + { + name: "status is not reset if mode changes", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeSuspended, + Strategy: fakeRolloutStrategyName, + }, + // status should NOT have been reset and still contain the old groups + expectedStatus: oldStatus, + expectedStrategyCalls: 1, + }, + { + name: "groups are unset if schedule is immediate", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleImmediate, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName, + }, + // groups should be unset + expectedStatus: &autoupdate.AutoUpdateAgentRolloutStatus{}, + expectedStrategyCalls: 0, + }, + { + name: "new groups are populated if previous ones were empty", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + // old groups were empty + Status: &autoupdate.AutoUpdateAgentRolloutStatus{}, + }, + // no spec change + newSpec: oldSpec, + // still, we have the new groups set + expectedStatus: newStatus, + expectedStrategyCalls: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategy := &fakeRolloutStrategy{strategyName: tt.newSpec.Strategy} + r := &reconciler{ + log: log, + clock: clock, + rolloutStrategies: []rolloutStrategy{strategy}, + mutex: sync.Mutex{}, + } + result, err := r.computeStatus(ctx, tt.existingRollout, tt.newSpec, schedules) + require.NoError(t, err) + require.Empty(t, cmp.Diff(tt.expectedStatus, result, protocmp.Transform())) + require.Equal(t, tt.expectedStrategyCalls, strategy.calls) + }) + } +} diff --git a/lib/autoupdate/rollout/strategy.go b/lib/autoupdate/rollout/strategy.go new file mode 100644 index 0000000000000..ee3c1aa9b4ea0 --- /dev/null +++ b/lib/autoupdate/rollout/strategy.go @@ -0,0 +1,72 @@ +package rollout + +import ( + "context" + "time" + + "github.com/gravitational/trace" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types" +) + +// rolloutStrategy is responsible for rolling out the update across groups. +// This interface allows us to inject dummy strategies for simpler testing. +type rolloutStrategy interface { + name() string + progressRollout(context.Context, []*autoupdate.AutoUpdateAgentRolloutStatusGroup) error +} + +func inWindow(group *autoupdate.AutoUpdateAgentRolloutStatusGroup, now time.Time) (bool, error) { + dayOK, err := canUpdateToday(group.ConfigDays, now) + if err != nil { + return false, trace.Wrap(err, "checking the day of the week") + } + if !dayOK { + return false, nil + } + return int(group.ConfigStartHour) == now.Hour(), nil +} + +func canUpdateToday(allowedDays []string, now time.Time) (bool, error) { + for _, allowedDay := range allowedDays { + if allowedDay == types.Wildcard { + return true, nil + } + weekday, ok := types.ParseWeekday(allowedDay) + if !ok { + return false, trace.BadParameter("failed to parse weekday %q", allowedDay) + } + if weekday == now.Weekday() { + return true, nil + } + } + return false, nil +} + +func setGroupState(group *autoupdate.AutoUpdateAgentRolloutStatusGroup, newState autoupdate.AutoUpdateAgentGroupState, reason string, now time.Time) { + changed := false + previousState := group.State + + // Check if there is a state transition + if previousState != newState { + group.State = newState + changed = true + // If we just started the group, also update the start time + if newState == autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE { + group.StartTime = timestamppb.New(now) + } + } + + // Check if there is a reason change. Even if the state did not change, we + // might expected to explain why. + if group.LastUpdateReason != reason { + group.LastUpdateReason = reason + changed = true + } + + if changed { + group.LastUpdateTime = timestamppb.New(now) + } +} diff --git a/lib/autoupdate/rollout/strategy_test.go b/lib/autoupdate/rollout/strategy_test.go new file mode 100644 index 0000000000000..b395bbe2d2b3d --- /dev/null +++ b/lib/autoupdate/rollout/strategy_test.go @@ -0,0 +1,287 @@ +package rollout + +import ( + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" +) + +var ( + // 2024-11-30 is a Saturday + testSaturday = time.Date(2024, 11, 30, 15, 30, 0, 0, time.UTC) + // 2024-12-01 is a Sunday + testSunday = time.Date(2024, 12, 1, 12, 30, 0, 0, time.UTC) + matchingStartHour = int32(12) + nonMatchingStartHour = int32(15) + everyWeekday = []string{"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"} + everyWeekdayButSunday = []string{"Mon", "Tue", "Wed", "Thu", "Fri", "Sat"} +) + +func Test_canUpdateToday(t *testing.T) { + tests := []struct { + name string + allowedDays []string + now time.Time + want bool + wantErr require.ErrorAssertionFunc + }{ + { + name: "Empty list", + allowedDays: []string{}, + now: time.Now(), + want: false, + wantErr: require.NoError, + }, + { + name: "Wildcard", + allowedDays: []string{"*"}, + now: time.Now(), + want: true, + wantErr: require.NoError, + }, + { + name: "Matching day", + allowedDays: everyWeekday, + now: testSunday, + want: true, + wantErr: require.NoError, + }, + { + name: "No matching day", + allowedDays: everyWeekdayButSunday, + now: testSunday, + want: false, + wantErr: require.NoError, + }, + { + name: "Malformed day", + allowedDays: []string{"Mon", "Tue", "HelloThereGeneralKenobi"}, + now: testSunday, + want: false, + wantErr: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := canUpdateToday(tt.allowedDays, tt.now) + tt.wantErr(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func Test_inWindow(t *testing.T) { + tests := []struct { + name string + group *autoupdate.AutoUpdateAgentRolloutStatusGroup + now time.Time + want bool + wantErr require.ErrorAssertionFunc + }{ + { + name: "out of window", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: everyWeekdayButSunday, + ConfigStartHour: matchingStartHour, + }, + now: testSunday, + want: false, + wantErr: require.NoError, + }, + { + name: "inside window, wrong hour", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: everyWeekday, + ConfigStartHour: nonMatchingStartHour, + }, + now: testSunday, + want: false, + wantErr: require.NoError, + }, + { + name: "inside window, correct hour", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: everyWeekday, + ConfigStartHour: matchingStartHour, + }, + now: testSunday, + want: true, + wantErr: require.NoError, + }, + { + name: "invalid weekdays", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: []string{"HelloThereGeneralKenobi"}, + ConfigStartHour: matchingStartHour, + }, + now: testSunday, + want: false, + wantErr: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := inWindow(tt.group, tt.now) + tt.wantErr(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func Test_setGroupState(t *testing.T) { + groupName := "test-group" + + clock := clockwork.NewFakeClock() + // oldUpdateTime is 5 minutes in the past + oldUpdateTime := clock.Now() + clock.Advance(5 * time.Minute) + + tests := []struct { + name string + group *autoupdate.AutoUpdateAgentRolloutStatusGroup + newState autoupdate.AutoUpdateAgentGroupState + reason string + now time.Time + expected *autoupdate.AutoUpdateAgentRolloutStatusGroup + }{ + { + name: "same state, no change", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + reason: updateReasonCannotStart, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + // update time has not been bumped as nothing changed + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + }, + { + name: "same state, reason change", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + reason: updateReasonReconcilerError, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + // update time has been bumped because reason changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonReconcilerError, + }, + }, + { + name: "new state, no reason change", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + reason: updateReasonCannotStart, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + // update time has been bumped because state changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonCannotStart, + }, + }, + { + name: "new state, reason change", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + reason: updateReasonReconcilerError, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + // update time has been bumped because state and reason changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonReconcilerError, + }, + }, + { + name: "new state, transition to active", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + reason: updateReasonCanStart, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + // We set start time during the transition + StartTime: timestamppb.New(clock.Now()), + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + // update time has been bumped because state and reason changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonCanStart, + }, + }, + { + name: "same state, transition from active to active", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: timestamppb.New(oldUpdateTime), + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCanStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + reason: updateReasonReconcilerError, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + // As the state was already active, the start time should not be refreshed + StartTime: timestamppb.New(oldUpdateTime), + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + // update time has been bumped because reason changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonReconcilerError, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setGroupState(tt.group, tt.newState, tt.reason, tt.now) + require.Equal(t, tt.expected, tt.group) + }) + } +}