From ac9a4e7e0cf78906dea7f1e35acf24f7015cb176 Mon Sep 17 00:00:00 2001 From: David Symons Date: Thu, 1 Jul 2021 17:41:44 +1000 Subject: [PATCH 1/7] Adds ability to configure frequency for pod termination via annotation --- chaoskube/chaoskube.go | 80 +++++++++++++----- chaoskube/chaoskube_test.go | 150 +++++++++++++++++++++++++--------- chart/chaoskube/values.yaml | 3 + main.go | 120 ++++++++++++++------------- notifier/slack_test.go | 6 +- terminator/delete_pod_test.go | 6 +- util/util.go | 141 +++++++++++++++++++++++--------- util/util_test.go | 37 ++++++++- 8 files changed, 378 insertions(+), 165 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index 1bb7a7ba..3da75cf1 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math/rand" "regexp" "time" @@ -32,6 +33,8 @@ import ( type Chaoskube struct { // a kubernetes client object Client kubernetes.Interface + // the interval Chaoskube is configured to run at + Interval time.Duration // a label selector which restricts the pods to choose from Labels labels.Selector // an annotation selector which restricts the pods to choose from @@ -56,6 +59,8 @@ type Chaoskube struct { Timezone *time.Location // minimum age of pods to consider MinimumAge time.Duration + // an annotation containing the frequency to kill a pod at + FrequencyAnnotation string // an instance of logrus.StdLogger to write log messages to Logger log.FieldLogger // a terminator that terminates victim pods @@ -68,7 +73,7 @@ type Chaoskube struct { EventRecorder record.EventRecorder // a function to retrieve the current time Now func() time.Time - + // the maximum number of pods to terminate per interval MaxKill int // chaos events notifier Notifier notifier.Notifier @@ -95,32 +100,34 @@ var ( // * a logger implementing logrus.FieldLogger to send log output to // * what specific terminator to use to imbue chaos on victim pods // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube { +func New(client kubernetes.Interface, interval time.Duration, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"}) return &Chaoskube{ - Client: client, - Labels: labels, - Annotations: annotations, - Kinds: kinds, - Namespaces: namespaces, - NamespaceLabels: namespaceLabels, - IncludedPodNames: includedPodNames, - ExcludedPodNames: excludedPodNames, - ExcludedWeekdays: excludedWeekdays, - ExcludedTimesOfDay: excludedTimesOfDay, - ExcludedDaysOfYear: excludedDaysOfYear, - Timezone: timezone, - MinimumAge: minimumAge, - Logger: logger, - DryRun: dryRun, - Terminator: terminator, - EventRecorder: recorder, - Now: time.Now, - MaxKill: maxKill, - Notifier: notifier, + Client: client, + Interval: interval, + Labels: labels, + Annotations: annotations, + Kinds: kinds, + Namespaces: namespaces, + NamespaceLabels: namespaceLabels, + IncludedPodNames: includedPodNames, + ExcludedPodNames: excludedPodNames, + ExcludedWeekdays: excludedWeekdays, + ExcludedTimesOfDay: excludedTimesOfDay, + ExcludedDaysOfYear: excludedDaysOfYear, + Timezone: timezone, + MinimumAge: minimumAge, + FrequencyAnnotation: frequencyAnnotation, + Logger: logger, + DryRun: dryRun, + Terminator: terminator, + EventRecorder: recorder, + Now: time.Now, + MaxKill: maxKill, + Notifier: notifier, } } @@ -238,6 +245,8 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) pods = filterByOwnerReference(pods) + pods = filterByFrequency(pods, c.FrequencyAnnotation, c.Interval, c.Logger) + return pods, nil } @@ -533,3 +542,30 @@ func filterByOwnerReference(pods []v1.Pod) []v1.Pod { return filteredList } + +func filterByFrequency(pods []v1.Pod, annotation string, interval time.Duration, logger log.FieldLogger) []v1.Pod { + if annotation == "" { + return pods + } + + filteredList := []v1.Pod{} + for _, pod := range pods { + text, ok := pod.Annotations[annotation] + + // Don't filter out pods missing frequency annotation + if !ok { + filteredList = append(filteredList, pod) + } + + chance, err := util.ParseFrequency(text, interval) + if err != nil { + logger.WithField("err", err).Warn("failed to parse frequency annotation") + } + + if chance > rand.Float64() { + filteredList = append(filteredList, pod) + } + } + + return filteredList +} diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 0175965f..1d5650b2 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -48,26 +48,29 @@ func (suite *Suite) SetupTest() { // TestNew tests that arguments are passed to the new instance correctly func (suite *Suite) TestNew() { var ( - client = fake.NewSimpleClientset() - labelSelector, _ = labels.Parse("foo=bar") - annotations, _ = labels.Parse("baz=waldo") - kinds, _ = labels.Parse("job") - namespaces, _ = labels.Parse("qux") - namespaceLabels, _ = labels.Parse("taz=wubble") - includedPodNames = regexp.MustCompile("foo") - excludedPodNames = regexp.MustCompile("bar") - excludedWeekdays = []time.Weekday{time.Friday} - excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}} - excludedDaysOfYear = []time.Time{time.Now()} - minimumAge = time.Duration(42) - dryRun = true - terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) - maxKill = 1 - notifier = testNotifier + client = fake.NewSimpleClientset() + interval = 10 * time.Minute + labelSelector, _ = labels.Parse("foo=bar") + annotations, _ = labels.Parse("baz=waldo") + kinds, _ = labels.Parse("job") + namespaces, _ = labels.Parse("qux") + namespaceLabels, _ = labels.Parse("taz=wubble") + includedPodNames = regexp.MustCompile("foo") + excludedPodNames = regexp.MustCompile("bar") + excludedWeekdays = []time.Weekday{time.Friday} + excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}} + excludedDaysOfYear = []time.Time{time.Now()} + minimumAge = time.Duration(42) + frequencyAnnotation = "chaos.alpha.kubernetes.io/frequency" + dryRun = true + terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) + maxKill = 1 + notifier = testNotifier ) chaoskube := New( client, + interval, labelSelector, annotations, kinds, @@ -80,6 +83,7 @@ func (suite *Suite) TestNew() { excludedDaysOfYear, time.UTC, minimumAge, + frequencyAnnotation, logger, dryRun, terminator, @@ -89,6 +93,7 @@ func (suite *Suite) TestNew() { suite.Require().NotNil(chaoskube) suite.Equal(client, chaoskube.Client) + suite.Equal(10*time.Minute, chaoskube.Interval) suite.Equal("foo=bar", chaoskube.Labels.String()) suite.Equal("baz=waldo", chaoskube.Annotations.String()) suite.Equal("job", chaoskube.Kinds.String()) @@ -101,6 +106,7 @@ func (suite *Suite) TestNew() { suite.Equal(excludedDaysOfYear, chaoskube.ExcludedDaysOfYear) suite.Equal(time.UTC, chaoskube.Timezone) suite.Equal(minimumAge, chaoskube.MinimumAge) + suite.Equal("chaos.alpha.kubernetes.io/frequency", chaoskube.FrequencyAnnotation) suite.Equal(logger, chaoskube.Logger) suite.Equal(dryRun, chaoskube.DryRun) suite.Equal(terminator, chaoskube.Terminator) @@ -109,6 +115,7 @@ func (suite *Suite) TestNew() { // TestRunContextCanceled tests that a canceled context will exit the Run function. func (suite *Suite) TestRunContextCanceled() { chaoskube := suite.setup( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -121,6 +128,7 @@ func (suite *Suite) TestRunContextCanceled() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, 1, @@ -165,6 +173,7 @@ func (suite *Suite) TestCandidates() { suite.Require().NoError(err) chaoskube := suite.setupWithPods( + 10*time.Minute, labelSelector, annotationSelector, labels.Everything(), @@ -177,6 +186,7 @@ func (suite *Suite) TestCandidates() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, ) @@ -210,6 +220,7 @@ func (suite *Suite) TestCandidatesNamespaceLabels() { suite.Require().NoError(err) chaoskube := suite.setupWithPods( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -222,6 +233,7 @@ func (suite *Suite) TestCandidatesNamespaceLabels() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, ) @@ -253,6 +265,7 @@ func (suite *Suite) TestCandidatesPodNameRegexp() { {regexp.MustCompile("fo.*"), regexp.MustCompile("f.*"), []map[string]string{}}, } { chaoskube := suite.setupWithPods( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -265,6 +278,7 @@ func (suite *Suite) TestCandidatesPodNameRegexp() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, ) @@ -293,6 +307,7 @@ func (suite *Suite) TestVictim() { suite.Require().NoError(err) chaoskube := suite.setupWithPods( + 10*time.Minute, labelSelector, labels.Everything(), labels.Everything(), @@ -305,6 +320,7 @@ func (suite *Suite) TestVictim() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, ) @@ -346,6 +362,7 @@ func (suite *Suite) TestVictims() { suite.Require().NoError(err) chaoskube := suite.setup( + 10*time.Minute, labelSelector, labels.Everything(), labels.Everything(), @@ -358,6 +375,7 @@ func (suite *Suite) TestVictims() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, tt.maxKill, @@ -371,6 +389,7 @@ func (suite *Suite) TestVictims() { // TestNoVictimReturnsError tests that on missing victim it returns a known error func (suite *Suite) TestNoVictimReturnsError() { chaoskube := suite.setup( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -383,6 +402,7 @@ func (suite *Suite) TestNoVictimReturnsError() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, 1, @@ -406,6 +426,7 @@ func (suite *Suite) TestDeletePod() { {true, []map[string]string{foo, bar}}, } { chaoskube := suite.setupWithPods( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -418,11 +439,12 @@ func (suite *Suite) TestDeletePod() { []time.Time{}, time.UTC, time.Duration(0), + "", tt.dryRun, 10, ) - victim := util.NewPod("default", "foo", v1.PodRunning) + victim := util.NewPodBuilder("default", "foo").Build() err := chaoskube.DeletePod(context.Background(), victim) suite.Require().NoError(err) @@ -435,6 +457,7 @@ func (suite *Suite) TestDeletePod() { // TestDeletePodNotFound tests missing target pod will return an error. func (suite *Suite) TestDeletePodNotFound() { chaoskube := suite.setup( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -447,12 +470,13 @@ func (suite *Suite) TestDeletePodNotFound() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, 1, ) - victim := util.NewPod("default", "foo", v1.PodRunning) + victim := util.NewPodBuilder("default", "foo").Build() err := chaoskube.DeletePod(context.Background(), victim) suite.EqualError(err, `pods "foo" not found`) @@ -667,6 +691,7 @@ func (suite *Suite) TestTerminateVictim() { }, } { chaoskube := suite.setupWithPods( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -679,6 +704,7 @@ func (suite *Suite) TestTerminateVictim() { tt.excludedDaysOfYear, tt.timezone, time.Duration(0), + "", false, 10, ) @@ -697,6 +723,7 @@ func (suite *Suite) TestTerminateVictim() { // TestTerminateNoVictimLogsInfo tests that missing victim prints a log message func (suite *Suite) TestTerminateNoVictimLogsInfo() { chaoskube := suite.setup( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -709,6 +736,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, 1, @@ -746,8 +774,9 @@ func (suite *Suite) assertNotified(notifier *notifier.Noop) { suite.Assert().Greater(notifier.Calls, 0) } -func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube { +func (suite *Suite) setupWithPods(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, dryRun bool, gracePeriod time.Duration) *Chaoskube { chaoskube := suite.setup( + 10*time.Minute, labelSelector, annotations, kinds, @@ -760,6 +789,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab excludedDaysOfYear, timezone, minimumAge, + frequencyAnnotation, dryRun, gracePeriod, 1, @@ -774,9 +804,9 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab } pods := []v1.Pod{ - util.NewPod("default", "foo", v1.PodRunning), - util.NewPod("testing", "bar", v1.PodRunning), - util.NewPod("testing", "baz", v1.PodPending), // Non-running pods are ignored + util.NewPodBuilder("default", "foo").Build(), + util.NewPodBuilder("testing", "bar").Build(), + util.NewPodBuilder("testing", "baz").Build(), // Non-running pods are ignored } for _, pod := range pods { @@ -792,13 +822,13 @@ func (suite *Suite) createPods(client kubernetes.Interface, podsInfo []podInfo) namespace := util.NewNamespace(p.Namespace) _, err := client.CoreV1().Namespaces().Create(context.Background(), &namespace, metav1.CreateOptions{}) suite.Require().NoError(err) - pod := util.NewPod(p.Namespace, p.Name, v1.PodRunning) + pod := util.NewPodBuilder(p.Namespace, p.Name).Build() _, err = client.CoreV1().Pods(p.Namespace).Create(context.Background(), &pod, metav1.CreateOptions{}) suite.Require().NoError(err) } } -func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int) *Chaoskube { +func (suite *Suite) setup(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, dryRun bool, gracePeriod time.Duration, maxKill int) *Chaoskube { logOutput.Reset() client := fake.NewSimpleClientset() @@ -806,6 +836,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele return New( client, + interval, labelSelector, annotations, kinds, @@ -818,6 +849,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele excludedDaysOfYear, timezone, minimumAge, + frequencyAnnotation, logger, dryRun, terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod), @@ -913,6 +945,7 @@ func (suite *Suite) TestMinimumAge() { }, } { chaoskube := suite.setup( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -925,6 +958,7 @@ func (suite *Suite) TestMinimumAge() { []time.Time{}, time.UTC, tt.minimumAge, + "", false, 10, 1, @@ -932,7 +966,7 @@ func (suite *Suite) TestMinimumAge() { chaoskube.Now = tt.now for _, p := range tt.pods { - pod := util.NewPod(p.namespace, p.name, v1.PodRunning) + pod := util.NewPodBuilder(p.namespace, p.name).Build() pod.ObjectMeta.CreationTimestamp = metav1.Time{Time: p.creationTime} _, err := chaoskube.Client.CoreV1().Pods(pod.Namespace).Create(context.Background(), &pod, metav1.CreateOptions{}) suite.Require().NoError(err) @@ -946,11 +980,11 @@ func (suite *Suite) TestMinimumAge() { } func (suite *Suite) TestFilterDeletedPods() { - deletedPod := util.NewPod("default", "deleted", v1.PodRunning) + deletedPod := util.NewPodBuilder("default", "deleted").Build() now := metav1.NewTime(time.Now()) deletedPod.SetDeletionTimestamp(&now) - runningPod := util.NewPod("default", "running", v1.PodRunning) + runningPod := util.NewPodBuilder("default", "running").Build() pods := []v1.Pod{runningPod, deletedPod} @@ -960,11 +994,11 @@ func (suite *Suite) TestFilterDeletedPods() { } func (suite *Suite) TestFilterByKinds() { - foo := util.NewPodWithOwner("default", "foo", v1.PodRunning, "parent-1") - foo1 := util.NewPodWithOwner("default", "foo-1", v1.PodRunning, "parent-2") - bar := util.NewPodWithOwner("default", "bar", v1.PodRunning, "other-parent") - baz := util.NewPod("default", "baz", v1.PodRunning) - baz1 := util.NewPod("default", "baz-1", v1.PodRunning) + foo := util.NewPodBuilder("default", "foo").WithOwnerUID("parent-1").Build() + foo1 := util.NewPodBuilder("default", "foo-1").WithOwnerUID("parent-2").Build() + bar := util.NewPodBuilder("default", "bar").WithOwnerUID("other-parent").Build() + baz := util.NewPodBuilder("default", "baz").Build() + baz1 := util.NewPodBuilder("default", "baz-1").Build() for _, tt := range []struct { name string @@ -1035,11 +1069,11 @@ func (suite *Suite) TestFilterByKinds() { } func (suite *Suite) TestFilterByOwnerReference() { - foo := util.NewPodWithOwner("default", "foo", v1.PodRunning, "parent") - foo1 := util.NewPodWithOwner("default", "foo-1", v1.PodRunning, "parent") - bar := util.NewPodWithOwner("default", "bar", v1.PodRunning, "other-parent") - baz := util.NewPod("default", "baz", v1.PodRunning) - baz1 := util.NewPod("default", "baz-1", v1.PodRunning) + foo := util.NewPodBuilder("default", "foo").WithOwnerUID("parent").Build() + foo1 := util.NewPodBuilder("default", "foo-1").WithOwnerUID("parent").Build() + bar := util.NewPodBuilder("default", "bar").WithOwnerUID("other-parent").Build() + baz := util.NewPodBuilder("default", "baz").Build() + baz1 := util.NewPodBuilder("default", "baz-1").Build() for _, tt := range []struct { seed int64 @@ -1094,8 +1128,47 @@ func (suite *Suite) TestFilterByOwnerReference() { } } +func (suite *Suite) TestFilterByFrequency() { + interval := 10 * time.Minute + logger, _ := test.NewNullLogger() + + foo := util.NewPodBuilder("default", "foo").WithFrequency("1 / hour").Build() + foo1 := util.NewPodBuilder("default", "foo-1").WithFrequency("1 / minute").Build() + bar := util.NewPodBuilder("default", "bar").WithFrequency("2.5 / hour").Build() + baz := util.NewPodBuilder("default", "baz").Build() + + pods := []v1.Pod{foo, foo1, bar, baz} + alwaysExpected := []v1.Pod{foo1, baz} + + for _, tt := range []struct { + seed int64 + expected []v1.Pod + }{ + { + seed: 1000, + expected: []v1.Pod{}, + }, + { + seed: 3000, + expected: []v1.Pod{foo}, + }, + { + seed: 4000, + expected: []v1.Pod{bar}, + }, + } { + expected := append(tt.expected, alwaysExpected...) + + rand.Seed(tt.seed) + results := filterByFrequency(pods, "chaos.alpha.kubernetes.io/frequency", interval, logger) + + suite.Assert().ElementsMatch(results, expected) + } +} + func (suite *Suite) TestNotifierCall() { chaoskube := suite.setupWithPods( + 10*time.Minute, labels.Everything(), labels.Everything(), labels.Everything(), @@ -1108,11 +1181,12 @@ func (suite *Suite) TestNotifierCall() { []time.Time{}, time.UTC, time.Duration(0), + "", false, 10, ) - victim := util.NewPod("default", "foo", v1.PodRunning) + victim := util.NewPodBuilder("default", "foo").Build() err := chaoskube.DeletePod(context.Background(), victim) suite.Require().NoError(err) diff --git a/chart/chaoskube/values.yaml b/chart/chaoskube/values.yaml index c2440a47..fed1b825 100644 --- a/chart/chaoskube/values.yaml +++ b/chart/chaoskube/values.yaml @@ -33,6 +33,9 @@ chaoskube: timezone: "UTC" # exclude all pods that haven't been running for at least one hour minimum-age: "1h" + # checks for "chaos.alpha.kubernetes.io/frequency" annotation on pods to determine frequency to terminate + # eg. setting "chaos.alpha.kubernetes.io/frequency=1/hour" will terminate the pod approximately once per hour + termination-frequency-annotation: "chaos.alpha.kubernetes.io/frequency" # terminate pods for real: this disables dry-run mode which is on by default no-dry-run: "" diff --git a/main.go b/main.go index 29830d02..ffce7d4c 100644 --- a/main.go +++ b/main.go @@ -15,9 +15,9 @@ import ( "syscall" "time" - "gopkg.in/alecthomas/kingpin.v2" "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" + "gopkg.in/alecthomas/kingpin.v2" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" @@ -36,30 +36,31 @@ var ( ) var ( - labelString string - annString string - kindsString string - nsString string - nsLabelString string - includedPodNames *regexp.Regexp - excludedPodNames *regexp.Regexp - excludedWeekdays string - excludedTimesOfDay string - excludedDaysOfYear string - timezone string - minimumAge time.Duration - maxRuntime time.Duration - maxKill int - master string - kubeconfig string - interval time.Duration - dryRun bool - debug bool - metricsAddress string - gracePeriod time.Duration - logFormat string - logCaller bool - slackWebhook string + labelString string + annString string + kindsString string + nsString string + nsLabelString string + includedPodNames *regexp.Regexp + excludedPodNames *regexp.Regexp + excludedWeekdays string + excludedTimesOfDay string + excludedDaysOfYear string + timezone string + minimumAge time.Duration + frequencyAnnotation string + maxRuntime time.Duration + maxKill int + master string + kubeconfig string + interval time.Duration + dryRun bool + debug bool + metricsAddress string + gracePeriod time.Duration + logFormat string + logCaller bool + slackWebhook string ) func init() { @@ -78,6 +79,7 @@ func init() { kingpin.Flag("excluded-days-of-year", "A list of days of a year when termination is suspended, e.g. Apr1,Dec24").StringVar(&excludedDaysOfYear) kingpin.Flag("timezone", "The timezone by which to interpret the excluded weekdays and times of day, e.g. UTC, Local, Europe/Berlin. Defaults to UTC.").Default("UTC").StringVar(&timezone) kingpin.Flag("minimum-age", "Minimum age of pods to consider for termination").Default("0s").DurationVar(&minimumAge) + kingpin.Flag("termination-frequency-annotation", "Annotation to look for on pods describing how frequently a pod should be terminated.").StringVar(&frequencyAnnotation) kingpin.Flag("max-runtime", "Maximum runtime before chaoskube exits").Default("-1s").DurationVar(&maxRuntime) kingpin.Flag("max-kill", "Specifies the maximum number of pods to be terminated per interval.").Default("1").IntVar(&maxKill) kingpin.Flag("master", "The address of the Kubernetes cluster to target").StringVar(&master) @@ -110,29 +112,30 @@ func main() { log.SetReportCaller(logCaller) log.WithFields(log.Fields{ - "labels": labelString, - "annotations": annString, - "kinds": kindsString, - "namespaces": nsString, - "namespaceLabels": nsLabelString, - "includedPodNames": includedPodNames, - "excludedPodNames": excludedPodNames, - "excludedWeekdays": excludedWeekdays, - "excludedTimesOfDay": excludedTimesOfDay, - "excludedDaysOfYear": excludedDaysOfYear, - "timezone": timezone, - "minimumAge": minimumAge, - "maxRuntime": maxRuntime, - "maxKill": maxKill, - "master": master, - "kubeconfig": kubeconfig, - "interval": interval, - "dryRun": dryRun, - "debug": debug, - "metricsAddress": metricsAddress, - "gracePeriod": gracePeriod, - "logFormat": logFormat, - "slackWebhook": slackWebhook, + "labels": labelString, + "annotations": annString, + "kinds": kindsString, + "namespaces": nsString, + "namespaceLabels": nsLabelString, + "includedPodNames": includedPodNames, + "excludedPodNames": excludedPodNames, + "excludedWeekdays": excludedWeekdays, + "excludedTimesOfDay": excludedTimesOfDay, + "excludedDaysOfYear": excludedDaysOfYear, + "timezone": timezone, + "minimumAge": minimumAge, + "frequencyAnnotation": frequencyAnnotation, + "maxRuntime": maxRuntime, + "maxKill": maxKill, + "master": master, + "kubeconfig": kubeconfig, + "interval": interval, + "dryRun": dryRun, + "debug": debug, + "metricsAddress": metricsAddress, + "gracePeriod": gracePeriod, + "logFormat": logFormat, + "slackWebhook": slackWebhook, }).Debug("reading config") log.WithFields(log.Fields{ @@ -156,15 +159,16 @@ func main() { ) log.WithFields(log.Fields{ - "labels": labelSelector, - "annotations": annotations, - "kinds": kinds, - "namespaces": namespaces, - "namespaceLabels": namespaceLabels, - "includedPodNames": includedPodNames, - "excludedPodNames": excludedPodNames, - "minimumAge": minimumAge, - "maxKill": maxKill, + "labels": labelSelector, + "annotations": annotations, + "kinds": kinds, + "namespaces": namespaces, + "namespaceLabels": namespaceLabels, + "includedPodNames": includedPodNames, + "excludedPodNames": excludedPodNames, + "minimumAge": minimumAge, + "frequencyAnnotation": frequencyAnnotation, + "maxKill": maxKill, }).Info("setting pod filter") parsedWeekdays := util.ParseWeekdays(excludedWeekdays) @@ -208,6 +212,7 @@ func main() { chaoskube := chaoskube.New( client, + interval, labelSelector, annotations, kinds, @@ -220,6 +225,7 @@ func main() { parsedDaysOfYear, parsedTimezone, minimumAge, + frequencyAnnotation, log.StandardLogger(), dryRun, terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod), diff --git a/notifier/slack_test.go b/notifier/slack_test.go index a001b7bb..cfa2799a 100644 --- a/notifier/slack_test.go +++ b/notifier/slack_test.go @@ -5,8 +5,6 @@ import ( "net/http/httptest" "testing" - v1 "k8s.io/api/core/v1" - "github.com/linki/chaoskube/internal/testutil" "github.com/linki/chaoskube/util" @@ -28,7 +26,7 @@ func (suite *SlackSuite) TestSlackNotificationForTerminationStatusOk() { })) defer testServer.Close() - testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) + testPod := util.NewPodBuilder("chaos", "chaos-57df4db6b-h9ktj").Build() slack := NewSlackNotifier(testServer.URL + webhookPath) err := slack.NotifyPodTermination(testPod) @@ -47,7 +45,7 @@ func (suite *SlackSuite) TestSlackNotificationForTerminationStatus500() { })) defer testServer.Close() - testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) + testPod := util.NewPodBuilder("chaos", "chaos-57df4db6b-h9ktj").Build() slack := NewSlackNotifier(testServer.URL + webhookPath) err := slack.NotifyPodTermination(testPod) diff --git a/terminator/delete_pod_test.go b/terminator/delete_pod_test.go index 369092a9..bcaa9be7 100644 --- a/terminator/delete_pod_test.go +++ b/terminator/delete_pod_test.go @@ -41,8 +41,8 @@ func (suite *DeletePodTerminatorSuite) TestTerminate() { terminator := NewDeletePodTerminator(client, logger, 10*time.Second) pods := []v1.Pod{ - util.NewPod("default", "foo", v1.PodRunning), - util.NewPod("testing", "bar", v1.PodRunning), + util.NewPodBuilder("default", "foo").Build(), + util.NewPodBuilder("testing", "bar").Build(), } for _, pod := range pods { @@ -50,7 +50,7 @@ func (suite *DeletePodTerminatorSuite) TestTerminate() { suite.Require().NoError(err) } - victim := util.NewPod("default", "foo", v1.PodRunning) + victim := util.NewPodBuilder("default", "foo").Build() err := terminator.Terminate(context.Background(), victim) suite.Require().NoError(err) diff --git a/util/util.go b/util/util.go index a17ec54b..868dbe98 100644 --- a/util/util.go +++ b/util/util.go @@ -3,6 +3,7 @@ package util import ( "fmt" "math/rand" + "strconv" "strings" "time" @@ -120,6 +121,35 @@ func ParseDays(days string) ([]time.Time, error) { return parsedDays, nil } +// Parses a "frequency" annotation in the form "[number] / [period]" (eg. 1/day) +// and converts it into a chance of occurrence in any given interval (eg. ~0.007) +func ParseFrequency(text string, interval time.Duration) (float64, error) { + parseablePeriods := map[string]time.Duration{ + "minute": 1 * time.Minute, + "hour": 1 * time.Hour, + "day": 24 * time.Hour, + "week": 24 * 7 * time.Hour, + } + + parts := strings.SplitN(text, "/", 2) + for i, p := range parts { + parts[i] = strings.TrimSpace(p) + } + + frequency, err := strconv.ParseFloat(parts[0], 64) + if err != nil { + return 0, err + } + + period, ok := parseablePeriods[parts[1]] + if !ok { + return 0, fmt.Errorf("unknown time period, %v", parts[1]) + } + + chance := (float64(interval) / float64(period)) * frequency + return chance, nil +} + // TimeOfDay normalizes the given point in time by returning a time object that represents the same // time of day of the given time but on the very first day (day 0). func TimeOfDay(pointInTime time.Time) time.Time { @@ -135,43 +165,6 @@ func FormatDays(days []time.Time) []string { return formattedDays } -// NewPod returns a new pod instance for testing purposes. -func NewPod(namespace, name string, phase v1.PodPhase) v1.Pod { - return NewPodWithOwner(namespace, name, phase, "") -} - -// NewPodWithOwner returns a new pod instance for testing purposes with a given owner UID -func NewPodWithOwner(namespace, name string, phase v1.PodPhase, owner types.UID) v1.Pod { - pod := v1.Pod{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Pod", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - Labels: map[string]string{ - "app": name, - }, - Annotations: map[string]string{ - "chaos": name, - }, - SelfLink: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", namespace, name), - }, - Status: v1.PodStatus{ - Phase: phase, - }, - } - - if owner != "" { - pod.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ - {UID: owner, Kind: "testkind"}, - } - } - - return pod -} - // NewNamespace returns a new namespace instance for testing purposes. func NewNamespace(name string) v1.Namespace { return v1.Namespace{ @@ -195,3 +188,77 @@ func RandomPodSubSlice(pods []v1.Pod, count int) []v1.Pod { res := pods[0:count] return res } + +type PodBuilder struct { + Name string + Namespace string + Phase v1.PodPhase + OwnerReference *metav1.OwnerReference + Labels map[string]string + Annotations map[string]string +} + +func NewPodBuilder(namespace string, name string) PodBuilder { + return PodBuilder{ + Name: name, + Namespace: namespace, + Phase: v1.PodRunning, + OwnerReference: nil, + Annotations: make(map[string]string), + Labels: make(map[string]string), + } +} + +func (b PodBuilder) Build() v1.Pod { + pod := v1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + Name: b.Name, + Labels: b.Labels, + Annotations: b.Annotations, + SelfLink: fmt.Sprintf( + "/api/v1/namespaces/%s/pods/%s", + b.Namespace, + b.Name, + ), + }, + Status: v1.PodStatus{ + Phase: b.Phase, + }, + } + + if b.OwnerReference != nil { + pod.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*b.OwnerReference} + } + + return pod +} + +func (b PodBuilder) WithPhase(phase v1.PodPhase) PodBuilder { + b.Phase = phase + return b +} +func (b PodBuilder) WithOwnerReference(ownerReference metav1.OwnerReference) PodBuilder { + b.OwnerReference = &ownerReference + return b +} +func (b PodBuilder) WithOwnerUID(owner types.UID) PodBuilder { + b.OwnerReference = &metav1.OwnerReference{UID: owner, Kind: "testkind"} + return b +} +func (b PodBuilder) WithAnnotations(annotations map[string]string) PodBuilder { + b.Annotations = annotations + return b +} +func (b PodBuilder) WithLabels(labels map[string]string) PodBuilder { + b.Labels = labels + return b +} +func (b PodBuilder) WithFrequency(text string) PodBuilder { + b.Annotations["chaos.alpha.kubernetes.io/frequency"] = text + return b +} diff --git a/util/util_test.go b/util/util_test.go index 98e073de..40b59ad0 100644 --- a/util/util_test.go +++ b/util/util_test.go @@ -366,6 +366,34 @@ func (suite *Suite) TestParseDates() { } } +func (suite *Suite) TestParseFrequency() { + interval := 10 * time.Minute + + for _, tt := range []struct { + given string + expectedApprox float64 + }{ + { + "1 / hour", + 0.166666667, + }, { + "1 / minute", + 10.0, + }, { + "2.5 / hour", + 0.416666667, + }, { + "60 / day", + 0.416666667, + }, + } { + result, err := ParseFrequency(tt.given, interval) + suite.Require().NoError(err) + + suite.Assert().InDelta(tt.expectedApprox, result, 0.001) + } +} + func (suite *Suite) TestFormatDays() { for _, tt := range []struct { given []time.Time @@ -389,7 +417,8 @@ func (suite *Suite) TestFormatDays() { } func (suite *Suite) TestNewPod() { - pod := NewPod("namespace", "name", "phase") + pod := NewPodBuilder("namespace", "name"). + WithPhase("phase").Build() suite.Equal("v1", pod.APIVersion) suite.Equal("Pod", pod.Kind) @@ -410,9 +439,9 @@ func (suite *Suite) TestNewNamespace() { func (suite *Suite) TestRandomPodSublice() { pods := []v1.Pod{ - NewPod("default", "foo", v1.PodRunning), - NewPod("testing", "bar", v1.PodRunning), - NewPod("test", "baz", v1.PodRunning), + NewPodBuilder("default", "foo").Build(), + NewPodBuilder("testing", "bar").Build(), + NewPodBuilder("test", "baz").Build(), } for _, tt := range []struct { From 7a7bfabe1b036fbed30b5356381bc3bee8ab1c4e Mon Sep 17 00:00:00 2001 From: David Symons Date: Thu, 1 Jul 2021 18:51:47 +1000 Subject: [PATCH 2/7] Add ability to configure default frequency for pods lacking annotation --- chaoskube/chaoskube.go | 17 ++++++++---- chaoskube/chaoskube_test.go | 55 +++++++++++++++++++++++++++---------- main.go | 4 +++ 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index 3da75cf1..bba637a1 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -61,6 +61,8 @@ type Chaoskube struct { MinimumAge time.Duration // an annotation containing the frequency to kill a pod at FrequencyAnnotation string + // default frequency for pods lacking annotation + DefaultFrequency string // an instance of logrus.StdLogger to write log messages to Logger log.FieldLogger // a terminator that terminates victim pods @@ -100,7 +102,7 @@ var ( // * a logger implementing logrus.FieldLogger to send log output to // * what specific terminator to use to imbue chaos on victim pods // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, interval time.Duration, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube { +func New(client kubernetes.Interface, interval time.Duration, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation, defaultFrequency string, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"}) @@ -121,6 +123,7 @@ func New(client kubernetes.Interface, interval time.Duration, labels, annotation Timezone: timezone, MinimumAge: minimumAge, FrequencyAnnotation: frequencyAnnotation, + DefaultFrequency: defaultFrequency, Logger: logger, DryRun: dryRun, Terminator: terminator, @@ -245,7 +248,7 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) pods = filterByOwnerReference(pods) - pods = filterByFrequency(pods, c.FrequencyAnnotation, c.Interval, c.Logger) + pods = filterByFrequency(pods, c.FrequencyAnnotation, c.DefaultFrequency, c.Interval, c.Logger) return pods, nil } @@ -543,8 +546,8 @@ func filterByOwnerReference(pods []v1.Pod) []v1.Pod { return filteredList } -func filterByFrequency(pods []v1.Pod, annotation string, interval time.Duration, logger log.FieldLogger) []v1.Pod { - if annotation == "" { +func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string, interval time.Duration, logger log.FieldLogger) []v1.Pod { + if annotation == "" && defaultFrequency == "" { return pods } @@ -554,7 +557,11 @@ func filterByFrequency(pods []v1.Pod, annotation string, interval time.Duration, // Don't filter out pods missing frequency annotation if !ok { - filteredList = append(filteredList, pod) + if defaultFrequency == "" { + filteredList = append(filteredList, pod) + } else { + text = defaultFrequency + } } chance, err := util.ParseFrequency(text, interval) diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 1d5650b2..3708f60e 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -62,6 +62,7 @@ func (suite *Suite) TestNew() { excludedDaysOfYear = []time.Time{time.Now()} minimumAge = time.Duration(42) frequencyAnnotation = "chaos.alpha.kubernetes.io/frequency" + defaultFrequency = "1 / hour" dryRun = true terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) maxKill = 1 @@ -84,6 +85,7 @@ func (suite *Suite) TestNew() { time.UTC, minimumAge, frequencyAnnotation, + defaultFrequency, logger, dryRun, terminator, @@ -107,6 +109,7 @@ func (suite *Suite) TestNew() { suite.Equal(time.UTC, chaoskube.Timezone) suite.Equal(minimumAge, chaoskube.MinimumAge) suite.Equal("chaos.alpha.kubernetes.io/frequency", chaoskube.FrequencyAnnotation) + suite.Equal("1 / hour", chaoskube.DefaultFrequency) suite.Equal(logger, chaoskube.Logger) suite.Equal(dryRun, chaoskube.DryRun) suite.Equal(terminator, chaoskube.Terminator) @@ -129,6 +132,7 @@ func (suite *Suite) TestRunContextCanceled() { time.UTC, time.Duration(0), "", + "", false, 10, 1, @@ -187,6 +191,7 @@ func (suite *Suite) TestCandidates() { time.UTC, time.Duration(0), "", + "", false, 10, ) @@ -234,6 +239,7 @@ func (suite *Suite) TestCandidatesNamespaceLabels() { time.UTC, time.Duration(0), "", + "", false, 10, ) @@ -279,6 +285,7 @@ func (suite *Suite) TestCandidatesPodNameRegexp() { time.UTC, time.Duration(0), "", + "", false, 10, ) @@ -321,6 +328,7 @@ func (suite *Suite) TestVictim() { time.UTC, time.Duration(0), "", + "", false, 10, ) @@ -376,6 +384,7 @@ func (suite *Suite) TestVictims() { time.UTC, time.Duration(0), "", + "", false, 10, tt.maxKill, @@ -403,6 +412,7 @@ func (suite *Suite) TestNoVictimReturnsError() { time.UTC, time.Duration(0), "", + "", false, 10, 1, @@ -440,6 +450,7 @@ func (suite *Suite) TestDeletePod() { time.UTC, time.Duration(0), "", + "", tt.dryRun, 10, ) @@ -471,6 +482,7 @@ func (suite *Suite) TestDeletePodNotFound() { time.UTC, time.Duration(0), "", + "", false, 10, 1, @@ -705,6 +717,7 @@ func (suite *Suite) TestTerminateVictim() { tt.timezone, time.Duration(0), "", + "", false, 10, ) @@ -737,6 +750,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() { time.UTC, time.Duration(0), "", + "", false, 10, 1, @@ -774,7 +788,7 @@ func (suite *Suite) assertNotified(notifier *notifier.Noop) { suite.Assert().Greater(notifier.Calls, 0) } -func (suite *Suite) setupWithPods(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, dryRun bool, gracePeriod time.Duration) *Chaoskube { +func (suite *Suite) setupWithPods(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, defaultFrequency string, dryRun bool, gracePeriod time.Duration) *Chaoskube { chaoskube := suite.setup( 10*time.Minute, labelSelector, @@ -790,6 +804,7 @@ func (suite *Suite) setupWithPods(interval time.Duration, labelSelector labels.S timezone, minimumAge, frequencyAnnotation, + defaultFrequency, dryRun, gracePeriod, 1, @@ -828,7 +843,7 @@ func (suite *Suite) createPods(client kubernetes.Interface, podsInfo []podInfo) } } -func (suite *Suite) setup(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, dryRun bool, gracePeriod time.Duration, maxKill int) *Chaoskube { +func (suite *Suite) setup(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, defaultFrequency string, dryRun bool, gracePeriod time.Duration, maxKill int) *Chaoskube { logOutput.Reset() client := fake.NewSimpleClientset() @@ -850,6 +865,7 @@ func (suite *Suite) setup(interval time.Duration, labelSelector labels.Selector, timezone, minimumAge, frequencyAnnotation, + defaultFrequency, logger, dryRun, terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod), @@ -959,6 +975,7 @@ func (suite *Suite) TestMinimumAge() { time.UTC, tt.minimumAge, "", + "", false, 10, 1, @@ -1138,31 +1155,38 @@ func (suite *Suite) TestFilterByFrequency() { baz := util.NewPodBuilder("default", "baz").Build() pods := []v1.Pod{foo, foo1, bar, baz} - alwaysExpected := []v1.Pod{foo1, baz} for _, tt := range []struct { - seed int64 - expected []v1.Pod + seed int64 + expected []v1.Pod + defaultFrequency string }{ { - seed: 1000, - expected: []v1.Pod{}, + seed: 1000, + expected: []v1.Pod{foo1, baz}, + defaultFrequency: "", }, { - seed: 3000, - expected: []v1.Pod{foo}, + seed: 3000, + expected: []v1.Pod{foo, foo1, baz}, + defaultFrequency: "", + }, + { + seed: 4000, + expected: []v1.Pod{foo1, bar, baz}, + defaultFrequency: "", }, { - seed: 4000, - expected: []v1.Pod{bar}, + seed: 1000, + expected: []v1.Pod{foo1}, + defaultFrequency: "0.1 / hour", // Should force pod to be terminated every interval }, } { - expected := append(tt.expected, alwaysExpected...) - rand.Seed(tt.seed) - results := filterByFrequency(pods, "chaos.alpha.kubernetes.io/frequency", interval, logger) + results := filterByFrequency(pods, "chaos.alpha.kubernetes.io/frequency", + tt.defaultFrequency, interval, logger) - suite.Assert().ElementsMatch(results, expected) + suite.Assert().ElementsMatch(tt.expected, results) } } @@ -1182,6 +1206,7 @@ func (suite *Suite) TestNotifierCall() { time.UTC, time.Duration(0), "", + "", false, 10, ) diff --git a/main.go b/main.go index ffce7d4c..c498b054 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ var ( timezone string minimumAge time.Duration frequencyAnnotation string + defaultFrequency string maxRuntime time.Duration maxKill int master string @@ -80,6 +81,7 @@ func init() { kingpin.Flag("timezone", "The timezone by which to interpret the excluded weekdays and times of day, e.g. UTC, Local, Europe/Berlin. Defaults to UTC.").Default("UTC").StringVar(&timezone) kingpin.Flag("minimum-age", "Minimum age of pods to consider for termination").Default("0s").DurationVar(&minimumAge) kingpin.Flag("termination-frequency-annotation", "Annotation to look for on pods describing how frequently a pod should be terminated.").StringVar(&frequencyAnnotation) + kingpin.Flag("default-termination-frequency", "Default termination frequency to apply to pods without the annotation.").StringVar(&defaultFrequency) kingpin.Flag("max-runtime", "Maximum runtime before chaoskube exits").Default("-1s").DurationVar(&maxRuntime) kingpin.Flag("max-kill", "Specifies the maximum number of pods to be terminated per interval.").Default("1").IntVar(&maxKill) kingpin.Flag("master", "The address of the Kubernetes cluster to target").StringVar(&master) @@ -125,6 +127,7 @@ func main() { "timezone": timezone, "minimumAge": minimumAge, "frequencyAnnotation": frequencyAnnotation, + "defaultFrequency": defaultFrequency, "maxRuntime": maxRuntime, "maxKill": maxKill, "master": master, @@ -226,6 +229,7 @@ func main() { parsedTimezone, minimumAge, frequencyAnnotation, + defaultFrequency, log.StandardLogger(), dryRun, terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod), From 6e9f615e6be09897fc012e07a7edad81b628c5ba Mon Sep 17 00:00:00 2001 From: David Symons Date: Fri, 2 Jul 2021 11:30:58 +1000 Subject: [PATCH 3/7] Create common annotation prefix for overriding config via pod annotations --- chaoskube/chaoskube.go | 66 +++++++++++-------- chaoskube/chaoskube_test.go | 55 ++++++++-------- chart/chaoskube/values.yaml | 6 +- main.go | 126 ++++++++++++++++++------------------ util/util.go | 6 +- 5 files changed, 138 insertions(+), 121 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index bba637a1..aee665e5 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "regexp" + "strings" "time" multierror "github.com/hashicorp/go-multierror" @@ -59,8 +60,9 @@ type Chaoskube struct { Timezone *time.Location // minimum age of pods to consider MinimumAge time.Duration - // an annotation containing the frequency to kill a pod at - FrequencyAnnotation string + // the annotation prefix (eg. "chaos.alpha.kubernetes.io") to use when + // looking for configuration overrides in pod annotations + ConfigAnnotationPrefix string // default frequency for pods lacking annotation DefaultFrequency string // an instance of logrus.StdLogger to write log messages to @@ -102,35 +104,35 @@ var ( // * a logger implementing logrus.FieldLogger to send log output to // * what specific terminator to use to imbue chaos on victim pods // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, interval time.Duration, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation, defaultFrequency string, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube { +func New(client kubernetes.Interface, interval time.Duration, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, configAnnotationPrefix, defaultFrequency string, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"}) return &Chaoskube{ - Client: client, - Interval: interval, - Labels: labels, - Annotations: annotations, - Kinds: kinds, - Namespaces: namespaces, - NamespaceLabels: namespaceLabels, - IncludedPodNames: includedPodNames, - ExcludedPodNames: excludedPodNames, - ExcludedWeekdays: excludedWeekdays, - ExcludedTimesOfDay: excludedTimesOfDay, - ExcludedDaysOfYear: excludedDaysOfYear, - Timezone: timezone, - MinimumAge: minimumAge, - FrequencyAnnotation: frequencyAnnotation, - DefaultFrequency: defaultFrequency, - Logger: logger, - DryRun: dryRun, - Terminator: terminator, - EventRecorder: recorder, - Now: time.Now, - MaxKill: maxKill, - Notifier: notifier, + Client: client, + Interval: interval, + Labels: labels, + Annotations: annotations, + Kinds: kinds, + Namespaces: namespaces, + NamespaceLabels: namespaceLabels, + IncludedPodNames: includedPodNames, + ExcludedPodNames: excludedPodNames, + ExcludedWeekdays: excludedWeekdays, + ExcludedTimesOfDay: excludedTimesOfDay, + ExcludedDaysOfYear: excludedDaysOfYear, + Timezone: timezone, + MinimumAge: minimumAge, + ConfigAnnotationPrefix: configAnnotationPrefix, + DefaultFrequency: defaultFrequency, + Logger: logger, + DryRun: dryRun, + Terminator: terminator, + EventRecorder: recorder, + Now: time.Now, + MaxKill: maxKill, + Notifier: notifier, } } @@ -248,7 +250,13 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) pods = filterByOwnerReference(pods) - pods = filterByFrequency(pods, c.FrequencyAnnotation, c.DefaultFrequency, c.Interval, c.Logger) + pods = filterByFrequency( + pods, + strings.Join([]string{c.ConfigAnnotationPrefix, "frequency"}, "/"), + c.DefaultFrequency, + c.Interval, + c.Logger, + ) return pods, nil } @@ -559,6 +567,7 @@ func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string if !ok { if defaultFrequency == "" { filteredList = append(filteredList, pod) + continue } else { text = defaultFrequency } @@ -566,7 +575,8 @@ func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string chance, err := util.ParseFrequency(text, interval) if err != nil { - logger.WithField("err", err).Warn("failed to parse frequency annotation") + logger.WithField("err", err).Warn("failed to parse frequency annotation, excluding from candidates") + continue } if chance > rand.Float64() { diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 3708f60e..421ec2b0 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -5,6 +5,7 @@ import ( "math/rand" "regexp" "sort" + "strings" "testing" "time" @@ -48,25 +49,25 @@ func (suite *Suite) SetupTest() { // TestNew tests that arguments are passed to the new instance correctly func (suite *Suite) TestNew() { var ( - client = fake.NewSimpleClientset() - interval = 10 * time.Minute - labelSelector, _ = labels.Parse("foo=bar") - annotations, _ = labels.Parse("baz=waldo") - kinds, _ = labels.Parse("job") - namespaces, _ = labels.Parse("qux") - namespaceLabels, _ = labels.Parse("taz=wubble") - includedPodNames = regexp.MustCompile("foo") - excludedPodNames = regexp.MustCompile("bar") - excludedWeekdays = []time.Weekday{time.Friday} - excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}} - excludedDaysOfYear = []time.Time{time.Now()} - minimumAge = time.Duration(42) - frequencyAnnotation = "chaos.alpha.kubernetes.io/frequency" - defaultFrequency = "1 / hour" - dryRun = true - terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) - maxKill = 1 - notifier = testNotifier + client = fake.NewSimpleClientset() + interval = 10 * time.Minute + labelSelector, _ = labels.Parse("foo=bar") + annotations, _ = labels.Parse("baz=waldo") + kinds, _ = labels.Parse("job") + namespaces, _ = labels.Parse("qux") + namespaceLabels, _ = labels.Parse("taz=wubble") + includedPodNames = regexp.MustCompile("foo") + excludedPodNames = regexp.MustCompile("bar") + excludedWeekdays = []time.Weekday{time.Friday} + excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}} + excludedDaysOfYear = []time.Time{time.Now()} + minimumAge = time.Duration(42) + configAnnotationPrefix = "chaos.alpha.kubernetes.io" + defaultFrequency = "1 / hour" + dryRun = true + terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) + maxKill = 1 + notifier = testNotifier ) chaoskube := New( @@ -84,7 +85,7 @@ func (suite *Suite) TestNew() { excludedDaysOfYear, time.UTC, minimumAge, - frequencyAnnotation, + configAnnotationPrefix, defaultFrequency, logger, dryRun, @@ -108,7 +109,7 @@ func (suite *Suite) TestNew() { suite.Equal(excludedDaysOfYear, chaoskube.ExcludedDaysOfYear) suite.Equal(time.UTC, chaoskube.Timezone) suite.Equal(minimumAge, chaoskube.MinimumAge) - suite.Equal("chaos.alpha.kubernetes.io/frequency", chaoskube.FrequencyAnnotation) + suite.Equal("chaos.alpha.kubernetes.io", chaoskube.ConfigAnnotationPrefix) suite.Equal("1 / hour", chaoskube.DefaultFrequency) suite.Equal(logger, chaoskube.Logger) suite.Equal(dryRun, chaoskube.DryRun) @@ -788,7 +789,7 @@ func (suite *Suite) assertNotified(notifier *notifier.Noop) { suite.Assert().Greater(notifier.Calls, 0) } -func (suite *Suite) setupWithPods(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, defaultFrequency string, dryRun bool, gracePeriod time.Duration) *Chaoskube { +func (suite *Suite) setupWithPods(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, configAnnotationPrefix string, defaultFrequency string, dryRun bool, gracePeriod time.Duration) *Chaoskube { chaoskube := suite.setup( 10*time.Minute, labelSelector, @@ -803,7 +804,7 @@ func (suite *Suite) setupWithPods(interval time.Duration, labelSelector labels.S excludedDaysOfYear, timezone, minimumAge, - frequencyAnnotation, + configAnnotationPrefix, defaultFrequency, dryRun, gracePeriod, @@ -843,7 +844,7 @@ func (suite *Suite) createPods(client kubernetes.Interface, podsInfo []podInfo) } } -func (suite *Suite) setup(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, frequencyAnnotation string, defaultFrequency string, dryRun bool, gracePeriod time.Duration, maxKill int) *Chaoskube { +func (suite *Suite) setup(interval time.Duration, labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, configAnnotationPrefix string, defaultFrequency string, dryRun bool, gracePeriod time.Duration, maxKill int) *Chaoskube { logOutput.Reset() client := fake.NewSimpleClientset() @@ -864,7 +865,7 @@ func (suite *Suite) setup(interval time.Duration, labelSelector labels.Selector, excludedDaysOfYear, timezone, minimumAge, - frequencyAnnotation, + configAnnotationPrefix, defaultFrequency, logger, dryRun, @@ -1183,7 +1184,9 @@ func (suite *Suite) TestFilterByFrequency() { }, } { rand.Seed(tt.seed) - results := filterByFrequency(pods, "chaos.alpha.kubernetes.io/frequency", + + annotation := strings.Join([]string{util.DefaultBaseAnnotation, "frequency"}, "/") + results := filterByFrequency(pods, annotation, tt.defaultFrequency, interval, logger) suite.Assert().ElementsMatch(tt.expected, results) diff --git a/chart/chaoskube/values.yaml b/chart/chaoskube/values.yaml index fed1b825..53144e61 100644 --- a/chart/chaoskube/values.yaml +++ b/chart/chaoskube/values.yaml @@ -33,9 +33,9 @@ chaoskube: timezone: "UTC" # exclude all pods that haven't been running for at least one hour minimum-age: "1h" - # checks for "chaos.alpha.kubernetes.io/frequency" annotation on pods to determine frequency to terminate - # eg. setting "chaos.alpha.kubernetes.io/frequency=1/hour" will terminate the pod approximately once per hour - termination-frequency-annotation: "chaos.alpha.kubernetes.io/frequency" + # sets the annotation prefix to use when looking for configuration overrides in pod annotations + # eg. termination frequency will look for the annotation "chaos.alpha.kubernetes.io/frequency" + config-annotation-prefix: "chaos.alpha.kubernetes.io" # terminate pods for real: this disables dry-run mode which is on by default no-dry-run: "" diff --git a/main.go b/main.go index c498b054..2f31766e 100644 --- a/main.go +++ b/main.go @@ -36,32 +36,32 @@ var ( ) var ( - labelString string - annString string - kindsString string - nsString string - nsLabelString string - includedPodNames *regexp.Regexp - excludedPodNames *regexp.Regexp - excludedWeekdays string - excludedTimesOfDay string - excludedDaysOfYear string - timezone string - minimumAge time.Duration - frequencyAnnotation string - defaultFrequency string - maxRuntime time.Duration - maxKill int - master string - kubeconfig string - interval time.Duration - dryRun bool - debug bool - metricsAddress string - gracePeriod time.Duration - logFormat string - logCaller bool - slackWebhook string + labelString string + annString string + kindsString string + nsString string + nsLabelString string + includedPodNames *regexp.Regexp + excludedPodNames *regexp.Regexp + excludedWeekdays string + excludedTimesOfDay string + excludedDaysOfYear string + timezone string + minimumAge time.Duration + configAnnotationPrefix string + defaultFrequency string + maxRuntime time.Duration + maxKill int + master string + kubeconfig string + interval time.Duration + dryRun bool + debug bool + metricsAddress string + gracePeriod time.Duration + logFormat string + logCaller bool + slackWebhook string ) func init() { @@ -80,7 +80,7 @@ func init() { kingpin.Flag("excluded-days-of-year", "A list of days of a year when termination is suspended, e.g. Apr1,Dec24").StringVar(&excludedDaysOfYear) kingpin.Flag("timezone", "The timezone by which to interpret the excluded weekdays and times of day, e.g. UTC, Local, Europe/Berlin. Defaults to UTC.").Default("UTC").StringVar(&timezone) kingpin.Flag("minimum-age", "Minimum age of pods to consider for termination").Default("0s").DurationVar(&minimumAge) - kingpin.Flag("termination-frequency-annotation", "Annotation to look for on pods describing how frequently a pod should be terminated.").StringVar(&frequencyAnnotation) + kingpin.Flag("config-annotation-prefix", "Annotation prefix to use when looking for configuration overrides in pod annotations. Defaults to 'chaos.alpha.kubernetes.io'.").Default(util.DefaultBaseAnnotation).StringVar(&configAnnotationPrefix) kingpin.Flag("default-termination-frequency", "Default termination frequency to apply to pods without the annotation.").StringVar(&defaultFrequency) kingpin.Flag("max-runtime", "Maximum runtime before chaoskube exits").Default("-1s").DurationVar(&maxRuntime) kingpin.Flag("max-kill", "Specifies the maximum number of pods to be terminated per interval.").Default("1").IntVar(&maxKill) @@ -114,31 +114,31 @@ func main() { log.SetReportCaller(logCaller) log.WithFields(log.Fields{ - "labels": labelString, - "annotations": annString, - "kinds": kindsString, - "namespaces": nsString, - "namespaceLabels": nsLabelString, - "includedPodNames": includedPodNames, - "excludedPodNames": excludedPodNames, - "excludedWeekdays": excludedWeekdays, - "excludedTimesOfDay": excludedTimesOfDay, - "excludedDaysOfYear": excludedDaysOfYear, - "timezone": timezone, - "minimumAge": minimumAge, - "frequencyAnnotation": frequencyAnnotation, - "defaultFrequency": defaultFrequency, - "maxRuntime": maxRuntime, - "maxKill": maxKill, - "master": master, - "kubeconfig": kubeconfig, - "interval": interval, - "dryRun": dryRun, - "debug": debug, - "metricsAddress": metricsAddress, - "gracePeriod": gracePeriod, - "logFormat": logFormat, - "slackWebhook": slackWebhook, + "labels": labelString, + "annotations": annString, + "kinds": kindsString, + "namespaces": nsString, + "namespaceLabels": nsLabelString, + "includedPodNames": includedPodNames, + "excludedPodNames": excludedPodNames, + "excludedWeekdays": excludedWeekdays, + "excludedTimesOfDay": excludedTimesOfDay, + "excludedDaysOfYear": excludedDaysOfYear, + "timezone": timezone, + "minimumAge": minimumAge, + "configAnnotationPrefix": configAnnotationPrefix, + "defaultFrequency": defaultFrequency, + "maxRuntime": maxRuntime, + "maxKill": maxKill, + "master": master, + "kubeconfig": kubeconfig, + "interval": interval, + "dryRun": dryRun, + "debug": debug, + "metricsAddress": metricsAddress, + "gracePeriod": gracePeriod, + "logFormat": logFormat, + "slackWebhook": slackWebhook, }).Debug("reading config") log.WithFields(log.Fields{ @@ -162,16 +162,16 @@ func main() { ) log.WithFields(log.Fields{ - "labels": labelSelector, - "annotations": annotations, - "kinds": kinds, - "namespaces": namespaces, - "namespaceLabels": namespaceLabels, - "includedPodNames": includedPodNames, - "excludedPodNames": excludedPodNames, - "minimumAge": minimumAge, - "frequencyAnnotation": frequencyAnnotation, - "maxKill": maxKill, + "labels": labelSelector, + "annotations": annotations, + "kinds": kinds, + "namespaces": namespaces, + "namespaceLabels": namespaceLabels, + "includedPodNames": includedPodNames, + "excludedPodNames": excludedPodNames, + "minimumAge": minimumAge, + "configAnnotationPrefix": configAnnotationPrefix, + "maxKill": maxKill, }).Info("setting pod filter") parsedWeekdays := util.ParseWeekdays(excludedWeekdays) @@ -228,7 +228,7 @@ func main() { parsedDaysOfYear, parsedTimezone, minimumAge, - frequencyAnnotation, + configAnnotationPrefix, defaultFrequency, log.StandardLogger(), dryRun, diff --git a/util/util.go b/util/util.go index 868dbe98..60c6542c 100644 --- a/util/util.go +++ b/util/util.go @@ -17,6 +17,8 @@ const ( Kitchen24 = "15:04" // a time format that just cares about the day and month. YearDay = "Jan_2" + + DefaultBaseAnnotation = "chaos.alpha.kubernetes.io" ) // TimePeriod represents a time period with a single beginning and end. @@ -259,6 +261,8 @@ func (b PodBuilder) WithLabels(labels map[string]string) PodBuilder { return b } func (b PodBuilder) WithFrequency(text string) PodBuilder { - b.Annotations["chaos.alpha.kubernetes.io/frequency"] = text + annotation := strings.Join([]string{DefaultBaseAnnotation, "frequency"}, "/") + + b.Annotations[annotation] = text return b } From 4d234b2a59ef75f90bda504096356a671afa01b0 Mon Sep 17 00:00:00 2001 From: David Symons Date: Fri, 2 Jul 2021 11:31:33 +1000 Subject: [PATCH 4/7] Enable overriding minimum age via pod annotation --- chaoskube/chaoskube.go | 31 +++++++-- chaoskube/chaoskube_test.go | 123 +++++++++++------------------------- util/util.go | 16 +++++ 3 files changed, 79 insertions(+), 91 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index aee665e5..c8c06fd9 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -246,7 +246,15 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { pods = filterByAnnotations(pods, c.Annotations) pods = filterByPhase(pods, v1.PodRunning) pods = filterTerminatingPods(pods) - pods = filterByMinimumAge(pods, c.MinimumAge, c.Now()) + + pods = filterByMinimumAge( + pods, + strings.Join([]string{c.ConfigAnnotationPrefix, "minimum-age"}, "/"), + c.MinimumAge, + c.Now(), + c.Logger, + ) + pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) pods = filterByOwnerReference(pods) @@ -490,16 +498,29 @@ func filterTerminatingPods(pods []v1.Pod) []v1.Pod { // filterByMinimumAge filters pods by creation time. Only pods // older than minimumAge are returned -func filterByMinimumAge(pods []v1.Pod, minimumAge time.Duration, now time.Time) []v1.Pod { - if minimumAge <= time.Duration(0) { +func filterByMinimumAge(pods []v1.Pod, annotation string, minimumAge time.Duration, now time.Time, logger log.FieldLogger) []v1.Pod { + if annotation == "" && minimumAge <= time.Duration(0) { return pods } - creationTime := now.Add(-minimumAge) - + defaultCreationTime := now.Add(-minimumAge) filteredList := []v1.Pod{} for _, pod := range pods { + text, ok := pod.Annotations[annotation] + + // Don't filter out pods missing frequency annotation + creationTime := defaultCreationTime + if ok { + minimumAgeOverride, err := time.ParseDuration(text) + if err != nil { + logger.WithField("err", err).Warn("failed to parse frequency annotation, excluding from candidates") + continue + } + + creationTime = now.Add(-minimumAgeOverride) + } + if pod.ObjectMeta.CreationTimestamp.Time.Before(creationTime) { filteredList = append(filteredList, pod) } diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 421ec2b0..4351ce35 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -889,111 +889,62 @@ func (t ThankGodItsFriday) Now() time.Time { } func (suite *Suite) TestMinimumAge() { - type pod struct { - name string - namespace string - creationTime time.Time - } + logger, _ := test.NewNullLogger() + minimumAge := 1 * time.Hour + + now := time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) + cutoff := now.Add(-minimumAge) + + tooYoung := util.NewPodBuilder("default", "young").WithCreationTime(cutoff.Add(5 * time.Minute)).Build() + exactMatch := util.NewPodBuilder("default", "exact").WithCreationTime(cutoff).Build() + oldEnough := util.NewPodBuilder("default", "old").WithCreationTime(cutoff.Add(-5 * time.Minute)).Build() + + overriddenMinAge := util.NewPodBuilder("default", "overridden"). + WithCreationTime(now.Add(-30 * time.Minute)). + WithMinimumAge("10m"). + Build() for _, tt := range []struct { minimumAge time.Duration - now func() time.Time - pods []pod - candidates int + pods []v1.Pod + expected []v1.Pod }{ // no minimum age set { time.Duration(0), - func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, - []pod{ - { - name: "test1", - namespace: "test", - creationTime: time.Date(0, 10, 24, 9, 00, 00, 00, time.UTC), - }, - }, - 1, + []v1.Pod{tooYoung}, + []v1.Pod{tooYoung}, }, // minimum age set, but pod is too young { - time.Hour * 1, - func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, - []pod{ - { - name: "test1", - namespace: "test", - creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC), - }, - }, - 0, + minimumAge, + []v1.Pod{tooYoung}, + []v1.Pod{}, }, // one pod is too young, one matches { - time.Hour * 1, - func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, - []pod{ - // too young - { - name: "test1", - namespace: "test", - creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC), - }, - // matches - { - name: "test2", - namespace: "test", - creationTime: time.Date(0, 10, 23, 8, 00, 00, 00, time.UTC), - }, - }, - 1, + minimumAge, + []v1.Pod{tooYoung, oldEnough}, + []v1.Pod{oldEnough}, }, // exact time - should not match { - time.Hour * 1, - func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, - []pod{ - { - name: "test1", - namespace: "test", - creationTime: time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC), - }, - }, - 0, + minimumAge, + []v1.Pod{exactMatch}, + []v1.Pod{}, + }, + // overridden minimum age - should match + { + minimumAge, + []v1.Pod{overriddenMinAge}, + []v1.Pod{overriddenMinAge}, }, } { - chaoskube := suite.setup( - 10*time.Minute, - labels.Everything(), - labels.Everything(), - labels.Everything(), - labels.Everything(), - labels.Everything(), - ®exp.Regexp{}, - ®exp.Regexp{}, - []time.Weekday{}, - []util.TimePeriod{}, - []time.Time{}, - time.UTC, - tt.minimumAge, - "", - "", - false, - 10, - 1, - ) - chaoskube.Now = tt.now - - for _, p := range tt.pods { - pod := util.NewPodBuilder(p.namespace, p.name).Build() - pod.ObjectMeta.CreationTimestamp = metav1.Time{Time: p.creationTime} - _, err := chaoskube.Client.CoreV1().Pods(pod.Namespace).Create(context.Background(), &pod, metav1.CreateOptions{}) - suite.Require().NoError(err) - } - - pods, err := chaoskube.Candidates(context.Background()) - suite.Require().NoError(err) + annotation := strings.Join([]string{util.DefaultBaseAnnotation, "minimum-age"}, "/") + pods := filterByMinimumAge(tt.pods, annotation, + tt.minimumAge, now, logger) - suite.Len(pods, tt.candidates) + suite.Assert().ElementsMatch(tt.expected, pods) } } diff --git a/util/util.go b/util/util.go index 60c6542c..ceed4171 100644 --- a/util/util.go +++ b/util/util.go @@ -195,6 +195,7 @@ type PodBuilder struct { Name string Namespace string Phase v1.PodPhase + CreationTime *time.Time OwnerReference *metav1.OwnerReference Labels map[string]string Annotations map[string]string @@ -205,6 +206,7 @@ func NewPodBuilder(namespace string, name string) PodBuilder { Name: name, Namespace: namespace, Phase: v1.PodRunning, + CreationTime: nil, OwnerReference: nil, Annotations: make(map[string]string), Labels: make(map[string]string), @@ -233,6 +235,10 @@ func (b PodBuilder) Build() v1.Pod { }, } + if b.CreationTime != nil { + pod.ObjectMeta.CreationTimestamp = metav1.Time{Time: *b.CreationTime} + } + if b.OwnerReference != nil { pod.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*b.OwnerReference} } @@ -244,6 +250,10 @@ func (b PodBuilder) WithPhase(phase v1.PodPhase) PodBuilder { b.Phase = phase return b } +func (b PodBuilder) WithCreationTime(time time.Time) PodBuilder { + b.CreationTime = &time + return b +} func (b PodBuilder) WithOwnerReference(ownerReference metav1.OwnerReference) PodBuilder { b.OwnerReference = &ownerReference return b @@ -266,3 +276,9 @@ func (b PodBuilder) WithFrequency(text string) PodBuilder { b.Annotations[annotation] = text return b } +func (b PodBuilder) WithMinimumAge(text string) PodBuilder { + annotation := strings.Join([]string{DefaultBaseAnnotation, "minimum-age"}, "/") + + b.Annotations[annotation] = text + return b +} From 98c4db6ef00656efa5169f6463e8cfe0f6960cf2 Mon Sep 17 00:00:00 2001 From: David Symons Date: Fri, 2 Jul 2021 11:43:14 +1000 Subject: [PATCH 5/7] Filter by owner reference last --- chaoskube/chaoskube.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index c8c06fd9..0bced9b4 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -246,6 +246,7 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { pods = filterByAnnotations(pods, c.Annotations) pods = filterByPhase(pods, v1.PodRunning) pods = filterTerminatingPods(pods) + pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) pods = filterByMinimumAge( pods, @@ -255,9 +256,6 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { c.Logger, ) - pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) - pods = filterByOwnerReference(pods) - pods = filterByFrequency( pods, strings.Join([]string{c.ConfigAnnotationPrefix, "frequency"}, "/"), @@ -266,6 +264,8 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { c.Logger, ) + pods = filterByOwnerReference(pods) + return pods, nil } From 1ea5f2e5ea21f7d36486dd102dcc37ac2b4bcd5f Mon Sep 17 00:00:00 2001 From: David Symons Date: Fri, 2 Jul 2021 13:09:27 +1000 Subject: [PATCH 6/7] Add ability to exclude pods based on time via annotations --- chaoskube/chaoskube.go | 124 ++++++++++++++++++++++++++++-------- chaoskube/chaoskube_test.go | 87 +++++++++++++++++++++---- util/util.go | 28 ++++++++ 3 files changed, 201 insertions(+), 38 deletions(-) diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index 0bced9b4..fa884320 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -6,7 +6,6 @@ import ( "fmt" "math/rand" "regexp" - "strings" "time" multierror "github.com/hashicorp/go-multierror" @@ -94,6 +93,8 @@ var ( msgTimeOfDayExcluded = "time of day excluded" // msgDayOfYearExcluded is the log message when termination is suspended due to the day of year filter msgDayOfYearExcluded = "day of year excluded" + // msgFailedToParseAnnotation is the log message when a filter fails to parse a pod annotation + msgFailedToParseAnnotation = "failed to parse annotation, '%v', excluding from candidates" ) // New returns a new instance of Chaoskube. It expects: @@ -181,7 +182,7 @@ func (c *Chaoskube) TerminateVictims(ctx context.Context) error { } } - victims, err := c.Victims(ctx) + victims, err := c.Victims(ctx, now) if err == errPodNotFound { c.Logger.Debug(msgVictimNotFound) return nil @@ -200,8 +201,8 @@ func (c *Chaoskube) TerminateVictims(ctx context.Context) error { } // Victims returns up to N pods as configured by MaxKill flag -func (c *Chaoskube) Victims(ctx context.Context) ([]v1.Pod, error) { - pods, err := c.Candidates(ctx) +func (c *Chaoskube) Victims(ctx context.Context, now time.Time) ([]v1.Pod, error) { + pods, err := c.Candidates(ctx, now) if err != nil { return []v1.Pod{}, err } @@ -220,7 +221,7 @@ func (c *Chaoskube) Victims(ctx context.Context) ([]v1.Pod, error) { // Candidates returns the list of pods that are available for termination. // It returns all pods that match the configured label, annotation and namespace selectors. -func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { +func (c *Chaoskube) Candidates(ctx context.Context, now time.Time) ([]v1.Pod, error) { listOptions := metav1.ListOptions{LabelSelector: c.Labels.String()} podList, err := c.Client.CoreV1().Pods(v1.NamespaceAll).List(ctx, listOptions) @@ -248,24 +249,11 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { pods = filterTerminatingPods(pods) pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) - pods = filterByMinimumAge( - pods, - strings.Join([]string{c.ConfigAnnotationPrefix, "minimum-age"}, "/"), - c.MinimumAge, - c.Now(), - c.Logger, - ) - - pods = filterByFrequency( - pods, - strings.Join([]string{c.ConfigAnnotationPrefix, "frequency"}, "/"), - c.DefaultFrequency, - c.Interval, - c.Logger, - ) + pods = filterByMinimumAge(pods, c.ConfigAnnotationPrefix, c.MinimumAge, c.Now(), c.Logger) + pods = filterByFrequency(pods, c.ConfigAnnotationPrefix, c.DefaultFrequency, c.Interval, c.Logger) + pods = filterByTime(pods, c.ConfigAnnotationPrefix, now, c.Logger) pods = filterByOwnerReference(pods) - return pods, nil } @@ -498,11 +486,13 @@ func filterTerminatingPods(pods []v1.Pod) []v1.Pod { // filterByMinimumAge filters pods by creation time. Only pods // older than minimumAge are returned -func filterByMinimumAge(pods []v1.Pod, annotation string, minimumAge time.Duration, now time.Time, logger log.FieldLogger) []v1.Pod { - if annotation == "" && minimumAge <= time.Duration(0) { +func filterByMinimumAge(pods []v1.Pod, annotationPrefix string, minimumAge time.Duration, now time.Time, logger log.FieldLogger) []v1.Pod { + if annotationPrefix == "" && minimumAge <= time.Duration(0) { return pods } + annotation := util.FormatAnnotation(annotationPrefix, "minimum-age") + defaultCreationTime := now.Add(-minimumAge) filteredList := []v1.Pod{} @@ -514,7 +504,7 @@ func filterByMinimumAge(pods []v1.Pod, annotation string, minimumAge time.Durati if ok { minimumAgeOverride, err := time.ParseDuration(text) if err != nil { - logger.WithField("err", err).Warn("failed to parse frequency annotation, excluding from candidates") + logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, annotation) continue } @@ -575,11 +565,13 @@ func filterByOwnerReference(pods []v1.Pod) []v1.Pod { return filteredList } -func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string, interval time.Duration, logger log.FieldLogger) []v1.Pod { - if annotation == "" && defaultFrequency == "" { +func filterByFrequency(pods []v1.Pod, annotationPrefix string, defaultFrequency string, interval time.Duration, logger log.FieldLogger) []v1.Pod { + if annotationPrefix == "" && defaultFrequency == "" { return pods } + annotation := util.FormatAnnotation(annotationPrefix, "frequency") + filteredList := []v1.Pod{} for _, pod := range pods { text, ok := pod.Annotations[annotation] @@ -596,7 +588,7 @@ func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string chance, err := util.ParseFrequency(text, interval) if err != nil { - logger.WithField("err", err).Warn("failed to parse frequency annotation, excluding from candidates") + logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, annotation) continue } @@ -607,3 +599,81 @@ func filterByFrequency(pods []v1.Pod, annotation string, defaultFrequency string return filteredList } + +func filterByTime(pods []v1.Pod, annotationPrefix string, now time.Time, logger log.FieldLogger) []v1.Pod { + if annotationPrefix == "" { + return pods + } + + timezoneAnnotation := util.FormatAnnotation(annotationPrefix, "timezone") + weekdaysAnnotation := util.FormatAnnotation(annotationPrefix, "excluded-weekdays") + timesOfDayAnnotation := util.FormatAnnotation(annotationPrefix, "excluded-times-of-day") + daysOfYearAnnotation := util.FormatAnnotation(annotationPrefix, "excluded-days-of-year") + + filteredList := []v1.Pod{} + +checkingPods: + for _, pod := range pods { + localNow := now + + text, ok := pod.Annotations[timezoneAnnotation] + if ok { + location, err := time.LoadLocation(text) + if err != nil { + logger.WithField("err", err).WithField("pod-name", pod.Name).WithField("pod-namespace", pod.Namespace). + Warnf(msgFailedToParseAnnotation, timezoneAnnotation) + + continue checkingPods + } + + localNow = localNow.In(location) + } + + // Weekdays + text, ok = pod.Annotations[weekdaysAnnotation] + if ok { + days := util.ParseWeekdays(text) + for _, wd := range days { + if wd == localNow.Weekday() { + continue checkingPods + } + } + } + + // Times of day + text, ok = pod.Annotations[timesOfDayAnnotation] + if ok { + periods, err := util.ParseTimePeriods(text) + if err != nil { + logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, timesOfDayAnnotation) + continue checkingPods + } + + for _, tp := range periods { + if tp.Includes(localNow) { + continue checkingPods + } + } + } + + // Days of year + text, ok = pod.Annotations[daysOfYearAnnotation] + if ok { + days, err := util.ParseDays(text) + if err != nil { + logger.WithField("err", err).Warnf(msgFailedToParseAnnotation, daysOfYearAnnotation) + continue checkingPods + } + + for _, d := range days { + if d.Day() == localNow.Day() && d.Month() == localNow.Month() { + continue checkingPods + } + } + } + + filteredList = append(filteredList, pod) + } + + return filteredList +} diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 4351ce35..551491d1 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "regexp" "sort" - "strings" "testing" "time" @@ -419,7 +418,7 @@ func (suite *Suite) TestNoVictimReturnsError() { 1, ) - _, err := chaoskube.Victims(context.Background()) + _, err := chaoskube.Victims(context.Background(), time.Now()) suite.Equal(err, errPodNotFound) suite.EqualError(err, "pod not found") } @@ -727,7 +726,7 @@ func (suite *Suite) TestTerminateVictim() { err := chaoskube.TerminateVictims(context.Background()) suite.Require().NoError(err) - pods, err := chaoskube.Candidates(context.Background()) + pods, err := chaoskube.Candidates(context.Background(), time.Now()) suite.Require().NoError(err) suite.Len(pods, tt.remainingPodCount) @@ -766,14 +765,14 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() { // helper functions func (suite *Suite) assertCandidates(chaoskube *Chaoskube, expected []map[string]string) { - pods, err := chaoskube.Candidates(context.Background()) + pods, err := chaoskube.Candidates(context.Background(), time.Now()) suite.Require().NoError(err) suite.AssertPods(pods, expected) } func (suite *Suite) assertVictims(chaoskube *Chaoskube, expected []map[string]string) { - victims, err := chaoskube.Victims(context.Background()) + victims, err := chaoskube.Victims(context.Background(), time.Now()) suite.Require().NoError(err) for i, victim := range victims { @@ -940,11 +939,11 @@ func (suite *Suite) TestMinimumAge() { []v1.Pod{overriddenMinAge}, }, } { - annotation := strings.Join([]string{util.DefaultBaseAnnotation, "minimum-age"}, "/") - pods := filterByMinimumAge(tt.pods, annotation, + pods := filterByMinimumAge(tt.pods, util.DefaultBaseAnnotation, tt.minimumAge, now, logger) - suite.Assert().ElementsMatch(tt.expected, pods) + suite.Assert().ElementsMatch(tt.expected, pods, + "minimum-age: %v", tt.minimumAge) } } @@ -1136,11 +1135,77 @@ func (suite *Suite) TestFilterByFrequency() { } { rand.Seed(tt.seed) - annotation := strings.Join([]string{util.DefaultBaseAnnotation, "frequency"}, "/") - results := filterByFrequency(pods, annotation, + results := filterByFrequency(pods, util.DefaultBaseAnnotation, tt.defaultFrequency, interval, logger) - suite.Assert().ElementsMatch(tt.expected, results) + suite.Assert().ElementsMatch(tt.expected, results, + "seed: %v, default: %v", tt.seed, tt.defaultFrequency) + } +} + +func (suite *Suite) TestFilterByWeekdays() { + logger, _ := test.NewNullLogger() + now := ThankGodItsFriday{}.Now() + + brisbane := "Australia/Brisbane" + brisbaneTimezone, _ := time.LoadLocation(brisbane) + + noExcludes := util.NewPodBuilder("default", "no-excludes").Build() + + neverFriday := util.NewPodBuilder("default", "never-friday"). + WithExcludedWeekdays("Fri").Build() + neverBeforeFridayBrisbane := util.NewPodBuilder("default", "never-before-friday"). + WithExcludedWeekdays("Mon,Tue,Wed,Thu").WithTimezone(brisbane).Build() + + neverAt3pm := util.NewPodBuilder("default", "never-at-3pm"). + WithExcludedTimesOfDay("15:00-16:00").Build() + neverAt8amBrisbane := util.NewPodBuilder("default", "never-at-8am-brisbane"). + WithExcludedTimesOfDay("08:00-09:00").WithTimezone(brisbane).Build() + + neverOnSept24th := util.NewPodBuilder("default", "never-on-sept-24"). + WithExcludedDaysOfYear("Sep24").Build() + neverOnSept28thBrisbane := util.NewPodBuilder("default", "never-on-sept-28-brisbane"). + WithExcludedDaysOfYear("Sep28").WithTimezone(brisbane).Build() + + pods := []v1.Pod{ + noExcludes, + neverFriday, + neverBeforeFridayBrisbane, + neverAt3pm, + neverAt8amBrisbane, + neverOnSept24th, + neverOnSept28thBrisbane, + } + + for _, tt := range []struct { + now time.Time + expected []v1.Pod + }{ + { + now: now, + expected: []v1.Pod{noExcludes, neverBeforeFridayBrisbane, neverAt8amBrisbane, neverOnSept28thBrisbane}, + }, + { + now: now.Add(2 * time.Hour), + expected: []v1.Pod{noExcludes, neverBeforeFridayBrisbane, neverAt3pm, neverAt8amBrisbane, neverOnSept28thBrisbane}, + }, + { + now: now.Add(7 * time.Hour), + expected: []v1.Pod{noExcludes, neverBeforeFridayBrisbane, neverAt3pm, neverOnSept28thBrisbane}, + }, + { + now: now.AddDate(0, 0, 1), + expected: []v1.Pod{noExcludes, neverFriday, neverBeforeFridayBrisbane, neverAt8amBrisbane, neverOnSept24th, neverOnSept28thBrisbane}, + }, + { + now: now.AddDate(0, 0, 4), + expected: []v1.Pod{noExcludes, neverFriday, neverAt8amBrisbane, neverOnSept24th, neverOnSept28thBrisbane}, + }, + } { + results := filterByTime(pods, util.DefaultBaseAnnotation, tt.now, logger) + + suite.Assert().ElementsMatch(tt.expected, results, + "now: %v, now (brisbane): %v, weekday; %v", tt.now, tt.now.In(brisbaneTimezone), tt.now.Weekday().String()) } } diff --git a/util/util.go b/util/util.go index ceed4171..2e1b79de 100644 --- a/util/util.go +++ b/util/util.go @@ -167,6 +167,10 @@ func FormatDays(days []time.Time) []string { return formattedDays } +func FormatAnnotation(prefix, name string) string { + return strings.Join([]string{prefix, name}, "/") +} + // NewNamespace returns a new namespace instance for testing purposes. func NewNamespace(name string) v1.Namespace { return v1.Namespace{ @@ -282,3 +286,27 @@ func (b PodBuilder) WithMinimumAge(text string) PodBuilder { b.Annotations[annotation] = text return b } +func (b PodBuilder) WithTimezone(text string) PodBuilder { + annotation := strings.Join([]string{DefaultBaseAnnotation, "timezone"}, "/") + + b.Annotations[annotation] = text + return b +} +func (b PodBuilder) WithExcludedWeekdays(text string) PodBuilder { + annotation := strings.Join([]string{DefaultBaseAnnotation, "excluded-weekdays"}, "/") + + b.Annotations[annotation] = text + return b +} +func (b PodBuilder) WithExcludedTimesOfDay(text string) PodBuilder { + annotation := strings.Join([]string{DefaultBaseAnnotation, "excluded-times-of-day"}, "/") + + b.Annotations[annotation] = text + return b +} +func (b PodBuilder) WithExcludedDaysOfYear(text string) PodBuilder { + annotation := strings.Join([]string{DefaultBaseAnnotation, "excluded-days-of-year"}, "/") + + b.Annotations[annotation] = text + return b +} From 6b7de055c81c09bd9a51ecd804bf6cfcadd7b204 Mon Sep 17 00:00:00 2001 From: David Symons Date: Fri, 2 Jul 2021 14:18:30 +1000 Subject: [PATCH 7/7] Add expected annotations/labels to PodBuilder --- chaoskube/chaoskube_test.go | 2 +- util/util.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 551491d1..956530d2 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -821,7 +821,7 @@ func (suite *Suite) setupWithPods(interval time.Duration, labelSelector labels.S pods := []v1.Pod{ util.NewPodBuilder("default", "foo").Build(), util.NewPodBuilder("testing", "bar").Build(), - util.NewPodBuilder("testing", "baz").Build(), // Non-running pods are ignored + util.NewPodBuilder("testing", "baz").WithPhase(v1.PodPending).Build(), // Non-running pods are ignored } for _, pod := range pods { diff --git a/util/util.go b/util/util.go index 2e1b79de..5aa6f1dc 100644 --- a/util/util.go +++ b/util/util.go @@ -17,7 +17,7 @@ const ( Kitchen24 = "15:04" // a time format that just cares about the day and month. YearDay = "Jan_2" - + // default annotation prefix for configuration overrides DefaultBaseAnnotation = "chaos.alpha.kubernetes.io" ) @@ -212,8 +212,8 @@ func NewPodBuilder(namespace string, name string) PodBuilder { Phase: v1.PodRunning, CreationTime: nil, OwnerReference: nil, - Annotations: make(map[string]string), - Labels: make(map[string]string), + Annotations: map[string]string{"chaos": name}, + Labels: map[string]string{"app": name}, } }