feat(retention): added image retention policies

feat(metaDB): add more image statistics info

Signed-off-by: Petu Eusebiu <[email protected]>

errors/errors.go

@@ -163,4 +163,5 @@ var (
 	ErrInvalidOutputFormat            = errors.New("cli: invalid output format")
 	ErrFlagValueUnsupported           = errors.New("supported values ")
 	ErrUnknownSubcommand              = errors.New("cli: unknown subcommand")
+	ErrGCPolicyNotFound               = errors.New("gc: repo/tag policy not found")
 )

examples/config-gc-periodic.json

@@ -4,13 +4,13 @@
         "rootDirectory": "/tmp/zot",
         "gc": true,
         "gcDelay": "1h",
-        "gcInterval": "24h",
+        "gcInterval": "1h",
         "subPaths": {
             "/a": {
                 "rootDirectory": "/tmp/zot1",
                 "gc": true,
                 "gcDelay": "1h",
-                "gcInterval": "24h"
+                "gcInterval": "1h"
             }
         }
     },

examples/config-gc.json

@@ -3,10 +3,47 @@
     "storage": {
         "rootDirectory": "/tmp/zot",
         "gc": true,
-        "gcReferrers": true,
         "gcDelay": "2h",
         "untaggedImageRetentionDelay": "4h",
-        "gcInterval": "1h"
+        "gcInterval": "1h",
+        "retention": {
+            "dryRun": false,
+            "policies": [
+                {
+                    "repoNames": ["infra/*", "prod/*"],
+                    "deleteReferrers": false,
+                    "deleteUntagged": true,
+                    "tagsRetention": [{
+                        "names": ["v2.*", ".*-prod"]
+                    },
+                    {
+                        "names": ["v3.*", ".*-prod"],
+                        "pulledWithinLastNrDays": 7
+                    }]
+                },
+                {
+                    "repoNames": ["tmp/**"],
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "tagsRetention": [{
+                        "names": ["v1.*"],
+                        "pulledWithinLastNrDays": 7,
+                        "pushedWithinLastNrDays": 7   
+                    }]
+                },
+                {
+                    "repoNames": ["**"],
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "tagsRetention": [{
+                        "mostRecentlyPushedCount": 10,
+                        "mostRecentlyPulledCount": 10,
+                        "pulledWithinLastNrDays": 30,
+                        "pushedWithinLastNrDays": 30
+                    }]
+                }
+            ]
+        }
     },
     "http": {
         "address": "127.0.0.1",

examples/config-sync.json

@@ -35,12 +35,12 @@
 							}
 						},
 						{
-							"prefix": "/repo1/repo",
+							"prefix": "/repo2/repo",
 							"destination": "/repo",
 							"stripPrefix": true
 						},
 						{
-							"prefix": "/repo2/repo"
+							"prefix": "/repo3/**"
 						}
 					]
 				},
@@ -54,7 +54,7 @@
 					"onDemand": false,
 					"content": [
 						{
-							"prefix": "/repo2",
+							"prefix": "**",
 							"tags": {
 								"semver": true
 							}

pkg/api/config/config.go

@@ -30,12 +30,32 @@ type StorageConfig struct {
 	Commit                      bool
 	GCDelay                     time.Duration
 	GCInterval                  time.Duration
-	GCReferrers                 bool
 	UntaggedImageRetentionDelay time.Duration
+	Retention                   ImageRetention
 	StorageDriver               map[string]interface{} `mapstructure:",omitempty"`
 	CacheDriver                 map[string]interface{} `mapstructure:",omitempty"`
 }
 
+type ImageRetention struct {
+	DryRun   bool
+	Policies []GCPolicy
+}
+
+type GCPolicy struct {
+	RepoNames       []string
+	DeleteReferrers bool
+	DeleteUntagged  bool
+	TagsRetention   []TagsRetentionPolicy
+}
+
+type TagsRetentionPolicy struct {
+	Names                   []string
+	PulledWithinLastNrDays  int
+	PushedWithinLastNrDays  int
+	MostRecentlyPushedCount int
+	MostRecentlyPulledCount int
+}
+
 type TLSConfig struct {
 	Cert   string
 	Key    string
@@ -190,9 +210,12 @@ func New() *Config {
 		BinaryType:      BinaryType,
 		Storage: GlobalStorageConfig{
 			StorageConfig: StorageConfig{
-				GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay,
+				Dedupe:                      true,
+				GC:                          true,
+				GCDelay:                     storageConstants.DefaultGCDelay,
 				UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
-				GCInterval:                  storageConstants.DefaultGCInterval, Dedupe: true,
+				GCInterval:                  storageConstants.DefaultGCInterval,
+				Retention:                   ImageRetention{},
 			},
 		},
 		HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}},
@@ -202,7 +225,8 @@ func New() *Config {
 
 func (expConfig StorageConfig) ParamsEqual(actConfig StorageConfig) bool {
 	return expConfig.GC == actConfig.GC && expConfig.Dedupe == actConfig.Dedupe &&
-		expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval
+		expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval &&
+		expConfig.UntaggedImageRetentionDelay == actConfig.UntaggedImageRetentionDelay
 }
 
 // SameFile compare two files.
@@ -368,6 +392,42 @@ func (c *Config) IsImageTrustEnabled() bool {
 	return c.Extensions != nil && c.Extensions.Trust != nil && *c.Extensions.Trust.Enable
 }
 
+// check if metaDB statistics is need for image retention policy.
+func (c *Config) IsMetaDBNeededForRetention() bool {
+	var needsMetaDB bool
+
+	for _, retentionPolicy := range c.Storage.Retention.Policies {
+		for _, tagRetentionPolicy := range retentionPolicy.TagsRetention {
+			if c.isMetaDBNeededInTagRetentionPolicy(tagRetentionPolicy) {
+				needsMetaDB = true
+			}
+		}
+	}
+
+	for _, subpath := range c.Storage.SubPaths {
+		for _, retentionPolicy := range subpath.Retention.Policies {
+			for _, tagRetentionPolicy := range retentionPolicy.TagsRetention {
+				if c.isMetaDBNeededInTagRetentionPolicy(tagRetentionPolicy) {
+					needsMetaDB = true
+				}
+			}
+		}
+	}
+
+	return needsMetaDB
+}
+
+func (c *Config) isMetaDBNeededInTagRetentionPolicy(tagRetentionPolicy TagsRetentionPolicy) bool {
+	if tagRetentionPolicy.MostRecentlyPulledCount != 0 ||
+		tagRetentionPolicy.MostRecentlyPushedCount != 0 ||
+		tagRetentionPolicy.PulledWithinLastNrDays != 0 ||
+		tagRetentionPolicy.PushedWithinLastNrDays != 0 {
+		return true
+	}
+
+	return false
+}
+
 func (c *Config) IsCosignEnabled() bool {
 	return c.IsImageTrustEnabled() && c.Extensions.Trust.Cosign
 }

pkg/api/config/config_test.go

@@ -65,6 +65,7 @@ func TestConfig(t *testing.T) {
 		So(err, ShouldBeNil)
 		So(isSame, ShouldBeTrue)
 	})
+
 	Convey("Test DeepCopy() & Sanitize()", t, func() {
 		conf := config.New()
 		So(conf, ShouldNotBeNil)
@@ -81,4 +82,48 @@ func TestConfig(t *testing.T) {
 		err = config.DeepCopy(obj, conf)
 		So(err, ShouldNotBeNil)
 	})
+
+	Convey("Test IsMetaDBNeededForRetention()", t, func() {
+		conf := config.New()
+		So(conf.IsMetaDBNeededForRetention(), ShouldBeFalse)
+
+		conf.Storage.Retention.Policies = []config.GCPolicy{
+			{
+				RepoNames: []string{"repo"},
+			},
+		}
+
+		So(conf.IsMetaDBNeededForRetention(), ShouldBeFalse)
+
+		policies := []config.GCPolicy{
+			{
+				RepoNames: []string{"repo"},
+				TagsRetention: []config.TagsRetentionPolicy{
+					{
+						Names:                   []string{"tag"},
+						MostRecentlyPulledCount: 2,
+					},
+				},
+			},
+		}
+
+		conf.Storage.Retention = config.ImageRetention{
+			Policies: policies,
+		}
+
+		So(conf.IsMetaDBNeededForRetention(), ShouldBeTrue)
+
+		subPaths := make(map[string]config.StorageConfig)
+
+		subPaths["/a"] = config.StorageConfig{
+			GC: true,
+			Retention: config.ImageRetention{
+				Policies: policies,
+			},
+		}
+
+		conf.Storage.SubPaths = subPaths
+
+		So(conf.IsMetaDBNeededForRetention(), ShouldBeTrue)
+	})
 }

pkg/api/controller.go

@@ -261,7 +261,8 @@ func (c *Controller) InitImageStore() error {
 
 func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
 	// init metaDB if search is enabled or we need to store user profiles, api keys or signatures
-	if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() {
+	if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() ||
+		c.Config.IsMetaDBNeededForRetention() {
 		driver, err := meta.New(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck
 		if err != nil {
 			return err
@@ -277,7 +278,7 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
 			return err
 		}
 
-		err = meta.ParseStorage(driver, c.StoreController, c.Log)
+		err = meta.ParseStorage(driver, c.StoreController, c.Log) //nolint: contextcheck
 		if err != nil {
 			return err
 		}
@@ -293,10 +294,32 @@ func (c *Controller) LoadNewConfig(reloadCtx context.Context, newConfig *config.
 	c.Config.HTTP.AccessControl = newConfig.HTTP.AccessControl
 
 	// reload periodical gc config
-	c.Config.Storage.GCInterval = newConfig.Storage.GCInterval
 	c.Config.Storage.GC = newConfig.Storage.GC
+	c.Config.Storage.Dedupe = newConfig.Storage.Dedupe
 	c.Config.Storage.GCDelay = newConfig.Storage.GCDelay
-	c.Config.Storage.GCReferrers = newConfig.Storage.GCReferrers
+	c.Config.Storage.GCInterval = newConfig.Storage.GCInterval
+	c.Config.Storage.UntaggedImageRetentionDelay = newConfig.Storage.UntaggedImageRetentionDelay
+	// only if we have a metaDB already in place
+	if c.Config.IsMetaDBNeededForRetention() {
+		c.Config.Storage.Retention = newConfig.Storage.Retention
+	}
+
+	for subPath, storageConfig := range newConfig.Storage.SubPaths {
+		subPathConfig, ok := c.Config.Storage.SubPaths[subPath]
+		if ok {
+			subPathConfig.GC = storageConfig.GC
+			subPathConfig.Dedupe = storageConfig.Dedupe
+			subPathConfig.GCDelay = storageConfig.GCDelay
+			subPathConfig.GCInterval = storageConfig.GCInterval
+			subPathConfig.UntaggedImageRetentionDelay = storageConfig.UntaggedImageRetentionDelay
+			// only if we have a metaDB already in place
+			if c.Config.IsMetaDBNeededForRetention() {
+				subPathConfig.Retention = storageConfig.Retention
+			}
+
+			c.Config.Storage.SubPaths[subPath] = subPathConfig
+		}
+	}
 
 	// reload background tasks
 	if newConfig.Extensions != nil {
@@ -340,10 +363,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
 	// Enable running garbage-collect periodically for DefaultStore
 	if c.Config.Storage.GC {
 		gc := gc.NewGarbageCollect(c.StoreController.DefaultStore, c.MetaDB, gc.Options{
-			Referrers:      c.Config.Storage.GCReferrers,
 			Delay:          c.Config.Storage.GCDelay,
-			RetentionDelay: c.Config.Storage.UntaggedImageRetentionDelay,
-		}, c.Log)
+			UntaggedDelay:  c.Config.Storage.UntaggedImageRetentionDelay,
+			ImageRetention: c.Config.Storage.Retention,
+		}, c.Audit, c.Log)
 
 		gc.CleanImageStorePeriodically(c.Config.Storage.GCInterval, taskScheduler)
 	}
@@ -363,10 +386,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
 			if storageConfig.GC {
 				gc := gc.NewGarbageCollect(c.StoreController.SubStore[route], c.MetaDB,
 					gc.Options{
-						Referrers:      storageConfig.GCReferrers,
 						Delay:          storageConfig.GCDelay,
-						RetentionDelay: storageConfig.UntaggedImageRetentionDelay,
-					}, c.Log)
+						UntaggedDelay:  storageConfig.UntaggedImageRetentionDelay,
+						ImageRetention: storageConfig.Retention,
+					}, c.Audit, c.Log)
 
 				gc.CleanImageStorePeriodically(storageConfig.GCInterval, taskScheduler)
 			}