feat(retention): added image retention policies

feat(metaDB): add more image statistics info

Signed-off-by: Petu Eusebiu <[email protected]>

errors/errors.go

@@ -163,4 +163,5 @@ var (
 	ErrInvalidOutputFormat            = errors.New("cli: invalid output format")
 	ErrFlagValueUnsupported           = errors.New("supported values ")
 	ErrUnknownSubcommand              = errors.New("cli: unknown subcommand")
+	ErrGCPolicyNotFound               = errors.New("gc: repo/tag policy not found")
 )

examples/config-gc-periodic.json

@@ -4,13 +4,13 @@
         "rootDirectory": "/tmp/zot",
         "gc": true,
         "gcDelay": "1h",
-        "gcInterval": "24h",
+        "gcInterval": "1h",
         "subPaths": {
             "/a": {
                 "rootDirectory": "/tmp/zot1",
                 "gc": true,
                 "gcDelay": "1h",
-                "gcInterval": "24h"
+                "gcInterval": "1h"
             }
         }
     },

examples/config-gc.json

@@ -3,10 +3,47 @@
     "storage": {
         "rootDirectory": "/tmp/zot",
         "gc": true,
-        "gcReferrers": true,
         "gcDelay": "2h",
         "untaggedImageRetentionDelay": "4h",
-        "gcInterval": "1h"
+        "gcInterval": "1h",
+        "retention": {
+            "dryRun": false,
+            "policies": [
+                {
+                    "repoNames": ["infra/*", "prod/*"],
+                    "deleteReferrers": false,
+                    "deleteUntagged": true,
+                    "tagsRetention": [{
+                        "names": ["v2.*", ".*-prod"]
+                    },
+                    {
+                        "names": ["v3.*", ".*-prod"],
+                        "pulledWithinLastNrDays": 7
+                    }]
+                },
+                {
+                    "repoNames": ["tmp/**"],
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "tagsRetention": [{
+                        "names": ["v1.*"],
+                        "pulledWithinLastNrDays": 7,
+                        "pushedWithinLastNrDays": 7   
+                    }]
+                },
+                {
+                    "repoNames": ["**"],
+                    "deleteReferrers": true,
+                    "deleteUntagged": true,
+                    "tagsRetention": [{
+                        "mostRecentlyPushedCount": 10,
+                        "mostRecentlyPulledCount": 10,
+                        "pulledWithinLastNrDays": 30,
+                        "pushedWithinLastNrDays": 30
+                    }]
+                }
+            ]
+        }
     },
     "http": {
         "address": "127.0.0.1",

examples/config-sync.json

@@ -35,12 +35,12 @@
 							}
 						},
 						{
-							"prefix": "/repo1/repo",
+							"prefix": "/repo2/repo",
 							"destination": "/repo",
 							"stripPrefix": true
 						},
 						{
-							"prefix": "/repo2/repo"
+							"prefix": "/repo3/**"
 						}
 					]
 				},
@@ -54,7 +54,7 @@
 					"onDemand": false,
 					"content": [
 						{
-							"prefix": "/repo2",
+							"prefix": "**",
 							"tags": {
 								"semver": true
 							}

pkg/api/config/config.go

@@ -30,12 +30,32 @@ type StorageConfig struct {
 	Commit                      bool
 	GCDelay                     time.Duration
 	GCInterval                  time.Duration
-	GCReferrers                 bool
 	UntaggedImageRetentionDelay time.Duration
+	Retention                   ImageRetention
 	StorageDriver               map[string]interface{} `mapstructure:",omitempty"`
 	CacheDriver                 map[string]interface{} `mapstructure:",omitempty"`
 }
 
+type ImageRetention struct {
+	DryRun   bool
+	Policies []GCPolicy
+}
+
+type GCPolicy struct {
+	RepoNames       []string
+	DeleteReferrers bool
+	DeleteUntagged  bool
+	TagsRetention   []TagsRetentionPolicy
+}
+
+type TagsRetentionPolicy struct {
+	Names                   []string
+	PulledWithinLastNrDays  int
+	PushedWithinLastNrDays  int
+	MostRecentlyPushedCount int
+	MostRecentlyPulledCount int
+}
+
 type TLSConfig struct {
 	Cert   string
 	Key    string
@@ -190,9 +210,12 @@ func New() *Config {
 		BinaryType:      BinaryType,
 		Storage: GlobalStorageConfig{
 			StorageConfig: StorageConfig{
-				GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay,
+				Dedupe:                      true,
+				GC:                          true,
+				GCDelay:                     storageConstants.DefaultGCDelay,
 				UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
-				GCInterval:                  storageConstants.DefaultGCInterval, Dedupe: true,
+				GCInterval:                  storageConstants.DefaultGCInterval,
+				Retention:                   ImageRetention{},
 			},
 		},
 		HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}},
@@ -202,7 +225,8 @@ func New() *Config {
 
 func (expConfig StorageConfig) ParamsEqual(actConfig StorageConfig) bool {
 	return expConfig.GC == actConfig.GC && expConfig.Dedupe == actConfig.Dedupe &&
-		expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval
+		expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval &&
+		expConfig.UntaggedImageRetentionDelay == actConfig.UntaggedImageRetentionDelay
 }
 
 // SameFile compare two files.
@@ -368,6 +392,42 @@ func (c *Config) IsImageTrustEnabled() bool {
 	return c.Extensions != nil && c.Extensions.Trust != nil && *c.Extensions.Trust.Enable
 }
 
+// check if metaDB statistics is need for image retention policy.
+func (c *Config) IsMetaDBNeededForRetention() bool {
+	var needsMetaDB bool
+
+	for _, retentionPolicy := range c.Storage.Retention.Policies {
+		for _, tagRetentionPolicy := range retentionPolicy.TagsRetention {
+			if c.isMetaDBNeededInTagRetentionPolicy(tagRetentionPolicy) {
+				needsMetaDB = true
+			}
+		}
+	}
+
+	for _, subpath := range c.Storage.SubPaths {
+		for _, retentionPolicy := range subpath.Retention.Policies {
+			for _, tagRetentionPolicy := range retentionPolicy.TagsRetention {
+				if c.isMetaDBNeededInTagRetentionPolicy(tagRetentionPolicy) {
+					needsMetaDB = true
+				}
+			}
+		}
+	}
+
+	return needsMetaDB
+}
+
+func (c *Config) isMetaDBNeededInTagRetentionPolicy(tagRetentionPolicy TagsRetentionPolicy) bool {
+	if tagRetentionPolicy.MostRecentlyPulledCount != 0 ||
+		tagRetentionPolicy.MostRecentlyPushedCount != 0 ||
+		tagRetentionPolicy.PulledWithinLastNrDays != 0 ||
+		tagRetentionPolicy.PushedWithinLastNrDays != 0 {
+		return true
+	}
+
+	return false
+}
+
 func (c *Config) IsCosignEnabled() bool {
 	return c.IsImageTrustEnabled() && c.Extensions.Trust.Cosign
 }

pkg/api/controller.go

@@ -261,7 +261,8 @@ func (c *Controller) InitImageStore() error {
 
 func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
 	// init metaDB if search is enabled or we need to store user profiles, api keys or signatures
-	if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() {
+	if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() ||
+		c.Config.IsMetaDBNeededForRetention() {
 		driver, err := meta.New(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck
 		if err != nil {
 			return err
@@ -277,7 +278,7 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error {
 			return err
 		}
 
-		err = meta.ParseStorage(driver, c.StoreController, c.Log)
+		err = meta.ParseStorage(driver, c.StoreController, c.Log) //nolint: contextcheck
 		if err != nil {
 			return err
 		}
@@ -293,10 +294,7 @@ func (c *Controller) LoadNewConfig(reloadCtx context.Context, newConfig *config.
 	c.Config.HTTP.AccessControl = newConfig.HTTP.AccessControl
 
 	// reload periodical gc config
-	c.Config.Storage.GCInterval = newConfig.Storage.GCInterval
 	c.Config.Storage.GC = newConfig.Storage.GC
-	c.Config.Storage.GCDelay = newConfig.Storage.GCDelay
-	c.Config.Storage.GCReferrers = newConfig.Storage.GCReferrers
 
 	// reload background tasks
 	if newConfig.Extensions != nil {
@@ -340,10 +338,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
 	// Enable running garbage-collect periodically for DefaultStore
 	if c.Config.Storage.GC {
 		gc := gc.NewGarbageCollect(c.StoreController.DefaultStore, c.MetaDB, gc.Options{
-			Referrers:      c.Config.Storage.GCReferrers,
 			Delay:          c.Config.Storage.GCDelay,
-			RetentionDelay: c.Config.Storage.UntaggedImageRetentionDelay,
-		}, c.Log)
+			UntaggedDelay:  c.Config.Storage.UntaggedImageRetentionDelay,
+			ImageRetention: c.Config.Storage.Retention,
+		}, c.Audit, c.Log)
 
 		gc.CleanImageStorePeriodically(c.Config.Storage.GCInterval, taskScheduler)
 	}
@@ -363,10 +361,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
 			if storageConfig.GC {
 				gc := gc.NewGarbageCollect(c.StoreController.SubStore[route], c.MetaDB,
 					gc.Options{
-						Referrers:      storageConfig.GCReferrers,
 						Delay:          storageConfig.GCDelay,
-						RetentionDelay: storageConfig.UntaggedImageRetentionDelay,
-					}, c.Log)
+						UntaggedDelay:  storageConfig.UntaggedImageRetentionDelay,
+						ImageRetention: storageConfig.Retention,
+					}, c.Audit, c.Log)
 
 				gc.CleanImageStorePeriodically(storageConfig.GCInterval, taskScheduler)
 			}

pkg/api/controller_test.go

@@ -4999,6 +4999,7 @@ func TestHardLink(t *testing.T) {
 		port := test.GetFreePort()
 		conf := config.New()
 		conf.HTTP.Port = port
+		conf.Storage.GC = false
 
 		dir := t.TempDir()
 
@@ -7682,6 +7683,15 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			ctlr.Config.Storage.GC = true
 			ctlr.Config.Storage.GCDelay = 1 * time.Millisecond
 			ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Millisecond
+			ctlr.Config.Storage.Retention = config.ImageRetention{
+				Policies: []config.GCPolicy{
+					{
+						RepoNames:       []string{"**"},
+						DeleteReferrers: true,
+						DeleteUntagged:  true,
+					},
+				},
+			}
 
 			ctlr.Config.Storage.Dedupe = false
 
@@ -7697,11 +7707,10 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 
 			gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB,
 				gc.Options{
-					Referrers:      ctlr.Config.Storage.GCReferrers,
 					Delay:          ctlr.Config.Storage.GCDelay,
-					RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay,
-				},
-				ctlr.Log)
+					UntaggedDelay:  ctlr.Config.Storage.UntaggedImageRetentionDelay,
+					ImageRetention: ctlr.Config.Storage.Retention,
+				}, ctlr.Audit, ctlr.Log)
 
 			resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag))
 			So(err, ShouldBeNil)
@@ -7927,6 +7936,15 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			ctlr.Config.Storage.GC = true
 			ctlr.Config.Storage.GCDelay = 1 * time.Second
 			ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Second
+			ctlr.Config.Storage.Retention = config.ImageRetention{
+				Policies: []config.GCPolicy{
+					{
+						RepoNames:       []string{"**"},
+						DeleteReferrers: true,
+						DeleteUntagged:  true,
+					},
+				},
+			}
 
 			err := WriteImageToFileSystem(CreateDefaultImage(), repoName, tag,
 				ociutils.GetDefaultStoreController(dir, ctlr.Log))
@@ -7938,10 +7956,10 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 
 			gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB,
 				gc.Options{
-					Referrers:      ctlr.Config.Storage.GCReferrers,
 					Delay:          ctlr.Config.Storage.GCDelay,
-					RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay,
-				}, ctlr.Log)
+					UntaggedDelay:  ctlr.Config.Storage.UntaggedImageRetentionDelay,
+					ImageRetention: ctlr.Config.Storage.Retention,
+				}, ctlr.Audit, ctlr.Log)
 
 			resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag))
 			So(err, ShouldBeNil)
@@ -8071,8 +8089,13 @@ func TestPeriodicGC(t *testing.T) {
 		subPaths := make(map[string]config.StorageConfig)
 
 		subPaths["/a"] = config.StorageConfig{
-			RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second,
-			UntaggedImageRetentionDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false,
+			RootDirectory:               subDir,
+			GC:                          true,
+			GCDelay:                     1 * time.Second,
+			UntaggedImageRetentionDelay: 1 * time.Second,
+			GCInterval:                  24 * time.Hour,
+			RemoteCache:                 false,
+			Dedupe:                      false,
 		} //nolint:lll // gofumpt conflicts with lll
 		ctlr.Config.Storage.Dedupe = false
 		ctlr.Config.Storage.SubPaths = subPaths

pkg/api/routes.go

@@ -721,8 +721,8 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
 	}
 
 	if rh.c.MetaDB != nil {
-		err := meta.OnUpdateManifest(name, reference, mediaType, digest, body, rh.c.StoreController, rh.c.MetaDB,
-			rh.c.Log)
+		err := meta.OnUpdateManifest(request.Context(), name, reference, mediaType,
+			digest, body, rh.c.StoreController, rh.c.MetaDB, rh.c.Log)
 		if err != nil {
 			response.WriteHeader(http.StatusInternalServerError)