diff --git a/errors/errors.go b/errors/errors.go index ed2dcf42cf..d240a970ad 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -163,4 +163,5 @@ var ( ErrInvalidOutputFormat = errors.New("cli: invalid output format") ErrFlagValueUnsupported = errors.New("supported values ") ErrUnknownSubcommand = errors.New("cli: unknown subcommand") + ErrGCPolicyNotFound = errors.New("gc: repo/tag policy not found") ) diff --git a/examples/config-gc-periodic.json b/examples/config-gc-periodic.json index 88e38cffa8..8e44166c4d 100644 --- a/examples/config-gc-periodic.json +++ b/examples/config-gc-periodic.json @@ -4,13 +4,13 @@ "rootDirectory": "/tmp/zot", "gc": true, "gcDelay": "1h", - "gcInterval": "24h", + "gcInterval": "1h", "subPaths": { "/a": { "rootDirectory": "/tmp/zot1", "gc": true, "gcDelay": "1h", - "gcInterval": "24h" + "gcInterval": "1h" } } }, diff --git a/examples/config-gc.json b/examples/config-gc.json index cad60105fd..6ddf6ac00f 100644 --- a/examples/config-gc.json +++ b/examples/config-gc.json @@ -3,10 +3,47 @@ "storage": { "rootDirectory": "/tmp/zot", "gc": true, - "gcReferrers": true, "gcDelay": "2h", "untaggedImageRetentionDelay": "4h", - "gcInterval": "1h" + "gcInterval": "1h", + "retention": { + "dryRun": false, + "policies": [ + { + "repoNames": ["infra/*", "prod/*"], + "deleteReferrers": false, + "deleteUntagged": true, + "tagsRetention": [{ + "names": ["v2.*", ".*-prod"] + }, + { + "names": ["v3.*", ".*-prod"], + "pulledWithinLastNrDays": 7 + }] + }, + { + "repoNames": ["tmp/**"], + "deleteReferrers": true, + "deleteUntagged": true, + "tagsRetention": [{ + "names": ["v1.*"], + "pulledWithinLastNrDays": 7, + "pushedWithinLastNrDays": 7 + }] + }, + { + "repoNames": ["**"], + "deleteReferrers": true, + "deleteUntagged": true, + "tagsRetention": [{ + "mostRecentlyPushedCount": 10, + "mostRecentlyPulledCount": 10, + "pulledWithinLastNrDays": 30, + "pushedWithinLastNrDays": 30 + }] + } + ] + } }, "http": { "address": "127.0.0.1", diff --git a/examples/config-sync-localhost.json b/examples/config-sync-localhost.json deleted file mode 100644 index fc545f7b98..0000000000 --- a/examples/config-sync-localhost.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "distspecversion":"1.1.0-dev", - "storage": { - "rootDirectory": "/tmp/zot_to_sync", - "dedupe": false, - "gc": false - }, - "http": { - "address": "127.0.0.1", - "port": "8081" - }, - "log": { - "level": "debug" - }, - "extensions": { - "sync": { - "registries": [ - { - "urls": [ - "http://localhost:8080" - ], - "onDemand": true, - "tlsVerify": false, - "PollInterval": "30s", - "content": [ - { - "prefix": "**" - } - ] - } - ] - }, - "scrub": { - "interval": "24h" - } - } -} \ No newline at end of file diff --git a/examples/config-sync.json b/examples/config-sync.json index 092e4b1e85..c993119711 100644 --- a/examples/config-sync.json +++ b/examples/config-sync.json @@ -35,12 +35,12 @@ } }, { - "prefix": "/repo1/repo", + "prefix": "/repo2/repo", "destination": "/repo", "stripPrefix": true }, { - "prefix": "/repo2/repo" + "prefix": "/repo3/**" } ] }, @@ -54,7 +54,7 @@ "onDemand": false, "content": [ { - "prefix": "/repo2", + "prefix": "**", "tags": { "semver": true } diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index 8ab156e328..4b87f39d28 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -30,12 +30,32 @@ type StorageConfig struct { Commit bool GCDelay time.Duration GCInterval time.Duration - GCReferrers bool UntaggedImageRetentionDelay time.Duration + Retention ImageRetention StorageDriver map[string]interface{} `mapstructure:",omitempty"` CacheDriver map[string]interface{} `mapstructure:",omitempty"` } +type ImageRetention struct { + DryRun bool + Policies []GCPolicy +} + +type GCPolicy struct { + RepoNames []string + DeleteReferrers bool + DeleteUntagged bool + TagsRetention []TagsRetentionPolicy +} + +type TagsRetentionPolicy struct { + Names []string + PulledWithinLastNrDays int + PushedWithinLastNrDays int + MostRecentlyPushedCount int + MostRecentlyPulledCount int +} + type TLSConfig struct { Cert string Key string @@ -190,9 +210,12 @@ func New() *Config { BinaryType: BinaryType, Storage: GlobalStorageConfig{ StorageConfig: StorageConfig{ - GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay, + Dedupe: true, + GC: true, + GCDelay: storageConstants.DefaultGCDelay, UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - GCInterval: storageConstants.DefaultGCInterval, Dedupe: true, + GCInterval: storageConstants.DefaultGCInterval, + Retention: ImageRetention{}, }, }, HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}}, @@ -202,7 +225,8 @@ func New() *Config { func (expConfig StorageConfig) ParamsEqual(actConfig StorageConfig) bool { return expConfig.GC == actConfig.GC && expConfig.Dedupe == actConfig.Dedupe && - expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval + expConfig.GCDelay == actConfig.GCDelay && expConfig.GCInterval == actConfig.GCInterval && + expConfig.UntaggedImageRetentionDelay == actConfig.UntaggedImageRetentionDelay } // SameFile compare two files. @@ -368,6 +392,42 @@ func (c *Config) IsImageTrustEnabled() bool { return c.Extensions != nil && c.Extensions.Trust != nil && *c.Extensions.Trust.Enable } +// check if metaDB statistics is need for image retention policy. +func (c *Config) IsMetaDBNeededForRetention() bool { + var needsMetaDB bool + + for _, retentionPolicy := range c.Storage.Retention.Policies { + for _, tagRetentionPolicy := range retentionPolicy.TagsRetention { + if c.isMetaDBNeededInTagRetentionPolicy(tagRetentionPolicy) { + needsMetaDB = true + } + } + } + + for _, subpath := range c.Storage.SubPaths { + for _, retentionPolicy := range subpath.Retention.Policies { + for _, tagRetentionPolicy := range retentionPolicy.TagsRetention { + if c.isMetaDBNeededInTagRetentionPolicy(tagRetentionPolicy) { + needsMetaDB = true + } + } + } + } + + return needsMetaDB +} + +func (c *Config) isMetaDBNeededInTagRetentionPolicy(tagRetentionPolicy TagsRetentionPolicy) bool { + if tagRetentionPolicy.MostRecentlyPulledCount != 0 || + tagRetentionPolicy.MostRecentlyPushedCount != 0 || + tagRetentionPolicy.PulledWithinLastNrDays != 0 || + tagRetentionPolicy.PushedWithinLastNrDays != 0 { + return true + } + + return false +} + func (c *Config) IsCosignEnabled() bool { return c.IsImageTrustEnabled() && c.Extensions.Trust.Cosign } diff --git a/pkg/api/config/config_test.go b/pkg/api/config/config_test.go index 9d23e0ce27..a884243fae 100644 --- a/pkg/api/config/config_test.go +++ b/pkg/api/config/config_test.go @@ -65,6 +65,7 @@ func TestConfig(t *testing.T) { So(err, ShouldBeNil) So(isSame, ShouldBeTrue) }) + Convey("Test DeepCopy() & Sanitize()", t, func() { conf := config.New() So(conf, ShouldNotBeNil) @@ -81,4 +82,48 @@ func TestConfig(t *testing.T) { err = config.DeepCopy(obj, conf) So(err, ShouldNotBeNil) }) + + Convey("Test IsMetaDBNeededForRetention()", t, func() { + conf := config.New() + So(conf.IsMetaDBNeededForRetention(), ShouldBeFalse) + + conf.Storage.Retention.Policies = []config.GCPolicy{ + { + RepoNames: []string{"repo"}, + }, + } + + So(conf.IsMetaDBNeededForRetention(), ShouldBeFalse) + + policies := []config.GCPolicy{ + { + RepoNames: []string{"repo"}, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{"tag"}, + MostRecentlyPulledCount: 2, + }, + }, + }, + } + + conf.Storage.Retention = config.ImageRetention{ + Policies: policies, + } + + So(conf.IsMetaDBNeededForRetention(), ShouldBeTrue) + + subPaths := make(map[string]config.StorageConfig) + + subPaths["/a"] = config.StorageConfig{ + GC: true, + Retention: config.ImageRetention{ + Policies: policies, + }, + } + + conf.Storage.SubPaths = subPaths + + So(conf.IsMetaDBNeededForRetention(), ShouldBeTrue) + }) } diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 9df5054784..205558f254 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -261,7 +261,8 @@ func (c *Controller) InitImageStore() error { func (c *Controller) InitMetaDB(reloadCtx context.Context) error { // init metaDB if search is enabled or we need to store user profiles, api keys or signatures - if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() { + if c.Config.IsSearchEnabled() || c.Config.IsBasicAuthnEnabled() || c.Config.IsImageTrustEnabled() || + c.Config.IsMetaDBNeededForRetention() { driver, err := meta.New(c.Config.Storage.StorageConfig, c.Log) //nolint:contextcheck if err != nil { return err @@ -277,7 +278,7 @@ func (c *Controller) InitMetaDB(reloadCtx context.Context) error { return err } - err = meta.ParseStorage(driver, c.StoreController, c.Log) + err = meta.ParseStorage(driver, c.StoreController, c.Log) //nolint: contextcheck if err != nil { return err } @@ -293,10 +294,32 @@ func (c *Controller) LoadNewConfig(reloadCtx context.Context, newConfig *config. c.Config.HTTP.AccessControl = newConfig.HTTP.AccessControl // reload periodical gc config - c.Config.Storage.GCInterval = newConfig.Storage.GCInterval c.Config.Storage.GC = newConfig.Storage.GC + c.Config.Storage.Dedupe = newConfig.Storage.Dedupe c.Config.Storage.GCDelay = newConfig.Storage.GCDelay - c.Config.Storage.GCReferrers = newConfig.Storage.GCReferrers + c.Config.Storage.GCInterval = newConfig.Storage.GCInterval + c.Config.Storage.UntaggedImageRetentionDelay = newConfig.Storage.UntaggedImageRetentionDelay + // only if we have a metaDB already in place + if c.Config.IsMetaDBNeededForRetention() { + c.Config.Storage.Retention = newConfig.Storage.Retention + } + + for subPath, storageConfig := range newConfig.Storage.SubPaths { + subPathConfig, ok := c.Config.Storage.SubPaths[subPath] + if ok { + subPathConfig.GC = storageConfig.GC + subPathConfig.Dedupe = storageConfig.Dedupe + subPathConfig.GCDelay = storageConfig.GCDelay + subPathConfig.GCInterval = storageConfig.GCInterval + subPathConfig.UntaggedImageRetentionDelay = storageConfig.UntaggedImageRetentionDelay + // only if we have a metaDB already in place + if c.Config.IsMetaDBNeededForRetention() { + subPathConfig.Retention = storageConfig.Retention + } + + c.Config.Storage.SubPaths[subPath] = subPathConfig + } + } // reload background tasks if newConfig.Extensions != nil { @@ -340,10 +363,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) { // Enable running garbage-collect periodically for DefaultStore if c.Config.Storage.GC { gc := gc.NewGarbageCollect(c.StoreController.DefaultStore, c.MetaDB, gc.Options{ - Referrers: c.Config.Storage.GCReferrers, Delay: c.Config.Storage.GCDelay, - RetentionDelay: c.Config.Storage.UntaggedImageRetentionDelay, - }, c.Log) + UntaggedDelay: c.Config.Storage.UntaggedImageRetentionDelay, + ImageRetention: c.Config.Storage.Retention, + }, c.Audit, c.Log) gc.CleanImageStorePeriodically(c.Config.Storage.GCInterval, taskScheduler) } @@ -363,10 +386,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) { if storageConfig.GC { gc := gc.NewGarbageCollect(c.StoreController.SubStore[route], c.MetaDB, gc.Options{ - Referrers: storageConfig.GCReferrers, Delay: storageConfig.GCDelay, - RetentionDelay: storageConfig.UntaggedImageRetentionDelay, - }, c.Log) + UntaggedDelay: storageConfig.UntaggedImageRetentionDelay, + ImageRetention: storageConfig.Retention, + }, c.Audit, c.Log) gc.CleanImageStorePeriodically(storageConfig.GCInterval, taskScheduler) } diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index a7c0bb726e..9e2ddd1781 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -4999,6 +4999,7 @@ func TestHardLink(t *testing.T) { port := test.GetFreePort() conf := config.New() conf.HTTP.Port = port + conf.Storage.GC = false dir := t.TempDir() @@ -7665,6 +7666,11 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { conf := config.New() conf.HTTP.Port = port + logFile, err := os.CreateTemp("", "zot-log*.txt") + So(err, ShouldBeNil) + + conf.Log.Audit = logFile.Name() + value := true searchConfig := &extconf.SearchConfig{ BaseConfig: extconf.BaseConfig{Enable: &value}, @@ -7682,6 +7688,20 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { ctlr.Config.Storage.GC = true ctlr.Config.Storage.GCDelay = 1 * time.Millisecond ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Millisecond + ctlr.Config.Storage.Retention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{".*"}, // just for coverage + }, + }, + }, + }, + } ctlr.Config.Storage.Dedupe = false @@ -7692,16 +7712,15 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { img := CreateDefaultImage() - err := UploadImage(img, baseURL, repoName, tag) + err = UploadImage(img, baseURL, repoName, tag) So(err, ShouldBeNil) gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB, gc.Options{ - Referrers: ctlr.Config.Storage.GCReferrers, Delay: ctlr.Config.Storage.GCDelay, - RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay, - }, - ctlr.Log) + UntaggedDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay, + ImageRetention: ctlr.Config.Storage.Retention, + }, ctlr.Audit, ctlr.Log) resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag)) So(err, ShouldBeNil) @@ -7927,6 +7946,15 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { ctlr.Config.Storage.GC = true ctlr.Config.Storage.GCDelay = 1 * time.Second ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Second + ctlr.Config.Storage.Retention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + }, + }, + } err := WriteImageToFileSystem(CreateDefaultImage(), repoName, tag, ociutils.GetDefaultStoreController(dir, ctlr.Log)) @@ -7938,10 +7966,10 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { gc := gc.NewGarbageCollect(ctlr.StoreController.DefaultStore, ctlr.MetaDB, gc.Options{ - Referrers: ctlr.Config.Storage.GCReferrers, Delay: ctlr.Config.Storage.GCDelay, - RetentionDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay, - }, ctlr.Log) + UntaggedDelay: ctlr.Config.Storage.UntaggedImageRetentionDelay, + ImageRetention: ctlr.Config.Storage.Retention, + }, ctlr.Audit, ctlr.Log) resp, err := resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, tag)) So(err, ShouldBeNil) @@ -8071,8 +8099,13 @@ func TestPeriodicGC(t *testing.T) { subPaths := make(map[string]config.StorageConfig) subPaths["/a"] = config.StorageConfig{ - RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second, - UntaggedImageRetentionDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false, + RootDirectory: subDir, + GC: true, + GCDelay: 1 * time.Second, + UntaggedImageRetentionDelay: 1 * time.Second, + GCInterval: 24 * time.Hour, + RemoteCache: false, + Dedupe: false, } //nolint:lll // gofumpt conflicts with lll ctlr.Config.Storage.Dedupe = false ctlr.Config.Storage.SubPaths = subPaths diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 1db9a8044b..d995ee1bd1 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -721,8 +721,8 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht } if rh.c.MetaDB != nil { - err := meta.OnUpdateManifest(name, reference, mediaType, digest, body, rh.c.StoreController, rh.c.MetaDB, - rh.c.Log) + err := meta.OnUpdateManifest(request.Context(), name, reference, mediaType, + digest, body, rh.c.StoreController, rh.c.MetaDB, rh.c.Log) if err != nil { response.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/cli/server/config_reloader_test.go b/pkg/cli/server/config_reloader_test.go index f22781f167..1148e5dc47 100644 --- a/pkg/cli/server/config_reloader_test.go +++ b/pkg/cli/server/config_reloader_test.go @@ -166,6 +166,112 @@ func TestConfigReloader(t *testing.T) { So(string(data), ShouldContainSubstring, "\"Actions\":[\"read\",\"create\",\"update\",\"delete\"]") }) + Convey("reload gc config", t, func(c C) { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + + logFile, err := os.CreateTemp("", "zot-log*.txt") + So(err, ShouldBeNil) + + defer os.Remove(logFile.Name()) // clean up + + content := fmt.Sprintf(`{ + "distSpecVersion": "1.1.0-dev", + "storage": { + "rootDirectory": "%s", + "gc": false, + "dedupe": false, + "subPaths": { + "/a": { + "rootDirectory": "%s", + "gc": false, + "dedupe": false + } + } + }, + "http": { + "address": "127.0.0.1", + "port": "%s" + }, + "log": { + "level": "debug", + "output": "%s" + } + }`, t.TempDir(), t.TempDir(), port, logFile.Name()) + + cfgfile, err := os.CreateTemp("", "zot-test*.json") + So(err, ShouldBeNil) + + defer os.Remove(cfgfile.Name()) // clean up + + _, err = cfgfile.Write([]byte(content)) + So(err, ShouldBeNil) + + // err = cfgfile.Close() + // So(err, ShouldBeNil) + + os.Args = []string{"cli_test", "serve", cfgfile.Name()} + go func() { + err = cli.NewServerRootCmd().Execute() + So(err, ShouldBeNil) + }() + + test.WaitTillServerReady(baseURL) + + content = fmt.Sprintf(`{ + "distSpecVersion": "1.1.0-dev", + "storage": { + "rootDirectory": "%s", + "gc": true, + "dedupe": true, + "subPaths": { + "/a": { + "rootDirectory": "%s", + "gc": true, + "dedupe": true + } + } + }, + "http": { + "address": "127.0.0.1", + "port": "%s" + }, + "log": { + "level": "debug", + "output": "%s" + } + }`, t.TempDir(), t.TempDir(), port, logFile.Name()) + + err = cfgfile.Truncate(0) + So(err, ShouldBeNil) + + _, err = cfgfile.Seek(0, io.SeekStart) + So(err, ShouldBeNil) + + // truncate log before changing config, for the ShouldNotContainString + So(logFile.Truncate(0), ShouldBeNil) + + _, err = cfgfile.WriteString(content) + So(err, ShouldBeNil) + + err = cfgfile.Close() + So(err, ShouldBeNil) + + // wait for config reload + time.Sleep(2 * time.Second) + + data, err := os.ReadFile(logFile.Name()) + So(err, ShouldBeNil) + t.Logf("log file: %s", data) + + So(string(data), ShouldContainSubstring, "reloaded params") + So(string(data), ShouldContainSubstring, "loaded new configuration settings") + So(string(data), ShouldContainSubstring, "\"GC\":true") + So(string(data), ShouldContainSubstring, "\"Dedupe\":true") + So(string(data), ShouldNotContainSubstring, "\"GC\":false") + So(string(data), ShouldNotContainSubstring, "\"Dedupe\":false") + }) + Convey("reload sync config", t, func(c C) { port := test.GetFreePort() baseURL := test.GetBaseURL(port) diff --git a/pkg/cli/server/extensions_test.go b/pkg/cli/server/extensions_test.go index bf2c6a61d6..5804888185 100644 --- a/pkg/cli/server/extensions_test.go +++ b/pkg/cli/server/extensions_test.go @@ -30,9 +30,9 @@ func TestVerifyExtensionsConfig(t *testing.T) { So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{ + content := fmt.Sprintf(`{ "storage":{ - "rootDirectory":"/tmp/zot", + "rootDirectory":"%s", "dedupe":true, "remoteCache":false, "storageDriver":{ @@ -56,21 +56,22 @@ func TestVerifyExtensionsConfig(t *testing.T) { } } } - }`) - err = os.WriteFile(tmpfile.Name(), content, 0o0600) + }`, t.TempDir()) + + err = os.WriteFile(tmpfile.Name(), []byte(content), 0o0600) So(err, ShouldBeNil) os.Args = []string{"cli_test", "verify", tmpfile.Name()} So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) - content = []byte(`{ + content = fmt.Sprintf(`{ "storage":{ - "rootDirectory":"/tmp/zot", + "rootDirectory":"%s", "dedupe":true, "remoteCache":false, "subPaths":{ "/a": { - "rootDirectory": "/tmp/zot1", + "rootDirectory": "%s", "dedupe": false, "storageDriver":{ "name":"s3", @@ -95,8 +96,8 @@ func TestVerifyExtensionsConfig(t *testing.T) { } } } - }`) - err = os.WriteFile(tmpfile.Name(), content, 0o0600) + }`, t.TempDir(), t.TempDir()) + err = os.WriteFile(tmpfile.Name(), []byte(content), 0o0600) So(err, ShouldBeNil) os.Args = []string{"cli_test", "verify", tmpfile.Name()} @@ -107,12 +108,12 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot", "storageDriver": {"name": "s3"}}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s", "storageDriver": {"name": "s3"}}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], - "maxRetries": 1, "retryDelay": "10s"}]}}}`) - _, err = tmpfile.Write(content) + "maxRetries": 1, "retryDelay": "10s"}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -124,12 +125,12 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], - "maxRetries": 1, "retryDelay": "10s"}]}}}`) - _, err = tmpfile.Write(content) + "maxRetries": 1, "retryDelay": "10s"}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -141,13 +142,13 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], "maxRetries": 1, "retryDelay": "10s", - "content": [{"prefix":"[repo%^&"}]}]}}}`) - _, err = tmpfile.Write(content) + "content": [{"prefix":"[repo^&["}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -159,13 +160,13 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], "maxRetries": 1, "retryDelay": "10s", - "content": [{"prefix":"zot-repo","stripPrefix":true,"destination":"/"}]}]}}}`) - _, err = tmpfile.Write(content) + "content": [{"prefix":"zot-repo","stripPrefix":true,"destination":"/"}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -177,13 +178,13 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], "maxRetries": 1, "retryDelay": "10s", - "content": [{"prefix":"zot-repo/*","stripPrefix":true,"destination":"/"}]}]}}}`) - _, err = tmpfile.Write(content) + "content": [{"prefix":"zot-repo/*","stripPrefix":true,"destination":"/"}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -196,13 +197,13 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], "maxRetries": 1, "retryDelay": "10s", - "content": [{"prefix":"repo**"}]}]}}}`) - _, err = tmpfile.Write(content) + "content": [{"prefix":"repo**"}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -215,12 +216,12 @@ func TestVerifyExtensionsConfig(t *testing.T) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + content := fmt.Sprintf(`{"storage":{"rootDirectory":"%s"}, "http":{"address":"127.0.0.1","port":"8080","realm":"zot", "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, "extensions":{"sync": {"registries": [{"urls":["localhost:9999"], - "maxRetries": 10, "content": [{"prefix":"repo**"}]}]}}}`) - _, err = tmpfile.Write(content) + "maxRetries": 10, "content": [{"prefix":"repo**"}]}]}}}`, t.TempDir()) + _, err = tmpfile.Write([]byte(content)) So(err, ShouldBeNil) err = tmpfile.Close() So(err, ShouldBeNil) @@ -377,7 +378,7 @@ func TestServeExtensions(t *testing.T) { content := fmt.Sprintf(`{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -387,7 +388,7 @@ func TestServeExtensions(t *testing.T) { "level": "debug", "output": "%s" } - }`, port, logFile.Name()) + }`, t.TempDir(), port, logFile.Name()) cfgfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) @@ -418,7 +419,7 @@ func TestServeExtensions(t *testing.T) { content := fmt.Sprintf(`{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -430,7 +431,7 @@ func TestServeExtensions(t *testing.T) { }, "extensions": { } - }`, port, logFile.Name()) + }`, t.TempDir(), port, logFile.Name()) cfgfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) @@ -454,7 +455,7 @@ func TestServeExtensions(t *testing.T) { }) } -func testWithMetricsEnabled(cfgContentFormat string) { +func testWithMetricsEnabled(rootDir string, cfgContentFormat string) { port := GetFreePort() baseURL := GetBaseURL(port) logFile, err := os.CreateTemp("", "zot-log*.txt") @@ -462,7 +463,7 @@ func testWithMetricsEnabled(cfgContentFormat string) { defer os.Remove(logFile.Name()) // clean up - content := fmt.Sprintf(cfgContentFormat, port, logFile.Name()) + content := fmt.Sprintf(cfgContentFormat, rootDir, port, logFile.Name()) cfgfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) @@ -502,7 +503,7 @@ func TestServeMetricsExtension(t *testing.T) { Convey("no explicit enable", t, func(c C) { content := `{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -517,13 +518,13 @@ func TestServeMetricsExtension(t *testing.T) { } } }` - testWithMetricsEnabled(content) + testWithMetricsEnabled(t.TempDir(), content) }) Convey("no explicit enable but with prometheus parameter", t, func(c C) { content := `{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -541,13 +542,13 @@ func TestServeMetricsExtension(t *testing.T) { } } }` - testWithMetricsEnabled(content) + testWithMetricsEnabled(t.TempDir(), content) }) Convey("with explicit enable, but without prometheus parameter", t, func(c C) { content := `{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -563,7 +564,7 @@ func TestServeMetricsExtension(t *testing.T) { } } }` - testWithMetricsEnabled(content) + testWithMetricsEnabled(t.TempDir(), content) }) Convey("with explicit disable", t, func(c C) { @@ -575,7 +576,7 @@ func TestServeMetricsExtension(t *testing.T) { content := fmt.Sprintf(`{ "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "%s" }, "http": { "address": "127.0.0.1", @@ -590,7 +591,7 @@ func TestServeMetricsExtension(t *testing.T) { "enable": false } } - }`, port, logFile.Name()) + }`, t.TempDir(), port, logFile.Name()) cfgfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) diff --git a/pkg/cli/server/root.go b/pkg/cli/server/root.go index c4088fa806..fa9ae94aac 100644 --- a/pkg/cli/server/root.go +++ b/pkg/cli/server/root.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "path" + "regexp" "strconv" "strings" "time" @@ -615,7 +616,7 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z config.Storage.RemoteCache = true } - // s3 dedup=false, check for previous dedup usage and set to true if cachedb found + // s3 dedup=false, check for previous dedupe usage and set to true if cachedb found if !config.Storage.Dedupe && config.Storage.StorageDriver != nil { cacheDir, _ := config.Storage.StorageDriver["rootdirectory"].(string) cachePath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) @@ -651,11 +652,6 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z // if gc is enabled if storageConfig.GC { - // and gcReferrers is not set, it is set to default value - if !viperInstance.IsSet("storage::subpaths::" + name + "::gcreferrers") { - storageConfig.GCReferrers = true - } - // and gcDelay is not set, it is set to default value if !viperInstance.IsSet("storage::subpaths::" + name + "::gcdelay") { storageConfig.GCDelay = storageConstants.DefaultGCDelay @@ -851,6 +847,10 @@ func validateGC(config *config.Config, log zlog.Logger) error { } } + if err := validateGCRules(config.Storage.Retention, log); err != nil { + return err + } + // subpaths for name, subPath := range config.Storage.SubPaths { if subPath.GC && subPath.GCDelay <= 0 { @@ -861,6 +861,37 @@ func validateGC(config *config.Config, log zlog.Logger) error { return zerr.ErrBadConfig } + + if err := validateGCRules(subPath.Retention, log); err != nil { + return err + } + } + + return nil +} + +func validateGCRules(retention config.ImageRetention, log zlog.Logger) error { + for _, policy := range retention.Policies { + for _, pattern := range policy.RepoNames { + if ok := glob.ValidatePattern(pattern); !ok { + log.Error().Err(glob.ErrBadPattern).Str("pattern", pattern). + Msg("retention repo glob pattern could not be compiled") + + return zerr.ErrBadConfig + } + } + + for _, tagRule := range policy.TagsRetention { + for _, regex := range tagRule.Names { + _, err := regexp.Compile(regex) + if err != nil { + log.Error().Err(glob.ErrBadPattern).Str("regex", regex). + Msg("retention tag regex could not be compiled") + + return zerr.ErrBadConfig + } + } + } } return nil @@ -882,9 +913,20 @@ func validateSync(config *config.Config, log zlog.Logger) error { for _, content := range regCfg.Content { ok := glob.ValidatePattern(content.Prefix) if !ok { - log.Error().Err(glob.ErrBadPattern).Str("prefix", content.Prefix).Msg("sync prefix could not be compiled") + log.Error().Err(glob.ErrBadPattern).Str("prefix", content.Prefix). + Msg("sync prefix could not be compiled") + + return zerr.ErrBadConfig + } + + if content.Tags != nil && content.Tags.Regex != nil { + _, err := regexp.Compile(*content.Tags.Regex) + if err != nil { + log.Error().Err(glob.ErrBadPattern).Str("regex", *content.Tags.Regex). + Msg("sync content regex could not be compiled") - return glob.ErrBadPattern + return zerr.ErrBadConfig + } } if content.StripPrefix && !strings.Contains(content.Prefix, "/*") && content.Destination == "/" { diff --git a/pkg/cli/server/root_test.go b/pkg/cli/server/root_test.go index 390e4bfb1c..d9b771cbe3 100644 --- a/pkg/cli/server/root_test.go +++ b/pkg/cli/server/root_test.go @@ -417,6 +417,82 @@ func TestVerify(t *testing.T) { So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldNotPanic) }) + Convey("Test verify with bad gc retention repo patterns", t, func(c C) { + tmpfile, err := os.CreateTemp("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(tmpfile.Name()) // clean up + content := []byte(`{ + "distSpecVersion": "1.1.0-dev", + "storage": { + "rootDirectory": "/tmp/zot", + "gc": true, + "retention": { + "policies": [ + { + "repoNames": ["["], + "deleteReferrers": false, + "deleteUntagged": true + } + ] + } + }, + "http": { + "address": "127.0.0.1", + "port": "8080" + }, + "log": { + "level": "debug" + } + }`) + + _, err = tmpfile.Write(content) + So(err, ShouldBeNil) + err = tmpfile.Close() + So(err, ShouldBeNil) + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) + }) + + Convey("Test verify with bad gc image retention tag regex", t, func(c C) { + tmpfile, err := os.CreateTemp("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(tmpfile.Name()) // clean up + content := []byte(`{ + "distSpecVersion": "1.1.0-dev", + "storage": { + "rootDirectory": "/tmp/zot", + "gc": true, + "retention": { + "dryRun": false, + "policies": [ + { + "repoNames": ["infra/*"], + "deleteReferrers": false, + "deleteUntagged": true, + "tagsRetention": [{ + "names": ["["] + }] + } + ] + } + }, + "http": { + "address": "127.0.0.1", + "port": "8080" + }, + "log": { + "level": "debug" + } + }`) + + _, err = tmpfile.Write(content) + So(err, ShouldBeNil) + err = tmpfile.Close() + So(err, ShouldBeNil) + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) + }) + Convey("Test apply defaults cache db", t, func(c C) { tmpfile, err := os.CreateTemp("", "zot-test*.json") So(err, ShouldBeNil) diff --git a/pkg/cli/server/stress_test.go b/pkg/cli/server/stress_test.go index a12113ca9f..61500e96aa 100644 --- a/pkg/cli/server/stress_test.go +++ b/pkg/cli/server/stress_test.go @@ -25,7 +25,7 @@ const ( WorkerRunningTime = 60 * time.Second ) -func TestSressTooManyOpenFiles(t *testing.T) { +func TestStressTooManyOpenFiles(t *testing.T) { oldArgs := os.Args defer func() { os.Args = oldArgs }() diff --git a/pkg/common/common.go b/pkg/common/common.go index 1aa354a448..978c1e5e5f 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -6,6 +6,7 @@ import ( "fmt" "io/fs" "os" + "regexp" "strings" "syscall" "time" @@ -119,3 +120,22 @@ func ContainsStringIgnoreCase(strSlice []string, str string) bool { return false } + +// MatchesListOfRegex is used by retention, it return true if list of regexes is empty. +func MatchesListOfRegex(name string, regexes []string) bool { + if len(regexes) == 0 { + // empty regexes matches everything in retention logic + return true + } + + for _, regex := range regexes { + // all are compilable because they are checked at startup + if tagReg, err := regexp.Compile(regex); err == nil { + if tagReg.MatchString(name) { + return true + } + } + } + + return false +} diff --git a/pkg/extensions/search/convert/convert_internal_test.go b/pkg/extensions/search/convert/convert_internal_test.go index fe99ae68c7..85948f85d4 100644 --- a/pkg/extensions/search/convert/convert_internal_test.go +++ b/pkg/extensions/search/convert/convert_internal_test.go @@ -56,7 +56,7 @@ func TestCVEConvert(t *testing.T) { digest11 := godigest.FromString("abc1") err = metaDB.SetManifestMeta("repo1", digest11, repoMeta11) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) reposMeta, manifestMetaMap, _, err := metaDB.SearchRepos(context.Background(), "") diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 32d9059150..ace30a86ef 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -5,6 +5,7 @@ package cveinfo_test import ( + "context" "encoding/json" "fmt" "io" @@ -773,7 +774,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo1, image11.ManifestDescriptor.Digest, repoMeta11) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image12 := CreateImageWith().DefaultLayers(). @@ -788,7 +789,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo1, image12.ManifestDescriptor.Digest, repoMeta12) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image13 := CreateImageWith().DefaultLayers(). @@ -803,7 +804,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo1, image13.ManifestDescriptor.Digest, repoMeta13) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image14 := CreateImageWith().DefaultLayers(). @@ -816,7 +817,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo1, image14.ManifestDescriptor.Digest, repoMeta14) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for scannable image with no vulnerabilities @@ -830,7 +831,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo6, image61.ManifestDescriptor.Digest, repoMeta61) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo6, "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo6, "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for image not supporting scanning @@ -847,7 +848,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo2, image21.ManifestDescriptor.Digest, repoMeta21) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for invalid images/negative tests @@ -861,7 +862,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo digest31 := godigest.FromBytes(manifestBlob31) err = metaDB.SetManifestMeta(repo3, digest31, repoMeta31) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, "invalid-manifest", digest31, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo3, "invalid-manifest", digest31, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image41 := CreateImageWith().DefaultLayers(). @@ -874,12 +875,12 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo4, image41.ManifestDescriptor.Digest, repoMeta41) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo4, "invalid-config", image41.ManifestDescriptor.Digest, + err = metaDB.SetRepoReference(context.Background(), repo4, "invalid-config", image41.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) digest51 := godigest.FromString("abc8") - err = metaDB.SetRepoReference(repo5, "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo5, "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for scannable image which errors during scan @@ -893,7 +894,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetManifestMeta(repo7, image71.ManifestDescriptor.Digest, repoMeta71) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo7, "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo7, "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // create multiarch image with vulnerabilities @@ -933,6 +934,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo So(err, ShouldBeNil) err = metaDB.SetRepoReference( + context.Background(), repoMultiarch, "tagIndex", multiarchImage.IndexDescriptor.Digest, diff --git a/pkg/extensions/search/cve/pagination_test.go b/pkg/extensions/search/cve/pagination_test.go index ab518c2cd0..fad034bf5a 100644 --- a/pkg/extensions/search/cve/pagination_test.go +++ b/pkg/extensions/search/cve/pagination_test.go @@ -4,6 +4,7 @@ package cveinfo_test import ( + "context" "encoding/json" "fmt" "sort" @@ -65,7 +66,7 @@ func TestCVEPagination(t *testing.T) { digest11 := godigest.FromBytes(manifestBlob11) err = metaDB.SetManifestMeta("repo1", digest11, repoMeta11) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) @@ -99,7 +100,7 @@ func TestCVEPagination(t *testing.T) { digest12 := godigest.FromBytes(manifestBlob12) err = metaDB.SetManifestMeta("repo1", digest12, repoMeta12) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // MetaDB loaded with initial data, mock the scanner diff --git a/pkg/extensions/search/cve/scan_test.go b/pkg/extensions/search/cve/scan_test.go index ccb45dfc8b..a9ded0fcc5 100644 --- a/pkg/extensions/search/cve/scan_test.go +++ b/pkg/extensions/search/cve/scan_test.go @@ -82,7 +82,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo1", image11.ManifestDescriptor.Digest, repoMeta11) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image12 := CreateImageWith().DefaultLayers(). @@ -97,7 +98,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo1", image12.ManifestDescriptor.Digest, repoMeta12) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image13 := CreateImageWith().DefaultLayers(). @@ -112,7 +114,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo1", image13.ManifestDescriptor.Digest, repoMeta13) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image14 := CreateImageWith().DefaultLayers(). @@ -125,7 +128,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo1", image14.ManifestDescriptor.Digest, repoMeta14) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for scannable image with no vulnerabilities @@ -139,7 +143,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo6", image61.ManifestDescriptor.Digest, repoMeta61) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo6", "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo6", + "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for image not supporting scanning @@ -156,7 +161,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo2", image21.ManifestDescriptor.Digest, repoMeta21) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo2", "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo2", + "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for invalid images/negative tests @@ -170,7 +176,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo digest31 := godigest.FromBytes(manifestBlob31) err = metaDB.SetManifestMeta("repo3", digest31, repoMeta31) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo3", "invalid-manifest", digest31, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo3", + "invalid-manifest", digest31, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) image41 := CreateImageWith().DefaultLayers(). @@ -183,12 +190,13 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo4", image41.ManifestDescriptor.Digest, repoMeta41) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo4", "invalid-config", image41.ManifestDescriptor.Digest, - ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo4", + "invalid-config", image41.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) digest51 := godigest.FromString("abc8") - err = metaDB.SetRepoReference("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo5", + "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create metadb data for scannable image which errors during scan @@ -202,7 +210,8 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetManifestMeta("repo7", image71.ManifestDescriptor.Digest, repoMeta71) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo7", "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo7", + "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create multiarch image with vulnerabilities @@ -242,6 +251,7 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo So(err, ShouldBeNil) err = metaDB.SetRepoReference( + context.Background(), repoIndex, "tagIndex", multiarchImage.IndexDescriptor.Digest, diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 4d81ef9778..8842b810c2 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -5,6 +5,7 @@ package trivy import ( "bytes" + "context" "encoding/json" "os" "path" @@ -325,7 +326,8 @@ func TestImageScannable(t *testing.T) { panic(err) } - err = metaDB.SetRepoReference("repo1", "valid", digestValidManifest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "valid", digestValidManifest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } @@ -361,7 +363,7 @@ func TestImageScannable(t *testing.T) { panic(err) } - err = metaDB.SetRepoReference("repo1", "unscannable-layer", digestManifestUnscannableLayer, + err = metaDB.SetRepoReference(context.Background(), "repo1", "unscannable-layer", digestManifestUnscannableLayer, ispec.MediaTypeImageManifest) if err != nil { panic(err) @@ -381,7 +383,8 @@ func TestImageScannable(t *testing.T) { panic(err) } - err = metaDB.SetRepoReference("repo1", "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } @@ -389,13 +392,15 @@ func TestImageScannable(t *testing.T) { // Manifest meta cannot be found digestMissingManifest := godigest.FromBytes([]byte("Some other string")) - err = metaDB.SetRepoReference("repo1", "missing", digestMissingManifest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "missing", digestMissingManifest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } // RepoMeta contains invalid digest - err = metaDB.SetRepoReference("repo1", "invalid-digest", "invalid", ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo1", + "invalid-digest", "invalid", ispec.MediaTypeImageManifest) if err != nil { panic(err) } diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index eedcce38b3..dae30468f2 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -2072,7 +2072,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo for image, digest := range tagsMap { repo, tag := common.GetImageDirAndTag(image) - err := metaDB.SetRepoReference(repo, tag, digest, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo, tag, digest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 1ed410162f..58d3831a60 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -4530,7 +4530,9 @@ func TestMetaDBWhenPushingImages(t *testing.T) { Convey("SetManifestMeta succeeds but SetRepoReference fails", func() { ctlr.MetaDB = mocks.MetaDBMock{ - SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, mediaType string) error { + SetRepoReferenceFn: func(ctx context.Context, repo, + reference string, manifestDigest godigest.Digest, mediaType string, + ) error { return ErrTestError }, } @@ -5258,7 +5260,7 @@ func TestMetaDBWhenReadingImages(t *testing.T) { Convey("Error when incrementing", func() { ctlr.MetaDB = mocks.MetaDBMock{ - IncrementImageDownloadsFn: func(repo string, tag string) error { + UpdateStatsOnDownloadFn: func(repo string, tag string) error { return ErrTestError }, } diff --git a/pkg/extensions/sync/local.go b/pkg/extensions/sync/local.go index 9bf1717901..86ccf357c5 100644 --- a/pkg/extensions/sync/local.go +++ b/pkg/extensions/sync/local.go @@ -4,6 +4,7 @@ package sync import ( + "context" "encoding/json" "errors" "fmt" @@ -164,7 +165,7 @@ func (registry *LocalRegistry) CommitImage(imageReference types.ImageReference, } if registry.metaDB != nil { - err = meta.SetImageMetaFromInput(repo, reference, mediaType, + err = meta.SetImageMetaFromInput(context.Background(), repo, reference, mediaType, manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log) if err != nil { return fmt.Errorf("metaDB: failed to set metadata for image '%s %s': %w", repo, reference, err) @@ -222,7 +223,7 @@ func (registry *LocalRegistry) copyManifest(repo string, manifestContent []byte, } if registry.metaDB != nil { - err = meta.SetImageMetaFromInput(repo, reference, ispec.MediaTypeImageManifest, + err = meta.SetImageMetaFromInput(context.Background(), repo, reference, ispec.MediaTypeImageManifest, digest, manifestContent, imageStore, registry.metaDB, registry.log) if err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). diff --git a/pkg/extensions/sync/references/cosign.go b/pkg/extensions/sync/references/cosign.go index 74d399f99d..6867849e06 100644 --- a/pkg/extensions/sync/references/cosign.go +++ b/pkg/extensions/sync/references/cosign.go @@ -166,7 +166,8 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote SignatureDigest: referenceDigest.String(), }) } else { - err = meta.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest, + err = meta.SetImageMetaFromInput(context.Background(), localRepo, //nolint:contextcheck + cosignTag, ispec.MediaTypeImageManifest, referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) } diff --git a/pkg/extensions/sync/references/oci.go b/pkg/extensions/sync/references/oci.go index 4a98a33e6b..7985214d9d 100644 --- a/pkg/extensions/sync/references/oci.go +++ b/pkg/extensions/sync/references/oci.go @@ -150,7 +150,8 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe SignatureDigest: referenceDigest.String(), }) } else { - err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, + err = meta.SetImageMetaFromInput(context.Background(), localRepo, //nolint:contextcheck + referenceDigest.String(), referrer.MediaType, referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) } diff --git a/pkg/extensions/sync/references/oras.go b/pkg/extensions/sync/references/oras.go index c5a208d613..12ba3a8e68 100644 --- a/pkg/extensions/sync/references/oras.go +++ b/pkg/extensions/sync/references/oras.go @@ -154,7 +154,8 @@ func (ref ORASReferences) SyncReferences(ctx context.Context, localRepo, remoteR ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("metaDB: trying to sync oras artifact for image") - err := meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, + err := meta.SetImageMetaFromInput(context.Background(), localRepo, //nolint:contextcheck + referenceDigest.String(), referrer.MediaType, referenceDigest, orasBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) if err != nil { diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 9b6c2968a8..29ef4e7939 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -336,7 +336,9 @@ func TestLocalRegistry(t *testing.T) { Convey("trigger metaDB error on index manifest in CommitImage()", func() { registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{ - SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { + SetRepoReferenceFn: func(ctx context.Context, repo, + Reference string, manifestDigest godigest.Digest, mediaType string, + ) error { if Reference == "1.0" { return zerr.ErrRepoMetaNotFound } @@ -351,7 +353,9 @@ func TestLocalRegistry(t *testing.T) { Convey("trigger metaDB error on image manifest in CommitImage()", func() { registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{ - SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { + SetRepoReferenceFn: func(ctx context.Context, repo, + Reference string, manifestDigest godigest.Digest, mediaType string, + ) error { return zerr.ErrRepoMetaNotFound }, }, log) diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 20a57c014b..128c1d6bda 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -875,7 +875,7 @@ func TestOnDemand(t *testing.T) { return nil }, - SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, + SetRepoReferenceFn: func(ctx context.Context, repo, reference string, manifestDigest godigest.Digest, mediaType string, ) error { if strings.HasPrefix(reference, "sha256-") && diff --git a/pkg/log/log.go b/pkg/log/log.go index 72b534ac8e..e3b42aaef5 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -52,7 +52,7 @@ func NewLogger(level, output string) Logger { return Logger{Logger: log.Hook(goroutineHook{}).With().Caller().Timestamp().Logger()} } -func NewAuditLogger(level, audit string) *Logger { +func NewAuditLogger(level, output string) *Logger { loggerSetTimeFormat.Do(func() { zerolog.TimeFieldFormat = time.RFC3339Nano }) @@ -66,12 +66,16 @@ func NewAuditLogger(level, audit string) *Logger { var auditLog zerolog.Logger - auditFile, err := os.OpenFile(audit, os.O_APPEND|os.O_WRONLY|os.O_CREATE, defaultPerms) - if err != nil { - panic(err) - } + if output == "" { + auditLog = zerolog.New(os.Stdout) + } else { + auditFile, err := os.OpenFile(output, os.O_APPEND|os.O_WRONLY|os.O_CREATE, defaultPerms) + if err != nil { + panic(err) + } - auditLog = zerolog.New(auditFile) + auditLog = zerolog.New(auditFile) + } return &Logger{Logger: auditLog.With().Timestamp().Logger()} } diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index ca14e456da..eb245285de 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -471,14 +471,21 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go return err } -func (bdw *BoltDB) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, +func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference string, manifestDigest godigest.Digest, mediaType string, ) error { if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil { return err } - err := bdw.DB.Update(func(tx *bbolt.Tx) error { + var userid string + + userAc, err := reqCtx.UserAcFromContext(ctx) + if err == nil { + userid = userAc.GetUsername() + } + + err = bdw.DB.Update(func(tx *bbolt.Tx) error { buck := tx.Bucket([]byte(RepoMetadataBucket)) repoMetaBlob := buck.Get([]byte(repo)) @@ -507,7 +514,13 @@ func (bdw *BoltDB) SetRepoReference(repo string, reference string, manifestDiges } if _, ok := repoMeta.Statistics[manifestDigest.String()]; !ok { - repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{DownloadCount: 0} + repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{ + DownloadCount: 0, + LastPullTimestamp: time.Time{}, + PushTimestamp: time.Now().Local(), + // Q do we care if it was pushed by sync or by metaDB.ParseStorage()? + PushedBy: userid, + } } if _, ok := repoMeta.Signatures[manifestDigest.String()]; !ok { @@ -752,7 +765,7 @@ func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta return foundRepos, err } -func (bdw *BoltDB) IncrementImageDownloads(repo string, reference string) error { +func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { buck := tx.Bucket([]byte(RepoMetadataBucket)) @@ -783,6 +796,7 @@ func (bdw *BoltDB) IncrementImageDownloads(repo string, reference string) error manifestStatistics := repoMeta.Statistics[manifestDigest] manifestStatistics.DownloadCount++ + manifestStatistics.LastPullTimestamp = time.Now().Local() repoMeta.Statistics[manifestDigest] = manifestStatistics repoMetaBlob, err = json.Marshal(repoMeta) diff --git a/pkg/meta/boltdb/boltdb_test.go b/pkg/meta/boltdb/boltdb_test.go index 4729e5a892..eb7fc4b43b 100644 --- a/pkg/meta/boltdb/boltdb_test.go +++ b/pkg/meta/boltdb/boltdb_test.go @@ -335,7 +335,7 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = boltdbWrapper.SetRepoReference("repo1", "tag", "digest", ispec.MediaTypeImageManifest) + err = boltdbWrapper.SetRepoReference(context.Background(), "repo1", "tag", "digest", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) @@ -434,7 +434,7 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("IncrementImageDownloads", func() { + Convey("UpdateStatsOnDownload", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) @@ -442,10 +442,10 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = boltdbWrapper.IncrementImageDownloads("repo2", "tag") + err = boltdbWrapper.UpdateStatsOnDownload("repo2", "tag") So(err, ShouldNotBeNil) - err = boltdbWrapper.IncrementImageDownloads("repo1", "tag") + err = boltdbWrapper.UpdateStatsOnDownload("repo1", "tag") So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { @@ -455,7 +455,7 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = boltdbWrapper.IncrementImageDownloads("repo1", "tag") + err = boltdbWrapper.UpdateStatsOnDownload("repo1", "tag") So(err, ShouldNotBeNil) }) @@ -648,7 +648,8 @@ func TestWrapperErrors(t *testing.T) { Convey("Bad index data", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) @@ -664,7 +665,8 @@ func TestWrapperErrors(t *testing.T) { Convey("Bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{ @@ -778,7 +780,8 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) @@ -792,7 +795,8 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{ @@ -812,7 +816,8 @@ func TestWrapperErrors(t *testing.T) { manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") ) - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) indexBlob, err := GetIndexBlobWithManifests([]digest.Digest{ @@ -979,7 +984,8 @@ func TestWrapperErrors(t *testing.T) { Convey("Unsuported type", func() { digest := digest.FromString("digest") - err := boltdbWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck + err := boltdbWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", digest, "invalid type") So(err, ShouldBeNil) _, _, _, err = boltdbWrapper.SearchRepos(ctx, "") diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index d96c2b1dfa..89d068c672 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -520,14 +520,21 @@ func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest return err } -func (dwr *DynamoDB) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, - mediaType string, +func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, reference string, + manifestDigest godigest.Digest, mediaType string, ) error { if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil { return err } - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + var userid string + + userAc, err := reqCtx.UserAcFromContext(ctx) + if err == nil { + userid = userAc.GetUsername() + } + + resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ //nolint:contextcheck TableName: aws.String(dwr.RepoMetaTablename), Key: map[string]types.AttributeValue{ "RepoName": &types.AttributeValueMemberS{Value: repo}, @@ -560,7 +567,13 @@ func (dwr *DynamoDB) SetRepoReference(repo string, reference string, manifestDig } if _, ok := repoMeta.Statistics[manifestDigest.String()]; !ok { - repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{DownloadCount: 0} + repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{ + DownloadCount: 0, + LastPullTimestamp: time.Time{}, + PushTimestamp: time.Now().Local(), + // Q do we care if it was pushed by sync or by metaDB.ParseStorage()? + PushedBy: userid, + } } if _, ok := repoMeta.Signatures[manifestDigest.String()]; !ok { @@ -571,7 +584,7 @@ func (dwr *DynamoDB) SetRepoReference(repo string, reference string, manifestDig repoMeta.Referrers[manifestDigest.String()] = []mTypes.ReferrerInfo{} } - err = dwr.SetRepoMeta(repo, repoMeta) + err = dwr.SetRepoMeta(repo, repoMeta) //nolint: contextcheck return err } @@ -682,7 +695,7 @@ func (dwr *DynamoDB) GetUserRepoMeta(ctx context.Context, repo string) (mTypes.R return repoMeta, nil } -func (dwr *DynamoDB) IncrementImageDownloads(repo string, reference string) error { +func (dwr *DynamoDB) UpdateStatsOnDownload(repo string, reference string) error { repoMeta, err := dwr.GetRepoMeta(repo) if err != nil { return err @@ -703,6 +716,7 @@ func (dwr *DynamoDB) IncrementImageDownloads(repo string, reference string) erro manifestStatistics := repoMeta.Statistics[descriptorDigest] manifestStatistics.DownloadCount++ + manifestStatistics.LastPullTimestamp = time.Now().Local() repoMeta.Statistics[descriptorDigest] = manifestStatistics return dwr.SetRepoMeta(repo, repoMeta) diff --git a/pkg/meta/dynamodb/dynamodb_test.go b/pkg/meta/dynamodb/dynamodb_test.go index 4901d8ec12..63adfab733 100644 --- a/pkg/meta/dynamodb/dynamodb_test.go +++ b/pkg/meta/dynamodb/dynamodb_test.go @@ -68,13 +68,13 @@ func TestIterator(t *testing.T) { So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) - err = dynamoWrapper.SetRepoReference("repo1", "tag1", "manifestType", "manifestDigest1") + err = dynamoWrapper.SetRepoReference(context.Background(), "repo1", "tag1", "manifestType", "manifestDigest1") So(err, ShouldBeNil) - err = dynamoWrapper.SetRepoReference("repo2", "tag2", "manifestType", "manifestDigest2") + err = dynamoWrapper.SetRepoReference(context.Background(), "repo2", "tag2", "manifestType", "manifestDigest2") So(err, ShouldBeNil) - err = dynamoWrapper.SetRepoReference("repo3", "tag3", "manifestType", "manifestDigest3") + err = dynamoWrapper.SetRepoReference(context.Background(), "repo3", "tag3", "manifestType", "manifestDigest3") So(err, ShouldBeNil) repoMetaAttributeIterator := mdynamodb.NewBaseDynamoAttributesIterator( @@ -522,7 +522,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("GetManifestMeta GetManifestData not found error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) _, err = dynamoWrapper.GetManifestMeta("repo", "dig") @@ -551,7 +551,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SetRepoReference client error", func() { dynamoWrapper.RepoMetaTablename = badTablename digest := digest.FromString("str") - err := dynamoWrapper.SetRepoReference("repo", digest.String(), digest, ispec.MediaTypeImageManifest) + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", + digest.String(), digest, ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) @@ -685,16 +686,16 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("IncrementImageDownloads GetRepoMeta error", func() { - err = dynamoWrapper.IncrementImageDownloads("repoNotFound", "") + Convey("UpdateStatsOnDownload GetRepoMeta error", func() { + err = dynamoWrapper.UpdateStatsOnDownload("repoNotFound", "") So(err, ShouldNotBeNil) }) - Convey("IncrementImageDownloads tag not found error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + Convey("UpdateStatsOnDownload tag not found error", func() { + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) - err = dynamoWrapper.IncrementImageDownloads("repo", "notFoundTag") + err = dynamoWrapper.UpdateStatsOnDownload("repo", "notFoundTag") So(err, ShouldNotBeNil) }) @@ -721,7 +722,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("AddManifestSignature GetRepoMeta error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.AddManifestSignature("repoNotFound", "tag", mTypes.SignatureMetadata{}) @@ -729,7 +730,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("AddManifestSignature ManifestSignatures signedManifestDigest not found error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", mTypes.SignatureMetadata{}) @@ -737,7 +738,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("AddManifestSignature SignatureType metadb.NotationType", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", mTypes.SignatureMetadata{ @@ -834,7 +835,8 @@ func TestWrapperErrors(t *testing.T) { }) Convey("SearchRepos GetManifestMeta error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "notFoundDigest", //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag1", //nolint:contextcheck + "notFoundDigest", ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -846,7 +848,8 @@ func TestWrapperErrors(t *testing.T) { Convey("Unsuported type", func() { digest := digest.FromString("digest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", digest, "invalid type") So(err, ShouldBeNil) _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") @@ -863,7 +866,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchRepos bad index data", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck @@ -876,7 +880,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchRepos bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck @@ -898,7 +903,8 @@ func TestWrapperErrors(t *testing.T) { }) Convey("SearchTags GetManifestMeta error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", "manifestNotFound", ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -910,7 +916,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchTags bad index data", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck @@ -923,7 +930,8 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchTags bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck @@ -983,7 +991,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("FilterTags manifestMeta not found", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", "tag1", "manifestNotFound", //nolint:contextcheck ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -996,7 +1004,8 @@ func TestWrapperErrors(t *testing.T) { }) Convey("FilterTags manifestMeta unmarshal error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig", ispec.MediaTypeImageManifest) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", "dig", ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck @@ -1014,7 +1023,8 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck @@ -1028,7 +1038,8 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck @@ -1048,7 +1059,8 @@ func TestWrapperErrors(t *testing.T) { manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") ) - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag1", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) indexBlob, err := GetIndexBlobWithManifests([]digest.Digest{ @@ -1126,7 +1138,8 @@ func TestWrapperErrors(t *testing.T) { }) Convey("GetUserRepoMeta userMeta not found", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", digest.FromString("1"), ispec.MediaTypeImageManifest) + err := dynamoWrapper.SetRepoReference(context.Background(), "repo", + "tag", digest.FromString("1"), ispec.MediaTypeImageManifest) So(err, ShouldBeNil) dynamoWrapper.UserDataTablename = badTablename diff --git a/pkg/meta/hooks.go b/pkg/meta/hooks.go index 3a37814eb4..d782dc9fa3 100644 --- a/pkg/meta/hooks.go +++ b/pkg/meta/hooks.go @@ -1,6 +1,8 @@ package meta import ( + "context" + godigest "github.com/opencontainers/go-digest" "zotregistry.io/zot/pkg/log" @@ -12,7 +14,7 @@ import ( // OnUpdateManifest is called when a new manifest is added. It updates metadb according to the type // of image pushed(normal images, signatues, etc.). In care of any errors, it makes sure to keep // consistency between metadb and the image store. -func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest, body []byte, +func OnUpdateManifest(ctx context.Context, repo, reference, mediaType string, digest godigest.Digest, body []byte, storeController storage.StoreController, metaDB mTypes.MetaDB, log log.Logger, ) error { imgStore := storeController.GetImageStore(repo) @@ -58,16 +60,17 @@ func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest, } } } else { - err = SetImageMetaFromInput(repo, reference, mediaType, digest, body, + err = SetImageMetaFromInput(ctx, repo, reference, mediaType, digest, body, imgStore, metaDB, log) if err != nil { + log.Info().Err(err).Str("tag", reference).Str("repository", repo).Bytes("body", body). + Msg("uploading image meta was unsuccessful for tag in repo") + metadataSuccessfullySet = false } } if !metadataSuccessfullySet { - log.Info().Str("tag", reference).Str("repository", repo).Msg("uploading image meta was unsuccessful for tag in repo") - if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil { log.Error().Err(err).Str("reference", reference).Str("repository", repo). Msg("couldn't remove image manifest in repo") @@ -155,7 +158,7 @@ func OnGetManifest(name, reference string, body []byte, } if !isSignature { - err := metaDB.IncrementImageDownloads(name, reference) + err := metaDB.UpdateStatsOnDownload(name, reference) if err != nil { log.Error().Err(err).Str("repository", name).Str("reference", reference). Msg("unexpected error for image") diff --git a/pkg/meta/hooks_test.go b/pkg/meta/hooks_test.go index 0530f3e498..096b976a0d 100644 --- a/pkg/meta/hooks_test.go +++ b/pkg/meta/hooks_test.go @@ -1,6 +1,7 @@ package meta_test import ( + "context" "encoding/json" "errors" "testing" @@ -56,7 +57,8 @@ func TestOnUpdateManifest(t *testing.T) { digest := godigest.FromBytes(manifestBlob) - err = meta.OnUpdateManifest("repo", "tag1", "", digest, manifestBlob, storeController, metaDB, log) + err = meta.OnUpdateManifest(context.Background(), "repo", + "tag1", "", digest, manifestBlob, storeController, metaDB, log) So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta("repo") @@ -78,7 +80,7 @@ func TestOnUpdateManifest(t *testing.T) { }, } - err := meta.OnUpdateManifest("repo", "tag1", ispec.MediaTypeImageManifest, "digest", + err := meta.OnUpdateManifest(context.Background(), "repo", "tag1", ispec.MediaTypeImageManifest, "digest", []byte("{}"), storeController, metaDB, log) So(err, ShouldNotBeNil) }) @@ -103,7 +105,7 @@ func TestUpdateErrors(t *testing.T) { return nil } - err := meta.OnUpdateManifest("repo", "tag1", "digest", "media", badManifestBlob, + err := meta.OnUpdateManifest(context.Background(), "repo", "tag1", "digest", "media", badManifestBlob, storeController, metaDB, log) So(err, ShouldNotBeNil) }) @@ -124,7 +126,7 @@ func TestUpdateErrors(t *testing.T) { return badNotationManifestBlob, "", "", nil } - err = meta.OnUpdateManifest("repo", "tag1", "", "digest", badNotationManifestBlob, + err = meta.OnUpdateManifest(context.Background(), "repo", "tag1", "", "digest", badNotationManifestBlob, storeController, metaDB, log) So(err, ShouldNotBeNil) }) @@ -156,7 +158,7 @@ func TestUpdateErrors(t *testing.T) { return ErrTestError } - err = meta.OnUpdateManifest("repo", "tag1", "", "digest", notationManifestBlob, + err = meta.OnUpdateManifest(context.Background(), "repo", "tag1", "", "digest", notationManifestBlob, storeController, metaDB, log) So(err, ShouldNotBeNil) }) @@ -216,7 +218,7 @@ func TestUpdateErrors(t *testing.T) { metaDB := mocks.MetaDBMock{} log := log.NewLogger("debug", "") - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", + err := meta.SetImageMetaFromInput(context.Background(), "repo", "ref", ispec.MediaTypeImageManifest, "digest", []byte("BadManifestBlob"), imageStore, metaDB, log) So(err, ShouldNotBeNil) @@ -233,8 +235,8 @@ func TestUpdateErrors(t *testing.T) { return []byte("{}"), nil } - err = meta.SetImageMetaFromInput("repo", string(godigest.FromString("reference")), "", "digest", - manifestBlob, imageStore, metaDB, log) + err = meta.SetImageMetaFromInput(context.Background(), "repo", + string(godigest.FromString("reference")), "", "digest", manifestBlob, imageStore, metaDB, log) So(err, ShouldBeNil) }) @@ -247,7 +249,7 @@ func TestUpdateErrors(t *testing.T) { return ErrTestError }, } - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", + err := meta.SetImageMetaFromInput(context.Background(), "repo", "ref", ispec.MediaTypeImageManifest, "digest", []byte("{}"), imageStore, metaDB, log) So(err, ShouldNotBeNil) }) @@ -261,7 +263,7 @@ func TestUpdateErrors(t *testing.T) { return ErrTestError }, } - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest", + err := meta.SetImageMetaFromInput(context.Background(), "repo", "ref", ispec.MediaTypeImageIndex, "digest", []byte("{}"), imageStore, metaDB, log) So(err, ShouldNotBeNil) }) @@ -280,7 +282,7 @@ func TestUpdateErrors(t *testing.T) { }, } - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", + err := meta.SetImageMetaFromInput(context.Background(), "repo", "ref", ispec.MediaTypeImageManifest, "digest", []byte(`{"subject": {"digest": "subjDigest"}}`), imageStore, metaDB, log) So(err, ShouldNotBeNil) }) diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index 7e9996df49..c5f5a58f70 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -529,7 +529,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ) Convey("Setting a good repo", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta(repo1) @@ -555,7 +555,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldNotBeNil) digest := godigest.FromString("digest") - err = metaDB.SetRepoReference(repo1, digest.String(), digest, + err = metaDB.SetRepoReference(context.Background(), repo1, digest.String(), digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -565,9 +565,9 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Set multiple tags for repo", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta(repo1) @@ -577,9 +577,9 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Set multiple repos", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMeta1, err := metaDB.GetRepoMeta(repo1) @@ -593,17 +593,17 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func Convey("Setting a repo with invalid fields", func() { Convey("Repo name is not valid", func() { - err := metaDB.SetRepoReference("", tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), "", tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) Convey("Tag is not valid", func() { - err := metaDB.SetRepoReference(repo1, "", manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, "", manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) Convey("Manifest Digest is not valid", func() { - err := metaDB.SetRepoReference(repo1, tag1, "", ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, "", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) }) @@ -622,10 +622,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func InexistentRepo = "InexistentRepo" ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) Convey("Get a existent repo", func() { @@ -654,10 +654,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest2 = godigest.FromString("fake-manifest2") ) - err := metaDB.SetRepoReference(repo, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) Convey("Delete reference from repo", func() { @@ -764,13 +764,13 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest2 = godigest.FromString("fake-manifest2") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) Convey("Get all Repometa", func() { @@ -804,7 +804,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) @@ -836,7 +836,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) @@ -871,7 +871,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) @@ -931,10 +931,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func // anonymous user ctx3 := userAc.DeriveContext(context.Background()) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) starCount, err := metaDB.GetRepoStars(repo1) @@ -1168,10 +1168,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func // anonymous user ctx3 := userAc.DeriveContext(context.Background()) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, err := metaDB.GetBookmarkedRepos(ctx1) @@ -1275,7 +1275,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(len(repos), ShouldEqual, 0) }) - Convey("Test IncrementImageDownloads", func() { + Convey("Test UpdateStatsOnDownload", func() { var ( repo1 = "repo1" tag1 = "0.0.1" @@ -1286,7 +1286,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest := godigest.FromBytes(manifestBlob) - err = metaDB.SetRepoReference(repo1, tag1, manifestDigest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest, mTypes.ManifestMetadata{ @@ -1295,7 +1295,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.IncrementImageDownloads(repo1, tag1) + err = metaDB.UpdateStatsOnDownload(repo1, tag1) So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta(repo1) @@ -1303,13 +1303,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 1) - err = metaDB.IncrementImageDownloads(repo1, tag1) + err = metaDB.UpdateStatsOnDownload(repo1, tag1) So(err, ShouldBeNil) repoMeta, err = metaDB.GetRepoMeta(repo1) So(err, ShouldBeNil) So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 2) + So(time.Now().Local(), ShouldHappenAfter, repoMeta.Statistics[manifestDigest.String()].LastPullTimestamp) _, err = metaDB.GetManifestMeta(repo1, "badManiestDigest") So(err, ShouldNotBeNil) @@ -1322,7 +1323,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{}) @@ -1351,7 +1352,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("dig") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{ @@ -1386,7 +1387,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func repo := "repo" tag := "0.0.1" - err := metaDB.SetRepoReference(repo, tag, manifestDigest, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo, tag, manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo, manifestDigest, mTypes.ManifestMetadata{ @@ -1497,7 +1498,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestData(manifestDigest1, mTypes.ManifestData{}) @@ -1519,7 +1520,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestData(manifestDigest1, mTypes.ManifestData{}) @@ -1580,11 +1581,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func } Convey("Search all repos", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + tag3, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1604,7 +1608,8 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search a repo by name", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1618,10 +1623,12 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search non-existing repo by name", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "RepoThatDoesntExist") @@ -1631,11 +1638,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search with partial match", func() { - err := metaDB.SetRepoReference("alpine", tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), "alpine", //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("pine", tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "pine", //nolint:contextcheck + tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("golang", tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "golang", //nolint:contextcheck + tag3, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta("alpine", manifestDigest1, emptyRepoMeta) @@ -1654,11 +1664,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search multiple repos that share manifests", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo3, //nolint:contextcheck + tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1675,11 +1688,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Search repos with access control", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo3, //nolint:contextcheck + tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1764,10 +1780,12 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag4, indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag5, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, "repo") @@ -1807,17 +1825,23 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ConfigBlob: emptyConfigBlob, } - err = metaDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1907,11 +1931,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func tag3 = "0.0.3" ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo3, //nolint:contextcheck + tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) config := ispec.Image{} @@ -2004,13 +2031,16 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag4, indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag5, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", tag6, manifestDigest4, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + tag6, manifestDigest4, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, manifestMetaMap, indexDataMap, err := metaDB.SearchTags(ctx, "repo:0.0") @@ -2040,15 +2070,20 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func tag5 = "0.0.5" ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag4, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag4, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag5, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + tag5, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) config := ispec.Image{} @@ -2120,7 +2155,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ConfigBlob: emptyConfigBlob, } - err = metaDB.SetRepoReference(repo1, "2.0.0", indexDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetRepoReference(context.Background(), repo1, "2.0.0", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) indexBlob, err := GetIndexBlobWithManifests([]godigest.Digest{ @@ -2134,17 +2169,23 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo1, //nolint:contextcheck + "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo2, //nolint:contextcheck + "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyManifestMeta) @@ -2290,7 +2331,8 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetManifestData(referredDigest, manifestData) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", "tag", referredDigest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + "tag", referredDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // ------- Add Artifact 1 @@ -2464,10 +2506,12 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) } - err = metaDB.SetRepoReference("repo", img.DigestStr(), imgDigest, img.Manifest.MediaType) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + img.DigestStr(), imgDigest, img.Manifest.MediaType) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", multiarch.DigestStr(), multiarchDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetRepoReference(context.Background(), "repo", //nolint:contextcheck + multiarch.DigestStr(), multiarchDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) repoMetas, _, _, err := metaDB.FilterRepos(context.Background(), @@ -2493,7 +2537,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo99, "tag", manifestDigest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo99, "tag", manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMetas, _, _, err := metaDB.SearchRepos(ctx, repo99) @@ -2568,7 +2612,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func digest := godigest.FromString("1") - err := metaDB.SetRepoReference("repo", "tag", digest, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(context.Background(), "repo", "tag", digest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) _, err = metaDB.ToggleBookmarkRepo(ctx, "repo") diff --git a/pkg/meta/parse.go b/pkg/meta/parse.go index d9317f65cf..777b28b9fc 100644 --- a/pkg/meta/parse.go +++ b/pkg/meta/parse.go @@ -1,6 +1,7 @@ package meta import ( + "context" "encoding/json" "errors" "fmt" @@ -134,8 +135,8 @@ func ParseRepo(repo string, metaDB mTypes.MetaDB, storeController storage.StoreC reference = descriptor.Digest.String() } - err = SetImageMetaFromInput(repo, reference, descriptor.MediaType, descriptor.Digest, descriptorBlob, - imageStore, metaDB, log) + err = SetImageMetaFromInput(context.Background(), repo, reference, descriptor.MediaType, + descriptor.Digest, descriptorBlob, imageStore, metaDB, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("tag", tag). Msg("load-repo: failed to set metadata for image") @@ -333,7 +334,7 @@ func getNotationSignatureLayersInfo( } // NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object. -func NewManifestData(repoName string, manifestBlob []byte, imageStore storageTypes.ImageStore, +func NewManifestData(repoName string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger, ) (mTypes.ManifestData, error) { var ( manifestContent ispec.Manifest @@ -380,19 +381,21 @@ func NewIndexData(repoName string, indexBlob []byte, imageStore storageTypes.Ima // SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag // (in case the reference is a tag). The function expects image manifests and indexes (multi arch images). -func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte, - imageStore storageTypes.ImageStore, metaDB mTypes.MetaDB, log log.Logger, +func SetImageMetaFromInput(ctx context.Context, repo, reference, mediaType string, digest godigest.Digest, + descriptorBlob []byte, imageStore storageTypes.ImageStore, metaDB mTypes.MetaDB, log log.Logger, ) error { switch mediaType { case ispec.MediaTypeImageManifest: - imageData, err := NewManifestData(repo, descriptorBlob, imageStore) + imageData, err := NewManifestData(repo, descriptorBlob, imageStore, log) if err != nil { + log.Error().Err(err).Msg("metadb: error while getting image data") + return err } err = metaDB.SetManifestData(digest, imageData) if err != nil { - log.Error().Err(err).Msg("metadb: error while putting manifest meta") + log.Error().Err(err).Msg("metadb: error while setting manifest meta") return err } @@ -401,7 +404,7 @@ func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Di err := metaDB.SetIndexData(digest, indexData) if err != nil { - log.Error().Err(err).Msg("metadb: error while putting index data") + log.Error().Err(err).Msg("metadb: error while setting index data") return err } @@ -411,15 +414,15 @@ func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Di if hasSubject && err == nil { err := metaDB.SetReferrer(repo, referredDigest, referrerInfo) if err != nil { - log.Error().Err(err).Msg("metadb: error while settingg referrer") + log.Error().Err(err).Msg("metadb: error while setting referrer") return err } } - err = metaDB.SetRepoReference(repo, reference, digest, mediaType) + err = metaDB.SetRepoReference(ctx, repo, reference, digest, mediaType) if err != nil { - log.Error().Err(err).Msg("metadb: error while putting repo meta") + log.Error().Err(err).Msg("metadb: error while setting repo meta") return err } diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index 21ae002001..47f8253ffe 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -8,6 +8,7 @@ import ( "os" "path" "testing" + "time" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -161,7 +162,9 @@ func TestParseStorageErrors(t *testing.T) { } Convey("metaDB.SetRepoReference", func() { - metaDB.SetRepoReferenceFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error { + metaDB.SetRepoReferenceFn = func(ctx context.Context, repo, //nolint:contextcheck + tag string, manifestDigest godigest.Digest, mediaType string, + ) error { return ErrTestError } @@ -558,16 +561,16 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { err = WriteImageToFileSystem(image, repo, "tag", storeController) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo, "tag", manifestDigest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(context.Background(), repo, "tag", manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo) So(err, ShouldBeNil) - err = metaDB.IncrementImageDownloads(repo, "tag") + err = metaDB.UpdateStatsOnDownload(repo, "tag") So(err, ShouldBeNil) - err = metaDB.IncrementImageDownloads(repo, "tag") + err = metaDB.UpdateStatsOnDownload(repo, "tag") So(err, ShouldBeNil) - err = metaDB.IncrementImageDownloads(repo, "tag") + err = metaDB.UpdateStatsOnDownload(repo, "tag") So(err, ShouldBeNil) repoMeta, err := metaDB.GetRepoMeta(repo) @@ -575,6 +578,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3) So(repoMeta.Stars, ShouldEqual, 1) + So(time.Now().Local(), ShouldHappenAfter, repoMeta.Statistics[manifestDigest.String()].LastPullTimestamp) err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", "")) So(err, ShouldBeNil) diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index 6e64280bef..4b036a0fbc 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -43,7 +43,8 @@ type MetaDB interface { //nolint:interfacebloat GetRepoStars(repo string) (int, error) // SetRepoReference sets the reference of a manifest in the tag list of a repo - SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string) error + SetRepoReference(ctx context.Context, repo string, reference string, + manifestDigest godigest.Digest, mediaType string) error /* RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag, @@ -102,8 +103,8 @@ type MetaDB interface { //nolint:interfacebloat GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) ( []ReferrerInfo, error) - // IncrementManifestDownloads adds 1 to the download count of a manifest - IncrementImageDownloads(repo string, reference string) error + // UpdateStatsOnDownload adds 1 to the download count of a manifest and sets the timestamp of download + UpdateStatsOnDownload(repo string, reference string) error // AddManifestSignature adds signature metadata to a given manifest in the database AddManifestSignature(repo string, signedManifestDigest godigest.Digest, sm SignatureMetadata) error @@ -212,7 +213,10 @@ type Descriptor struct { } type DescriptorStatistics struct { - DownloadCount int + DownloadCount int + LastPullTimestamp time.Time + PushTimestamp time.Time + PushedBy string } type ManifestSignatures map[string][]SignatureInfo diff --git a/pkg/storage/gc/gc.go b/pkg/storage/gc/gc.go index 220bfd6894..92695a2587 100644 --- a/pkg/storage/gc/gc.go +++ b/pkg/storage/gc/gc.go @@ -15,6 +15,7 @@ import ( oras "github.com/oras-project/artifacts-spec/specs-go/v1" zerr "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" zcommon "zotregistry.io/zot/pkg/common" zlog "zotregistry.io/zot/pkg/log" mTypes "zotregistry.io/zot/pkg/meta/types" @@ -30,27 +31,36 @@ const ( ) type Options struct { - // will garbage collect referrers with missing subject older than Delay - Referrers bool // will garbage collect blobs older than Delay Delay time.Duration // will garbage collect untagged manifests older than RetentionDelay - RetentionDelay time.Duration + UntaggedDelay time.Duration + + ImageRetention config.ImageRetention } type GarbageCollect struct { - imgStore types.ImageStore - opts Options - metaDB mTypes.MetaDB - log zlog.Logger + imgStore types.ImageStore + opts Options + metaDB mTypes.MetaDB + policyMgr policyManager + auditLog *zlog.Logger + log zlog.Logger } -func NewGarbageCollect(imgStore types.ImageStore, metaDB mTypes.MetaDB, opts Options, log zlog.Logger, +func NewGarbageCollect(imgStore types.ImageStore, metaDB mTypes.MetaDB, opts Options, + auditLog *zlog.Logger, log zlog.Logger, ) GarbageCollect { return GarbageCollect{ imgStore: imgStore, metaDB: metaDB, opts: opts, + policyMgr: policyManager{ + opts.ImageRetention, + auditLog, + log, + }, + auditLog: auditLog, log: log, } } @@ -75,17 +85,20 @@ It also gc referrers with missing subject if the Referrer Option is enabled It also gc untagged manifests. */ func (gc GarbageCollect) CleanRepo(repo string) error { - gc.log.Info().Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(gc.imgStore.RootDir(), repo))) + gc.log.Info().Str("module", "gc"). + Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(gc.imgStore.RootDir(), repo))) if err := gc.cleanRepo(repo); err != nil { errMessage := fmt.Sprintf("error while running GC for %s", path.Join(gc.imgStore.RootDir(), repo)) - gc.log.Error().Err(err).Msg(errMessage) - gc.log.Info().Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(gc.imgStore.RootDir(), repo))) + gc.log.Error().Err(err).Str("module", "gc").Msg(errMessage) + gc.log.Info().Str("module", "gc"). + Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(gc.imgStore.RootDir(), repo))) return err } - gc.log.Info().Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(gc.imgStore.RootDir(), repo))) + gc.log.Info().Str("module", "gc"). + Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(gc.imgStore.RootDir(), repo))) return nil } @@ -112,28 +125,39 @@ func (gc GarbageCollect) cleanRepo(repo string) error { */ index, err := common.GetIndex(gc.imgStore, repo, gc.log) if err != nil { + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to read index.json in repo") + return err } // gc referrers manifests with missing subject and untagged manifests - if err := gc.cleanManifests(repo, &index); err != nil { + if err := gc.removeManifestsPerRepoPolicy(repo, &index); err != nil { return err } - // update repos's index.json in storage - if err := gc.imgStore.PutIndexContent(repo, index); err != nil { + // apply tags retention + if err := gc.removeTagsPerRetentionPolicy(repo, &index); err != nil { return err } + // update repos's index.json in storage + if !gc.opts.ImageRetention.DryRun { + /* this will update the index.json with manifests deleted above + and the manifests blobs will be removed by gc.removeUnreferencedBlobs()*/ + if err := gc.imgStore.PutIndexContent(repo, index); err != nil { + return err + } + } + // gc unreferenced blobs - if err := gc.cleanBlobs(repo, index, gc.opts.Delay, gc.log); err != nil { + if err := gc.removeUnreferencedBlobs(repo, gc.opts.Delay, gc.log); err != nil { return err } return nil } -func (gc GarbageCollect) cleanManifests(repo string, index *ispec.Index) error { +func (gc GarbageCollect) removeManifestsPerRepoPolicy(repo string, index *ispec.Index) error { var err error /* gc all manifests that have a missing subject, stop when neither gc(referrer and untagged) @@ -142,32 +166,36 @@ func (gc GarbageCollect) cleanManifests(repo string, index *ispec.Index) error { for !stop { var gcedReferrer bool - if gc.opts.Referrers { - gc.log.Debug().Str("repository", repo).Msg("gc: manifests with missing referrers") + var gcedUntagged bool - gcedReferrer, err = gc.cleanIndexReferrers(repo, index, *index) + if gc.policyMgr.hasDeleteReferrer(repo) { + gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("manifests with missing referrers") + + gcedReferrer, err = gc.removeIndexReferrers(repo, index, *index) if err != nil { return err } } - referenced := make(map[godigest.Digest]bool, 0) + if gc.policyMgr.hasDeleteUntagged(repo) { + referenced := make(map[godigest.Digest]bool, 0) - /* gather all manifests referenced in multiarch images/by other manifests - so that we can skip them in cleanUntaggedManifests */ - if err := gc.identifyManifestsReferencedInIndex(*index, repo, referenced); err != nil { - return err - } + /* gather all manifests referenced in multiarch images/by other manifests + so that we can skip them in cleanUntaggedManifests */ + if err := gc.identifyManifestsReferencedInIndex(*index, repo, referenced); err != nil { + return err + } - // apply image retention policy - gcedManifest, err := gc.cleanUntaggedManifests(repo, index, referenced) - if err != nil { - return err + // apply image retention policy + gcedUntagged, err = gc.removeUntaggedManifests(repo, index, referenced) + if err != nil { + return err + } } /* if we gced any manifest then loop again and gc manifests with a subject pointing to the last ones which were gced. */ - stop = !gcedReferrer && !gcedManifest + stop = !gcedReferrer && !gcedUntagged } return nil @@ -179,7 +207,7 @@ garbageCollectIndexReferrers will gc all referrers with a missing subject recurs rootIndex is indexJson, need to pass it down to garbageCollectReferrer() rootIndex is the place we look for referrers. */ -func (gc GarbageCollect) cleanIndexReferrers(repo string, rootIndex *ispec.Index, index ispec.Index, +func (gc GarbageCollect) removeIndexReferrers(repo string, rootIndex *ispec.Index, index ispec.Index, ) (bool, error) { var count int @@ -190,13 +218,13 @@ func (gc GarbageCollect) cleanIndexReferrers(repo string, rootIndex *ispec.Index case ispec.MediaTypeImageIndex: indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log) if err != nil { - gc.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read multiarch(index) image") + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("failed to read multiarch(index) image") return false, err } - gced, err := gc.cleanReferrer(repo, rootIndex, desc, indexImage.Subject, indexImage.ArtifactType) + gced, err := gc.removeReferrer(repo, rootIndex, desc, indexImage.Subject, indexImage.ArtifactType) if err != nil { return false, err } @@ -208,7 +236,7 @@ func (gc GarbageCollect) cleanIndexReferrers(repo string, rootIndex *ispec.Index return true, nil } - gced, err = gc.cleanIndexReferrers(repo, rootIndex, indexImage) + gced, err = gc.removeIndexReferrers(repo, rootIndex, indexImage) if err != nil { return false, err } @@ -219,15 +247,15 @@ func (gc GarbageCollect) cleanIndexReferrers(repo string, rootIndex *ispec.Index case ispec.MediaTypeImageManifest, oras.MediaTypeArtifactManifest: image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log) if err != nil { - gc.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read manifest image") + gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo).Str("digest", desc.Digest.String()). + Msg("failed to read manifest image") return false, err } artifactType := zcommon.GetManifestArtifactType(image) - gced, err := gc.cleanReferrer(repo, rootIndex, desc, image.Subject, artifactType) + gced, err := gc.removeReferrer(repo, rootIndex, desc, image.Subject, artifactType) if err != nil { return false, err } @@ -241,7 +269,7 @@ func (gc GarbageCollect) cleanIndexReferrers(repo string, rootIndex *ispec.Index return count > 0, err } -func (gc GarbageCollect) cleanReferrer(repo string, index *ispec.Index, manifestDesc ispec.Descriptor, +func (gc GarbageCollect) removeReferrer(repo string, index *ispec.Index, manifestDesc ispec.Descriptor, subject *ispec.Descriptor, artifactType string, ) (bool, error) { var gced bool @@ -263,14 +291,31 @@ func (gc GarbageCollect) cleanReferrer(repo string, index *ispec.Index, manifest if err != nil { return false, err } + + if gced { + gc.log.Info().Str("module", "gc"). + Str("repository", repo). + Str("reference", manifestDesc.Digest.String()). + Str("subject", subject.Digest.String()). + Str("decision", "delete"). + Str("reason", "deleteReferrers").Msg("removed manifest without reference") + + if gc.auditLog != nil { + gc.auditLog.Info().Str("module", "gc"). + Str("repository", repo). + Str("reference", manifestDesc.Digest.String()). + Str("subject", subject.Digest.String()). + Str("decision", "delete"). + Str("reason", "deleteReferrers").Msg("removed manifest without reference") + } + } } } // cosign - tag, ok := manifestDesc.Annotations[ispec.AnnotationRefName] + tag, ok := getDescriptorTag(manifestDesc) if ok { - if strings.HasPrefix(tag, "sha256-") && (strings.HasSuffix(tag, cosignSignatureTagSuffix) || - strings.HasSuffix(tag, SBOMTagSuffix)) { + if isCosignTag(tag) { subjectDigest := getSubjectFromCosignTag(tag) referenced := isManifestReferencedInIndex(index, subjectDigest) @@ -279,6 +324,26 @@ func (gc GarbageCollect) cleanReferrer(repo string, index *ispec.Index, manifest if err != nil { return false, err } + + if gced { + gc.log.Info().Str("module", "gc"). + Bool("dry-run", gc.opts.ImageRetention.DryRun). + Str("repository", repo). + Str("reference", tag). + Str("subject", subjectDigest.String()). + Str("decision", "delete"). + Str("reason", "deleteReferrers").Msg("removed cosign manifest without reference") + + if gc.auditLog != nil { + gc.auditLog.Info().Str("module", "gc"). + Bool("dry-run", gc.opts.ImageRetention.DryRun). + Str("repository", repo). + Str("reference", tag). + Str("subject", subjectDigest.String()). + Str("decision", "delete"). + Str("reason", "deleteReferrers").Msg("removed cosign manifest without reference") + } + } } } } @@ -286,6 +351,36 @@ func (gc GarbageCollect) cleanReferrer(repo string, index *ispec.Index, manifest return gced, nil } +func (gc GarbageCollect) removeTagsPerRetentionPolicy(repo string, index *ispec.Index) error { + if !gc.policyMgr.hasTagRetention(repo) { + return nil + } + + repoMeta, err := gc.metaDB.GetRepoMeta(repo) + if err != nil { + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("can't retrieve repoMeta for repo") + + return err + } + + retainTags := gc.policyMgr.getRetainedTags(repoMeta, *index) + + // remove + for _, desc := range index.Manifests { + // check tag + tag, ok := getDescriptorTag(desc) + if ok && !zcommon.Contains(retainTags, tag) { + // remove tags which should not be retained + _, err := gc.removeManifest(repo, index, desc, tag, "", "") + if err != nil && !errors.Is(err, zerr.ErrManifestNotFound) { + return err + } + } + } + + return nil +} + // gcManifest removes a manifest entry from an index and syncs metaDB accordingly if the blob is older than gc.Delay. func (gc GarbageCollect) gcManifest(repo string, index *ispec.Index, desc ispec.Descriptor, signatureType string, subjectDigest godigest.Digest, delay time.Duration, @@ -294,14 +389,14 @@ func (gc GarbageCollect) gcManifest(repo string, index *ispec.Index, desc ispec. canGC, err := isBlobOlderThan(gc.imgStore, repo, desc.Digest, delay, gc.log) if err != nil { - gc.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Str("delay", gc.opts.Delay.String()).Msg("gc: failed to check if blob is older than delay") + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", desc.Digest.String()). + Str("delay", delay.String()).Msg("failed to check if blob is older than delay") return false, err } if canGC { - if gced, err = gc.removeManifest(repo, index, desc, signatureType, subjectDigest); err != nil { + if gced, err = gc.removeManifest(repo, index, desc, desc.Digest.String(), signatureType, subjectDigest); err != nil { return false, err } } @@ -311,12 +406,9 @@ func (gc GarbageCollect) gcManifest(repo string, index *ispec.Index, desc ispec. // removeManifest removes a manifest entry from an index and syncs metaDB accordingly. func (gc GarbageCollect) removeManifest(repo string, index *ispec.Index, - desc ispec.Descriptor, signatureType string, subjectDigest godigest.Digest, + desc ispec.Descriptor, reference string, signatureType string, subjectDigest godigest.Digest, ) (bool, error) { - gc.log.Debug().Str("repository", repo).Str("digest", desc.Digest.String()).Msg("gc: removing manifest") - - // remove from index - _, err := common.RemoveManifestDescByReference(index, desc.Digest.String(), true) + _, err := common.RemoveManifestDescByReference(index, reference, true) if err != nil { if errors.Is(err, zerr.ErrManifestConflict) { return false, nil @@ -325,6 +417,10 @@ func (gc GarbageCollect) removeManifest(repo string, index *ispec.Index, return false, err } + if gc.opts.ImageRetention.DryRun { + return true, nil + } + // sync metaDB if gc.metaDB != nil { if signatureType != "" { @@ -333,14 +429,14 @@ func (gc GarbageCollect) removeManifest(repo string, index *ispec.Index, SignatureType: signatureType, }) if err != nil { - gc.log.Error().Err(err).Msg("gc,metadb: unable to remove signature in metaDB") + gc.log.Error().Err(err).Str("module", "gc").Msg("metadb: unable to remove signature in metaDB") return false, err } } else { - err := gc.metaDB.RemoveRepoReference(repo, desc.Digest.String(), desc.Digest) + err := gc.metaDB.RemoveRepoReference(repo, reference, desc.Digest) if err != nil { - gc.log.Error().Err(err).Msg("gc, metadb: unable to remove repo reference in metaDB") + gc.log.Error().Err(err).Str("module", "gc").Msg("metadb: unable to remove repo reference in metaDB") return false, err } @@ -350,16 +446,15 @@ func (gc GarbageCollect) removeManifest(repo string, index *ispec.Index, return true, nil } -func (gc GarbageCollect) cleanUntaggedManifests(repo string, index *ispec.Index, +func (gc GarbageCollect) removeUntaggedManifests(repo string, index *ispec.Index, referenced map[godigest.Digest]bool, ) (bool, error) { var gced bool var err error - gc.log.Debug().Str("repository", repo).Msg("gc: manifests without tags") + gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("manifests without tags") - // first gather manifests part of image indexes and referrers, we want to skip checking them for _, desc := range index.Manifests { // skip manifests referenced in image indexes if _, referenced := referenced[desc.Digest]; referenced { @@ -368,12 +463,30 @@ func (gc GarbageCollect) cleanUntaggedManifests(repo string, index *ispec.Index, // remove untagged images if desc.MediaType == ispec.MediaTypeImageManifest || desc.MediaType == ispec.MediaTypeImageIndex { - _, ok := desc.Annotations[ispec.AnnotationRefName] + _, ok := getDescriptorTag(desc) if !ok { - gced, err = gc.gcManifest(repo, index, desc, "", "", gc.opts.RetentionDelay) + gced, err = gc.gcManifest(repo, index, desc, "", "", gc.opts.UntaggedDelay) if err != nil { return false, err } + + if gced { + gc.log.Info().Str("module", "gc"). + Bool("dry-run", gc.opts.ImageRetention.DryRun). + Str("repository", repo). + Str("reference", desc.Digest.String()). + Str("decision", "delete"). + Str("reason", "deleteUntagged").Msg("removed untagged manifest") + + if gc.auditLog != nil { + gc.auditLog.Info().Str("module", "gc"). + Bool("dry-run", gc.opts.ImageRetention.DryRun). + Str("repository", repo). + Str("reference", desc.Digest.String()). + Str("decision", "delete"). + Str("reason", "deleteUntagged").Msg("removed untagged manifest") + } + } } } } @@ -390,8 +503,8 @@ func (gc GarbageCollect) identifyManifestsReferencedInIndex(index ispec.Index, r case ispec.MediaTypeImageIndex: indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log) if err != nil { - gc.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read multiarch(index) image") + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Str("digest", desc.Digest.String()).Msg("failed to read multiarch(index) image") return err } @@ -410,8 +523,8 @@ func (gc GarbageCollect) identifyManifestsReferencedInIndex(index ispec.Index, r case ispec.MediaTypeImageManifest, oras.MediaTypeArtifactManifest: image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log) if err != nil { - gc.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read manifest image") + gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo). + Str("digest", desc.Digest.String()).Msg("failed to read manifest image") return err } @@ -425,17 +538,23 @@ func (gc GarbageCollect) identifyManifestsReferencedInIndex(index ispec.Index, r return nil } -// cleanBlobs gc all blobs which are not referenced by any manifest found in repo's index.json. -func (gc GarbageCollect) cleanBlobs(repo string, index ispec.Index, - delay time.Duration, log zlog.Logger, +// removeUnreferencedBlobs gc all blobs which are not referenced by any manifest found in repo's index.json. +func (gc GarbageCollect) removeUnreferencedBlobs(repo string, delay time.Duration, log zlog.Logger, ) error { - gc.log.Debug().Str("repository", repo).Msg("gc: blobs") + gc.log.Debug().Str("module", "gc").Str("repository", repo).Msg("cleaning orphan blobs") refBlobs := map[string]bool{} - err := gc.addIndexBlobsToReferences(repo, index, refBlobs) + index, err := common.GetIndex(gc.imgStore, repo, gc.log) if err != nil { - log.Error().Err(err).Str("repository", repo).Msg("gc: unable to get referenced blobs in repo") + log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to read index.json in repo") + + return err + } + + err = gc.addIndexBlobsToReferences(repo, index, refBlobs) + if err != nil { + log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to get referenced blobs in repo") return err } @@ -447,7 +566,7 @@ func (gc GarbageCollect) cleanBlobs(repo string, index ispec.Index, return nil } - log.Error().Err(err).Str("repository", repo).Msg("gc: unable to get all blobs") + log.Error().Err(err).Str("module", "gc").Str("repository", repo).Msg("unable to get all blobs") return err } @@ -457,7 +576,8 @@ func (gc GarbageCollect) cleanBlobs(repo string, index ispec.Index, for _, blob := range allBlobs { digest := godigest.NewDigestFromEncoded(godigest.SHA256, blob) if err = digest.Validate(); err != nil { - log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("gc: unable to parse digest") + log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob). + Msg("unable to parse digest") return err } @@ -465,7 +585,8 @@ func (gc GarbageCollect) cleanBlobs(repo string, index ispec.Index, if _, ok := refBlobs[digest.String()]; !ok { canGC, err := isBlobOlderThan(gc.imgStore, repo, digest, delay, log) if err != nil { - log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("gc: unable to determine GC delay") + log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob). + Msg("unable to determine GC delay") return err } @@ -484,11 +605,13 @@ func (gc GarbageCollect) cleanBlobs(repo string, index ispec.Index, return err } - log.Info().Str("repository", repo).Int("count", reaped).Msg("gc: garbage collected blobs") + log.Info().Str("module", "gc").Str("repository", repo).Int("count", reaped). + Msg("garbage collected blobs") return nil } +// used by removeUnreferencedBlobs() // addIndexBlobsToReferences adds referenced blobs found in referenced manifests (index.json) in refblobs map. func (gc GarbageCollect) addIndexBlobsToReferences(repo string, index ispec.Index, refBlobs map[string]bool, ) error { @@ -496,22 +619,22 @@ func (gc GarbageCollect) addIndexBlobsToReferences(repo string, index ispec.Inde switch desc.MediaType { case ispec.MediaTypeImageIndex: if err := gc.addImageIndexBlobsToReferences(repo, desc.Digest, refBlobs); err != nil { - gc.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read blobs in multiarch(index) image") + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Str("digest", desc.Digest.String()).Msg("failed to read blobs in multiarch(index) image") return err } case ispec.MediaTypeImageManifest: if err := gc.addImageManifestBlobsToReferences(repo, desc.Digest, refBlobs); err != nil { - gc.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read blobs in image manifest") + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Str("digest", desc.Digest.String()).Msg("failed to read blobs in image manifest") return err } case oras.MediaTypeArtifactManifest: if err := gc.addORASImageManifestBlobsToReferences(repo, desc.Digest, refBlobs); err != nil { - gc.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read blobs in ORAS image manifest") + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Str("digest", desc.Digest.String()).Msg("failed to read blobs in ORAS image manifest") return err } @@ -525,8 +648,8 @@ func (gc GarbageCollect) addImageIndexBlobsToReferences(repo string, mdigest god ) error { index, err := common.GetImageIndex(gc.imgStore, repo, mdigest, gc.log) if err != nil { - gc.log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). - Msg("gc: failed to read manifest image") + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", mdigest.String()). + Msg("failed to read manifest image") return err } @@ -550,8 +673,8 @@ func (gc GarbageCollect) addImageManifestBlobsToReferences(repo string, mdigest ) error { manifestContent, err := common.GetImageManifest(gc.imgStore, repo, mdigest, gc.log) if err != nil { - gc.log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). - Msg("gc: failed to read manifest image") + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Str("digest", mdigest.String()).Msg("failed to read manifest image") return err } @@ -576,8 +699,8 @@ func (gc GarbageCollect) addORASImageManifestBlobsToReferences(repo string, mdig ) error { manifestContent, err := common.GetOrasManifestByDigest(gc.imgStore, repo, mdigest, gc.log) if err != nil { - gc.log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). - Msg("gc: failed to read manifest image") + gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Str("digest", mdigest.String()).Msg("failed to read manifest image") return err } @@ -611,8 +734,8 @@ func isBlobOlderThan(imgStore types.ImageStore, repo string, ) (bool, error) { _, _, modtime, err := imgStore.StatBlob(repo, digest) if err != nil { - log.Error().Err(err).Str("repository", repo).Str("digest", digest.String()). - Msg("gc: failed to stat blob") + log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", digest.String()). + Msg("failed to stat blob") return false, err } @@ -631,6 +754,35 @@ func getSubjectFromCosignTag(tag string) godigest.Digest { return godigest.NewDigestFromEncoded(godigest.Algorithm(alg), encoded) } +func getDescriptorTag(desc ispec.Descriptor) (string, bool) { + tag, ok := desc.Annotations[ispec.AnnotationRefName] + + return tag, ok +} + +func getIndexTags(index ispec.Index) []string { + tags := make([]string, 0) + + for _, desc := range index.Manifests { + tag, ok := getDescriptorTag(desc) + if ok { + tags = append(tags, tag) + } + } + + return tags +} + +// this function will check if tag is a cosign tag (signature or sbom). +func isCosignTag(tag string) bool { + if strings.HasPrefix(tag, "sha256-") && + (strings.HasSuffix(tag, cosignSignatureTagSuffix) || strings.HasSuffix(tag, SBOMTagSuffix)) { + return true + } + + return false +} + /* GCTaskGenerator takes all repositories found in the storage.imagestore @@ -704,5 +856,5 @@ func NewGCTask(imgStore types.ImageStore, gc GarbageCollect, repo string, func (gct *gcTask) DoWork(ctx context.Context) error { // run task - return gct.gc.CleanRepo(gct.repo) + return gct.gc.CleanRepo(gct.repo) //nolint: contextcheck } diff --git a/pkg/storage/gc/gc_internal_test.go b/pkg/storage/gc/gc_internal_test.go index 9618f32331..6b188ce054 100644 --- a/pkg/storage/gc/gc_internal_test.go +++ b/pkg/storage/gc/gc_internal_test.go @@ -12,12 +12,12 @@ import ( godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" - "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" + "zotregistry.io/zot/pkg/api/config" zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" - "zotregistry.io/zot/pkg/log" + zlog "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" @@ -37,8 +37,11 @@ func TestGarbageCollectManifestErrors(t *testing.T) { Convey("Make imagestore and upload manifest", t, func(c C) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, Name: "cache", @@ -47,10 +50,17 @@ func TestGarbageCollectManifestErrors(t *testing.T) { imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + }, + }, + }, + }, audit, log) Convey("trigger repo not found in addImageIndexBlobsToReferences()", func() { err := gc.addIndexBlobsToReferences(repoName, ispec.Index{ @@ -164,7 +174,9 @@ func TestGarbageCollectIndexErrors(t *testing.T) { Convey("Make imagestore and upload manifest", t, func(c C) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -174,10 +186,17 @@ func TestGarbageCollectIndexErrors(t *testing.T) { imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + }, + }, + }, + }, audit, log) content := []byte("this is a blob") bdgst := godigest.FromBytes(content) @@ -271,7 +290,82 @@ func TestGarbageCollectIndexErrors(t *testing.T) { func TestGarbageCollectWithMockedImageStore(t *testing.T) { Convey("Cover gc error paths", t, func(c C) { - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "") + + gcOptions := Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + }, + }, + }, + } + + Convey("Error on GetIndex in gc.cleanRepo()", func() { + gc := NewGarbageCollect(mocks.MockedImageStore{}, mocks.MetaDBMock{ + GetRepoMetaFn: func(repo string) (types.RepoMetadata, error) { + return types.RepoMetadata{}, errGC + }, + }, gcOptions, audit, log) + + err := gc.cleanRepo(repoName) + So(err, ShouldNotBeNil) + }) + + Convey("Error on GetIndex in gc.removeUnreferencedBlobs()", func() { + gc := NewGarbageCollect(mocks.MockedImageStore{}, mocks.MetaDBMock{ + GetRepoMetaFn: func(repo string) (types.RepoMetadata, error) { + return types.RepoMetadata{}, errGC + }, + }, gcOptions, audit, log) + + err := gc.removeUnreferencedBlobs("repo", time.Hour, log) + So(err, ShouldNotBeNil) + }) + + Convey("Error on gc.removeManifest()", func() { + gc := NewGarbageCollect(mocks.MockedImageStore{}, mocks.MetaDBMock{ + GetRepoMetaFn: func(repo string) (types.RepoMetadata, error) { + return types.RepoMetadata{}, errGC + }, + }, gcOptions, audit, log) + + _, err := gc.removeManifest("", &ispec.Index{}, ispec.DescriptorEmptyJSON, "tag", "", "") + So(err, ShouldNotBeNil) + }) + + Convey("Error on metaDB in gc.cleanRepo()", func() { + gcOptions := Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{".*"}, + }, + }, + }, + }, + }, + } + + gc := NewGarbageCollect(mocks.MockedImageStore{}, mocks.MetaDBMock{ + GetRepoMetaFn: func(repo string) (types.RepoMetadata, error) { + return types.RepoMetadata{}, errGC + }, + }, gcOptions, audit, log) + + err := gc.removeTagsPerRetentionPolicy("name", &ispec.Index{}) + So(err, ShouldNotBeNil) + }) Convey("Error on PutIndexContent in gc.cleanRepo()", func() { returnedIndexJSON := ispec.Index{} @@ -288,11 +382,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, audit, log) err = gc.cleanRepo(repoName) So(err, ShouldNotBeNil) @@ -316,11 +406,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, audit, log) err = gc.cleanRepo(repoName) So(err, ShouldNotBeNil) @@ -333,11 +419,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, audit, log) err := gc.cleanRepo(repoName) So(err, ShouldNotBeNil) @@ -369,13 +451,17 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: false, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gcOptions.ImageRetention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteUntagged: true, + }, + }, + } + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, audit, log) - err = gc.cleanManifests(repoName, &ispec.Index{ + err = gc.removeManifestsPerRepoPolicy(repoName, &ispec.Index{ Manifests: []ispec.Descriptor{ { MediaType: ispec.MediaTypeImageIndex, @@ -393,13 +479,18 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: false, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gcOptions.ImageRetention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteUntagged: true, + }, + }, + } + + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, audit, log) - err := gc.cleanManifests(repoName, &ispec.Index{ + err := gc.removeManifestsPerRepoPolicy(repoName, &ispec.Index{ Manifests: []ispec.Descriptor{ { MediaType: ispec.MediaTypeImageManifest, @@ -430,13 +521,17 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, metaDB, Options{ - Referrers: false, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gcOptions.ImageRetention = config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteUntagged: true, + }, + }, + } + gc := NewGarbageCollect(imgStore, metaDB, gcOptions, audit, log) - err = gc.cleanManifests(repoName, &ispec.Index{ + err = gc.removeManifestsPerRepoPolicy(repoName, &ispec.Index{ Manifests: []ispec.Descriptor{ { MediaType: ispec.MediaTypeImageManifest, @@ -467,11 +562,8 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, metaDB, Options{ - Referrers: false, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gcOptions.ImageRetention = config.ImageRetention{} + gc := NewGarbageCollect(imgStore, metaDB, gcOptions, audit, log) desc := ispec.Descriptor{ MediaType: ispec.MediaTypeImageManifest, @@ -481,7 +573,7 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { index := &ispec.Index{ Manifests: []ispec.Descriptor{desc}, } - _, err = gc.removeManifest(repoName, index, desc, storage.NotationType, + _, err = gc.removeManifest(repoName, index, desc, desc.Digest.String(), storage.NotationType, godigest.FromBytes([]byte("digest2"))) So(err, ShouldNotBeNil) @@ -515,13 +607,9 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, audit, log) - err = gc.cleanManifests(repoName, &returnedIndexImage) + err = gc.removeManifestsPerRepoPolicy(repoName, &returnedIndexImage) So(err, ShouldNotBeNil) }) @@ -550,13 +638,9 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { }, } - gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gcOptions, audit, log) - err = gc.cleanManifests(repoName, &ispec.Index{ + err = gc.removeManifestsPerRepoPolicy(repoName, &ispec.Index{ Manifests: []ispec.Descriptor{ manifestDesc, }, diff --git a/pkg/storage/gc/gc_test.go b/pkg/storage/gc/gc_test.go new file mode 100644 index 0000000000..12b232de61 --- /dev/null +++ b/pkg/storage/gc/gc_test.go @@ -0,0 +1,868 @@ +package gc_test + +import ( + "context" + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/docker/distribution/registry/storage/driver/factory" + _ "github.com/docker/distribution/registry/storage/driver/s3-aws" + guuid "github.com/gofrs/uuid" + . "github.com/smartystreets/goconvey/convey" + "gopkg.in/resty.v1" + + "zotregistry.io/zot/pkg/api/config" + "zotregistry.io/zot/pkg/extensions/monitoring" + zlog "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta" + "zotregistry.io/zot/pkg/meta/boltdb" + "zotregistry.io/zot/pkg/meta/dynamodb" + mTypes "zotregistry.io/zot/pkg/meta/types" + "zotregistry.io/zot/pkg/storage" + storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/gc" + "zotregistry.io/zot/pkg/storage/local" + "zotregistry.io/zot/pkg/storage/s3" + storageTypes "zotregistry.io/zot/pkg/storage/types" + . "zotregistry.io/zot/pkg/test/image-utils" + tskip "zotregistry.io/zot/pkg/test/skip" +) + +const ( + region = "us-east-2" +) + +//nolint:gochecknoglobals +var testCases = []struct { + testCaseName string + storageType string +}{ + { + testCaseName: "S3APIs", + storageType: storageConstants.S3StorageDriverName, + }, + { + testCaseName: "LocalAPIs", + storageType: storageConstants.LocalStorageDriverName, + }, +} + +func TestGarbageCollectAndRetention(t *testing.T) { + log := zlog.NewLogger("info", "") + audit := zlog.NewAuditLogger("debug", "") + + metrics := monitoring.NewMetricsServer(false, log) + + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + var imgStore storageTypes.ImageStore + + var metaDB mTypes.MetaDB + + if testcase.storageType == storageConstants.S3StorageDriverName { + tskip.SkipDynamo(t) + tskip.SkipS3(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + rootDir := path.Join("/oci-repo-test", uuid.String()) + cacheDir := t.TempDir() + + bucket := "zot-storage-test" + + storageDriverParams := map[string]interface{}{ + "rootDir": rootDir, + "name": "s3", + "region": region, + "bucket": bucket, + "regionendpoint": os.Getenv("S3MOCK_ENDPOINT"), + "accesskey": "minioadmin", + "secretkey": "minioadmin", + "secure": false, + "skipverify": false, + } + + storeName := fmt.Sprintf("%v", storageDriverParams["name"]) + + store, err := factory.Create(storeName, storageDriverParams) + if err != nil { + panic(err) + } + + defer store.Delete(context.Background(), rootDir) //nolint: errcheck + + // create bucket if it doesn't exists + _, err = resty.R().Put("http://" + os.Getenv("S3MOCK_ENDPOINT") + "/" + bucket) + if err != nil { + panic(err) + } + + uuid, err = guuid.NewV4() + if err != nil { + panic(err) + } + + params := dynamodb.DBDriverParameters{ //nolint:contextcheck + Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), + Region: region, + RepoMetaTablename: "repo" + uuid.String(), + ManifestDataTablename: "manifest" + uuid.String(), + IndexDataTablename: "index" + uuid.String(), + UserDataTablename: "user" + uuid.String(), + APIKeyTablename: "apiKey" + uuid.String(), + VersionTablename: "version" + uuid.String(), + } + + client, err := dynamodb.GetDynamoClient(params) + if err != nil { + panic(err) + } + + metaDB, err = dynamodb.New(client, params, log) + if err != nil { + panic(err) + } + + imgStore = s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, nil) + } else { + // Create temporary directory + rootDir := t.TempDir() + + // Create ImageStore + imgStore = local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + + // init metaDB + params := boltdb.DBParameters{ + RootDir: rootDir, + } + + boltDriver, err := boltdb.GetBoltDriver(params) + if err != nil { + panic(err) + } + + metaDB, err = boltdb.New(boltDriver, log) + if err != nil { + panic(err) + } + } + + storeController := storage.StoreController{} + storeController.DefaultStore = imgStore + + Convey("setup gc images", t, func() { + // for gc testing + // basic images + gcTest1 := CreateRandomImage() + err := WriteImageToFileSystem(gcTest1, "gc-test1", "0.0.1", storeController) + So(err, ShouldBeNil) + + // also add same image(same digest) with another tag + err = WriteImageToFileSystem(gcTest1, "gc-test1", "0.0.2", storeController) + So(err, ShouldBeNil) + + gcTest2 := CreateRandomImage() + err = WriteImageToFileSystem(gcTest2, "gc-test2", "0.0.1", storeController) + So(err, ShouldBeNil) + + gcTest3 := CreateRandomImage() + err = WriteImageToFileSystem(gcTest3, "gc-test3", "0.0.1", storeController) + So(err, ShouldBeNil) + + // referrers + ref1 := CreateRandomImageWith().Subject(gcTest1.DescriptorRef()).Build() + err = WriteImageToFileSystem(ref1, "gc-test1", ref1.DigestStr(), storeController) + So(err, ShouldBeNil) + + ref2 := CreateRandomImageWith().Subject(gcTest2.DescriptorRef()).Build() + err = WriteImageToFileSystem(ref2, "gc-test2", ref2.DigestStr(), storeController) + So(err, ShouldBeNil) + + ref3 := CreateRandomImageWith().Subject(gcTest3.DescriptorRef()).Build() + err = WriteImageToFileSystem(ref3, "gc-test3", ref3.DigestStr(), storeController) + So(err, ShouldBeNil) + + // referrers pointing to referrers + refOfRef1 := CreateRandomImageWith().Subject(ref1.DescriptorRef()).Build() + err = WriteImageToFileSystem(refOfRef1, "gc-test1", refOfRef1.DigestStr(), storeController) + So(err, ShouldBeNil) + + refOfRef2 := CreateRandomImageWith().Subject(ref2.DescriptorRef()).Build() + err = WriteImageToFileSystem(refOfRef2, "gc-test2", refOfRef2.DigestStr(), storeController) + So(err, ShouldBeNil) + + refOfRef3 := CreateRandomImageWith().Subject(ref3.DescriptorRef()).Build() + err = WriteImageToFileSystem(refOfRef3, "gc-test3", refOfRef3.DigestStr(), storeController) + So(err, ShouldBeNil) + + // untagged images + gcUntagged1 := CreateRandomImage() + err = WriteImageToFileSystem(gcUntagged1, "gc-test1", gcUntagged1.DigestStr(), storeController) + So(err, ShouldBeNil) + + gcUntagged2 := CreateRandomImage() + err = WriteImageToFileSystem(gcUntagged2, "gc-test2", gcUntagged2.DigestStr(), storeController) + So(err, ShouldBeNil) + + gcUntagged3 := CreateRandomImage() + err = WriteImageToFileSystem(gcUntagged3, "gc-test3", gcUntagged3.DigestStr(), storeController) + So(err, ShouldBeNil) + + // for image retention testing + // old images + gcOld1 := CreateRandomImage() + err = WriteImageToFileSystem(gcOld1, "retention", "0.0.1", storeController) + So(err, ShouldBeNil) + + gcOld2 := CreateRandomImage() + err = WriteImageToFileSystem(gcOld2, "retention", "0.0.2", storeController) + So(err, ShouldBeNil) + + gcOld3 := CreateRandomImage() + err = WriteImageToFileSystem(gcOld3, "retention", "0.0.3", storeController) + So(err, ShouldBeNil) + + // new images + gcNew1 := CreateRandomImage() + err = WriteImageToFileSystem(gcNew1, "retention", "0.0.4", storeController) + So(err, ShouldBeNil) + + gcNew2 := CreateRandomImage() + err = WriteImageToFileSystem(gcNew2, "retention", "0.0.5", storeController) + So(err, ShouldBeNil) + + gcNew3 := CreateRandomImage() + err = WriteImageToFileSystem(gcNew3, "retention", "0.0.6", storeController) + So(err, ShouldBeNil) + + err = meta.ParseStorage(metaDB, storeController, log) + So(err, ShouldBeNil) + + retentionMeta, err := metaDB.GetRepoMeta("retention") + So(err, ShouldBeNil) + + // update timestamps for image retention + gcOld1Stats := retentionMeta.Statistics[gcOld1.DigestStr()] + gcOld1Stats.PushTimestamp = time.Now().Add(-10 * 24 * time.Hour) + gcOld1Stats.LastPullTimestamp = time.Now().Add(-10 * 24 * time.Hour) + + gcOld2Stats := retentionMeta.Statistics[gcOld2.DigestStr()] + gcOld2Stats.PushTimestamp = time.Now().Add(-11 * 24 * time.Hour) + gcOld2Stats.LastPullTimestamp = time.Now().Add(-11 * 24 * time.Hour) + + gcOld3Stats := retentionMeta.Statistics[gcOld3.DigestStr()] + gcOld3Stats.PushTimestamp = time.Now().Add(-12 * 24 * time.Hour) + gcOld3Stats.LastPullTimestamp = time.Now().Add(-12 * 24 * time.Hour) + + gcNew1Stats := retentionMeta.Statistics[gcNew1.DigestStr()] + gcNew1Stats.PushTimestamp = time.Now().Add(-1 * 24 * time.Hour) + gcNew1Stats.LastPullTimestamp = time.Now().Add(-1 * 24 * time.Hour) + + gcNew2Stats := retentionMeta.Statistics[gcNew2.DigestStr()] + gcNew2Stats.PushTimestamp = time.Now().Add(-2 * 24 * time.Hour) + gcNew2Stats.LastPullTimestamp = time.Now().Add(-2 * 24 * time.Hour) + + gcNew3Stats := retentionMeta.Statistics[gcNew3.DigestStr()] + gcNew3Stats.PushTimestamp = time.Now().Add(-3 * 24 * time.Hour) + gcNew3Stats.LastPullTimestamp = time.Now().Add(-2 * 24 * time.Hour) + + retentionMeta.Statistics[gcOld1.DigestStr()] = gcOld1Stats + retentionMeta.Statistics[gcOld2.DigestStr()] = gcOld2Stats + retentionMeta.Statistics[gcOld3.DigestStr()] = gcOld3Stats + + retentionMeta.Statistics[gcNew1.DigestStr()] = gcNew1Stats + retentionMeta.Statistics[gcNew2.DigestStr()] = gcNew2Stats + retentionMeta.Statistics[gcNew3.DigestStr()] = gcNew3Stats + + // update repo meta + err = metaDB.SetRepoMeta("retention", retentionMeta) + So(err, ShouldBeNil) + + Convey("should not gc anything", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + {}, + }, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo("gc-test1") + So(err, ShouldBeNil) + + err = gc.CleanRepo("gc-test2") + So(err, ShouldBeNil) + + err = gc.CleanRepo("gc-test3") + So(err, ShouldBeNil) + + err = gc.CleanRepo("retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcTest1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcTest2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcUntagged2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", ref2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", refOfRef2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcTest3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcUntagged3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", ref3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", refOfRef3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + }) + + Convey("gc untagged manifests", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{}, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo("gc-test1") + So(err, ShouldBeNil) + + err = gc.CleanRepo("gc-test2") + So(err, ShouldBeNil) + + err = gc.CleanRepo("gc-test3") + So(err, ShouldBeNil) + + err = gc.CleanRepo("retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcTest1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcTest2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", gcUntagged2.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", ref2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", refOfRef2.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcTest3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", gcUntagged3.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", ref3.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", refOfRef3.DigestStr()) + So(err, ShouldBeNil) + }) + + Convey("gc all tags, untagged, and afterwards referrers", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: 1 * time.Millisecond, + UntaggedDelay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"gc-test1"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{"v1"}, // should not match any tag + }, + }, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo("gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldNotBeNil) + + // although we have two tags both should be deleted + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcTest1.DigestStr()) + So(err, ShouldNotBeNil) + + // refs should be preserved because retain tags policy runs after removing referrers + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + err = gc.CleanRepo("gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldNotBeNil) + + // now repo should get gc'ed + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldNotContain, "gc-test1") + So(repos, ShouldContain, "gc-test2") + So(repos, ShouldContain, "gc-test3") + So(repos, ShouldContain, "retention") + }) + + Convey("gc with dry-run all tags, untagged, and afterwards referrers", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: 1 * time.Millisecond, + UntaggedDelay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + DryRun: true, + Policies: []config.GCPolicy{ + { + RepoNames: []string{"gc-test1"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{"v1"}, // should not match any tag + }, + }, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo("gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + // now repo should not be gc'ed + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldContain, "gc-test1") + So(repos, ShouldContain, "gc-test2") + So(repos, ShouldContain, "gc-test3") + So(repos, ShouldContain, "retention") + + tags, err := imgStore.GetImageTags("gc-test1") + So(err, ShouldBeNil) + So(tags, ShouldContain, "0.0.1") + So(tags, ShouldContain, "0.0.2") + }) + + Convey("all tags matches for retention", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{"0.0.*"}, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo("retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test2", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test3", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldBeNil) + }) + + Convey("retain new tags", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{".*"}, + PulledWithinLastNrDays: 7, + PushedWithinLastNrDays: 7, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo("retention") + So(err, ShouldBeNil) + + tags, err := imgStore.GetImageTags("retention") + So(err, ShouldBeNil) + + So(tags, ShouldContain, "0.0.4") + So(tags, ShouldContain, "0.0.5") + So(tags, ShouldContain, "0.0.6") + + So(tags, ShouldNotContain, "0.0.1") + So(tags, ShouldNotContain, "0.0.2") + So(tags, ShouldNotContain, "0.0.3") + }) + + Convey("retain 3 most recently pushed images", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{".*"}, + MostRecentlyPushedCount: 3, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo("retention") + So(err, ShouldBeNil) + + tags, err := imgStore.GetImageTags("retention") + So(err, ShouldBeNil) + + So(tags, ShouldContain, "0.0.4") + So(tags, ShouldContain, "0.0.5") + So(tags, ShouldContain, "0.0.6") + + So(tags, ShouldNotContain, "0.0.1") + So(tags, ShouldNotContain, "0.0.2") + So(tags, ShouldNotContain, "0.0.3") + }) + + Convey("retain 3 most recently pulled images", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{".*"}, + MostRecentlyPulledCount: 3, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo("retention") + So(err, ShouldBeNil) + + tags, err := imgStore.GetImageTags("retention") + So(err, ShouldBeNil) + + So(tags, ShouldContain, "0.0.4") + So(tags, ShouldContain, "0.0.5") + So(tags, ShouldContain, "0.0.6") + + So(tags, ShouldNotContain, "0.0.1") + So(tags, ShouldNotContain, "0.0.2") + So(tags, ShouldNotContain, "0.0.3") + }) + + Convey("retain 3 most recently pulled OR 4 most recently pushed images", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{".*"}, + MostRecentlyPulledCount: 3, + MostRecentlyPushedCount: 4, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo("retention") + So(err, ShouldBeNil) + + tags, err := imgStore.GetImageTags("retention") + So(err, ShouldBeNil) + + So(tags, ShouldContain, "0.0.1") + So(tags, ShouldContain, "0.0.4") + So(tags, ShouldContain, "0.0.5") + So(tags, ShouldContain, "0.0.6") + + So(tags, ShouldNotContain, "0.0.2") + So(tags, ShouldNotContain, "0.0.3") + }) + + Convey("test if first match rule logic works", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{"0.0.1"}, + }, + { + Names: []string{"0.0.2"}, + }, + { + Names: []string{".*"}, + PulledWithinLastNrDays: 2, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo("retention") + So(err, ShouldBeNil) + + tags, err := imgStore.GetImageTags("retention") + So(err, ShouldBeNil) + t.Log(tags) + So(tags, ShouldContain, "0.0.1") + So(tags, ShouldContain, "0.0.2") + So(tags, ShouldContain, "0.0.4") + + So(tags, ShouldNotContain, "0.0.3") + So(tags, ShouldNotContain, "0.0.5") + So(tags, ShouldNotContain, "0.0.6") + }) + + Convey("gc - do not match any repo", func() { + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: 1 * time.Millisecond, + UntaggedDelay: 1 * time.Millisecond, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"no-match"}, + DeleteReferrers: true, + DeleteUntagged: true, + }, + }, + }, + }, audit, log) + + err := gc.CleanRepo("gc-test1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", gcUntagged1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", ref1.DigestStr()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("gc-test1", refOfRef1.DigestStr()) + So(err, ShouldBeNil) + + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(repos, ShouldContain, "gc-test1") + So(repos, ShouldContain, "gc-test2") + So(repos, ShouldContain, "gc-test3") + So(repos, ShouldContain, "retention") + }) + + Convey("remove one tag because it didn't match, preserve tags without statistics in metaDB", func() { + // add new tag in retention repo which can not be found in metaDB, should be always retained + err = WriteImageToFileSystem(CreateRandomImage(), "retention", "0.0.7", storeController) + So(err, ShouldBeNil) + + gc := gc.NewGarbageCollect(imgStore, metaDB, gc.Options{ + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + TagsRetention: []config.TagsRetentionPolicy{ + { + Names: []string{"0.0.[1-5]"}, + }, + }, + }, + }, + }, + }, audit, log) + + err = gc.CleanRepo("retention") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.1") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.2") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.3") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.4") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.5") + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.6") + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest("retention", "0.0.7") + So(err, ShouldBeNil) + }) + }) + }) + } +} diff --git a/pkg/storage/gc/policy.go b/pkg/storage/gc/policy.go new file mode 100644 index 0000000000..64e7f97560 --- /dev/null +++ b/pkg/storage/gc/policy.go @@ -0,0 +1,264 @@ +package gc + +import ( + "fmt" + + glob "github.com/bmatcuk/doublestar/v4" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + + zerr "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" + zcommon "zotregistry.io/zot/pkg/common" + zlog "zotregistry.io/zot/pkg/log" + mTypes "zotregistry.io/zot/pkg/meta/types" + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" + "zotregistry.io/zot/pkg/storage/gc/retention/rules" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/dayspull" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/dayspush" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/latestpull" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/latestpush" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/name" + "zotregistry.io/zot/pkg/storage/gc/retention/rules/retainall" +) + +type candidatesRules struct { + candidates []candidate.Candidate + nameRule rules.Rule + // tag retention rules + rules []rules.Rule +} + +type policyManager struct { + retentionConfig config.ImageRetention + auditLog *zlog.Logger + log zlog.Logger +} + +func (p policyManager) hasDeleteUntagged(repo string) bool { + if policy, err := p.getRepoPolicy(repo); err == nil { + return policy.DeleteUntagged + } + + // default + return false +} + +func (p policyManager) hasDeleteReferrer(repo string) bool { + if policy, err := p.getRepoPolicy(repo); err == nil { + return policy.DeleteReferrers + } + + // default + return false +} + +func (p policyManager) hasTagRetention(repo string) bool { + if policy, err := p.getRepoPolicy(repo); err == nil { + return len(policy.TagsRetention) > 0 + } + + // default + return false +} + +func (p policyManager) getNameRule(repo, tag string) rules.Rule { + repoPolicy, err := p.getRepoPolicy(repo) + if err != nil { + // if repo policy not found, then retain all + return retainall.New(true) + } + + tagPolicy, err := p.getTagPolicy(tag, repoPolicy.TagsRetention) + if err != nil { + // if repo policy found but tag not found, then retain all tags + return retainall.New(false) + } + + // name rule + return name.New(tagPolicy.Names) +} + +func (p policyManager) getRules(repo string, tag string) []rules.Rule { + rules := make([]rules.Rule, 0) + + repoPolicy, err := p.getRepoPolicy(repo) + if err != nil { + // if repo policy not found, then do not apply any tag retention policy + return rules + } + + tagPolicy, err := p.getTagPolicy(tag, repoPolicy.TagsRetention) + if err != nil { + // if tag policy not found, retain all tags by default + return append(rules, retainall.New(false)) + } + + if tagPolicy.MostRecentlyPulledCount != 0 { + rules = append(rules, latestpull.New(tagPolicy.MostRecentlyPulledCount)) + } + + if tagPolicy.MostRecentlyPushedCount != 0 { + rules = append(rules, latestpush.New(tagPolicy.MostRecentlyPushedCount)) + } + + if tagPolicy.PulledWithinLastNrDays != 0 { + rules = append(rules, dayspull.New(tagPolicy.PulledWithinLastNrDays)) + } + + if tagPolicy.PushedWithinLastNrDays != 0 { + rules = append(rules, dayspush.New(tagPolicy.PushedWithinLastNrDays)) + } + + return rules +} + +func (p policyManager) getRepoPolicy(repo string) (config.GCPolicy, error) { + for _, policy := range p.retentionConfig.Policies { + for _, pattern := range policy.RepoNames { + matched, err := glob.Match(pattern, repo) + if err == nil && matched { + return policy, nil + } + } + } + + return config.GCPolicy{}, zerr.ErrGCPolicyNotFound +} + +func (p policyManager) getTagPolicy(tag string, tagPolicies []config.TagsRetentionPolicy, +) (config.TagsRetentionPolicy, error) { + for _, tagPolicy := range tagPolicies { + if zcommon.MatchesListOfRegex(tag, tagPolicy.Names) { + return tagPolicy, nil + } + } + + return config.TagsRetentionPolicy{}, zerr.ErrGCPolicyNotFound +} + +func (p policyManager) getRetainedTags(repoMeta mTypes.RepoMetadata, index ispec.Index) []string { + repo := repoMeta.Name + + matchedByName := make([]string, 0) + + candidates := candidate.GetCandidates(repoMeta) + retainTags := make([]string, 0) + + // we need to make sure tags for which we can not find statistics in repoDB are not removed + actualTags := getIndexTags(index) + + // find tags which are not in candidates list, if they are not in repoDB we want to keep them + for _, tag := range actualTags { + found := false + + for _, candidate := range candidates { + if candidate.Tag == tag { + found = true + } + } + + if !found { + p.log.Info().Str("module", "gc"). + Bool("dry-run", p.retentionConfig.DryRun). + Str("repository", repo). + Str("tag", tag). + Str("decision", "keep"). + Str("reason", "tag statistics not found").Msg("will keep tag") + + retainTags = append(retainTags, tag) + } + } + + grouped := p.groupCandidatesByNameRule(repo, candidates) + + for _, candidates := range grouped { + retainCandidates := candidates.candidates // copy + rules := candidates.rules + nameRule := candidates.nameRule + + // first filter candidates by tag glob pattern + retainCandidates = nameRule.Perform(retainCandidates) + + for _, retainedByName := range retainCandidates { + matchedByName = append(matchedByName, retainedByName.Tag) + } + + rulesCandidates := make([]candidate.Candidate, 0) + + // we retain candidates if any of the below rules are met (OR logic between rules) + for _, rule := range rules { + ruleCandidates := rule.Perform(retainCandidates) + + rulesCandidates = append(rulesCandidates, ruleCandidates...) + } + + // if we applied any rule + if len(rules) > 0 { + retainCandidates = rulesCandidates + } // else we retain just the one matching name rule + + for _, retainCandidate := range retainCandidates { + // there may be duplicates + if !zcommon.Contains(retainTags, retainCandidate.Tag) { + // unique tags + logAction(repo, "keep", retainCandidate.RetainedBy, retainCandidate, p.retentionConfig.DryRun, &p.log) + + retainTags = append(retainTags, retainCandidate.Tag) + } + } + } + + // log tags which will be removed + for _, candidateInfo := range candidates { + if !zcommon.Contains(retainTags, candidateInfo.Tag) { + var reason string + if zcommon.Contains(matchedByName, candidateInfo.Tag) { + reason = "didn't meet any tag retention rule" + } else { + reason = "filtered out by tag names rules" + } + + logAction(repo, "delete", reason, candidateInfo, p.retentionConfig.DryRun, &p.log) + + if p.auditLog != nil { + logAction(repo, "delete", reason, candidateInfo, p.retentionConfig.DryRun, p.auditLog) + } + } + } + + return retainTags +} + +func (p policyManager) groupCandidatesByNameRule(repo string, candidates []candidate.Candidate, +) map[string]candidatesRules { + candidatesByNameRule := make(map[string]candidatesRules) + + for _, candidateInfo := range candidates { + nameRule := p.getNameRule(repo, candidateInfo.Tag) + if _, ok := candidatesByNameRule[nameRule.Name()]; !ok { + candidatesRules := candidatesRules{candidates: []candidate.Candidate{candidateInfo}} + candidatesRules.nameRule = nameRule + candidatesRules.rules = p.getRules(repo, candidateInfo.Tag) + candidatesByNameRule[nameRule.Name()] = candidatesRules + } else { + candidatesRules := candidatesByNameRule[nameRule.Name()] + candidatesRules.candidates = append(candidatesRules.candidates, candidateInfo) + candidatesByNameRule[nameRule.Name()] = candidatesRules + } + } + + return candidatesByNameRule +} + +func logAction(repo, decision, reason string, candidate candidate.Candidate, dryRun bool, log *zlog.Logger) { + log.Info().Str("module", "gc"). + Bool("dry-run", dryRun). + Str("repository", repo). + Str("mediaType", candidate.MediaType). + Str("digest", candidate.DigestStr). + Str("tag", candidate.Tag). + Str("lastPullTimestamp", candidate.PullTimestamp.String()). + Str("pushTimestamp", candidate.PushTimestamp.String()). + Str("decision", decision). + Str("reason", fmt.Sprintf("matched %s policy", reason)).Msgf("will %s tag", decision) +} diff --git a/pkg/storage/gc/retention/candidate/candidate.go b/pkg/storage/gc/retention/candidate/candidate.go new file mode 100644 index 0000000000..045c4e2c86 --- /dev/null +++ b/pkg/storage/gc/retention/candidate/candidate.go @@ -0,0 +1,40 @@ +package candidate + +import ( + "time" + + mTypes "zotregistry.io/zot/pkg/meta/types" +) + +// helper struct. +type Candidate struct { + DigestStr string + MediaType string + Tag string + PushTimestamp time.Time + PullTimestamp time.Time + RetainedBy string +} + +func GetCandidates(repoMeta mTypes.RepoMetadata) []Candidate { + candidates := make([]Candidate, 0) + + // collect all statistic of repo's manifests + for tag, desc := range repoMeta.Tags { + for digestStr, stats := range repoMeta.Statistics { + if digestStr == desc.Digest { + candidate := Candidate{ + MediaType: desc.MediaType, + DigestStr: digestStr, + Tag: tag, + PushTimestamp: stats.PushTimestamp, + PullTimestamp: stats.LastPullTimestamp, + } + + candidates = append(candidates, candidate) + } + } + } + + return candidates +} diff --git a/pkg/storage/gc/retention/rules/dayspull/dayspull.go b/pkg/storage/gc/retention/rules/dayspull/dayspull.go new file mode 100644 index 0000000000..100a43c3ee --- /dev/null +++ b/pkg/storage/gc/retention/rules/dayspull/dayspull.go @@ -0,0 +1,40 @@ +package dayspull + +import ( + "fmt" + "time" + + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ( + ruleName = "mostRecentlyPulledCount" + day = time.Hour * 24 +) + +type rule struct { + count int +} + +func New(count int) rule { + return rule{count: count} +} + +func (r rule) Name() string { + return fmt.Sprintf("%s:%d", ruleName, r.count) +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + filtered := make([]candidate.Candidate, 0) + + timestamp := time.Now().Add(time.Duration(-r.count) * day) + + for _, candidate := range candidates { + if candidate.PullTimestamp.After(timestamp) { + candidate.RetainedBy = r.Name() + filtered = append(filtered, candidate) + } + } + + return filtered +} diff --git a/pkg/storage/gc/retention/rules/dayspush/dayspush.go b/pkg/storage/gc/retention/rules/dayspush/dayspush.go new file mode 100644 index 0000000000..779445b4b5 --- /dev/null +++ b/pkg/storage/gc/retention/rules/dayspush/dayspush.go @@ -0,0 +1,41 @@ +package dayspush + +import ( + "fmt" + "time" + + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ( + ruleName = "mostRecentlyPushedCount" + day = time.Hour * 24 +) + +type rule struct { + count int +} + +func New(count int) rule { + return rule{count: count} +} + +func (r rule) Name() string { + return fmt.Sprintf("%s:%d", ruleName, r.count) +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + filtered := make([]candidate.Candidate, 0) + + timestamp := time.Now().Add(time.Duration(-r.count) * day) + + for _, candidate := range candidates { + if candidate.PushTimestamp.After(timestamp) { + candidate.RetainedBy = r.Name() + + filtered = append(filtered, candidate) + } + } + + return filtered +} diff --git a/pkg/storage/gc/retention/rules/latestpull/latestpull.go b/pkg/storage/gc/retention/rules/latestpull/latestpull.go new file mode 100644 index 0000000000..c5a3c4cfaf --- /dev/null +++ b/pkg/storage/gc/retention/rules/latestpull/latestpull.go @@ -0,0 +1,42 @@ +package latestpull + +import ( + "fmt" + "sort" + + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ruleName = "pulledWithinLastNrDays" + +type rule struct { + count int +} + +func New(count int) rule { + return rule{count: count} +} + +func (r rule) Name() string { + return fmt.Sprintf("%s:%d", ruleName, r.count) +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].PullTimestamp.After(candidates[j].PullTimestamp) + }) + + // take top count candidates + upper := r.count + if r.count > len(candidates) { + upper = len(candidates) + } + + candidates = candidates[:upper] + + for _, candidate := range candidates { + candidate.RetainedBy = r.Name() + } + + return candidates +} diff --git a/pkg/storage/gc/retention/rules/latestpush/latestpush.go b/pkg/storage/gc/retention/rules/latestpush/latestpush.go new file mode 100644 index 0000000000..393b5f0113 --- /dev/null +++ b/pkg/storage/gc/retention/rules/latestpush/latestpush.go @@ -0,0 +1,42 @@ +package latestpush + +import ( + "fmt" + "sort" + + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ruleName = "pulledWithinLastNrDays" + +type rule struct { + count int +} + +func New(count int) rule { + return rule{count: count} +} + +func (r rule) Name() string { + return fmt.Sprintf("%s:%d", ruleName, r.count) +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].PushTimestamp.After(candidates[j].PushTimestamp) + }) + + // take top count candidates + upper := r.count + if r.count > len(candidates) { + upper = len(candidates) + } + + candidates = candidates[:upper] + + for _, candidate := range candidates { + candidate.RetainedBy = r.Name() + } + + return candidates +} diff --git a/pkg/storage/gc/retention/rules/name/name.go b/pkg/storage/gc/retention/rules/name/name.go new file mode 100644 index 0000000000..96b0fc8fec --- /dev/null +++ b/pkg/storage/gc/retention/rules/name/name.go @@ -0,0 +1,35 @@ +package name + +import ( + "fmt" + + "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ruleName = "name" + +type rule struct { + names []string +} + +func New(names []string) rule { + return rule{names: names} +} + +func (r rule) Name() string { + return fmt.Sprintf("%s:%v", ruleName, r.names) +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + filtered := make([]candidate.Candidate, 0) + + for _, candidate := range candidates { + if common.MatchesListOfRegex(candidate.Tag, r.names) { + candidate.RetainedBy = r.Name() + filtered = append(filtered, candidate) + } + } + + return filtered +} diff --git a/pkg/storage/gc/retention/rules/retainall/retainall.go b/pkg/storage/gc/retention/rules/retainall/retainall.go new file mode 100644 index 0000000000..57bd1a69a6 --- /dev/null +++ b/pkg/storage/gc/retention/rules/retainall/retainall.go @@ -0,0 +1,31 @@ +package retainall + +import ( + "zotregistry.io/zot/pkg/storage/gc/retention/candidate" +) + +const ruleName = "retainByDefault" + +type rule struct { + retain bool +} + +func New(retain bool) rule { + return rule{retain} +} + +func (r rule) Name() string { + return ruleName +} + +func (r rule) Perform(candidates []candidate.Candidate) []candidate.Candidate { + if r.retain { + for _, candidate := range candidates { + candidate.RetainedBy = r.Name() + } + + return candidates + } + + return []candidate.Candidate{} +} diff --git a/pkg/storage/gc/retention/rules/rule.go b/pkg/storage/gc/retention/rules/rule.go new file mode 100644 index 0000000000..f18a3dbf65 --- /dev/null +++ b/pkg/storage/gc/retention/rules/rule.go @@ -0,0 +1,8 @@ +package rules + +import "zotregistry.io/zot/pkg/storage/gc/retention/candidate" + +type Rule interface { + Name() string + Perform(candidates []candidate.Candidate) []candidate.Candidate +} diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index a65762c10b..9dcb684de0 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -1139,7 +1139,7 @@ func (is *ImageStore) CheckBlob(repo string, digest godigest.Digest) (bool, int6 // Check blobs in cache dstRecord, err := is.checkCacheBlob(digest) if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + is.log.Debug().Err(err).Str("digest", digest.String()).Msg("cache: not found") return false, -1, zerr.ErrBlobNotFound } @@ -1185,7 +1185,7 @@ func (is *ImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64 // Check blobs in cache dstRecord, err := is.checkCacheBlob(digest) if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + is.log.Debug().Err(err).Str("digest", digest.String()).Msg("cache: not found") return false, -1, time.Time{}, zerr.ErrBlobNotFound } @@ -1512,7 +1512,8 @@ func (is *ImageStore) CleanupRepo(repo string, blobs []godigest.Digest, removeRe count := 0 for _, digest := range blobs { - is.log.Debug().Str("repository", repo).Str("digest", digest.String()).Msg("perform GC on blob") + is.log.Debug().Str("repository", repo). + Str("digest", digest.String()).Msg("perform GC on blob") if err := is.deleteBlob(repo, digest); err != nil { if errors.Is(err, zerr.ErrBlobReferenced) { @@ -1544,6 +1545,8 @@ func (is *ImageStore) CleanupRepo(repo string, blobs []godigest.Digest, removeRe // if removeRepo flag is true and we cleanup all blobs and there are no blobs currently being uploaded. if removeRepo && count == len(blobs) && count > 0 && len(blobUploads) == 0 { + is.log.Info().Str("repository", repo).Msg("removed all blobs, removing repo") + if err := is.storeDriver.Delete(path.Join(is.rootDir, repo)); err != nil { is.log.Error().Err(err).Str("repository", repo).Msg("unable to remove repo") diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index 571475be31..f73755628b 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -29,7 +29,7 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" - "zotregistry.io/zot/pkg/log" + zlog "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" @@ -47,10 +47,20 @@ const ( repoName = "test" ) +var DeleteReferrers = config.ImageRetention{ //nolint: gochecknoglobals + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + }, + }, +} + var errCache = errors.New("new cache error") func runAndGetScheduler() (*scheduler.Scheduler, context.CancelFunc) { - taskScheduler := scheduler.NewScheduler(config.New(), log.Logger{}) + taskScheduler := scheduler.NewScheduler(config.New(), zlog.Logger{}) taskScheduler.RateLimit = 50 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) @@ -62,7 +72,7 @@ func runAndGetScheduler() (*scheduler.Scheduler, context.CancelFunc) { func TestStorageFSAPIs(t *testing.T) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -199,7 +209,7 @@ func TestStorageFSAPIs(t *testing.T) { func TestGetOrasReferrers(t *testing.T) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -258,7 +268,7 @@ func FuzzNewBlobUpload(f *testing.F) { dir := t.TempDir() defer os.RemoveAll(dir) t.Logf("Input argument is %s", data) - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -284,7 +294,8 @@ func FuzzPutBlobChunk(f *testing.F) { dir := t.TempDir() defer os.RemoveAll(dir) t.Logf("Input argument is %s", data) - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -317,7 +328,7 @@ func FuzzPutBlobChunkStreamed(f *testing.F) { dir := t.TempDir() defer os.RemoveAll(dir) t.Logf("Input argument is %s", data) - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -349,7 +360,7 @@ func FuzzGetBlobUpload(f *testing.F) { f.Fuzz(func(t *testing.T, data1 string, data2 string) { dir := t.TempDir() defer os.RemoveAll(dir) - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -371,7 +382,7 @@ func FuzzGetBlobUpload(f *testing.F) { func FuzzTestPutGetImageManifest(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -422,7 +433,7 @@ func FuzzTestPutGetImageManifest(f *testing.F) { func FuzzTestPutDeleteImageManifest(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -480,7 +491,7 @@ func FuzzTestPutDeleteImageManifest(f *testing.F) { // no integration with PutImageManifest, just throw fuzz data. func FuzzTestDeleteImageManifest(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -515,7 +526,7 @@ func FuzzDirExists(f *testing.F) { func FuzzInitRepo(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -539,7 +550,7 @@ func FuzzInitRepo(f *testing.F) { func FuzzInitValidateRepo(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -570,7 +581,7 @@ func FuzzInitValidateRepo(f *testing.F) { func FuzzGetImageTags(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -594,7 +605,7 @@ func FuzzGetImageTags(f *testing.F) { func FuzzBlobUploadPath(f *testing.F) { f.Fuzz(func(t *testing.T, repo, uuid string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -613,7 +624,7 @@ func FuzzBlobUploadPath(f *testing.F) { func FuzzBlobUploadInfo(f *testing.F) { f.Fuzz(func(t *testing.T, data string, uuid string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -642,7 +653,7 @@ func FuzzTestGetImageManifest(f *testing.F) { dir := t.TempDir() defer os.RemoveAll(dir) - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -670,7 +681,7 @@ func FuzzFinishBlobUpload(f *testing.F) { dir := t.TempDir() defer os.RemoveAll(dir) - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -714,7 +725,7 @@ func FuzzFinishBlobUpload(f *testing.F) { func FuzzFullBlobUpload(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) repoName := "test" @@ -745,7 +756,7 @@ func FuzzFullBlobUpload(f *testing.F) { func TestStorageCacheErrors(t *testing.T) { Convey("get error in DedupeBlob() when cache.Put() deduped blob", t, func() { - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) dir := t.TempDir() @@ -787,7 +798,7 @@ func TestStorageCacheErrors(t *testing.T) { func FuzzDedupeBlob(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -827,7 +838,7 @@ func FuzzDedupeBlob(f *testing.F) { func FuzzDeleteBlobUpload(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) repoName := data @@ -858,7 +869,7 @@ func FuzzDeleteBlobUpload(f *testing.F) { func FuzzBlobPath(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) repoName := data @@ -879,7 +890,7 @@ func FuzzBlobPath(f *testing.F) { func FuzzCheckBlob(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) repoName := data @@ -910,7 +921,7 @@ func FuzzCheckBlob(f *testing.F) { func FuzzGetBlob(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) repoName := data @@ -948,7 +959,7 @@ func FuzzGetBlob(f *testing.F) { func FuzzDeleteBlob(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) repoName := data @@ -983,7 +994,7 @@ func FuzzDeleteBlob(f *testing.F) { func FuzzGetIndexContent(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) repoName := data @@ -1018,7 +1029,7 @@ func FuzzGetIndexContent(f *testing.F) { func FuzzGetBlobContent(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) repoName := data @@ -1053,7 +1064,7 @@ func FuzzGetBlobContent(f *testing.F) { func FuzzGetOrasReferrers(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := &log.Logger{Logger: zerolog.New(os.Stdout)} + log := &zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, *log) dir := t.TempDir() @@ -1116,7 +1127,9 @@ func FuzzGetOrasReferrers(f *testing.F) { func FuzzRunGCRepo(f *testing.F) { f.Fuzz(func(t *testing.T, data string) { - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) dir := t.TempDir() defer os.RemoveAll(dir) @@ -1129,10 +1142,10 @@ func FuzzRunGCRepo(f *testing.F) { imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) if err := gc.CleanRepo(data); err != nil { t.Error(err) @@ -1155,7 +1168,7 @@ func TestDedupeLinks(t *testing.T) { }, } - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) for _, testCase := range testCases { @@ -1520,7 +1533,7 @@ func TestDedupe(t *testing.T) { Convey("Valid ImageStore", func() { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -1540,7 +1553,7 @@ func TestNegativeCases(t *testing.T) { Convey("Invalid root dir", t, func(c C) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -1563,7 +1576,7 @@ func TestNegativeCases(t *testing.T) { Convey("Invalid init repo", t, func(c C) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -1613,7 +1626,7 @@ func TestNegativeCases(t *testing.T) { Convey("Invalid validate repo", t, func(c C) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -1725,7 +1738,7 @@ func TestNegativeCases(t *testing.T) { Convey("Invalid get image tags", t, func(c C) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -1750,7 +1763,7 @@ func TestNegativeCases(t *testing.T) { Convey("Invalid get image manifest", t, func(c C) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -1797,7 +1810,7 @@ func TestNegativeCases(t *testing.T) { Convey("Invalid new blob upload", t, func(c C) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -1968,7 +1981,7 @@ func TestInjectWriteFile(t *testing.T) { Convey("writeFile without commit", t, func() { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -1994,7 +2007,9 @@ func TestGarbageCollectForImageStore(t *testing.T) { defer os.Remove(logFile.Name()) // clean up - log := log.NewLogger("debug", logFile.Name()) + log := zlog.NewLogger("debug", logFile.Name()) + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2006,10 +2021,10 @@ func TestGarbageCollectForImageStore(t *testing.T) { repoName := "gc-all-repos-short" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 1 * time.Second, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) image := CreateDefaultVulnerableImage() err := WriteImageToFileSystem(image, repoName, "0.0.1", storage.StoreController{ @@ -2039,7 +2054,9 @@ func TestGarbageCollectForImageStore(t *testing.T) { defer os.Remove(logFile.Name()) // clean up - log := log.NewLogger("debug", logFile.Name()) + log := zlog.NewLogger("debug", logFile.Name()) + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2051,10 +2068,10 @@ func TestGarbageCollectForImageStore(t *testing.T) { repoName := "gc-all-repos-short" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 1 * time.Second, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) image := CreateDefaultVulnerableImage() err := WriteImageToFileSystem(image, repoName, "0.0.1", storage.StoreController{ @@ -2081,7 +2098,9 @@ func TestGarbageCollectForImageStore(t *testing.T) { defer os.Remove(logFile.Name()) // clean up - log := log.NewLogger("debug", logFile.Name()) + log := zlog.NewLogger("debug", logFile.Name()) + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2092,10 +2111,10 @@ func TestGarbageCollectForImageStore(t *testing.T) { repoName := "gc-sig" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 1 * time.Second, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) storeController := storage.StoreController{DefaultStore: imgStore} img := CreateRandomImage() @@ -2146,7 +2165,9 @@ func TestGarbageCollectImageUnknownManifest(t *testing.T) { Convey("Garbage collect with short delay", t, func() { dir := t.TempDir() - log := log.NewLogger("debug", "") + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2161,10 +2182,10 @@ func TestGarbageCollectImageUnknownManifest(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 1 * time.Second, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) unsupportedMediaType := "application/vnd.oci.artifact.manifest.v1+json" @@ -2324,7 +2345,9 @@ func TestGarbageCollectErrors(t *testing.T) { Convey("Make image store", t, func(c C) { dir := t.TempDir() - log := log.NewLogger("debug", "") + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2336,10 +2359,10 @@ func TestGarbageCollectErrors(t *testing.T) { repoName := "gc-index" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: 500 * time.Millisecond, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) // create a blob/layer upload, err := imgStore.NewBlobUpload(repoName) @@ -2538,7 +2561,7 @@ func TestGarbageCollectErrors(t *testing.T) { err = gc.CleanRepo(repoName) So(err, ShouldBeNil) - // blob shouldn't be gc'ed + // blob shouldn't be gc'ed //TODO check this one found, _, err := imgStore.CheckBlob(repoName, digest) So(err, ShouldBeNil) So(found, ShouldEqual, true) @@ -2566,7 +2589,7 @@ func TestInitRepo(t *testing.T) { Convey("Get error when creating BlobUploadDir subdir on initRepo", t, func() { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2588,7 +2611,7 @@ func TestValidateRepo(t *testing.T) { Convey("Get error when unable to read directory", t, func() { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2608,7 +2631,7 @@ func TestValidateRepo(t *testing.T) { Convey("Get error when repo name is not compliant with repo spec", t, func() { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2653,7 +2676,7 @@ func TestGetRepositories(t *testing.T) { Convey("Verify errors and repos returned by GetRepositories()", t, func() { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2748,7 +2771,7 @@ func TestGetRepositories(t *testing.T) { Convey("Verify GetRepositories() doesn't return '.' when having an oci layout as root directory ", t, func() { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2794,7 +2817,7 @@ func TestGetRepositories(t *testing.T) { err := os.Mkdir(rootDir, 0o755) So(err, ShouldBeNil) - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: rootDir, @@ -2838,7 +2861,7 @@ func TestGetRepositories(t *testing.T) { func TestGetNextRepository(t *testing.T) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2891,7 +2914,7 @@ func TestPutBlobChunkStreamed(t *testing.T) { Convey("Get error on opening file", t, func() { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -2918,7 +2941,7 @@ func TestPullRange(t *testing.T) { Convey("Repo layout", t, func(c C) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) Convey("Negative cases", func() { @@ -2968,7 +2991,7 @@ func TestPullRange(t *testing.T) { func TestStorageDriverErr(t *testing.T) { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 023990677a..13b031ddce 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -28,8 +28,9 @@ import ( "gopkg.in/resty.v1" zerr "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/extensions/monitoring" - "zotregistry.io/zot/pkg/log" + zlog "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" storageCommon "zotregistry.io/zot/pkg/storage/common" @@ -44,6 +45,16 @@ import ( tskip "zotregistry.io/zot/pkg/test/skip" ) +var DeleteReferrers = config.ImageRetention{ //nolint: gochecknoglobals + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + DeleteUntagged: true, + }, + }, +} + func cleanupStorage(store driver.StorageDriver, name string) { _ = store.Delete(context.Background(), name) } @@ -78,7 +89,7 @@ func createObjectsStore(rootDir string, cacheDir string) ( panic(err) } - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ @@ -129,7 +140,7 @@ func TestStorageAPIs(t *testing.T) { } else { dir := t.TempDir() - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, @@ -741,7 +752,7 @@ func TestMandatoryAnnotations(t *testing.T) { var testDir, tdir string var store driver.StorageDriver - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) if testcase.storageType == storageConstants.S3StorageDriverName { @@ -865,7 +876,7 @@ func TestDeleteBlobsInUse(t *testing.T) { var testDir, tdir string var store driver.StorageDriver - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) if testcase.storageType == storageConstants.S3StorageDriverName { @@ -1190,7 +1201,7 @@ func TestStorageHandler(t *testing.T) { secondRootDir = t.TempDir() thirdRootDir = t.TempDir() - log := log.NewLogger("debug", "") + log := zlog.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) @@ -1249,7 +1260,9 @@ func TestGarbageCollectImageManifest(t *testing.T) { for _, testcase := range testCases { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) Convey("Repo layout", t, func(c C) { @@ -1284,10 +1297,17 @@ func TestGarbageCollectImageManifest(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, - Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + Delay: storageConstants.DefaultGCDelay, + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: config.ImageRetention{ + Policies: []config.GCPolicy{ + { + RepoNames: []string{"**"}, + DeleteReferrers: true, + }, + }, + }, + }, audit, log) repoName := "gc-long" @@ -1447,10 +1467,10 @@ func TestGarbageCollectImageManifest(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: gcDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) // upload orphan blob upload, err := imgStore.NewBlobUpload(repoName) @@ -1757,10 +1777,10 @@ func TestGarbageCollectImageManifest(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: gcDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) // first upload an image to the first repo and wait for GC timeout @@ -1958,7 +1978,9 @@ func TestGarbageCollectImageIndex(t *testing.T) { for _, testcase := range testCases { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) Convey("Repo layout", t, func(c C) { @@ -1993,10 +2015,10 @@ func TestGarbageCollectImageIndex(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: storageConstants.DefaultGCDelay, - RetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, - }, log) + UntaggedDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) repoName := "gc-long" @@ -2123,10 +2145,10 @@ func TestGarbageCollectImageIndex(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: gcDelay, - RetentionDelay: imageRetentionDelay, - }, log) + UntaggedDelay: imageRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) // upload orphan blob upload, err := imgStore.NewBlobUpload(repoName) @@ -2395,7 +2417,9 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { for _, testcase := range testCases { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { - log := log.Logger{Logger: zerolog.New(os.Stdout)} + log := zlog.NewLogger("debug", "") + audit := zlog.NewAuditLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) Convey("Garbage collect with short delay", t, func() { @@ -2433,10 +2457,10 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ - Referrers: true, Delay: gcDelay, - RetentionDelay: imageRetentionDelay, - }, log) + UntaggedDelay: imageRetentionDelay, + ImageRetention: DeleteReferrers, + }, audit, log) // upload orphan blob upload, err := imgStore.NewBlobUpload(repoName) diff --git a/pkg/test/image-utils/upload_test.go b/pkg/test/image-utils/upload_test.go index 178e08efaa..7291bc412f 100644 --- a/pkg/test/image-utils/upload_test.go +++ b/pkg/test/image-utils/upload_test.go @@ -81,17 +81,24 @@ func TestUploadImage(t *testing.T) { conf.HTTP.Port = port conf.Storage.RootDirectory = tempDir - err := os.Chmod(tempDir, 0o400) - if err != nil { - t.Fatal(err) - } - ctlr := api.NewController(conf) ctlrManager := tcommon.NewControllerManager(ctlr) ctlrManager.StartAndWait(port) defer ctlrManager.StopServer() + err := os.Chmod(tempDir, 0o400) + if err != nil { + t.Fatal(err) + } + + defer func() { + err := os.Chmod(tempDir, 0o700) + if err != nil { + t.Fatal(err) + } + }() + img := Image{ Layers: make([][]byte, 10), } diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index 9d40b4c1da..badfdba40b 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -19,7 +19,8 @@ type MetaDBMock struct { SetRepoLogoFn func(repo string, logoPath string) error - SetRepoReferenceFn func(repo string, Reference string, manifestDigest godigest.Digest, mediaType string) error + SetRepoReferenceFn func(ctx context.Context, repo string, Reference string, + manifestDigest godigest.Digest, mediaType string) error RemoveRepoReferenceFn func(repo, reference string, manifestDigest godigest.Digest) error @@ -55,7 +56,7 @@ type MetaDBMock struct { GetReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) ( []mTypes.ReferrerInfo, error) - IncrementImageDownloadsFn func(repo string, reference string) error + UpdateStatsOnDownloadFn func(repo string, reference string) error UpdateSignaturesValidityFn func(repo string, manifestDigest godigest.Digest) error @@ -160,11 +161,11 @@ func (sdm MetaDBMock) GetRepoStars(repo string) (int, error) { return 0, nil } -func (sdm MetaDBMock) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, - mediaType string, +func (sdm MetaDBMock) SetRepoReference(ctx context.Context, repo string, reference string, + manifestDigest godigest.Digest, mediaType string, ) error { if sdm.SetRepoReferenceFn != nil { - return sdm.SetRepoReferenceFn(repo, reference, manifestDigest, mediaType) + return sdm.SetRepoReferenceFn(ctx, repo, reference, manifestDigest, mediaType) } return nil @@ -251,9 +252,9 @@ func (sdm MetaDBMock) SetManifestMeta(repo string, manifestDigest godigest.Diges return nil } -func (sdm MetaDBMock) IncrementImageDownloads(repo string, reference string) error { - if sdm.IncrementImageDownloadsFn != nil { - return sdm.IncrementImageDownloadsFn(repo, reference) +func (sdm MetaDBMock) UpdateStatsOnDownload(repo string, reference string) error { + if sdm.UpdateStatsOnDownloadFn != nil { + return sdm.UpdateStatsOnDownloadFn(repo, reference) } return nil diff --git a/test/blackbox/garbage_collect.bats b/test/blackbox/garbage_collect.bats index 61ca08d5e3..5faeeaabc5 100644 --- a/test/blackbox/garbage_collect.bats +++ b/test/blackbox/garbage_collect.bats @@ -34,10 +34,18 @@ function setup_file() { "storage": { "rootDirectory": "${zot_root_dir}", "gc": true, - "gcReferrers": true, "gcDelay": "30s", "untaggedImageRetentionDelay": "40s", - "gcInterval": "1s" + "gcInterval": "1s", + "retention": { + "policies": [ + { + "repoNames": ["**"], + "deleteReferrers": true, + "deleteUntagged": true + } + ] + } }, "http": { "address": "0.0.0.0", diff --git a/test/cluster/config-minio.json b/test/cluster/config-minio.json index 618cf2cb1e..53464a7cfe 100644 --- a/test/cluster/config-minio.json +++ b/test/cluster/config-minio.json @@ -2,7 +2,7 @@ "distSpecVersion": "1.1.0-dev", "storage": { "rootDirectory": "/tmp/zot", - "gc": false, + "gc": true, "dedupe": false, "storageDriver": { "name": "s3", diff --git a/test/gc-stress/config-gc-bench-local.json b/test/gc-stress/config-gc-bench-local.json index ae1010c155..3d1dbb9901 100644 --- a/test/gc-stress/config-gc-bench-local.json +++ b/test/gc-stress/config-gc-bench-local.json @@ -3,7 +3,6 @@ "storage": { "rootDirectory": "/tmp/zot/local", "gc": true, - "gcReferrers": false, "gcDelay": "20s", "untaggedImageRetentionDelay": "20s", "gcInterval": "1s" diff --git a/test/gc-stress/config-gc-bench-s3-localstack.json b/test/gc-stress/config-gc-bench-s3-localstack.json index 11151bdce2..9bb3ad3636 100644 --- a/test/gc-stress/config-gc-bench-s3-localstack.json +++ b/test/gc-stress/config-gc-bench-s3-localstack.json @@ -3,7 +3,6 @@ "storage": { "rootDirectory": "/tmp/zot/s3", "gc": true, - "gcReferrers": false, "gcDelay": "50m", "untaggedImageRetentionDelay": "50m", "gcInterval": "2m", @@ -20,7 +19,13 @@ "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", - "cacheTablename": "BlobTable" + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "indexDataTablename": "IndexDataTable", + "manifestDataTablename": "ManifestDataTable", + "apikeytablename": "ApiKeyDataTable", + "userdatatablename": "UserDataTable", + "versionTablename": "VersionTable" } }, "http": { diff --git a/test/gc-stress/config-gc-bench-s3-minio.json b/test/gc-stress/config-gc-bench-s3-minio.json index 7c59e661ad..8fbeb0f549 100644 --- a/test/gc-stress/config-gc-bench-s3-minio.json +++ b/test/gc-stress/config-gc-bench-s3-minio.json @@ -3,7 +3,6 @@ "storage": { "rootDirectory": "/tmp/zot/s3", "gc": true, - "gcReferrers": false, "gcDelay": "3m", "untaggedImageRetentionDelay": "3m", "gcInterval": "1s", @@ -22,7 +21,13 @@ "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", - "cacheTablename": "BlobTable" + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "indexDataTablename": "IndexDataTable", + "manifestDataTablename": "ManifestDataTable", + "apikeytablename": "ApiKeyDataTable", + "userdatatablename": "UserDataTable", + "versionTablename": "VersionTable" } }, "http": { diff --git a/test/gc-stress/config-gc-referrers-bench-local.json b/test/gc-stress/config-gc-referrers-bench-local.json index 909be1a085..5263ce5791 100644 --- a/test/gc-stress/config-gc-referrers-bench-local.json +++ b/test/gc-stress/config-gc-referrers-bench-local.json @@ -3,10 +3,17 @@ "storage": { "rootDirectory": "/tmp/zot/local", "gc": true, - "gcReferrers": true, "gcDelay": "20s", "untaggedImageRetentionDelay": "20s", - "gcInterval": "1s" + "gcInterval": "1s", + "retention": { + "policies": [ + { + "repoNames": ["**"], + "deleteReferrers": true + } + ] + } }, "http": { "address": "127.0.0.1", diff --git a/test/gc-stress/config-gc-referrers-bench-s3-localstack.json b/test/gc-stress/config-gc-referrers-bench-s3-localstack.json index 98cee6bc48..bdfb5ff765 100644 --- a/test/gc-stress/config-gc-referrers-bench-s3-localstack.json +++ b/test/gc-stress/config-gc-referrers-bench-s3-localstack.json @@ -3,10 +3,17 @@ "storage": { "rootDirectory": "/tmp/zot/s3", "gc": true, - "gcReferrers": true, "gcDelay": "50m", "untaggedImageRetentionDelay": "50m", "gcInterval": "2m", + "retention": { + "policies": [ + { + "repoNames": ["**"], + "deleteReferrers": true + } + ] + }, "storageDriver": { "name": "s3", "rootdirectory": "/zot", @@ -20,7 +27,13 @@ "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", - "cacheTablename": "BlobTable" + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "indexDataTablename": "IndexDataTable", + "manifestDataTablename": "ManifestDataTable", + "apikeytablename": "ApiKeyDataTable", + "userdatatablename": "UserDataTable", + "versionTablename": "VersionTable" } }, "http": { diff --git a/test/gc-stress/config-gc-referrers-bench-s3-minio.json b/test/gc-stress/config-gc-referrers-bench-s3-minio.json index 58ba09934d..93e6a176c2 100644 --- a/test/gc-stress/config-gc-referrers-bench-s3-minio.json +++ b/test/gc-stress/config-gc-referrers-bench-s3-minio.json @@ -3,10 +3,17 @@ "storage": { "rootDirectory": "/tmp/zot/s3", "gc": true, - "gcReferrers": true, "gcDelay": "3m", "untaggedImageRetentionDelay": "3m", "gcInterval": "1s", + "retention": { + "policies": [ + { + "repoNames": ["**"], + "deleteReferrers": true + } + ] + }, "storageDriver": { "name": "s3", "rootdirectory": "/zot", @@ -22,7 +29,13 @@ "name": "dynamodb", "endpoint": "http://localhost:4566", "region": "us-east-2", - "cacheTablename": "BlobTable" + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "indexDataTablename": "IndexDataTable", + "manifestDataTablename": "ManifestDataTable", + "apikeytablename": "ApiKeyDataTable", + "userdatatablename": "UserDataTable", + "versionTablename": "VersionTable" } }, "http": {