From 79418df4eac1f6a2a079443d91eb7ddaed6446b0 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Mon, 16 Oct 2023 05:22:25 +0000 Subject: [PATCH 01/10] another rebase attempt Signed-off-by: Xiaoxuan Wang --- content/oci/deletableoci.go | 355 +++++++++++++++++++++++++++++++ content/oci/deletableoci_test.go | 83 ++++++++ 2 files changed, 438 insertions(+) create mode 100644 content/oci/deletableoci.go create mode 100644 content/oci/deletableoci_test.go diff --git a/content/oci/deletableoci.go b/content/oci/deletableoci.go new file mode 100644 index 00000000..ac18d19f --- /dev/null +++ b/content/oci/deletableoci.go @@ -0,0 +1,355 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package oci provides access to an OCI content store. +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +package oci + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/internal/container/set" + "oras.land/oras-go/v2/internal/descriptor" + "oras.land/oras-go/v2/internal/graph" + "oras.land/oras-go/v2/internal/resolver" +) + +// DeletableStore implements `oras.Target`, and represents a content store +// extended with the delete operation. +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +type DeletableStore struct { + // AutoSaveIndex controls if the OCI store will automatically save the index + // file on each Tag() call. + // - If AutoSaveIndex is set to true, the OCI store will automatically call + // this method on each Tag() call. + // - If AutoSaveIndex is set to false, it's the caller's responsibility + // to manually call SaveIndex() when needed. + // - Default value: true. + AutoSaveIndex bool + root string + indexPath string + index *ocispec.Index + lock sync.RWMutex + + storage *Storage + tagResolver *resolver.Memory + graph *graph.Memory +} + +// NewDeletableStore returns a new DeletableStore. +func NewDeletableStore(root string) (*DeletableStore, error) { + return NewDeletableStoreWithContext(context.Background(), root) +} + +// NewDeletableStoreWithContext creates a new DeletableStore. +func NewDeletableStoreWithContext(ctx context.Context, root string) (*DeletableStore, error) { + rootAbs, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", root, err) + } + storage, err := NewStorage(rootAbs) + if err != nil { + return nil, fmt.Errorf("failed to create storage: %w", err) + } + store := &DeletableStore{ + AutoSaveIndex: true, + root: rootAbs, + indexPath: filepath.Join(rootAbs, ocispec.ImageIndexFile), + storage: storage, + tagResolver: resolver.NewMemory(), + graph: graph.NewMemory(), + } + if err := ensureDir(filepath.Join(rootAbs, ocispec.ImageBlobsDir)); err != nil { + return nil, err + } + if err := store.ensureOCILayoutFile(); err != nil { + return nil, fmt.Errorf("invalid OCI Image Layout: %w", err) + } + if err := store.loadIndexFile(ctx); err != nil { + return nil, fmt.Errorf("invalid OCI Image Index: %w", err) + } + return store, nil +} + +// Fetch fetches the content identified by the descriptor. +func (ds *DeletableStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + return ds.storage.Fetch(ctx, target) +} + +// Push pushes the content, matching the expected descriptor. +func (ds *DeletableStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error { + ds.lock.Lock() + defer ds.lock.Unlock() + if err := ds.storage.Push(ctx, expected, reader); err != nil { + return err + } + if err := ds.graph.Index(ctx, ds.storage, expected); err != nil { + return err + } + if descriptor.IsManifest(expected) { + // tag by digest + return ds.tag(ctx, expected, expected.Digest.String()) + } + return nil +} + +// Delete removes the content matching the descriptor from the store. +func (ds *DeletableStore) Delete(ctx context.Context, target ocispec.Descriptor) error { + ds.lock.Lock() + defer ds.lock.Unlock() + resolvers := ds.tagResolver.Map() + for reference, desc := range resolvers { + if content.Equal(desc, target) { + ds.tagResolver.Untag(reference) + } + } + if err := ds.graph.Remove(ctx, target); err != nil { + return err + } + if ds.AutoSaveIndex { + err := ds.saveIndex() + if err != nil { + return err + } + } + return ds.storage.Delete(ctx, target) +} + +// Exists returns true if the described content exists. +func (ds *DeletableStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + return ds.storage.Exists(ctx, target) +} + +// Tag tags a descriptor with a reference string. +// reference should be a valid tag (e.g. "latest"). +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md#indexjson-file +func (ds *DeletableStore) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + ds.lock.Lock() + defer ds.lock.Unlock() + if err := validateReference(reference); err != nil { + return err + } + exists, err := ds.storage.Exists(ctx, desc) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("%s: %s: %w", desc.Digest, desc.MediaType, errdef.ErrNotFound) + } + return ds.tag(ctx, desc, reference) +} + +// tag tags a descriptor with a reference string. +func (ds *DeletableStore) tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + dgst := desc.Digest.String() + if reference != dgst { + // also tag desc by its digest + if err := ds.tagResolver.Tag(ctx, desc, dgst); err != nil { + return err + } + } + if err := ds.tagResolver.Tag(ctx, desc, reference); err != nil { + return err + } + if ds.AutoSaveIndex { + return ds.saveIndex() + } + return nil +} + +// Resolve resolves a reference to a descriptor. If the reference to be resolved +// is a tag, the returned descriptor will be a full descriptor declared by +// github.com/opencontainers/image-spec/specs-go/v1. If the reference is a +// digest the returned descriptor will be a plain descriptor (containing only +// the digest, media type and size). +func (ds *DeletableStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + if reference == "" { + return ocispec.Descriptor{}, errdef.ErrMissingReference + } + // attempt resolving manifest + desc, err := ds.tagResolver.Resolve(ctx, reference) + if err != nil { + if errors.Is(err, errdef.ErrNotFound) { + // attempt resolving blob + return resolveBlob(os.DirFS(ds.root), reference) + } + return ocispec.Descriptor{}, err + } + if reference == desc.Digest.String() { + return descriptor.Plain(desc), nil + } + return desc, nil +} + +// Predecessors returns the nodes directly pointing to the current node. +// Predecessors returns nil without error if the node does not exists in the +// store. +func (ds *DeletableStore) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + return ds.graph.Predecessors(ctx, node) +} + +// Tags lists the tags presented in the `index.json` file of the OCI layout, +// returned in ascending order. +// If `last` is NOT empty, the entries in the response start after the tag +// specified by `last`. Otherwise, the response starts from the top of the tags +// list. +// +// See also `Tags()` in the package `registry`. +func (ds *DeletableStore) Tags(ctx context.Context, last string, fn func(tags []string) error) error { + ds.lock.RLock() + defer ds.lock.RUnlock() + return listTags(ctx, ds.tagResolver, last, fn) +} + +// ensureOCILayoutFile ensures the `oci-layout` file. +func (ds *DeletableStore) ensureOCILayoutFile() error { + layoutFilePath := filepath.Join(ds.root, ocispec.ImageLayoutFile) + layoutFile, err := os.Open(layoutFilePath) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to open OCI layout file: %w", err) + } + layout := ocispec.ImageLayout{ + Version: ocispec.ImageLayoutVersion, + } + layoutJSON, err := json.Marshal(layout) + if err != nil { + return fmt.Errorf("failed to marshal OCI layout file: %w", err) + } + return os.WriteFile(layoutFilePath, layoutJSON, 0666) + } + defer layoutFile.Close() + var layout ocispec.ImageLayout + err = json.NewDecoder(layoutFile).Decode(&layout) + if err != nil { + return fmt.Errorf("failed to decode OCI layout file: %w", err) + } + return validateOCILayout(&layout) +} + +// loadIndexFile reads index.json from the file system. +// Create index.json if it does not exist. +func (ds *DeletableStore) loadIndexFile(ctx context.Context) error { + indexFile, err := os.Open(ds.indexPath) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to open index file: %w", err) + } + // write index.json if it does not exist + ds.index = &ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value + }, + Manifests: []ocispec.Descriptor{}, + } + return ds.writeIndexFile() + } + defer indexFile.Close() + var index ocispec.Index + if err := json.NewDecoder(indexFile).Decode(&index); err != nil { + return fmt.Errorf("failed to decode index file: %w", err) + } + ds.index = &index + return loadIndexInDeletableMemory(ctx, ds.index, ds.storage, ds.tagResolver, ds.graph) +} + +// SaveIndex writes the `index.json` file to the file system. +// - If AutoSaveIndex is set to true (default value), +// the OCI store will automatically save the index on each Tag() call. +// - If AutoSaveIndex is set to false, it's the caller's responsibility +// to manually call this method when needed. +func (ds *DeletableStore) SaveIndex() error { + ds.lock.Lock() + defer ds.lock.Unlock() + return ds.saveIndex() +} + +func (ds *DeletableStore) saveIndex() error { + var manifests []ocispec.Descriptor + tagged := set.New[digest.Digest]() + refMap := ds.tagResolver.Map() + + // 1. Add descriptors that are associated with tags + // Note: One descriptor can be associated with multiple tags. + for ref, desc := range refMap { + if ref != desc.Digest.String() { + annotations := make(map[string]string, len(desc.Annotations)+1) + for k, v := range desc.Annotations { + annotations[k] = v + } + annotations[ocispec.AnnotationRefName] = ref + desc.Annotations = annotations + manifests = append(manifests, desc) + // mark the digest as tagged for deduplication in step 2 + tagged.Add(desc.Digest) + } + } + // 2. Add descriptors that are not associated with any tag + for ref, desc := range refMap { + if ref == desc.Digest.String() && !tagged.Contains(desc.Digest) { + // skip tagged ones since they have been added in step 1 + manifests = append(manifests, deleteAnnotationRefName(desc)) + } + } + + ds.index.Manifests = manifests + return ds.writeIndexFile() +} + +// writeIndexFile writes the `index.json` file. +func (ds *DeletableStore) writeIndexFile() error { + indexJSON, err := json.Marshal(ds.index) + if err != nil { + return fmt.Errorf("failed to marshal index file: %w", err) + } + return os.WriteFile(ds.indexPath, indexJSON, 0666) +} + +// loadIndexInDeletableMemory loads index into the memory. +func loadIndexInDeletableMemory(ctx context.Context, index *ocispec.Index, fetcher content.Fetcher, tagger content.Tagger, graph *graph.Memory) error { + for _, desc := range index.Manifests { + if err := tagger.Tag(ctx, deleteAnnotationRefName(desc), desc.Digest.String()); err != nil { + return err + } + if ref := desc.Annotations[ocispec.AnnotationRefName]; ref != "" { + if err := tagger.Tag(ctx, desc, ref); err != nil { + return err + } + } + plain := descriptor.Plain(desc) + if err := graph.IndexAll(ctx, fetcher, plain); err != nil { + return err + } + } + return nil +} diff --git a/content/oci/deletableoci_test.go b/content/oci/deletableoci_test.go new file mode 100644 index 00000000..75c73108 --- /dev/null +++ b/content/oci/deletableoci_test.go @@ -0,0 +1,83 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package oci provides access to an OCI content store. +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +package oci + +import ( + "bytes" + "context" + "reflect" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestDeletableStore(t *testing.T) { + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test-delete", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + ref := "latest" + + tempDir := t.TempDir() + s, err := NewDeletableStore(tempDir) + if err != nil { + t.Fatal("New() error =", err) + } + ctx := context.Background() + + err = s.Push(ctx, desc, bytes.NewReader(content)) + if err != nil { + t.Errorf("Store.Push() error = %v, wantErr %v", err, false) + } + + err = s.Tag(ctx, desc, ref) + if err != nil { + t.Errorf("error tagging descriptor error = %v, wantErr %v", err, false) + } + + exists, err := s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if !exists { + t.Errorf("Store.Exists() = %v, want %v", exists, true) + } + + resolvedDescr, err := s.Resolve(ctx, ref) + if err != nil { + t.Errorf("error resolving descriptor error = %v, wantErr %v", err, false) + } + + if !reflect.DeepEqual(resolvedDescr, desc) { + t.Errorf("Store.Resolve() = %v, want %v", resolvedDescr, desc) + } + + err = s.Delete(ctx, desc) + if err != nil { + t.Errorf("Store.Delete() = %v, wantErr %v", err, nil) + } + + exists, err = s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if exists { + t.Errorf("Store.Exists() = %v, want %v", exists, false) + } +} From 718d63c1bfe3ffdda71e40d3a54c470d15bd347c Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Mon, 16 Oct 2023 06:14:10 +0000 Subject: [PATCH 02/10] squashed commits to rebased on main Signed-off-by: Xiaoxuan Wang --- content/oci/deletableoci.go | 355 ------------------------------- content/oci/deletableoci_test.go | 83 -------- content/oci/oci.go | 58 ++++- content/oci/oci_test.go | 101 +++++++++ 4 files changed, 157 insertions(+), 440 deletions(-) delete mode 100644 content/oci/deletableoci.go delete mode 100644 content/oci/deletableoci_test.go diff --git a/content/oci/deletableoci.go b/content/oci/deletableoci.go deleted file mode 100644 index ac18d19f..00000000 --- a/content/oci/deletableoci.go +++ /dev/null @@ -1,355 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package oci provides access to an OCI content store. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md -package oci - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sync" - - "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/content" - "oras.land/oras-go/v2/errdef" - "oras.land/oras-go/v2/internal/container/set" - "oras.land/oras-go/v2/internal/descriptor" - "oras.land/oras-go/v2/internal/graph" - "oras.land/oras-go/v2/internal/resolver" -) - -// DeletableStore implements `oras.Target`, and represents a content store -// extended with the delete operation. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md -type DeletableStore struct { - // AutoSaveIndex controls if the OCI store will automatically save the index - // file on each Tag() call. - // - If AutoSaveIndex is set to true, the OCI store will automatically call - // this method on each Tag() call. - // - If AutoSaveIndex is set to false, it's the caller's responsibility - // to manually call SaveIndex() when needed. - // - Default value: true. - AutoSaveIndex bool - root string - indexPath string - index *ocispec.Index - lock sync.RWMutex - - storage *Storage - tagResolver *resolver.Memory - graph *graph.Memory -} - -// NewDeletableStore returns a new DeletableStore. -func NewDeletableStore(root string) (*DeletableStore, error) { - return NewDeletableStoreWithContext(context.Background(), root) -} - -// NewDeletableStoreWithContext creates a new DeletableStore. -func NewDeletableStoreWithContext(ctx context.Context, root string) (*DeletableStore, error) { - rootAbs, err := filepath.Abs(root) - if err != nil { - return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", root, err) - } - storage, err := NewStorage(rootAbs) - if err != nil { - return nil, fmt.Errorf("failed to create storage: %w", err) - } - store := &DeletableStore{ - AutoSaveIndex: true, - root: rootAbs, - indexPath: filepath.Join(rootAbs, ocispec.ImageIndexFile), - storage: storage, - tagResolver: resolver.NewMemory(), - graph: graph.NewMemory(), - } - if err := ensureDir(filepath.Join(rootAbs, ocispec.ImageBlobsDir)); err != nil { - return nil, err - } - if err := store.ensureOCILayoutFile(); err != nil { - return nil, fmt.Errorf("invalid OCI Image Layout: %w", err) - } - if err := store.loadIndexFile(ctx); err != nil { - return nil, fmt.Errorf("invalid OCI Image Index: %w", err) - } - return store, nil -} - -// Fetch fetches the content identified by the descriptor. -func (ds *DeletableStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { - ds.lock.RLock() - defer ds.lock.RUnlock() - return ds.storage.Fetch(ctx, target) -} - -// Push pushes the content, matching the expected descriptor. -func (ds *DeletableStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error { - ds.lock.Lock() - defer ds.lock.Unlock() - if err := ds.storage.Push(ctx, expected, reader); err != nil { - return err - } - if err := ds.graph.Index(ctx, ds.storage, expected); err != nil { - return err - } - if descriptor.IsManifest(expected) { - // tag by digest - return ds.tag(ctx, expected, expected.Digest.String()) - } - return nil -} - -// Delete removes the content matching the descriptor from the store. -func (ds *DeletableStore) Delete(ctx context.Context, target ocispec.Descriptor) error { - ds.lock.Lock() - defer ds.lock.Unlock() - resolvers := ds.tagResolver.Map() - for reference, desc := range resolvers { - if content.Equal(desc, target) { - ds.tagResolver.Untag(reference) - } - } - if err := ds.graph.Remove(ctx, target); err != nil { - return err - } - if ds.AutoSaveIndex { - err := ds.saveIndex() - if err != nil { - return err - } - } - return ds.storage.Delete(ctx, target) -} - -// Exists returns true if the described content exists. -func (ds *DeletableStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { - ds.lock.RLock() - defer ds.lock.RUnlock() - return ds.storage.Exists(ctx, target) -} - -// Tag tags a descriptor with a reference string. -// reference should be a valid tag (e.g. "latest"). -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md#indexjson-file -func (ds *DeletableStore) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { - ds.lock.Lock() - defer ds.lock.Unlock() - if err := validateReference(reference); err != nil { - return err - } - exists, err := ds.storage.Exists(ctx, desc) - if err != nil { - return err - } - if !exists { - return fmt.Errorf("%s: %s: %w", desc.Digest, desc.MediaType, errdef.ErrNotFound) - } - return ds.tag(ctx, desc, reference) -} - -// tag tags a descriptor with a reference string. -func (ds *DeletableStore) tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { - dgst := desc.Digest.String() - if reference != dgst { - // also tag desc by its digest - if err := ds.tagResolver.Tag(ctx, desc, dgst); err != nil { - return err - } - } - if err := ds.tagResolver.Tag(ctx, desc, reference); err != nil { - return err - } - if ds.AutoSaveIndex { - return ds.saveIndex() - } - return nil -} - -// Resolve resolves a reference to a descriptor. If the reference to be resolved -// is a tag, the returned descriptor will be a full descriptor declared by -// github.com/opencontainers/image-spec/specs-go/v1. If the reference is a -// digest the returned descriptor will be a plain descriptor (containing only -// the digest, media type and size). -func (ds *DeletableStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { - ds.lock.RLock() - defer ds.lock.RUnlock() - if reference == "" { - return ocispec.Descriptor{}, errdef.ErrMissingReference - } - // attempt resolving manifest - desc, err := ds.tagResolver.Resolve(ctx, reference) - if err != nil { - if errors.Is(err, errdef.ErrNotFound) { - // attempt resolving blob - return resolveBlob(os.DirFS(ds.root), reference) - } - return ocispec.Descriptor{}, err - } - if reference == desc.Digest.String() { - return descriptor.Plain(desc), nil - } - return desc, nil -} - -// Predecessors returns the nodes directly pointing to the current node. -// Predecessors returns nil without error if the node does not exists in the -// store. -func (ds *DeletableStore) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { - ds.lock.RLock() - defer ds.lock.RUnlock() - return ds.graph.Predecessors(ctx, node) -} - -// Tags lists the tags presented in the `index.json` file of the OCI layout, -// returned in ascending order. -// If `last` is NOT empty, the entries in the response start after the tag -// specified by `last`. Otherwise, the response starts from the top of the tags -// list. -// -// See also `Tags()` in the package `registry`. -func (ds *DeletableStore) Tags(ctx context.Context, last string, fn func(tags []string) error) error { - ds.lock.RLock() - defer ds.lock.RUnlock() - return listTags(ctx, ds.tagResolver, last, fn) -} - -// ensureOCILayoutFile ensures the `oci-layout` file. -func (ds *DeletableStore) ensureOCILayoutFile() error { - layoutFilePath := filepath.Join(ds.root, ocispec.ImageLayoutFile) - layoutFile, err := os.Open(layoutFilePath) - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("failed to open OCI layout file: %w", err) - } - layout := ocispec.ImageLayout{ - Version: ocispec.ImageLayoutVersion, - } - layoutJSON, err := json.Marshal(layout) - if err != nil { - return fmt.Errorf("failed to marshal OCI layout file: %w", err) - } - return os.WriteFile(layoutFilePath, layoutJSON, 0666) - } - defer layoutFile.Close() - var layout ocispec.ImageLayout - err = json.NewDecoder(layoutFile).Decode(&layout) - if err != nil { - return fmt.Errorf("failed to decode OCI layout file: %w", err) - } - return validateOCILayout(&layout) -} - -// loadIndexFile reads index.json from the file system. -// Create index.json if it does not exist. -func (ds *DeletableStore) loadIndexFile(ctx context.Context) error { - indexFile, err := os.Open(ds.indexPath) - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("failed to open index file: %w", err) - } - // write index.json if it does not exist - ds.index = &ocispec.Index{ - Versioned: specs.Versioned{ - SchemaVersion: 2, // historical value - }, - Manifests: []ocispec.Descriptor{}, - } - return ds.writeIndexFile() - } - defer indexFile.Close() - var index ocispec.Index - if err := json.NewDecoder(indexFile).Decode(&index); err != nil { - return fmt.Errorf("failed to decode index file: %w", err) - } - ds.index = &index - return loadIndexInDeletableMemory(ctx, ds.index, ds.storage, ds.tagResolver, ds.graph) -} - -// SaveIndex writes the `index.json` file to the file system. -// - If AutoSaveIndex is set to true (default value), -// the OCI store will automatically save the index on each Tag() call. -// - If AutoSaveIndex is set to false, it's the caller's responsibility -// to manually call this method when needed. -func (ds *DeletableStore) SaveIndex() error { - ds.lock.Lock() - defer ds.lock.Unlock() - return ds.saveIndex() -} - -func (ds *DeletableStore) saveIndex() error { - var manifests []ocispec.Descriptor - tagged := set.New[digest.Digest]() - refMap := ds.tagResolver.Map() - - // 1. Add descriptors that are associated with tags - // Note: One descriptor can be associated with multiple tags. - for ref, desc := range refMap { - if ref != desc.Digest.String() { - annotations := make(map[string]string, len(desc.Annotations)+1) - for k, v := range desc.Annotations { - annotations[k] = v - } - annotations[ocispec.AnnotationRefName] = ref - desc.Annotations = annotations - manifests = append(manifests, desc) - // mark the digest as tagged for deduplication in step 2 - tagged.Add(desc.Digest) - } - } - // 2. Add descriptors that are not associated with any tag - for ref, desc := range refMap { - if ref == desc.Digest.String() && !tagged.Contains(desc.Digest) { - // skip tagged ones since they have been added in step 1 - manifests = append(manifests, deleteAnnotationRefName(desc)) - } - } - - ds.index.Manifests = manifests - return ds.writeIndexFile() -} - -// writeIndexFile writes the `index.json` file. -func (ds *DeletableStore) writeIndexFile() error { - indexJSON, err := json.Marshal(ds.index) - if err != nil { - return fmt.Errorf("failed to marshal index file: %w", err) - } - return os.WriteFile(ds.indexPath, indexJSON, 0666) -} - -// loadIndexInDeletableMemory loads index into the memory. -func loadIndexInDeletableMemory(ctx context.Context, index *ocispec.Index, fetcher content.Fetcher, tagger content.Tagger, graph *graph.Memory) error { - for _, desc := range index.Manifests { - if err := tagger.Tag(ctx, deleteAnnotationRefName(desc), desc.Digest.String()); err != nil { - return err - } - if ref := desc.Annotations[ocispec.AnnotationRefName]; ref != "" { - if err := tagger.Tag(ctx, desc, ref); err != nil { - return err - } - } - plain := descriptor.Plain(desc) - if err := graph.IndexAll(ctx, fetcher, plain); err != nil { - return err - } - } - return nil -} diff --git a/content/oci/deletableoci_test.go b/content/oci/deletableoci_test.go deleted file mode 100644 index 75c73108..00000000 --- a/content/oci/deletableoci_test.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package oci provides access to an OCI content store. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md -package oci - -import ( - "bytes" - "context" - "reflect" - "testing" - - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" -) - -func TestDeletableStore(t *testing.T) { - content := []byte("test delete") - desc := ocispec.Descriptor{ - MediaType: "test-delete", - Digest: digest.FromBytes(content), - Size: int64(len(content)), - } - ref := "latest" - - tempDir := t.TempDir() - s, err := NewDeletableStore(tempDir) - if err != nil { - t.Fatal("New() error =", err) - } - ctx := context.Background() - - err = s.Push(ctx, desc, bytes.NewReader(content)) - if err != nil { - t.Errorf("Store.Push() error = %v, wantErr %v", err, false) - } - - err = s.Tag(ctx, desc, ref) - if err != nil { - t.Errorf("error tagging descriptor error = %v, wantErr %v", err, false) - } - - exists, err := s.Exists(ctx, desc) - if err != nil { - t.Fatal("Store.Exists() error =", err) - } - if !exists { - t.Errorf("Store.Exists() = %v, want %v", exists, true) - } - - resolvedDescr, err := s.Resolve(ctx, ref) - if err != nil { - t.Errorf("error resolving descriptor error = %v, wantErr %v", err, false) - } - - if !reflect.DeepEqual(resolvedDescr, desc) { - t.Errorf("Store.Resolve() = %v, want %v", resolvedDescr, desc) - } - - err = s.Delete(ctx, desc) - if err != nil { - t.Errorf("Store.Delete() = %v, wantErr %v", err, nil) - } - - exists, err = s.Exists(ctx, desc) - if err != nil { - t.Fatal("Store.Exists() error =", err) - } - if exists { - t.Errorf("Store.Exists() = %v, want %v", exists, false) - } -} diff --git a/content/oci/oci.go b/content/oci/oci.go index 27afde16..1182dad9 100644 --- a/content/oci/oci.go +++ b/content/oci/oci.go @@ -54,8 +54,9 @@ type Store struct { indexPath string index *ocispec.Index indexLock sync.Mutex + sync sync.RWMutex - storage content.Storage + storage *Storage tagResolver *resolver.Memory graph *graph.Memory } @@ -98,13 +99,21 @@ func NewWithContext(ctx context.Context, root string) (*Store, error) { return store, nil } -// Fetch fetches the content identified by the descriptor. +// Fetch fetches the content identified by the descriptor. It returns an io.ReadCloser. +// It's recommended to close the io.ReadCloser before a DeletableStore.Delete +// operation, otherwise Delete may fail (for example on NTFS file systems). func (s *Store) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + s.sync.RLock() + defer s.sync.RUnlock() + return s.storage.Fetch(ctx, target) } // Push pushes the content, matching the expected descriptor. func (s *Store) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error { + s.sync.RLock() + defer s.sync.RUnlock() + if err := s.storage.Push(ctx, expected, reader); err != nil { return err } @@ -120,13 +129,46 @@ func (s *Store) Push(ctx context.Context, expected ocispec.Descriptor, reader io // Exists returns true if the described content exists. func (s *Store) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + s.sync.RLock() + defer s.sync.RUnlock() + return s.storage.Exists(ctx, target) } +// Delete removes the content matching the descriptor from the store. Delete may +// fail on some systems (i.e. Windows), if there is a process (i.e. an unclosed +// Reader) using `target`. +func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error { + s.sync.Lock() + defer s.sync.Unlock() + + resolvers := s.tagResolver.Map() + untagged := false + for reference, desc := range resolvers { + if content.Equal(desc, target) { + s.tagResolver.Untag(reference) + untagged = true + } + } + if err := s.graph.Remove(ctx, target); err != nil { + return err + } + if untagged && s.AutoSaveIndex { + err := s.SaveIndex() + if err != nil { + return err + } + } + return s.storage.Delete(ctx, target) +} + // Tag tags a descriptor with a reference string. // reference should be a valid tag (e.g. "latest"). // Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md#indexjson-file func (s *Store) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + s.sync.RLock() + defer s.sync.RUnlock() + if err := validateReference(reference); err != nil { return err } @@ -166,6 +208,9 @@ func (s *Store) tag(ctx context.Context, desc ocispec.Descriptor, reference stri // digest the returned descriptor will be a plain descriptor (containing only // the digest, media type and size). func (s *Store) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + s.sync.RLock() + defer s.sync.RUnlock() + if reference == "" { return ocispec.Descriptor{}, errdef.ErrMissingReference } @@ -191,6 +236,9 @@ func (s *Store) Resolve(ctx context.Context, reference string) (ocispec.Descript // Predecessors returns nil without error if the node does not exists in the // store. func (s *Store) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + s.sync.RLock() + defer s.sync.RUnlock() + return s.graph.Predecessors(ctx, node) } @@ -202,6 +250,9 @@ func (s *Store) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]oc // // See also `Tags()` in the package `registry`. func (s *Store) Tags(ctx context.Context, last string, fn func(tags []string) error) error { + s.sync.RLock() + defer s.sync.RUnlock() + return listTags(ctx, s.tagResolver, last, fn) } @@ -267,6 +318,9 @@ func (s *Store) loadIndexFile(ctx context.Context) error { // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call this method when needed. func (s *Store) SaveIndex() error { + s.sync.RLock() + defer s.sync.RUnlock() + s.indexLock.Lock() defer s.indexLock.Unlock() diff --git a/content/oci/oci_test.go b/content/oci/oci_test.go index 43e57f0a..d7dec330 100644 --- a/content/oci/oci_test.go +++ b/content/oci/oci_test.go @@ -1975,6 +1975,107 @@ func TestStore_Tags(t *testing.T) { } } +func TestStore_BasicDelete(t *testing.T) { + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test-delete", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + ref := "latest" + + tempDir := t.TempDir() + s, err := New(tempDir) + if err != nil { + t.Fatal("NewDeletableStore() error =", err) + } + ctx := context.Background() + + err = s.Push(ctx, desc, bytes.NewReader(content)) + if err != nil { + t.Errorf("Store.Push() error = %v, wantErr %v", err, false) + } + + err = s.Tag(ctx, desc, ref) + if err != nil { + t.Errorf("error tagging descriptor error = %v, wantErr %v", err, false) + } + + exists, err := s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if !exists { + t.Errorf("Store.Exists() = %v, want %v", exists, true) + } + + resolvedDescr, err := s.Resolve(ctx, ref) + if err != nil { + t.Errorf("error resolving descriptor error = %v, wantErr %v", err, false) + } + + if !reflect.DeepEqual(resolvedDescr, desc) { + t.Errorf("Store.Resolve() = %v, want %v", resolvedDescr, desc) + } + + err = s.Delete(ctx, desc) + if err != nil { + t.Errorf("Store.Delete() = %v, wantErr %v", err, nil) + } + + exists, err = s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if exists { + t.Errorf("Store.Exists() = %v, want %v", exists, false) + } +} + +func TestStore_FetchAndDelete(t *testing.T) { + // create a store + tempDir := t.TempDir() + s, err := New(tempDir) + if err != nil { + t.Fatal("error =", err) + } + + // push a content + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test-delete", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + err = s.Push(context.Background(), desc, bytes.NewReader(content)) + if err != nil { + t.Fatal("error =", err) + } + + // fetch a content + rc, err := s.Fetch(context.Background(), desc) + if err != nil { + t.Fatal("error =", err) + } + + // read and verify the content + got, err := io.ReadAll(rc) + if err != nil { + t.Fatal("error =", err) + } + if !bytes.Equal(got, content) { + fmt.Println(got) + t.Fatal("wrong content") + } + rc.Close() + + // delete. If rc is not closed, Delete would fail on some systems. + err = s.Delete(context.Background(), desc) + if err != nil { + t.Fatal("error =", err) + } +} + func equalDescriptorSet(actual []ocispec.Descriptor, expected []ocispec.Descriptor) bool { if len(actual) != len(expected) { return false From d7953ae2cc6cade63ea49079007a8fd662c59172 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Wed, 25 Oct 2023 08:30:50 +0000 Subject: [PATCH 03/10] fixed the deadlock Signed-off-by: Xiaoxuan Wang --- content/oci/oci.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/content/oci/oci.go b/content/oci/oci.go index 1182dad9..10d3a0ab 100644 --- a/content/oci/oci.go +++ b/content/oci/oci.go @@ -136,7 +136,7 @@ func (s *Store) Exists(ctx context.Context, target ocispec.Descriptor) (bool, er } // Delete removes the content matching the descriptor from the store. Delete may -// fail on some systems (i.e. Windows), if there is a process (i.e. an unclosed +// fail on certain systems (i.e. NTFS), if there is a process (i.e. an unclosed // Reader) using `target`. func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error { s.sync.Lock() @@ -318,9 +318,6 @@ func (s *Store) loadIndexFile(ctx context.Context) error { // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call this method when needed. func (s *Store) SaveIndex() error { - s.sync.RLock() - defer s.sync.RUnlock() - s.indexLock.Lock() defer s.indexLock.Unlock() From 29ba2736b8d684f6810f341b2b4f0d86a974329d Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Wed, 25 Oct 2023 08:54:23 +0000 Subject: [PATCH 04/10] minor refinement Signed-off-by: Xiaoxuan Wang --- content/oci/oci.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/oci/oci.go b/content/oci/oci.go index 10d3a0ab..372e1d3e 100644 --- a/content/oci/oci.go +++ b/content/oci/oci.go @@ -14,7 +14,7 @@ limitations under the License. */ // Package oci provides access to an OCI content store. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md package oci import ( @@ -40,7 +40,7 @@ import ( // Store implements `oras.Target`, and represents a content store // based on file system with the OCI-Image layout. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md type Store struct { // AutoSaveIndex controls if the OCI store will automatically save the index // file on each Tag() call. @@ -135,7 +135,7 @@ func (s *Store) Exists(ctx context.Context, target ocispec.Descriptor) (bool, er return s.storage.Exists(ctx, target) } -// Delete removes the content matching the descriptor from the store. Delete may +// Delete deletes the content matching the descriptor from the store. Delete may // fail on certain systems (i.e. NTFS), if there is a process (i.e. an unclosed // Reader) using `target`. func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error { @@ -164,7 +164,7 @@ func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error { // Tag tags a descriptor with a reference string. // reference should be a valid tag (e.g. "latest"). -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md#indexjson-file +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md#indexjson-file func (s *Store) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { s.sync.RLock() defer s.sync.RUnlock() From 1b2c8748fbb149ff3af6e2270fed69ffb5fa74b3 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Thu, 26 Oct 2023 09:18:14 +0000 Subject: [PATCH 05/10] resolved comments Signed-off-by: Xiaoxuan Wang --- content/oci/oci.go | 31 +++++++++++++++++++++---------- content/oci/oci_test.go | 2 +- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/content/oci/oci.go b/content/oci/oci.go index 372e1d3e..a325c89d 100644 --- a/content/oci/oci.go +++ b/content/oci/oci.go @@ -53,12 +53,16 @@ type Store struct { root string indexPath string index *ocispec.Index - indexLock sync.Mutex - sync sync.RWMutex - - storage *Storage - tagResolver *resolver.Memory - graph *graph.Memory + storage *Storage + tagResolver *resolver.Memory + graph *graph.Memory + + // sync ensures that most operations can be done concurrently, while Delete + // has the exclusive access to Store if a delete operation is underway. Operations + // such as Fetch, Push use sync.RLock(), while Delete uses sync.Lock(). + sync sync.RWMutex + // indexLock ensures that only one process is writing to the index. + indexLock sync.Mutex } // New creates a new OCI store with context.Background(). @@ -100,8 +104,8 @@ func NewWithContext(ctx context.Context, root string) (*Store, error) { } // Fetch fetches the content identified by the descriptor. It returns an io.ReadCloser. -// It's recommended to close the io.ReadCloser before a DeletableStore.Delete -// operation, otherwise Delete may fail (for example on NTFS file systems). +// It's recommended to close the io.ReadCloser before a Delete operation, otherwise +// Delete may fail (for example on NTFS file systems). func (s *Store) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { s.sync.RLock() defer s.sync.RUnlock() @@ -154,7 +158,7 @@ func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error { return err } if untagged && s.AutoSaveIndex { - err := s.SaveIndex() + err := s.saveIndex() if err != nil { return err } @@ -197,7 +201,7 @@ func (s *Store) tag(ctx context.Context, desc ocispec.Descriptor, reference stri return err } if s.AutoSaveIndex { - return s.SaveIndex() + return s.saveIndex() } return nil } @@ -318,6 +322,13 @@ func (s *Store) loadIndexFile(ctx context.Context) error { // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call this method when needed. func (s *Store) SaveIndex() error { + s.sync.RLock() + defer s.sync.RUnlock() + + return s.saveIndex() +} + +func (s *Store) saveIndex() error { s.indexLock.Lock() defer s.indexLock.Unlock() diff --git a/content/oci/oci_test.go b/content/oci/oci_test.go index d7dec330..914b83b2 100644 --- a/content/oci/oci_test.go +++ b/content/oci/oci_test.go @@ -644,7 +644,7 @@ func TestStore_DisableAutoSaveIndex(t *testing.T) { if got, want := len(s.index.Manifests), 0; got != want { t.Errorf("len(index.Manifests) = %v, want %v", got, want) } - if err := s.SaveIndex(); err != nil { + if err := s.saveIndex(); err != nil { t.Fatal("Store.SaveIndex() error =", err) } // test index file again From f0b3ae96dfa7cda8fa1688b88024acd57ccb41fa Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Thu, 26 Oct 2023 10:10:07 +0000 Subject: [PATCH 06/10] updated AutoSaveIndex doc Signed-off-by: Xiaoxuan Wang --- content/oci/oci.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/content/oci/oci.go b/content/oci/oci.go index a325c89d..0d23c76c 100644 --- a/content/oci/oci.go +++ b/content/oci/oci.go @@ -43,12 +43,13 @@ import ( // Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md type Store struct { // AutoSaveIndex controls if the OCI store will automatically save the index - // file on each Tag() call. - // - If AutoSaveIndex is set to true, the OCI store will automatically call - // this method on each Tag() call. + // file when needed. + // - If AutoSaveIndex is set to true, the OCI store will automatically save the + // changes to `index.json` on Tag() and Delete() calls, and when pushing a manifest. // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call SaveIndex() when needed. // - Default value: true. + AutoSaveIndex bool root string indexPath string @@ -318,7 +319,8 @@ func (s *Store) loadIndexFile(ctx context.Context) error { // SaveIndex writes the `index.json` file to the file system. // - If AutoSaveIndex is set to true (default value), -// the OCI store will automatically call this method on each Tag() call. +// the OCI store will automatically save the changes to `index.json` +// on Tag() and Delete() calls, and when pushing a manifest. // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call this method when needed. func (s *Store) SaveIndex() error { From 703d1ff13f17e1042d36cc5ba017844276a9de2a Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Thu, 26 Oct 2023 10:17:36 +0000 Subject: [PATCH 07/10] removed extra line Signed-off-by: Xiaoxuan Wang --- content/oci/oci.go | 1 - 1 file changed, 1 deletion(-) diff --git a/content/oci/oci.go b/content/oci/oci.go index 0d23c76c..8570e54c 100644 --- a/content/oci/oci.go +++ b/content/oci/oci.go @@ -49,7 +49,6 @@ type Store struct { // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call SaveIndex() when needed. // - Default value: true. - AutoSaveIndex bool root string indexPath string From 410068dd15f9808ebaa38f4f799e3ff6330237c2 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Tue, 14 Nov 2023 04:16:02 +0000 Subject: [PATCH 08/10] added predecessors delete test Signed-off-by: Xiaoxuan Wang --- content/oci/oci_test.go | 162 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/content/oci/oci_test.go b/content/oci/oci_test.go index 914b83b2..c87dae47 100644 --- a/content/oci/oci_test.go +++ b/content/oci/oci_test.go @@ -2076,6 +2076,168 @@ func TestStore_FetchAndDelete(t *testing.T) { } } +func TestStore_PredecessorsAndDelete(t *testing.T) { + tempDir := t.TempDir() + s, err := New(tempDir) + if err != nil { + t.Fatal("New() error =", err) + } + ctx := context.Background() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) + } + generateIndex := func(manifests ...ocispec.Descriptor) { + index := ocispec.Index{ + Manifests: manifests, + } + indexJSON, err := json.Marshal(index) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageIndex, indexJSON) + } + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3 + generateManifest(descs[0], descs[1:3]...) // Blob 4 + generateManifest(descs[0], descs[3]) // Blob 5 + generateManifest(descs[0], descs[1:4]...) // Blob 6 + generateIndex(descs[4:6]...) // Blob 7 + generateIndex(descs[6]) // Blob 8 + + eg, egCtx := errgroup.WithContext(ctx) + for i := range blobs { + eg.Go(func(i int) func() error { + return func() error { + err := s.Push(egCtx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + return fmt.Errorf("failed to push test content to src: %d: %v", i, err) + } + return nil + } + }(i)) + } + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // verify predecessors + wants := [][]ocispec.Descriptor{ + descs[4:7], // Blob 0 + {descs[4], descs[6]}, // Blob 1 + {descs[4], descs[6]}, // Blob 2 + {descs[5], descs[6]}, // Blob 3 + {descs[7]}, // Blob 4 + {descs[7]}, // Blob 5 + {descs[8]}, // Blob 6 + nil, // Blob 7 + nil, // Blob 8 + } + for i, want := range wants { + predecessors, err := s.Predecessors(ctx, descs[i]) + if err != nil { + t.Errorf("Store.Predecessors(%d) error = %v", i, err) + } + if !equalDescriptorSet(predecessors, want) { + t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want) + } + } + + // delete a node and verify the result + s.Delete(egCtx, descs[6]) + // verify predecessors + wants = [][]ocispec.Descriptor{ + descs[4:6], // Blob 0 + {descs[4]}, // Blob 1 + {descs[4]}, // Blob 2 + {descs[5]}, // Blob 3 + {descs[7]}, // Blob 4 + {descs[7]}, // Blob 5 + {descs[8]}, // Blob 6 + nil, // Blob 7 + nil, // Blob 8 + } + for i, want := range wants { + predecessors, err := s.Predecessors(ctx, descs[i]) + if err != nil { + t.Errorf("Store.Predecessors(%d) error = %v", i, err) + } + if !equalDescriptorSet(predecessors, want) { + t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want) + } + } + + // delete a node and verify the result + s.Delete(egCtx, descs[8]) + // verify predecessors + wants = [][]ocispec.Descriptor{ + descs[4:6], // Blob 0 + {descs[4]}, // Blob 1 + {descs[4]}, // Blob 2 + {descs[5]}, // Blob 3 + {descs[7]}, // Blob 4 + {descs[7]}, // Blob 5 + nil, // Blob 6 + nil, // Blob 7 + nil, // Blob 8 + } + for i, want := range wants { + predecessors, err := s.Predecessors(ctx, descs[i]) + if err != nil { + t.Errorf("Store.Predecessors(%d) error = %v", i, err) + } + if !equalDescriptorSet(predecessors, want) { + t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want) + } + } + + // delete a node and verify the result + s.Delete(egCtx, descs[5]) + // verify predecessors + wants = [][]ocispec.Descriptor{ + {descs[4]}, // Blob 0 + {descs[4]}, // Blob 1 + {descs[4]}, // Blob 2 + nil, // Blob 3 + {descs[7]}, // Blob 4 + {descs[7]}, // Blob 5 + nil, // Blob 6 + nil, // Blob 7 + nil, // Blob 8 + } + for i, want := range wants { + predecessors, err := s.Predecessors(ctx, descs[i]) + if err != nil { + t.Errorf("Store.Predecessors(%d) error = %v", i, err) + } + if !equalDescriptorSet(predecessors, want) { + t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want) + } + } +} + func equalDescriptorSet(actual []ocispec.Descriptor, expected []ocispec.Descriptor) bool { if len(actual) != len(expected) { return false From fad8ed42c010addf192d9bf81d13fa2b053373f6 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Tue, 14 Nov 2023 04:30:32 +0000 Subject: [PATCH 09/10] resolve comment Signed-off-by: Xiaoxuan Wang --- content/oci/oci.go | 10 ++++++---- content/oci/oci_test.go | 3 +-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/content/oci/oci.go b/content/oci/oci.go index 8570e54c..5d4699a1 100644 --- a/content/oci/oci.go +++ b/content/oci/oci.go @@ -44,8 +44,10 @@ import ( type Store struct { // AutoSaveIndex controls if the OCI store will automatically save the index // file when needed. - // - If AutoSaveIndex is set to true, the OCI store will automatically save the - // changes to `index.json` on Tag() and Delete() calls, and when pushing a manifest. + // - If AutoSaveIndex is set to true, the OCI store will automatically save + // the changes to `index.json` when + // 1. pushing a manifest + // 2. calling Tag() or Delete() // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call SaveIndex() when needed. // - Default value: true. @@ -61,7 +63,7 @@ type Store struct { // has the exclusive access to Store if a delete operation is underway. Operations // such as Fetch, Push use sync.RLock(), while Delete uses sync.Lock(). sync sync.RWMutex - // indexLock ensures that only one process is writing to the index. + // indexLock ensures that only one go-routine is writing to the index. indexLock sync.Mutex } @@ -141,7 +143,7 @@ func (s *Store) Exists(ctx context.Context, target ocispec.Descriptor) (bool, er // Delete deletes the content matching the descriptor from the store. Delete may // fail on certain systems (i.e. NTFS), if there is a process (i.e. an unclosed -// Reader) using `target`. +// Reader) using target. func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error { s.sync.Lock() defer s.sync.Unlock() diff --git a/content/oci/oci_test.go b/content/oci/oci_test.go index c87dae47..048c649d 100644 --- a/content/oci/oci_test.go +++ b/content/oci/oci_test.go @@ -2064,8 +2064,7 @@ func TestStore_FetchAndDelete(t *testing.T) { t.Fatal("error =", err) } if !bytes.Equal(got, content) { - fmt.Println(got) - t.Fatal("wrong content") + t.Errorf("Store.Fetch() = %v, want %v", string(got), string(content)) } rc.Close() From 4b163db0c2bdb50781aa167df4102c88790cf970 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Mon, 20 Nov 2023 08:51:51 +0000 Subject: [PATCH 10/10] formatted Signed-off-by: Xiaoxuan Wang --- content/oci/oci_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/oci/oci_test.go b/content/oci/oci_test.go index 048c649d..01028b1a 100644 --- a/content/oci/oci_test.go +++ b/content/oci/oci_test.go @@ -2064,7 +2064,7 @@ func TestStore_FetchAndDelete(t *testing.T) { t.Fatal("error =", err) } if !bytes.Equal(got, content) { - t.Errorf("Store.Fetch() = %v, want %v", string(got), string(content)) + t.Errorf("Store.Fetch() = %v, want %v", string(got), string(content)) } rc.Close()