From ce5be5ff9ee1e8a68e83a549c75df01084edd4dc Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Mon, 16 Oct 2023 06:14:10 +0000 Subject: [PATCH] squashed commits to rebased on main Signed-off-by: Xiaoxuan Wang --- content/oci/deletableoci.go | 355 ------------------------------- content/oci/deletableoci_test.go | 83 -------- content/oci/oci.go | 58 ++++- content/oci/oci_test.go | 101 +++++++++ 4 files changed, 157 insertions(+), 440 deletions(-) delete mode 100644 content/oci/deletableoci.go delete mode 100644 content/oci/deletableoci_test.go diff --git a/content/oci/deletableoci.go b/content/oci/deletableoci.go deleted file mode 100644 index ac18d19f..00000000 --- a/content/oci/deletableoci.go +++ /dev/null @@ -1,355 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package oci provides access to an OCI content store. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md -package oci - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sync" - - "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/content" - "oras.land/oras-go/v2/errdef" - "oras.land/oras-go/v2/internal/container/set" - "oras.land/oras-go/v2/internal/descriptor" - "oras.land/oras-go/v2/internal/graph" - "oras.land/oras-go/v2/internal/resolver" -) - -// DeletableStore implements `oras.Target`, and represents a content store -// extended with the delete operation. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md -type DeletableStore struct { - // AutoSaveIndex controls if the OCI store will automatically save the index - // file on each Tag() call. - // - If AutoSaveIndex is set to true, the OCI store will automatically call - // this method on each Tag() call. - // - If AutoSaveIndex is set to false, it's the caller's responsibility - // to manually call SaveIndex() when needed. - // - Default value: true. - AutoSaveIndex bool - root string - indexPath string - index *ocispec.Index - lock sync.RWMutex - - storage *Storage - tagResolver *resolver.Memory - graph *graph.Memory -} - -// NewDeletableStore returns a new DeletableStore. -func NewDeletableStore(root string) (*DeletableStore, error) { - return NewDeletableStoreWithContext(context.Background(), root) -} - -// NewDeletableStoreWithContext creates a new DeletableStore. -func NewDeletableStoreWithContext(ctx context.Context, root string) (*DeletableStore, error) { - rootAbs, err := filepath.Abs(root) - if err != nil { - return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", root, err) - } - storage, err := NewStorage(rootAbs) - if err != nil { - return nil, fmt.Errorf("failed to create storage: %w", err) - } - store := &DeletableStore{ - AutoSaveIndex: true, - root: rootAbs, - indexPath: filepath.Join(rootAbs, ocispec.ImageIndexFile), - storage: storage, - tagResolver: resolver.NewMemory(), - graph: graph.NewMemory(), - } - if err := ensureDir(filepath.Join(rootAbs, ocispec.ImageBlobsDir)); err != nil { - return nil, err - } - if err := store.ensureOCILayoutFile(); err != nil { - return nil, fmt.Errorf("invalid OCI Image Layout: %w", err) - } - if err := store.loadIndexFile(ctx); err != nil { - return nil, fmt.Errorf("invalid OCI Image Index: %w", err) - } - return store, nil -} - -// Fetch fetches the content identified by the descriptor. -func (ds *DeletableStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { - ds.lock.RLock() - defer ds.lock.RUnlock() - return ds.storage.Fetch(ctx, target) -} - -// Push pushes the content, matching the expected descriptor. -func (ds *DeletableStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error { - ds.lock.Lock() - defer ds.lock.Unlock() - if err := ds.storage.Push(ctx, expected, reader); err != nil { - return err - } - if err := ds.graph.Index(ctx, ds.storage, expected); err != nil { - return err - } - if descriptor.IsManifest(expected) { - // tag by digest - return ds.tag(ctx, expected, expected.Digest.String()) - } - return nil -} - -// Delete removes the content matching the descriptor from the store. -func (ds *DeletableStore) Delete(ctx context.Context, target ocispec.Descriptor) error { - ds.lock.Lock() - defer ds.lock.Unlock() - resolvers := ds.tagResolver.Map() - for reference, desc := range resolvers { - if content.Equal(desc, target) { - ds.tagResolver.Untag(reference) - } - } - if err := ds.graph.Remove(ctx, target); err != nil { - return err - } - if ds.AutoSaveIndex { - err := ds.saveIndex() - if err != nil { - return err - } - } - return ds.storage.Delete(ctx, target) -} - -// Exists returns true if the described content exists. -func (ds *DeletableStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { - ds.lock.RLock() - defer ds.lock.RUnlock() - return ds.storage.Exists(ctx, target) -} - -// Tag tags a descriptor with a reference string. -// reference should be a valid tag (e.g. "latest"). -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md#indexjson-file -func (ds *DeletableStore) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { - ds.lock.Lock() - defer ds.lock.Unlock() - if err := validateReference(reference); err != nil { - return err - } - exists, err := ds.storage.Exists(ctx, desc) - if err != nil { - return err - } - if !exists { - return fmt.Errorf("%s: %s: %w", desc.Digest, desc.MediaType, errdef.ErrNotFound) - } - return ds.tag(ctx, desc, reference) -} - -// tag tags a descriptor with a reference string. -func (ds *DeletableStore) tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { - dgst := desc.Digest.String() - if reference != dgst { - // also tag desc by its digest - if err := ds.tagResolver.Tag(ctx, desc, dgst); err != nil { - return err - } - } - if err := ds.tagResolver.Tag(ctx, desc, reference); err != nil { - return err - } - if ds.AutoSaveIndex { - return ds.saveIndex() - } - return nil -} - -// Resolve resolves a reference to a descriptor. If the reference to be resolved -// is a tag, the returned descriptor will be a full descriptor declared by -// github.com/opencontainers/image-spec/specs-go/v1. If the reference is a -// digest the returned descriptor will be a plain descriptor (containing only -// the digest, media type and size). -func (ds *DeletableStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { - ds.lock.RLock() - defer ds.lock.RUnlock() - if reference == "" { - return ocispec.Descriptor{}, errdef.ErrMissingReference - } - // attempt resolving manifest - desc, err := ds.tagResolver.Resolve(ctx, reference) - if err != nil { - if errors.Is(err, errdef.ErrNotFound) { - // attempt resolving blob - return resolveBlob(os.DirFS(ds.root), reference) - } - return ocispec.Descriptor{}, err - } - if reference == desc.Digest.String() { - return descriptor.Plain(desc), nil - } - return desc, nil -} - -// Predecessors returns the nodes directly pointing to the current node. -// Predecessors returns nil without error if the node does not exists in the -// store. -func (ds *DeletableStore) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { - ds.lock.RLock() - defer ds.lock.RUnlock() - return ds.graph.Predecessors(ctx, node) -} - -// Tags lists the tags presented in the `index.json` file of the OCI layout, -// returned in ascending order. -// If `last` is NOT empty, the entries in the response start after the tag -// specified by `last`. Otherwise, the response starts from the top of the tags -// list. -// -// See also `Tags()` in the package `registry`. -func (ds *DeletableStore) Tags(ctx context.Context, last string, fn func(tags []string) error) error { - ds.lock.RLock() - defer ds.lock.RUnlock() - return listTags(ctx, ds.tagResolver, last, fn) -} - -// ensureOCILayoutFile ensures the `oci-layout` file. -func (ds *DeletableStore) ensureOCILayoutFile() error { - layoutFilePath := filepath.Join(ds.root, ocispec.ImageLayoutFile) - layoutFile, err := os.Open(layoutFilePath) - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("failed to open OCI layout file: %w", err) - } - layout := ocispec.ImageLayout{ - Version: ocispec.ImageLayoutVersion, - } - layoutJSON, err := json.Marshal(layout) - if err != nil { - return fmt.Errorf("failed to marshal OCI layout file: %w", err) - } - return os.WriteFile(layoutFilePath, layoutJSON, 0666) - } - defer layoutFile.Close() - var layout ocispec.ImageLayout - err = json.NewDecoder(layoutFile).Decode(&layout) - if err != nil { - return fmt.Errorf("failed to decode OCI layout file: %w", err) - } - return validateOCILayout(&layout) -} - -// loadIndexFile reads index.json from the file system. -// Create index.json if it does not exist. -func (ds *DeletableStore) loadIndexFile(ctx context.Context) error { - indexFile, err := os.Open(ds.indexPath) - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("failed to open index file: %w", err) - } - // write index.json if it does not exist - ds.index = &ocispec.Index{ - Versioned: specs.Versioned{ - SchemaVersion: 2, // historical value - }, - Manifests: []ocispec.Descriptor{}, - } - return ds.writeIndexFile() - } - defer indexFile.Close() - var index ocispec.Index - if err := json.NewDecoder(indexFile).Decode(&index); err != nil { - return fmt.Errorf("failed to decode index file: %w", err) - } - ds.index = &index - return loadIndexInDeletableMemory(ctx, ds.index, ds.storage, ds.tagResolver, ds.graph) -} - -// SaveIndex writes the `index.json` file to the file system. -// - If AutoSaveIndex is set to true (default value), -// the OCI store will automatically save the index on each Tag() call. -// - If AutoSaveIndex is set to false, it's the caller's responsibility -// to manually call this method when needed. -func (ds *DeletableStore) SaveIndex() error { - ds.lock.Lock() - defer ds.lock.Unlock() - return ds.saveIndex() -} - -func (ds *DeletableStore) saveIndex() error { - var manifests []ocispec.Descriptor - tagged := set.New[digest.Digest]() - refMap := ds.tagResolver.Map() - - // 1. Add descriptors that are associated with tags - // Note: One descriptor can be associated with multiple tags. - for ref, desc := range refMap { - if ref != desc.Digest.String() { - annotations := make(map[string]string, len(desc.Annotations)+1) - for k, v := range desc.Annotations { - annotations[k] = v - } - annotations[ocispec.AnnotationRefName] = ref - desc.Annotations = annotations - manifests = append(manifests, desc) - // mark the digest as tagged for deduplication in step 2 - tagged.Add(desc.Digest) - } - } - // 2. Add descriptors that are not associated with any tag - for ref, desc := range refMap { - if ref == desc.Digest.String() && !tagged.Contains(desc.Digest) { - // skip tagged ones since they have been added in step 1 - manifests = append(manifests, deleteAnnotationRefName(desc)) - } - } - - ds.index.Manifests = manifests - return ds.writeIndexFile() -} - -// writeIndexFile writes the `index.json` file. -func (ds *DeletableStore) writeIndexFile() error { - indexJSON, err := json.Marshal(ds.index) - if err != nil { - return fmt.Errorf("failed to marshal index file: %w", err) - } - return os.WriteFile(ds.indexPath, indexJSON, 0666) -} - -// loadIndexInDeletableMemory loads index into the memory. -func loadIndexInDeletableMemory(ctx context.Context, index *ocispec.Index, fetcher content.Fetcher, tagger content.Tagger, graph *graph.Memory) error { - for _, desc := range index.Manifests { - if err := tagger.Tag(ctx, deleteAnnotationRefName(desc), desc.Digest.String()); err != nil { - return err - } - if ref := desc.Annotations[ocispec.AnnotationRefName]; ref != "" { - if err := tagger.Tag(ctx, desc, ref); err != nil { - return err - } - } - plain := descriptor.Plain(desc) - if err := graph.IndexAll(ctx, fetcher, plain); err != nil { - return err - } - } - return nil -} diff --git a/content/oci/deletableoci_test.go b/content/oci/deletableoci_test.go deleted file mode 100644 index 75c73108..00000000 --- a/content/oci/deletableoci_test.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package oci provides access to an OCI content store. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md -package oci - -import ( - "bytes" - "context" - "reflect" - "testing" - - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" -) - -func TestDeletableStore(t *testing.T) { - content := []byte("test delete") - desc := ocispec.Descriptor{ - MediaType: "test-delete", - Digest: digest.FromBytes(content), - Size: int64(len(content)), - } - ref := "latest" - - tempDir := t.TempDir() - s, err := NewDeletableStore(tempDir) - if err != nil { - t.Fatal("New() error =", err) - } - ctx := context.Background() - - err = s.Push(ctx, desc, bytes.NewReader(content)) - if err != nil { - t.Errorf("Store.Push() error = %v, wantErr %v", err, false) - } - - err = s.Tag(ctx, desc, ref) - if err != nil { - t.Errorf("error tagging descriptor error = %v, wantErr %v", err, false) - } - - exists, err := s.Exists(ctx, desc) - if err != nil { - t.Fatal("Store.Exists() error =", err) - } - if !exists { - t.Errorf("Store.Exists() = %v, want %v", exists, true) - } - - resolvedDescr, err := s.Resolve(ctx, ref) - if err != nil { - t.Errorf("error resolving descriptor error = %v, wantErr %v", err, false) - } - - if !reflect.DeepEqual(resolvedDescr, desc) { - t.Errorf("Store.Resolve() = %v, want %v", resolvedDescr, desc) - } - - err = s.Delete(ctx, desc) - if err != nil { - t.Errorf("Store.Delete() = %v, wantErr %v", err, nil) - } - - exists, err = s.Exists(ctx, desc) - if err != nil { - t.Fatal("Store.Exists() error =", err) - } - if exists { - t.Errorf("Store.Exists() = %v, want %v", exists, false) - } -} diff --git a/content/oci/oci.go b/content/oci/oci.go index 27afde16..1182dad9 100644 --- a/content/oci/oci.go +++ b/content/oci/oci.go @@ -54,8 +54,9 @@ type Store struct { indexPath string index *ocispec.Index indexLock sync.Mutex + sync sync.RWMutex - storage content.Storage + storage *Storage tagResolver *resolver.Memory graph *graph.Memory } @@ -98,13 +99,21 @@ func NewWithContext(ctx context.Context, root string) (*Store, error) { return store, nil } -// Fetch fetches the content identified by the descriptor. +// Fetch fetches the content identified by the descriptor. It returns an io.ReadCloser. +// It's recommended to close the io.ReadCloser before a DeletableStore.Delete +// operation, otherwise Delete may fail (for example on NTFS file systems). func (s *Store) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + s.sync.RLock() + defer s.sync.RUnlock() + return s.storage.Fetch(ctx, target) } // Push pushes the content, matching the expected descriptor. func (s *Store) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error { + s.sync.RLock() + defer s.sync.RUnlock() + if err := s.storage.Push(ctx, expected, reader); err != nil { return err } @@ -120,13 +129,46 @@ func (s *Store) Push(ctx context.Context, expected ocispec.Descriptor, reader io // Exists returns true if the described content exists. func (s *Store) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + s.sync.RLock() + defer s.sync.RUnlock() + return s.storage.Exists(ctx, target) } +// Delete removes the content matching the descriptor from the store. Delete may +// fail on some systems (i.e. Windows), if there is a process (i.e. an unclosed +// Reader) using `target`. +func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error { + s.sync.Lock() + defer s.sync.Unlock() + + resolvers := s.tagResolver.Map() + untagged := false + for reference, desc := range resolvers { + if content.Equal(desc, target) { + s.tagResolver.Untag(reference) + untagged = true + } + } + if err := s.graph.Remove(ctx, target); err != nil { + return err + } + if untagged && s.AutoSaveIndex { + err := s.SaveIndex() + if err != nil { + return err + } + } + return s.storage.Delete(ctx, target) +} + // Tag tags a descriptor with a reference string. // reference should be a valid tag (e.g. "latest"). // Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md#indexjson-file func (s *Store) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + s.sync.RLock() + defer s.sync.RUnlock() + if err := validateReference(reference); err != nil { return err } @@ -166,6 +208,9 @@ func (s *Store) tag(ctx context.Context, desc ocispec.Descriptor, reference stri // digest the returned descriptor will be a plain descriptor (containing only // the digest, media type and size). func (s *Store) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + s.sync.RLock() + defer s.sync.RUnlock() + if reference == "" { return ocispec.Descriptor{}, errdef.ErrMissingReference } @@ -191,6 +236,9 @@ func (s *Store) Resolve(ctx context.Context, reference string) (ocispec.Descript // Predecessors returns nil without error if the node does not exists in the // store. func (s *Store) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + s.sync.RLock() + defer s.sync.RUnlock() + return s.graph.Predecessors(ctx, node) } @@ -202,6 +250,9 @@ func (s *Store) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]oc // // See also `Tags()` in the package `registry`. func (s *Store) Tags(ctx context.Context, last string, fn func(tags []string) error) error { + s.sync.RLock() + defer s.sync.RUnlock() + return listTags(ctx, s.tagResolver, last, fn) } @@ -267,6 +318,9 @@ func (s *Store) loadIndexFile(ctx context.Context) error { // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call this method when needed. func (s *Store) SaveIndex() error { + s.sync.RLock() + defer s.sync.RUnlock() + s.indexLock.Lock() defer s.indexLock.Unlock() diff --git a/content/oci/oci_test.go b/content/oci/oci_test.go index 43e57f0a..d7dec330 100644 --- a/content/oci/oci_test.go +++ b/content/oci/oci_test.go @@ -1975,6 +1975,107 @@ func TestStore_Tags(t *testing.T) { } } +func TestStore_BasicDelete(t *testing.T) { + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test-delete", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + ref := "latest" + + tempDir := t.TempDir() + s, err := New(tempDir) + if err != nil { + t.Fatal("NewDeletableStore() error =", err) + } + ctx := context.Background() + + err = s.Push(ctx, desc, bytes.NewReader(content)) + if err != nil { + t.Errorf("Store.Push() error = %v, wantErr %v", err, false) + } + + err = s.Tag(ctx, desc, ref) + if err != nil { + t.Errorf("error tagging descriptor error = %v, wantErr %v", err, false) + } + + exists, err := s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if !exists { + t.Errorf("Store.Exists() = %v, want %v", exists, true) + } + + resolvedDescr, err := s.Resolve(ctx, ref) + if err != nil { + t.Errorf("error resolving descriptor error = %v, wantErr %v", err, false) + } + + if !reflect.DeepEqual(resolvedDescr, desc) { + t.Errorf("Store.Resolve() = %v, want %v", resolvedDescr, desc) + } + + err = s.Delete(ctx, desc) + if err != nil { + t.Errorf("Store.Delete() = %v, wantErr %v", err, nil) + } + + exists, err = s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if exists { + t.Errorf("Store.Exists() = %v, want %v", exists, false) + } +} + +func TestStore_FetchAndDelete(t *testing.T) { + // create a store + tempDir := t.TempDir() + s, err := New(tempDir) + if err != nil { + t.Fatal("error =", err) + } + + // push a content + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test-delete", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + err = s.Push(context.Background(), desc, bytes.NewReader(content)) + if err != nil { + t.Fatal("error =", err) + } + + // fetch a content + rc, err := s.Fetch(context.Background(), desc) + if err != nil { + t.Fatal("error =", err) + } + + // read and verify the content + got, err := io.ReadAll(rc) + if err != nil { + t.Fatal("error =", err) + } + if !bytes.Equal(got, content) { + fmt.Println(got) + t.Fatal("wrong content") + } + rc.Close() + + // delete. If rc is not closed, Delete would fail on some systems. + err = s.Delete(context.Background(), desc) + if err != nil { + t.Fatal("error =", err) + } +} + func equalDescriptorSet(actual []ocispec.Descriptor, expected []ocispec.Descriptor) bool { if len(actual) != len(expected) { return false