From c006867c2ba7eb1ab53aae6fbf9d316f250999cf Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Mon, 16 Oct 2023 05:22:25 +0000 Subject: [PATCH] another rebase attempt Signed-off-by: Xiaoxuan Wang --- content/oci/deletableoci.go | 355 +++++++++++++++++++++++++++++++ content/oci/deletableoci_test.go | 83 ++++++++ 2 files changed, 438 insertions(+) create mode 100644 content/oci/deletableoci.go create mode 100644 content/oci/deletableoci_test.go diff --git a/content/oci/deletableoci.go b/content/oci/deletableoci.go new file mode 100644 index 00000000..ac18d19f --- /dev/null +++ b/content/oci/deletableoci.go @@ -0,0 +1,355 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package oci provides access to an OCI content store. +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +package oci + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/internal/container/set" + "oras.land/oras-go/v2/internal/descriptor" + "oras.land/oras-go/v2/internal/graph" + "oras.land/oras-go/v2/internal/resolver" +) + +// DeletableStore implements `oras.Target`, and represents a content store +// extended with the delete operation. +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +type DeletableStore struct { + // AutoSaveIndex controls if the OCI store will automatically save the index + // file on each Tag() call. + // - If AutoSaveIndex is set to true, the OCI store will automatically call + // this method on each Tag() call. + // - If AutoSaveIndex is set to false, it's the caller's responsibility + // to manually call SaveIndex() when needed. + // - Default value: true. + AutoSaveIndex bool + root string + indexPath string + index *ocispec.Index + lock sync.RWMutex + + storage *Storage + tagResolver *resolver.Memory + graph *graph.Memory +} + +// NewDeletableStore returns a new DeletableStore. +func NewDeletableStore(root string) (*DeletableStore, error) { + return NewDeletableStoreWithContext(context.Background(), root) +} + +// NewDeletableStoreWithContext creates a new DeletableStore. +func NewDeletableStoreWithContext(ctx context.Context, root string) (*DeletableStore, error) { + rootAbs, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", root, err) + } + storage, err := NewStorage(rootAbs) + if err != nil { + return nil, fmt.Errorf("failed to create storage: %w", err) + } + store := &DeletableStore{ + AutoSaveIndex: true, + root: rootAbs, + indexPath: filepath.Join(rootAbs, ocispec.ImageIndexFile), + storage: storage, + tagResolver: resolver.NewMemory(), + graph: graph.NewMemory(), + } + if err := ensureDir(filepath.Join(rootAbs, ocispec.ImageBlobsDir)); err != nil { + return nil, err + } + if err := store.ensureOCILayoutFile(); err != nil { + return nil, fmt.Errorf("invalid OCI Image Layout: %w", err) + } + if err := store.loadIndexFile(ctx); err != nil { + return nil, fmt.Errorf("invalid OCI Image Index: %w", err) + } + return store, nil +} + +// Fetch fetches the content identified by the descriptor. +func (ds *DeletableStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + return ds.storage.Fetch(ctx, target) +} + +// Push pushes the content, matching the expected descriptor. +func (ds *DeletableStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error { + ds.lock.Lock() + defer ds.lock.Unlock() + if err := ds.storage.Push(ctx, expected, reader); err != nil { + return err + } + if err := ds.graph.Index(ctx, ds.storage, expected); err != nil { + return err + } + if descriptor.IsManifest(expected) { + // tag by digest + return ds.tag(ctx, expected, expected.Digest.String()) + } + return nil +} + +// Delete removes the content matching the descriptor from the store. +func (ds *DeletableStore) Delete(ctx context.Context, target ocispec.Descriptor) error { + ds.lock.Lock() + defer ds.lock.Unlock() + resolvers := ds.tagResolver.Map() + for reference, desc := range resolvers { + if content.Equal(desc, target) { + ds.tagResolver.Untag(reference) + } + } + if err := ds.graph.Remove(ctx, target); err != nil { + return err + } + if ds.AutoSaveIndex { + err := ds.saveIndex() + if err != nil { + return err + } + } + return ds.storage.Delete(ctx, target) +} + +// Exists returns true if the described content exists. +func (ds *DeletableStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + return ds.storage.Exists(ctx, target) +} + +// Tag tags a descriptor with a reference string. +// reference should be a valid tag (e.g. "latest"). +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md#indexjson-file +func (ds *DeletableStore) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + ds.lock.Lock() + defer ds.lock.Unlock() + if err := validateReference(reference); err != nil { + return err + } + exists, err := ds.storage.Exists(ctx, desc) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("%s: %s: %w", desc.Digest, desc.MediaType, errdef.ErrNotFound) + } + return ds.tag(ctx, desc, reference) +} + +// tag tags a descriptor with a reference string. +func (ds *DeletableStore) tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + dgst := desc.Digest.String() + if reference != dgst { + // also tag desc by its digest + if err := ds.tagResolver.Tag(ctx, desc, dgst); err != nil { + return err + } + } + if err := ds.tagResolver.Tag(ctx, desc, reference); err != nil { + return err + } + if ds.AutoSaveIndex { + return ds.saveIndex() + } + return nil +} + +// Resolve resolves a reference to a descriptor. If the reference to be resolved +// is a tag, the returned descriptor will be a full descriptor declared by +// github.com/opencontainers/image-spec/specs-go/v1. If the reference is a +// digest the returned descriptor will be a plain descriptor (containing only +// the digest, media type and size). +func (ds *DeletableStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + if reference == "" { + return ocispec.Descriptor{}, errdef.ErrMissingReference + } + // attempt resolving manifest + desc, err := ds.tagResolver.Resolve(ctx, reference) + if err != nil { + if errors.Is(err, errdef.ErrNotFound) { + // attempt resolving blob + return resolveBlob(os.DirFS(ds.root), reference) + } + return ocispec.Descriptor{}, err + } + if reference == desc.Digest.String() { + return descriptor.Plain(desc), nil + } + return desc, nil +} + +// Predecessors returns the nodes directly pointing to the current node. +// Predecessors returns nil without error if the node does not exists in the +// store. +func (ds *DeletableStore) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + return ds.graph.Predecessors(ctx, node) +} + +// Tags lists the tags presented in the `index.json` file of the OCI layout, +// returned in ascending order. +// If `last` is NOT empty, the entries in the response start after the tag +// specified by `last`. Otherwise, the response starts from the top of the tags +// list. +// +// See also `Tags()` in the package `registry`. +func (ds *DeletableStore) Tags(ctx context.Context, last string, fn func(tags []string) error) error { + ds.lock.RLock() + defer ds.lock.RUnlock() + return listTags(ctx, ds.tagResolver, last, fn) +} + +// ensureOCILayoutFile ensures the `oci-layout` file. +func (ds *DeletableStore) ensureOCILayoutFile() error { + layoutFilePath := filepath.Join(ds.root, ocispec.ImageLayoutFile) + layoutFile, err := os.Open(layoutFilePath) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to open OCI layout file: %w", err) + } + layout := ocispec.ImageLayout{ + Version: ocispec.ImageLayoutVersion, + } + layoutJSON, err := json.Marshal(layout) + if err != nil { + return fmt.Errorf("failed to marshal OCI layout file: %w", err) + } + return os.WriteFile(layoutFilePath, layoutJSON, 0666) + } + defer layoutFile.Close() + var layout ocispec.ImageLayout + err = json.NewDecoder(layoutFile).Decode(&layout) + if err != nil { + return fmt.Errorf("failed to decode OCI layout file: %w", err) + } + return validateOCILayout(&layout) +} + +// loadIndexFile reads index.json from the file system. +// Create index.json if it does not exist. +func (ds *DeletableStore) loadIndexFile(ctx context.Context) error { + indexFile, err := os.Open(ds.indexPath) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to open index file: %w", err) + } + // write index.json if it does not exist + ds.index = &ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value + }, + Manifests: []ocispec.Descriptor{}, + } + return ds.writeIndexFile() + } + defer indexFile.Close() + var index ocispec.Index + if err := json.NewDecoder(indexFile).Decode(&index); err != nil { + return fmt.Errorf("failed to decode index file: %w", err) + } + ds.index = &index + return loadIndexInDeletableMemory(ctx, ds.index, ds.storage, ds.tagResolver, ds.graph) +} + +// SaveIndex writes the `index.json` file to the file system. +// - If AutoSaveIndex is set to true (default value), +// the OCI store will automatically save the index on each Tag() call. +// - If AutoSaveIndex is set to false, it's the caller's responsibility +// to manually call this method when needed. +func (ds *DeletableStore) SaveIndex() error { + ds.lock.Lock() + defer ds.lock.Unlock() + return ds.saveIndex() +} + +func (ds *DeletableStore) saveIndex() error { + var manifests []ocispec.Descriptor + tagged := set.New[digest.Digest]() + refMap := ds.tagResolver.Map() + + // 1. Add descriptors that are associated with tags + // Note: One descriptor can be associated with multiple tags. + for ref, desc := range refMap { + if ref != desc.Digest.String() { + annotations := make(map[string]string, len(desc.Annotations)+1) + for k, v := range desc.Annotations { + annotations[k] = v + } + annotations[ocispec.AnnotationRefName] = ref + desc.Annotations = annotations + manifests = append(manifests, desc) + // mark the digest as tagged for deduplication in step 2 + tagged.Add(desc.Digest) + } + } + // 2. Add descriptors that are not associated with any tag + for ref, desc := range refMap { + if ref == desc.Digest.String() && !tagged.Contains(desc.Digest) { + // skip tagged ones since they have been added in step 1 + manifests = append(manifests, deleteAnnotationRefName(desc)) + } + } + + ds.index.Manifests = manifests + return ds.writeIndexFile() +} + +// writeIndexFile writes the `index.json` file. +func (ds *DeletableStore) writeIndexFile() error { + indexJSON, err := json.Marshal(ds.index) + if err != nil { + return fmt.Errorf("failed to marshal index file: %w", err) + } + return os.WriteFile(ds.indexPath, indexJSON, 0666) +} + +// loadIndexInDeletableMemory loads index into the memory. +func loadIndexInDeletableMemory(ctx context.Context, index *ocispec.Index, fetcher content.Fetcher, tagger content.Tagger, graph *graph.Memory) error { + for _, desc := range index.Manifests { + if err := tagger.Tag(ctx, deleteAnnotationRefName(desc), desc.Digest.String()); err != nil { + return err + } + if ref := desc.Annotations[ocispec.AnnotationRefName]; ref != "" { + if err := tagger.Tag(ctx, desc, ref); err != nil { + return err + } + } + plain := descriptor.Plain(desc) + if err := graph.IndexAll(ctx, fetcher, plain); err != nil { + return err + } + } + return nil +} diff --git a/content/oci/deletableoci_test.go b/content/oci/deletableoci_test.go new file mode 100644 index 00000000..75c73108 --- /dev/null +++ b/content/oci/deletableoci_test.go @@ -0,0 +1,83 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package oci provides access to an OCI content store. +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +package oci + +import ( + "bytes" + "context" + "reflect" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestDeletableStore(t *testing.T) { + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test-delete", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + ref := "latest" + + tempDir := t.TempDir() + s, err := NewDeletableStore(tempDir) + if err != nil { + t.Fatal("New() error =", err) + } + ctx := context.Background() + + err = s.Push(ctx, desc, bytes.NewReader(content)) + if err != nil { + t.Errorf("Store.Push() error = %v, wantErr %v", err, false) + } + + err = s.Tag(ctx, desc, ref) + if err != nil { + t.Errorf("error tagging descriptor error = %v, wantErr %v", err, false) + } + + exists, err := s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if !exists { + t.Errorf("Store.Exists() = %v, want %v", exists, true) + } + + resolvedDescr, err := s.Resolve(ctx, ref) + if err != nil { + t.Errorf("error resolving descriptor error = %v, wantErr %v", err, false) + } + + if !reflect.DeepEqual(resolvedDescr, desc) { + t.Errorf("Store.Resolve() = %v, want %v", resolvedDescr, desc) + } + + err = s.Delete(ctx, desc) + if err != nil { + t.Errorf("Store.Delete() = %v, wantErr %v", err, nil) + } + + exists, err = s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if exists { + t.Errorf("Store.Exists() = %v, want %v", exists, false) + } +}