diff --git a/content/oci/deletableoci.go b/content/oci/deletableoci.go new file mode 100644 index 00000000..0333693a --- /dev/null +++ b/content/oci/deletableoci.go @@ -0,0 +1,357 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package oci provides access to an OCI content store. +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +package oci + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/internal/container/set" + "oras.land/oras-go/v2/internal/descriptor" + "oras.land/oras-go/v2/internal/graph" + "oras.land/oras-go/v2/internal/resolver" +) + +// DeletableStore implements `oras.Target`, and represents a content store +// extended with the delete operation. +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +type DeletableStore struct { + // AutoSaveIndex controls if the OCI store will automatically save the index + // file on each Tag() call. + // - If AutoSaveIndex is set to true, the OCI store will automatically call + // this method on each Tag() call. + // - If AutoSaveIndex is set to false, it's the caller's responsibility + // to manually call SaveIndex() when needed. + // - Default value: true. + AutoSaveIndex bool + root string + indexPath string + index *ocispec.Index + lock sync.RWMutex + + storage *Storage + tagResolver *resolver.Memory + graph *graph.DeletableMemory +} + +// NewDeletableStore returns a new DeletableStore. +func NewDeletableStore(root string) (*DeletableStore, error) { + return NewDeletableStoreWithContext(context.Background(), root) +} + +// NewDeletableStoreWithContext creates a new DeletableStore. +func NewDeletableStoreWithContext(ctx context.Context, root string) (*DeletableStore, error) { + rootAbs, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", root, err) + } + storage, err := NewStorage(rootAbs) + if err != nil { + return nil, fmt.Errorf("failed to create storage: %w", err) + } + store := &DeletableStore{ + AutoSaveIndex: true, + root: rootAbs, + indexPath: filepath.Join(rootAbs, ociImageIndexFile), + storage: storage, + tagResolver: resolver.NewMemory(), + graph: graph.NewDeletableMemory(), + } + if err := ensureDir(filepath.Join(rootAbs, ociBlobsDir)); err != nil { + return nil, err + } + if err := store.ensureOCILayoutFile(); err != nil { + return nil, fmt.Errorf("invalid OCI Image Layout: %w", err) + } + if err := store.loadIndexFile(ctx); err != nil { + return nil, fmt.Errorf("invalid OCI Image Index: %w", err) + } + return store, nil +} + +// Fetch fetches the content identified by the descriptor. +func (ds *DeletableStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + return ds.storage.Fetch(ctx, target) +} + +// Push pushes the content, matching the expected descriptor. +func (ds *DeletableStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error { + ds.lock.Lock() + defer ds.lock.Unlock() + if err := ds.storage.Push(ctx, expected, reader); err != nil { + return err + } + if err := ds.graph.Index(ctx, ds.storage, expected); err != nil { + return err + } + if descriptor.IsManifest(expected) { + // tag by digest + return ds.tag(ctx, expected, expected.Digest.String()) + } + return nil +} + +// Delete removes the content matching the descriptor from the store. +func (ds *DeletableStore) Delete(ctx context.Context, target ocispec.Descriptor) error { + ds.lock.Lock() + defer ds.lock.Unlock() + resolvers := ds.tagResolver.Map() + for reference, desc := range resolvers { + if content.Equal(desc, target) { + ds.tagResolver.Delete(reference) + } + } + if err := ds.graph.Remove(ctx, target); err != nil { + return err + } + if ds.AutoSaveIndex { + err := ds.saveIndex() + if err != nil { + return err + } + } + return ds.storage.Delete(ctx, target) +} + +// Exists returns true if the described content exists. +func (ds *DeletableStore) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + return ds.storage.Exists(ctx, target) +} + +// Tag tags a descriptor with a reference string. +// reference should be a valid tag (e.g. "latest"). +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md#indexjson-file +func (ds *DeletableStore) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + ds.lock.Lock() + defer ds.lock.Unlock() + if err := validateReference(reference); err != nil { + return err + } + exists, err := ds.storage.Exists(ctx, desc) + if err != nil { + return err + } + if !exists { + return fmt.Errorf("%s: %s: %w", desc.Digest, desc.MediaType, errdef.ErrNotFound) + } + return ds.tag(ctx, desc, reference) +} + +// tag tags a descriptor with a reference string. +func (ds *DeletableStore) tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + dgst := desc.Digest.String() + if reference != dgst { + // also tag desc by its digest + if err := ds.tagResolver.Tag(ctx, desc, dgst); err != nil { + return err + } + } + if err := ds.tagResolver.Tag(ctx, desc, reference); err != nil { + return err + } + if ds.AutoSaveIndex { + return ds.saveIndex() + } + return nil +} + +// Resolve resolves a reference to a descriptor. If the reference to be resolved +// is a tag, the returned descriptor will be a full descriptor declared by +// github.com/opencontainers/image-spec/specs-go/v1. If the reference is a +// digest the returned descriptor will be a plain descriptor (containing only +// the digest, media type and size). +func (ds *DeletableStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + if reference == "" { + return ocispec.Descriptor{}, errdef.ErrMissingReference + } + // attempt resolving manifest + desc, err := ds.tagResolver.Resolve(ctx, reference) + if err != nil { + if errors.Is(err, errdef.ErrNotFound) { + // attempt resolving blob + return resolveBlob(os.DirFS(ds.root), reference) + } + return ocispec.Descriptor{}, err + } + if reference == desc.Digest.String() { + return descriptor.Plain(desc), nil + } + return desc, nil +} + +// Predecessors returns the nodes directly pointing to the current node. +// Predecessors returns nil without error if the node does not exists in the +// store. +func (ds *DeletableStore) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + ds.lock.RLock() + defer ds.lock.RUnlock() + return ds.graph.Predecessors(ctx, node) +} + +// Tags lists the tags presented in the `index.json` file of the OCI layout, +// returned in ascending order. +// If `last` is NOT empty, the entries in the response start after the tag +// specified by `last`. Otherwise, the response starts from the top of the tags +// list. +// +// See also `Tags()` in the package `registry`. +func (ds *DeletableStore) Tags(ctx context.Context, last string, fn func(tags []string) error) error { + ds.lock.RLock() + defer ds.lock.RUnlock() + return listTags(ctx, ds.tagResolver, last, fn) +} + +// ensureOCILayoutFile ensures the `oci-layout` file. +func (ds *DeletableStore) ensureOCILayoutFile() error { + layoutFilePath := filepath.Join(ds.root, ocispec.ImageLayoutFile) + layoutFile, err := os.Open(layoutFilePath) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to open OCI layout file: %w", err) + } + layout := ocispec.ImageLayout{ + Version: ocispec.ImageLayoutVersion, + } + layoutJSON, err := json.Marshal(layout) + if err != nil { + return fmt.Errorf("failed to marshal OCI layout file: %w", err) + } + return os.WriteFile(layoutFilePath, layoutJSON, 0666) + } + defer layoutFile.Close() + var layout ocispec.ImageLayout + err = json.NewDecoder(layoutFile).Decode(&layout) + if err != nil { + return fmt.Errorf("failed to decode OCI layout file: %w", err) + } + return validateOCILayout(&layout) +} + +// loadIndexFile reads index.json from the file system. +// Create index.json if it does not exist. +func (ds *DeletableStore) loadIndexFile(ctx context.Context) error { + indexFile, err := os.Open(ds.indexPath) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to open index file: %w", err) + } + // write index.json if it does not exist + ds.index = &ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value + }, + Manifests: []ocispec.Descriptor{}, + } + return ds.writeIndexFile() + } + defer indexFile.Close() + var index ocispec.Index + if err := json.NewDecoder(indexFile).Decode(&index); err != nil { + return fmt.Errorf("failed to decode index file: %w", err) + } + ds.index = &index + return loadIndexInDeletableMemory(ctx, ds.index, ds.storage, ds.tagResolver, ds.graph) +} + +// SaveIndex writes the `index.json` file to the file system. +// - If AutoSaveIndex is set to true (default value), +// the OCI store will automatically save the index on each Tag() call. +// - If AutoSaveIndex is set to false, it's the caller's responsibility +// to manually call this method when needed. +func (ds *DeletableStore) SaveIndex() error { + ds.lock.Lock() + defer ds.lock.Unlock() + return ds.saveIndex() +} + +func (ds *DeletableStore) saveIndex() error { + var manifests []ocispec.Descriptor + tagged := set.New[digest.Digest]() + refMap := ds.tagResolver.Map() + + // 1. Add descriptors that are associated with tags + // Note: One descriptor can be associated with multiple tags. + for ref, desc := range refMap { + if ref != desc.Digest.String() { + annotations := make(map[string]string, len(desc.Annotations)+1) + for k, v := range desc.Annotations { + annotations[k] = v + } + annotations[ocispec.AnnotationRefName] = ref + desc.Annotations = annotations + manifests = append(manifests, desc) + // mark the digest as tagged for deduplication in step 2 + tagged.Add(desc.Digest) + } + } + // 2. Add descriptors that are not associated with any tag + for ref, desc := range refMap { + if ref == desc.Digest.String() && !tagged.Contains(desc.Digest) { + // skip tagged ones since they have been added in step 1 + manifests = append(manifests, deleteAnnotationRefName(desc)) + } + } + + ds.index.Manifests = manifests + return ds.writeIndexFile() +} + +// writeIndexFile writes the `index.json` file. +func (ds *DeletableStore) writeIndexFile() error { + indexJSON, err := json.Marshal(ds.index) + if err != nil { + return fmt.Errorf("failed to marshal index file: %w", err) + } + return os.WriteFile(ds.indexPath, indexJSON, 0666) +} + +// loadIndexInDeletableMemory loads index into the memory. +func loadIndexInDeletableMemory(ctx context.Context, index *ocispec.Index, fetcher content.Fetcher, tagger content.Tagger, graph *graph.DeletableMemory) error { + for _, desc := range index.Manifests { + if err := tagger.Tag(ctx, deleteAnnotationRefName(desc), desc.Digest.String()); err != nil { + return err + } + if ref := desc.Annotations[ocispec.AnnotationRefName]; ref != "" { + if err := tagger.Tag(ctx, desc, ref); err != nil { + return err + } + } + plain := descriptor.Plain(desc) + if err := graph.IndexAll(ctx, fetcher, plain); err != nil { + return err + } + } + return nil +} diff --git a/content/oci/deletableoci_test.go b/content/oci/deletableoci_test.go new file mode 100644 index 00000000..dca14a92 --- /dev/null +++ b/content/oci/deletableoci_test.go @@ -0,0 +1,85 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package oci provides access to an OCI content store. +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +package oci + +import ( + "bytes" + "context" + "reflect" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestDeletableStore(t *testing.T) { + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test-delete", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + ref := "latest" + + tempDir := t.TempDir() + s, err := NewDeletableStore(tempDir) + if err != nil { + t.Fatal("New() error =", err) + } + ctx := context.Background() + + err = s.Push(ctx, desc, bytes.NewReader(content)) + if err != nil { + t.Errorf("Store.Push() error = %v, wantErr %v", err, false) + } + + err = s.Tag(ctx, desc, ref) + if err != nil { + t.Errorf("error tagging descriptor error = %v, wantErr %v", err, false) + } + + exists, err := s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if !exists { + t.Errorf("Store.Exists() = %v, want %v", exists, true) + } + + resolvedDescr, err := s.Resolve(ctx, ref) + if err != nil { + t.Errorf("error resolving descriptor error = %v, wantErr %v", err, false) + } + + if !reflect.DeepEqual(resolvedDescr, desc) { + t.Errorf("Store.Resolve() = %v, want %v", resolvedDescr, desc) + } + + err = s.Delete(ctx, desc) + if err != nil { + t.Errorf("Store.Delete() = %v, wantErr %v", err, nil) + } + + exists, err = s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if exists { + t.Errorf("Store.Exists() = %v, want %v", exists, false) + } +} diff --git a/content/oci/storage.go b/content/oci/storage.go index 9d4d03b3..6acf5b34 100644 --- a/content/oci/storage.go +++ b/content/oci/storage.go @@ -106,6 +106,16 @@ func (s *Storage) Push(_ context.Context, expected ocispec.Descriptor, content i return nil } +// Delete removes the target from the system. +func (s *Storage) Delete(ctx context.Context, target ocispec.Descriptor) error { + path, err := blobPath(target.Digest) + if err != nil { + return fmt.Errorf("%s: %s: %w", target.Digest, target.MediaType, errdef.ErrInvalidDigest) + } + targetPath := filepath.Join(s.root, path) + return os.Remove(targetPath) +} + // ingest write the content into a temporary ingest file. func (s *Storage) ingest(expected ocispec.Descriptor, content io.Reader) (path string, ingestErr error) { if err := ensureDir(s.ingestRoot); err != nil { diff --git a/content/oci/storage_test.go b/content/oci/storage_test.go index ac6ae8dd..6c30cba2 100644 --- a/content/oci/storage_test.go +++ b/content/oci/storage_test.go @@ -377,3 +377,39 @@ func TestStorage_Fetch_Concurrent(t *testing.T) { t.Fatal(err) } } + +func TestStorage_Delete(t *testing.T) { + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + tempDir := t.TempDir() + s, err := NewStorage(tempDir) + if err != nil { + t.Fatal("New() error =", err) + } + ctx := context.Background() + if err := s.Push(ctx, desc, bytes.NewReader(content)); err != nil { + t.Fatal("Storage.Push() error =", err) + } + exists, err := s.Exists(ctx, desc) + if err != nil { + t.Fatal("Storage.Exists() error =", err) + } + if !exists { + t.Errorf("Storage.Exists() = %v, want %v", exists, true) + } + err = s.Delete(ctx, desc) + if err != nil { + t.Fatal("Storage.Delete() error =", err) + } + exists, err = s.Exists(ctx, desc) + if err != nil { + t.Fatal("Storage.Exists() error =", err) + } + if exists { + t.Errorf("Storage.Exists() = %v, want %v", exists, false) + } +} diff --git a/internal/container/set/set.go b/internal/container/set/set.go index a084e288..07c96d47 100644 --- a/internal/container/set/set.go +++ b/internal/container/set/set.go @@ -33,3 +33,8 @@ func (s Set[T]) Contains(item T) bool { _, ok := s[item] return ok } + +// Delete deletes an item from the set. +func (s Set[T]) Delete(item T) { + delete(s, item) +} diff --git a/internal/graph/deletablememory.go b/internal/graph/deletablememory.go new file mode 100644 index 00000000..54aa02a5 --- /dev/null +++ b/internal/graph/deletablememory.go @@ -0,0 +1,144 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graph + +import ( + "context" + "errors" + "sync" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/errdef" + "oras.land/oras-go/v2/internal/container/set" + "oras.land/oras-go/v2/internal/descriptor" + "oras.land/oras-go/v2/internal/status" + "oras.land/oras-go/v2/internal/syncutil" +) + +// DeletableMemory is a memory based PredecessorFinder. +type DeletableMemory struct { + nodes map[descriptor.Descriptor]ocispec.Descriptor // nodes saves the map keys of ocispec.Descriptor + predecessors map[descriptor.Descriptor]set.Set[descriptor.Descriptor] + successors map[descriptor.Descriptor]set.Set[descriptor.Descriptor] + lock sync.RWMutex +} + +// NewDeletableMemory creates a new DeletableMemory. +func NewDeletableMemory() *DeletableMemory { + return &DeletableMemory{ + nodes: make(map[descriptor.Descriptor]ocispec.Descriptor), + predecessors: make(map[descriptor.Descriptor]set.Set[descriptor.Descriptor]), + successors: make(map[descriptor.Descriptor]set.Set[descriptor.Descriptor]), + } +} + +// Index indexes predecessors for each direct successor of the given node. +func (m *DeletableMemory) Index(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) error { + _, err := m.index(ctx, fetcher, node) + return err +} + +// Index indexes predecessors for all the successors of the given node. +func (m *DeletableMemory) IndexAll(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) error { + // track content status + tracker := status.NewTracker() + var fn syncutil.GoFunc[ocispec.Descriptor] + fn = func(ctx context.Context, region *syncutil.LimitedRegion, desc ocispec.Descriptor) error { + // skip the node if other go routine is working on it + _, committed := tracker.TryCommit(desc) + if !committed { + return nil + } + successors, err := m.index(ctx, fetcher, desc) + if err != nil { + if errors.Is(err, errdef.ErrNotFound) { + // skip the node if it does not exist + return nil + } + return err + } + if len(successors) > 0 { + // traverse and index successors + return syncutil.Go(ctx, nil, fn, successors...) + } + return nil + } + return syncutil.Go(ctx, nil, fn, node) +} + +// Predecessors returns the nodes directly pointing to the current node. +// Predecessors returns nil without error if the node does not exists in the +// store. +// Like other operations, calling Predecessors() is go-routine safe. However, +// it does not necessarily correspond to any consistent snapshot of the stored +// contents. +func (m *DeletableMemory) Predecessors(_ context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + m.lock.RLock() + defer m.lock.RUnlock() + key := descriptor.FromOCI(node) + set, exists := m.predecessors[key] + if !exists { + return nil, nil + } + var res []ocispec.Descriptor + for k := range set { + res = append(res, m.nodes[k]) + } + return res, nil +} + +// Remove removes the node from its predecessors and successors. +func (m *DeletableMemory) Remove(ctx context.Context, node ocispec.Descriptor) error { + nodeKey := descriptor.FromOCI(node) + m.lock.Lock() + defer m.lock.Unlock() + // remove the node from its successors' predecessor list + for successorKey := range m.successors[nodeKey] { + m.predecessors[successorKey].Delete(nodeKey) + } + delete(m.successors, nodeKey) + return nil +} + +// index indexes predecessors for each direct successor of the given node. +func (m *DeletableMemory) index(ctx context.Context, fetcher content.Fetcher, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + successors, err := content.Successors(ctx, fetcher, node) + if err != nil { + return nil, err + } + m.lock.Lock() + defer m.lock.Unlock() + + // index the node + nodeKey := descriptor.FromOCI(node) + m.nodes[nodeKey] = node + + // index the successors and predecessors + successorSet := set.New[descriptor.Descriptor]() + m.successors[nodeKey] = successorSet + for _, successor := range successors { + successorKey := descriptor.FromOCI(successor) + successorSet.Add(successorKey) + predecessorSet, exists := m.predecessors[successorKey] + if !exists { + predecessorSet = set.New[descriptor.Descriptor]() + m.predecessors[successorKey] = predecessorSet + } + predecessorSet.Add(nodeKey) + } + return successors, nil +} diff --git a/internal/graph/deletablememory_test.go b/internal/graph/deletablememory_test.go new file mode 100644 index 00000000..ef2ee164 --- /dev/null +++ b/internal/graph/deletablememory_test.go @@ -0,0 +1,379 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package graph + +import ( + "bytes" + "context" + "encoding/json" + "io" + "reflect" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/internal/cas" + "oras.land/oras-go/v2/internal/descriptor" +) + +/* +A(manifest)-------------------+ + + | | + | | + | | + v | + +B(manifest)-----------------+ | + + | | | + | | | + v v v + +C(layer) D(layer) +*/ +func TestDeletableMemory_IndexAndRemove(t *testing.T) { + testFetcher := cas.NewMemory() + testDeletableMemory := NewDeletableMemory() + ctx := context.Background() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) ocispec.Descriptor { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + return descs[len(descs)-1] + } + generateManifest := func(layers ...ocispec.Descriptor) ocispec.Descriptor { + manifest := ocispec.Manifest{ + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + return appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) + } + descC := appendBlob("layer node C", []byte("Node C is a layer")) // blobs[0], layer "C" + descD := appendBlob("layer node D", []byte("Node D is a layer")) // blobs[1], layer "D" + descB := generateManifest(descs[0:2]...) // blobs[2], manifest "B" + descA := generateManifest(descs[1:3]...) // blobs[3], manifest "A" + testFetcher.Push(ctx, descA, bytes.NewReader(blobs[3])) + testFetcher.Push(ctx, descB, bytes.NewReader(blobs[2])) + testFetcher.Push(ctx, descC, bytes.NewReader(blobs[0])) + testFetcher.Push(ctx, descD, bytes.NewReader(blobs[1])) + + // make sure that testFetcher works + rc, err := testFetcher.Fetch(ctx, descA) + if err != nil { + t.Errorf("testFetcher.Fetch() error = %v", err) + } + got, err := io.ReadAll(rc) + if err != nil { + t.Errorf("testFetcher.Fetch().Read() error = %v", err) + } + err = rc.Close() + if err != nil { + t.Errorf("testFetcher.Fetch().Close() error = %v", err) + } + if !bytes.Equal(got, blobs[3]) { + t.Errorf("testFetcher.Fetch() = %v, want %v", got, blobs[4]) + } + + // index test content into graph.DeletableMemory + testDeletableMemory.Index(ctx, testFetcher, descA) + testDeletableMemory.Index(ctx, testFetcher, descB) + testDeletableMemory.Index(ctx, testFetcher, descC) + testDeletableMemory.Index(ctx, testFetcher, descD) + + // check that the content of testDeletableMemory.predecessors and successors + // are correct + nodeKeyA := descriptor.FromOCI(descA) + nodeKeyB := descriptor.FromOCI(descB) + nodeKeyC := descriptor.FromOCI(descC) + nodeKeyD := descriptor.FromOCI(descD) + + // check the information of node A + predecessorsA := testDeletableMemory.predecessors[nodeKeyA] + successorsA := testDeletableMemory.successors[nodeKeyA] + for range predecessorsA { + t.Errorf("predecessors of %s should be empty", "A") + } + if !successorsA.Contains(nodeKeyB) { + t.Errorf("successors of %s should contain %s", "A", "B") + } + if !successorsA.Contains(nodeKeyD) { + t.Errorf("successors of %s should contain %s", "A", "D") + } + + // check the information of node B + predecessorsB := testDeletableMemory.predecessors[nodeKeyB] + successorsB := testDeletableMemory.successors[nodeKeyB] + if !predecessorsB.Contains(nodeKeyA) { + t.Errorf("predecessors of %s should contain %s", "B", "A") + } + if !successorsB.Contains(nodeKeyC) { + t.Errorf("successors of %s should contain %s", "B", "C") + } + if !successorsB.Contains(nodeKeyD) { + t.Errorf("successors of %s should contain %s", "B", "D") + } + + // check the information of node C + predecessorsC := testDeletableMemory.predecessors[nodeKeyC] + successorsC := testDeletableMemory.successors[nodeKeyC] + if !predecessorsC.Contains(nodeKeyB) { + t.Errorf("predecessors of %s should contain %s", "C", "B") + } + for range successorsC { + t.Errorf("successors of %s should be empty", "C") + } + + // check the information of node D + predecessorsD := testDeletableMemory.predecessors[nodeKeyD] + successorsD := testDeletableMemory.successors[nodeKeyD] + if !predecessorsD.Contains(nodeKeyB) { + t.Errorf("predecessors of %s should contain %s", "D", "B") + } + if !predecessorsD.Contains(nodeKeyA) { + t.Errorf("predecessors of %s should contain %s", "D", "A") + } + for range successorsD { + t.Errorf("successors of %s should be empty", "C") + } + + // remove node B and check the stored information + testDeletableMemory.Remove(ctx, descB) + if predecessorsC.Contains(nodeKeyB) { + t.Errorf("predecessors of %s should not contain %s", "C", "B") + } + if predecessorsD.Contains(nodeKeyB) { + t.Errorf("predecessors of %s should not contain %s", "D", "B") + } + if !successorsA.Contains(nodeKeyB) { + t.Errorf("successors of %s should still contain %s", "A", "B") + } + if _, exists := testDeletableMemory.successors[nodeKeyB]; exists { + t.Errorf("testDeletableMemory.successors should not contain the entry of %v", "B") + } + + // remove node A and check the stored information + testDeletableMemory.Remove(ctx, descA) + if predecessorsD.Contains(nodeKeyA) { + t.Errorf("predecessors of %s should not contain %s", "D", "A") + } + if _, exists := testDeletableMemory.successors[nodeKeyA]; exists { + t.Errorf("testDeletableMemory.successors should not contain the entry of %v", "A") + } +} + +func TestDeletableMemory_IndexAllAndPredecessors(t *testing.T) { + testFetcher := cas.NewMemory() + testDeletableMemory := NewDeletableMemory() + ctx := context.Background() + + // generate test content + var blobs [][]byte + var descriptors []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) ocispec.Descriptor { + blobs = append(blobs, blob) + descriptors = append(descriptors, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + return descriptors[len(descriptors)-1] + } + generateManifest := func(layers ...ocispec.Descriptor) ocispec.Descriptor { + manifest := ocispec.Manifest{ + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + return appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) + } + generateIndex := func(manifests ...ocispec.Descriptor) ocispec.Descriptor { + index := ocispec.Index{ + Manifests: manifests, + } + indexJSON, err := json.Marshal(index) + if err != nil { + t.Fatal(err) + } + return appendBlob(ocispec.MediaTypeImageIndex, indexJSON) + } + descE := appendBlob("layer node E", []byte("Node E is a layer")) // blobs[0], layer "E" + descF := appendBlob("layer node F", []byte("Node F is a layer")) // blobs[1], layer "F" + descB := generateManifest(descriptors[0:1]...) // blobs[2], manifest "B" + descC := generateManifest(descriptors[0:2]...) // blobs[3], manifest "C" + descD := generateManifest(descriptors[1:2]...) // blobs[4], manifest "D" + descA := generateIndex(descriptors[2:5]...) // blobs[5], index "A" + testFetcher.Push(ctx, descA, bytes.NewReader(blobs[5])) + testFetcher.Push(ctx, descB, bytes.NewReader(blobs[2])) + testFetcher.Push(ctx, descC, bytes.NewReader(blobs[3])) + testFetcher.Push(ctx, descD, bytes.NewReader(blobs[4])) + testFetcher.Push(ctx, descE, bytes.NewReader(blobs[0])) + testFetcher.Push(ctx, descF, bytes.NewReader(blobs[1])) + + // make sure that testFetcher works + rc, err := testFetcher.Fetch(ctx, descA) + if err != nil { + t.Errorf("testFetcher.Fetch() error = %v", err) + } + got, err := io.ReadAll(rc) + if err != nil { + t.Errorf("testFetcher.Fetch().Read() error = %v", err) + } + err = rc.Close() + if err != nil { + t.Errorf("testFetcher.Fetch().Close() error = %v", err) + } + if !bytes.Equal(got, blobs[5]) { + t.Errorf("testFetcher.Fetch() = %v, want %v", got, blobs[4]) + } + + // index node A into graph.DeletableMemory using IndexAll + testDeletableMemory.IndexAll(ctx, testFetcher, descA) + + // check that the content of testDeletableMemory.predecessors and successors + // are correct + nodeKeyA := descriptor.FromOCI(descA) + nodeKeyB := descriptor.FromOCI(descB) + nodeKeyC := descriptor.FromOCI(descC) + nodeKeyD := descriptor.FromOCI(descD) + nodeKeyE := descriptor.FromOCI(descE) + nodeKeyF := descriptor.FromOCI(descF) + + // check the information of node A + predecessorsA := testDeletableMemory.predecessors[nodeKeyA] + successorsA := testDeletableMemory.successors[nodeKeyA] + for range predecessorsA { + t.Errorf("predecessors of %s should be empty", "A") + } + if !successorsA.Contains(nodeKeyB) { + t.Errorf("successors of %s should contain %s", "A", "B") + } + if !successorsA.Contains(nodeKeyC) { + t.Errorf("successors of %s should contain %s", "A", "C") + } + if !successorsA.Contains(nodeKeyD) { + t.Errorf("successors of %s should contain %s", "A", "D") + } + + // check the information of node B + predecessorsB := testDeletableMemory.predecessors[nodeKeyB] + successorsB := testDeletableMemory.successors[nodeKeyB] + if !predecessorsB.Contains(nodeKeyA) { + t.Errorf("predecessors of %s should contain %s", "B", "A") + } + if !successorsB.Contains(nodeKeyE) { + t.Errorf("successors of %s should contain %s", "B", "E") + } + + // check the information of node C + predecessorsC := testDeletableMemory.predecessors[nodeKeyC] + successorsC := testDeletableMemory.successors[nodeKeyC] + if !predecessorsC.Contains(nodeKeyA) { + t.Errorf("predecessors of %s should contain %s", "C", "A") + } + if !successorsC.Contains(nodeKeyE) { + t.Errorf("successors of %s should contain %s", "C", "E") + } + if !successorsC.Contains(nodeKeyF) { + t.Errorf("successors of %s should contain %s", "C", "F") + } + + // check the information of node D + predecessorsD := testDeletableMemory.predecessors[nodeKeyD] + successorsD := testDeletableMemory.successors[nodeKeyD] + if !predecessorsD.Contains(nodeKeyA) { + t.Errorf("predecessors of %s should contain %s", "D", "A") + } + if !successorsD.Contains(nodeKeyF) { + t.Errorf("successors of %s should contain %s", "D", "F") + } + + // check the information of node E + predecessorsE := testDeletableMemory.predecessors[nodeKeyE] + successorsE := testDeletableMemory.successors[nodeKeyE] + if !predecessorsE.Contains(nodeKeyB) { + t.Errorf("predecessors of %s should contain %s", "E", "B") + } + if !predecessorsE.Contains(nodeKeyC) { + t.Errorf("predecessors of %s should contain %s", "E", "C") + } + for range successorsE { + t.Errorf("successors of %s should be empty", "E") + } + + // check the information of node F + predecessorsF := testDeletableMemory.predecessors[nodeKeyF] + successorsF := testDeletableMemory.successors[nodeKeyF] + if !predecessorsF.Contains(nodeKeyC) { + t.Errorf("predecessors of %s should contain %s", "F", "C") + } + if !predecessorsF.Contains(nodeKeyD) { + t.Errorf("predecessors of %s should contain %s", "F", "D") + } + for range successorsF { + t.Errorf("successors of %s should be empty", "E") + } + + // check the Predecessors + predsC, err := testDeletableMemory.Predecessors(ctx, descC) + if err != nil { + t.Errorf("testFetcher.Predecessors() error = %v", err) + } + if len(predsC) != 1 { + t.Errorf("%s should have length %d", "predsC", 1) + } + if !reflect.DeepEqual(predsC[0], descA) { + t.Errorf("incorrect predecessor result") + } + + // TODO: change it to G + + // // remove node B and check the stored information + // testDeletableMemory.Remove(ctx, descB) + // if predecessorsC.Contains(nodeKeyB) { + // t.Errorf("predecessors of %s should not contain %s", "C", "B") + // } + // if predecessorsD.Contains(nodeKeyB) { + // t.Errorf("predecessors of %s should not contain %s", "D", "B") + // } + // if !successorsA.Contains(nodeKeyB) { + // t.Errorf("successors of %s should still contain %s", "A", "B") + // } + // if _, exists := testDeletableMemory.successors[nodeKeyB]; exists { + // t.Errorf("testDeletableMemory.successors should not contain the entry of %v", "B") + // } + + // // remove node A and check the stored information + // testDeletableMemory.Remove(ctx, descA) + // if predecessorsD.Contains(nodeKeyA) { + // t.Errorf("predecessors of %s should not contain %s", "D", "A") + // } + // if _, exists := testDeletableMemory.successors[nodeKeyA]; exists { + // t.Errorf("testDeletableMemory.successors should not contain the entry of %v", "A") + // } +} diff --git a/internal/resolver/memory.go b/internal/resolver/memory.go index 6fac5e2d..27d6a59d 100644 --- a/internal/resolver/memory.go +++ b/internal/resolver/memory.go @@ -48,6 +48,11 @@ func (m *Memory) Tag(_ context.Context, desc ocispec.Descriptor, reference strin return nil } +// Delete removes a reference from index map. +func (m *Memory) Delete(reference string) { + m.index.Delete(reference) +} + // Map dumps the memory into a built-in map structure. // Like other operations, calling Map() is go-routine safe. However, it does not // necessarily correspond to any consistent snapshot of the storage contents.