From e2b7b235582e6e008ad836ff8e19149d2ac36f0e Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Fri, 22 Jan 2021 22:04:10 +0100 Subject: [PATCH] Add load subcommand --- client/load.go | 135 +++++++++++++++++++++++++++++++++++ load.go | 107 +++++++++++++++++++++++++++ load_test.go | 30 ++++++++ main.go | 1 + testdata/img-load/docker.tar | Bin 0 -> 10752 bytes testdata/img-load/oci.tar | Bin 0 -> 9216 bytes 6 files changed, 273 insertions(+) create mode 100644 client/load.go create mode 100644 load.go create mode 100644 load_test.go create mode 100755 testdata/img-load/docker.tar create mode 100755 testdata/img-load/oci.tar diff --git a/client/load.go b/client/load.go new file mode 100644 index 000000000..c3f000d44 --- /dev/null +++ b/client/load.go @@ -0,0 +1,135 @@ +package client + +import ( + "context" + "encoding/json" + "io" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/archive" + "github.com/docker/distribution/reference" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// LoadProgress receives informational updates about the progress of image loading. +type LoadProgress interface { + // LoadImageCreated indicates that the given image has been created. + LoadImageCreated(images.Image) error + + // LoadImageUpdated indicates that the given image has been updated. + LoadImageUpdated(images.Image) error + + // LoadImageReplaced indicates that an image has been replaced, leaving the old image without a name. + LoadImageReplaced(before, after images.Image) error +} + +// LoadImage imports an image into the image store. +func (c *Client) LoadImage(ctx context.Context, reader io.Reader, mon LoadProgress) error { + // Create the worker opts. + opt, err := c.createWorkerOpt(false) + if err != nil { + return errors.Wrap(err, "creating worker opt failed") + } + + if opt.ImageStore == nil { + return errors.New("image store is nil") + } + + logrus.Debug("Importing index") + index, err := importIndex(ctx, opt.ContentStore, reader) + if err != nil { + return err + } + + logrus.Debug("Extracting manifests") + manifests := extractManifests(index) + + for _, imageSkel := range manifests { + err := load(ctx, opt.ImageStore, imageSkel, mon) + if err != nil { + return nil + } + } + + return nil +} + +func importIndex(ctx context.Context, store content.Store, reader io.Reader) (*ocispec.Index, error) { + d, err := archive.ImportIndex(ctx, store, reader) + if err != nil { + return nil, err + } + + indexBytes, err := content.ReadBlob(ctx, store, d) + if err != nil { + return nil, err + } + + var index ocispec.Index + err = json.Unmarshal(indexBytes, &index) + if err != nil { + return nil, err + } + + return &index, nil +} + +func extractManifests(index *ocispec.Index) []images.Image { + var result []images.Image + for _, m := range index.Manifests { + switch m.MediaType { + case images.MediaTypeDockerSchema2Manifest: + if name, ok := m.Annotations[images.AnnotationImageName]; ok { + if ref, ok := m.Annotations[ocispec.AnnotationRefName]; ok { + if normalized, err := reference.ParseNormalizedNamed(name); err == nil { + if normalizedWithTag, err := reference.WithTag(normalized, ref); err == nil { + if normalized == normalizedWithTag { + result = append(result, images.Image{ + Name: normalized.String(), + Target: m, + }) + continue + } + } + } + } + } + logrus.Debugf("Failed to extract image info from manifest: %v", m) + } + } + + return result +} + +func load(ctx context.Context, store images.Store, imageSkel images.Image, mon LoadProgress) error { + image, err := store.Get(ctx, imageSkel.Name) + + if errors.Cause(err) == errdefs.ErrNotFound { + image, err = store.Create(ctx, imageSkel) + if err != nil { + return err + } + return mon.LoadImageCreated(image) + } + + if err != nil { + return err + } + + updated, err := store.Update(ctx, imageSkel) + if err != nil { + return err + } + + if image.Target.Digest == updated.Target.Digest { + return mon.LoadImageUpdated(updated) + } + + image.Name = "" + return mon.LoadImageReplaced(image, updated) +} diff --git a/load.go b/load.go new file mode 100644 index 000000000..bbf6f9252 --- /dev/null +++ b/load.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" + + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/namespaces" + "github.com/genuinetools/img/client" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/session" +) + +const loadUsageShortHelp = `Load an image from a tar archive or STDIN.` +const loadUsageLongHelp = `Load an image from a tar archive or STDIN.` + +func newLoadCommand() *cobra.Command { + + load := &loadCommand{} + + cmd := &cobra.Command{ + Use: "load [OPTIONS]", + DisableFlagsInUseLine: true, + SilenceUsage: true, + Short: loadUsageShortHelp, + Long: loadUsageLongHelp, + // Args: load.ValidateArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return load.Run(args) + }, + } + + fs := cmd.Flags() + + fs.StringVarP(&load.input, "input", "i", "", "Read from tar archive file, instead of STDIN") + fs.BoolVarP(&load.quiet, "quiet", "q", false, "Suppress the load output") + + return cmd +} + +type loadCommand struct { + input string + quiet bool +} + +func (cmd *loadCommand) Run(args []string) (err error) { + reexec() + + // Create the context. + id := identity.NewID() + ctx := session.NewContext(context.Background(), id) + ctx = namespaces.WithNamespace(ctx, "buildkit") + + // Create the client. + c, err := client.New(stateDir, backend, nil) + if err != nil { + return err + } + defer c.Close() + + // Create the reader. + reader, err := cmd.reader() + if err != nil { + return err + } + defer reader.Close() + + // Load images. + return c.LoadImage(ctx, reader, cmd) +} + +func (cmd *loadCommand) reader() (io.ReadCloser, error) { + if cmd.input != "" { + return os.Open(cmd.input) + } + + return os.Stdin, nil +} + +func (cmd *loadCommand) LoadImageCreated(image images.Image) error { + if cmd.quiet { + return nil + } + _, err := fmt.Printf("Loaded image: %s\n", image.Name) + return err +} + +func (cmd *loadCommand) LoadImageUpdated(image images.Image) error { + return cmd.LoadImageCreated(image) +} + +func (cmd *loadCommand) LoadImageReplaced(before, after images.Image) error { + if !cmd.quiet { + _, err := fmt.Printf( + "The image %s already exists, leaving the old one with ID %s orphaned\n", + after.Name, before.Target.Digest) + if err != nil { + return err + } + } + + return cmd.LoadImageCreated(after) +} diff --git a/load_test.go b/load_test.go new file mode 100644 index 000000000..0f358a661 --- /dev/null +++ b/load_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "testing" +) + +func TestLoadImage(t *testing.T) { + output := run(t, "load", "-i", "testdata/img-load/oci.tar") + if output != "Loaded image: docker.io/library/dummy:latest\n" { + t.Fatalf("Unexpected output: %s", output) + } + + output = run(t, "load", "-i", "testdata/img-load/oci.tar") + if output != "Loaded image: docker.io/library/dummy:latest\n" { + t.Fatalf("Unexpected output: %s", output) + } + + output = run(t, "load", "-i", "testdata/img-load/docker.tar") + expected := `The image docker.io/library/dummy:latest already exists, leaving the old one with ID sha256:e08488191147b6fc575452dfac3238721aa5b86d2545a9edaa1c1d88632b2233 orphaned +Loaded image: docker.io/library/dummy:latest +` + if output != expected { + t.Fatalf("Unexpected output: %s", output) + } + + output = run(t, "load", "-i", "testdata/img-load/docker.tar") + if output != "Loaded image: docker.io/library/dummy:latest\n" { + t.Fatalf("Unexpected output: %s", output) + } +} diff --git a/main.go b/main.go index 5e0a9ae61..c60e047c0 100644 --- a/main.go +++ b/main.go @@ -75,6 +75,7 @@ func main() { newDiskUsageCommand(), newInspectCommand(), newListCommand(), + newLoadCommand(), newLoginCommand(), newLogoutCommand(), newPruneCommand(), diff --git a/testdata/img-load/docker.tar b/testdata/img-load/docker.tar new file mode 100755 index 0000000000000000000000000000000000000000..eed6591d6155e312cad902b425c7d1a659cddb73 GIT binary patch literal 10752 zcmeHNTTkRR6y}*txB~^w^9*8w&Os$xr`FeYDc5_?>)|3 z7N}rg!=P=61^9M+$#-(jvA+6Q+Jn`Hcr*BrLHsiokQ9@T-W(t zXnfdJ{~wRve>ge)?V;^@Ttpadpz(D`>pB$KjWV=TLKxc%I_u=2O|AQ0x&F!UMNnR& z-RIvK0-f#u@1`oZJd-!}2kf6nyzvcQ6zm^MiF^Cs0_(GM&rz137gCI^(B5gfB|9I_#gxb#2czr1KwI+c~SzJyYmU7>i6-d2q(RbgQ~NE}rK2z;&o z(b#&~bLgBh*B@Mxl_*YC-q-=!v{h60??zk0(3Rwwn{CRc6I&iq1~##hKQ2F` z_#FLyc=q;}(X^>Yd1X{SYIIh{H|m?}*N(PnS%o6H7@#)=fOC#UP#rbb@Q?=v{`~k; zSxtTp4v&t4G|O$w7*41d8*3ypj0mfh4QawOF*-6h5|ooD(jrYPrCJg$4Nr*{%rXgw zGZJKFFbe#7CxyCN!TGVur-f}$Tu$?Rfc~uNPg!|&l&vs5E?qsFR9Ok=EhSFNU#D3P z`(0MwDQ)xB@WlWnmH7n9`)unbt3VB)@gc!Ox>W7!Qs@$(-C?b2bDcSB+_d(}R7D~} zz_|i#pgb$5U(lk9Bj{@Qqg>5wJ@jYjd(#&GAGSW<3oVmB|4GvO|9kO#&W>bS6tkiG z;+}KTJ9A+I@n39W{Q9iF&f)k^_OQf^(U1rq{|TJ`d;Q-*aW~bIKox;116xeoEO54Q zZ5aAte>cnZv-Sw|2=oZ_2z)09Sgj@88+bx684GSTky3G?gyu4|9Jt|-Xgju%)Od_J zFvEs~(o#)QA+e&v?)QI6A=s}Ee~|n8zaTyT_abk|9Nlxlcy%}PU0>vS(&N8PdOU&O zdVn9f&Kg(MKti3*pTw9g$+b%jBi9U4aA1NRu}2G_k8Usw15{U)OM!BjBR89P9-_VW zH$X%rS(;vE3G|(#=2}rMVhkQ#hbm&yKu}3^m|_;$Bup@{RCvrRPq7RsPBdoLaFs~O zMKrdSMig6$(Z!+@r|}|=^MA5Pp~@_UTi;{i__sc?kL?fe|CqtY-dpjX^!(4aV*S){ z=lel0hu=OV0 literal 0 HcmV?d00001 diff --git a/testdata/img-load/oci.tar b/testdata/img-load/oci.tar new file mode 100755 index 0000000000000000000000000000000000000000..8181dba72bb45ad08964b57a906a5033b4d0d4cb GIT binary patch literal 9216 zcmeHL%WoS+7&os9RpM4vZ~)8zQlIwB?8~-PiBKo0Ep|gvE6%!#%IwP-dmquR-NYLr zQ7R!JPM}I$H~{U11BXhzAaSYSz@<_J95|Epk3{8=wRPuy&|8pa)_fD{Y!pwez-C7MGU8n;`1 z`!wI^u79+|s3h;zHc0%kv-Kwe8L$7o?0}K=C)k!PlNzE)9Vc%avg3%@f<6`$OHynq zQAS0DNk%SjTNafK!4_@VCXPr9Aa9~^vi_nd4nrg9LmTvoM26VVqf;spA(h^BOm3pj z2>B_G|24!ybBVWXv(*l5R6xwLWD%V}X5c%#jtXmN(!YQT=g|8Tl{2TZ?I_H;fyvx# zWN^Q*p+D^28|adHU3f){Cr~vAo7}G#c?hDiAZEsYx6NG(75ujAu7lXnW-S}~h+;}k zVsetw3Lym%7qFb6iXw?Jm6YR{6flPQOFU`?VFwgk+wt7Ktn&s;M-#>}ufVUPy0fclCBSMi zSkbuHnJKD*R&0uDtcPlGQdWx-+?mHw$(viQ(fQaqU6M<_7U-1{(Hg-@ty=2Trq#~O zbV<~zHLf*uNpI8zwb7(A(`By5W?ZY*T)i@h^>`NePYQaYB3K8;J|^uuV1`)?Tt$pf4F$<+UK8s^U9gqw|=?($5#_KKKjJ}_|mU`eep9o z@Z6Yc-vE5|NhJ^ba7tM;9%$HJGc|1@>u$jdKW z`QXs4L#by|sk?Wti1FL0z6y`kuraf3d0-^|ZJZNxIg%$tRG|1XCCIiCwH#&&RLCim zu%z^pEed1?6(D28Bvvjb3)G-g5H^vRY^E*ZpJ04{{YPZ1jN^Y3cRp6~+QD=>jXc}p ztkO-AX~dS7U2d`#5B%%}-|AO784D_{kQ?oUp7EH^9Xo1eF3{tRDDcsVE}sZ$9*@mK zl7){@Ax(SaK`oi?O53hLnAKw3x5EtgSRHC9vO!eB#Nu^O0mR(P;k!G(^sGo&M?8jt zA)OPY#IE(88U6C1kUp0Xp$*J|*R5R`n7nU9mj{@Vfm?^;^_VXYWhP7>v}l(Zd4zl4 zC?d#x{nwM{;=w1eZh5Rh@DwxSzGbgwk`UfICnWeUZ+`yAGEoM10m-K+Ob8vH|MzY^ zHgZlN3_TRK(E#J&l+;JO0p4v5cDH4iWA(AxXI_#7P~RXPWL<8AEbL^v7X<|uZYm)E444C#AgpJCW!oRJiu$ZV z+sO>r^`~vb;BhB_U%bEmBSZnx`2KGncECB!(S+XXUDXVb2fiIuYPWS!PZ<1PKe)L6 zQMV5ZyG;>LnVy82G-$WBIpGmLOX7cv^ItOfVE!lL_1=qf{24%aPRDf dNCsyx`ahD=j;$C^7<*vsfw2e19@wb|{so68XCVLp literal 0 HcmV?d00001