From 97a7fc50e08ab75081effa947500a8be56e7fcd9 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Tue, 13 Feb 2024 13:01:40 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B9=20=20split=20loading=20container?= =?UTF-8?q?=20images=20from=20opening=20connections=20(#3237)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧹 split loading container images from opening connections Signed-off-by: Ivan Milchev * fix log levels Signed-off-by: Ivan Milchev * fix linter Signed-off-by: Ivan Milchev * more linter fixes Signed-off-by: Ivan Milchev * fix container pull Signed-off-by: Ivan Milchev * remove logic for compressed image Signed-off-by: Ivan Milchev * commit change core manifest Signed-off-by: Ivan Milchev --------- Signed-off-by: Ivan Milchev --- providers/core/resources/core.resources.json | 2 +- .../os/connection/container/image/docker.go | 17 +++-- .../os/connection/container/image/registry.go | 19 ++---- providers/os/connection/docker_container.go | 46 ++++---------- providers/os/connection/tar.go | 63 +++++++++++++++++-- 5 files changed, 88 insertions(+), 59 deletions(-) diff --git a/providers/core/resources/core.resources.json b/providers/core/resources/core.resources.json index 0a6ed36117..1fae85ed3b 100644 --- a/providers/core/resources/core.resources.json +++ b/providers/core/resources/core.resources.json @@ -1 +1 @@ -{"resources":{"asset":{"id":"asset","name":"asset","fields":{"arch":{"name":"arch","type":"\u0007","is_mandatory":true,"title":"Architecture this OS is running on","provider":"go.mondoo.com/cnquery/v9/providers/core"},"build":{"name":"build","type":"\u0007","is_mandatory":true,"title":"Build version of the platform (optional)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"eol":{"name":"eol","type":"\u001basset.eol","title":"Information about the assets platform end-of-life.","is_implicit_resource":true,"provider":"go.mondoo.com/cnquery/v9/providers/core"},"family":{"name":"family","type":"\u0019\u0007","is_mandatory":true,"title":"List of platform families that this platform belongs to","provider":"go.mondoo.com/cnquery/v9/providers/core"},"fqdn":{"name":"fqdn","type":"\u0007","is_mandatory":true,"title":"Fully qualified domain name (optional)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"ids":{"name":"ids","type":"\u0019\u0007","is_mandatory":true,"title":"All identifiers for this asset","provider":"go.mondoo.com/cnquery/v9/providers/core"},"kind":{"name":"kind","type":"\u0007","is_mandatory":true,"title":"Kind of platform, for example:","desc":"api, baremetal, vm, vm-image, container, container-image, network, ...","provider":"go.mondoo.com/cnquery/v9/providers/core"},"labels":{"name":"labels","type":"\u001a\u0007\u0007","is_mandatory":true,"title":"Optional platform information","provider":"go.mondoo.com/cnquery/v9/providers/core"},"name":{"name":"name","type":"\u0007","is_mandatory":true,"title":"Human readable name of the asset","provider":"go.mondoo.com/cnquery/v9/providers/core"},"platform":{"name":"platform","type":"\u0007","is_mandatory":true,"title":"Platform for this asset (redhat, windows, k8s-pod)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"runtime":{"name":"runtime","type":"\u0007","is_mandatory":true,"title":"Runtime is the specific kind of the platform. Examples include:","desc":"docker-container, podman-container, aws-ec2-instance, ...","provider":"go.mondoo.com/cnquery/v9/providers/core"},"title":{"name":"title","type":"\u0007","is_mandatory":true,"title":"Human-readable title of the platform (e.g., \"Red Hat 8, Container\")","provider":"go.mondoo.com/cnquery/v9/providers/core"},"version":{"name":"version","type":"\u0007","is_mandatory":true,"title":"Version of the platform","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"General asset information","min_mondoo_version":"6.13.0","defaults":"name platform version","provider":"go.mondoo.com/cnquery/v9/providers/core"},"asset.eol":{"id":"asset.eol","name":"asset.eol","fields":{"date":{"name":"date","type":"\t","is_mandatory":true,"title":"End-of-Life date","provider":"go.mondoo.com/cnquery/v9/providers/core"},"docsUrl":{"name":"docsUrl","type":"\u0007","is_mandatory":true,"title":"Documentation URL","provider":"go.mondoo.com/cnquery/v9/providers/core"},"productUrl":{"name":"productUrl","type":"\u0007","is_mandatory":true,"title":"Product URL","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"Information about the assets platform end-of-life.","min_mondoo_version":"latest","defaults":"date","provider":"go.mondoo.com/cnquery/v9/providers/core"},"cpe":{"id":"cpe","name":"cpe","fields":{"edition":{"name":"edition","type":"\u0007","title":"Edition of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"language":{"name":"language","type":"\u0007","title":"Language of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"other":{"name":"other","type":"\u0007","title":"Other of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"part":{"name":"part","type":"\u0007","title":"Part of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"product":{"name":"product","type":"\u0007","title":"Product of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"swEdition":{"name":"swEdition","type":"\u0007","title":"Software edition of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"targetHw":{"name":"targetHw","type":"\u0007","title":"Target hardware of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"targetSw":{"name":"targetSw","type":"\u0007","title":"Target software of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"update":{"name":"update","type":"\u0007","title":"Update of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"uri":{"name":"uri","type":"\u0007","is_mandatory":true,"title":"URI binding of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"vendor":{"name":"vendor","type":"\u0007","title":"Vendor of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"version":{"name":"version","type":"\u0007","title":"Version of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"init":{"args":[{"name":"uri","type":"\u0007"}]},"title":"Common Platform Enumeration (CPE) identifiers","min_mondoo_version":"latest","defaults":"uri","provider":"go.mondoo.com/cnquery/v9/providers/core"},"mondoo":{"id":"mondoo","name":"mondoo","fields":{"arch":{"name":"arch","type":"\u0007","title":"Architecture of this client (e.g., linux-amd64)","min_mondoo_version":"latest","provider":"go.mondoo.com/cnquery/v9/providers/core"},"build":{"name":"build","type":"\u0007","title":"Build of the client (e.g., production, development)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"capabilities":{"name":"capabilities","type":"\u0019\u0007","title":"Connection capabilities","provider":"go.mondoo.com/cnquery/v9/providers/core"},"jobEnvironment":{"name":"jobEnvironment","type":"\n","title":"Agent execution environment","provider":"go.mondoo.com/cnquery/v9/providers/core"},"version":{"name":"version","type":"\u0007","title":"Version of the client running on the asset","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"Contextual information about MQL runtime and environment","min_mondoo_version":"5.15.0","defaults":"version","provider":"go.mondoo.com/cnquery/v9/providers/core"},"parse":{"id":"parse","name":"parse","title":"Provides common parsers (json, ini, certs, etc)","min_mondoo_version":"5.15.0","provider":"go.mondoo.com/cnquery/v9/providers/core"},"regex":{"id":"regex","name":"regex","fields":{"creditCard":{"name":"creditCard","type":"\u0008","title":"Matches credit card numbers","provider":"go.mondoo.com/cnquery/v9/providers/core"},"email":{"name":"email","type":"\u0008","title":"Matches email addresses","provider":"go.mondoo.com/cnquery/v9/providers/core"},"emoji":{"name":"emoji","type":"\u0008","title":"Matches emojis","provider":"go.mondoo.com/cnquery/v9/providers/core"},"ipv4":{"name":"ipv4","type":"\u0008","title":"Matches IPv4 addresses","provider":"go.mondoo.com/cnquery/v9/providers/core"},"ipv6":{"name":"ipv6","type":"\u0008","title":"Matches IPv6 addresses","provider":"go.mondoo.com/cnquery/v9/providers/core"},"mac":{"name":"mac","type":"\u0008","title":"Matches MAC addresses","provider":"go.mondoo.com/cnquery/v9/providers/core"},"semver":{"name":"semver","type":"\u0008","title":"Matches semantic version numbers","provider":"go.mondoo.com/cnquery/v9/providers/core"},"url":{"name":"url","type":"\u0008","title":"Matches URL addresses (HTTP/HTTPS)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"uuid":{"name":"uuid","type":"\u0008","title":"Matches hyphen-deliminated UUIDs","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"Built-in regular expression functions","min_mondoo_version":"5.15.0","provider":"go.mondoo.com/cnquery/v9/providers/core"},"time":{"id":"time","name":"time","fields":{"day":{"name":"day","type":"\t","title":"One day, used for durations","provider":"go.mondoo.com/cnquery/v9/providers/core"},"hour":{"name":"hour","type":"\t","title":"One hour, used for durations","provider":"go.mondoo.com/cnquery/v9/providers/core"},"minute":{"name":"minute","type":"\t","title":"One minute, used for durations","provider":"go.mondoo.com/cnquery/v9/providers/core"},"now":{"name":"now","type":"\t","title":"The current time on the local system","provider":"go.mondoo.com/cnquery/v9/providers/core"},"second":{"name":"second","type":"\t","title":"One second, used for durations","provider":"go.mondoo.com/cnquery/v9/providers/core"},"today":{"name":"today","type":"\t","title":"The current day starting at midnight","provider":"go.mondoo.com/cnquery/v9/providers/core"},"tomorrow":{"name":"tomorrow","type":"\t","title":"The next day starting at midnight","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"Date and time functions","min_mondoo_version":"5.15.0","provider":"go.mondoo.com/cnquery/v9/providers/core"},"uuid":{"id":"uuid","name":"uuid","fields":{"urn":{"name":"urn","type":"\u0007","title":"URN returns the RFC 2141 URN form of uuid","provider":"go.mondoo.com/cnquery/v9/providers/core"},"value":{"name":"value","type":"\u0007","is_mandatory":true,"title":"Canonical string representation xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","provider":"go.mondoo.com/cnquery/v9/providers/core"},"variant":{"name":"variant","type":"\u0007","title":"Variant encoded in UUID","provider":"go.mondoo.com/cnquery/v9/providers/core"},"version":{"name":"version","type":"\u0005","title":"Version of UUID","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"init":{"args":[{"name":"value","type":"\u0007"}]},"title":"UUIDs based on RFC 4122 and DCE 1.1","min_mondoo_version":"5.15.0","defaults":"value","provider":"go.mondoo.com/cnquery/v9/providers/core"}}} \ No newline at end of file +{"resources":{"asset":{"id":"asset","name":"asset","fields":{"arch":{"name":"arch","type":"\u0007","is_mandatory":true,"title":"Architecture this OS is running on","provider":"go.mondoo.com/cnquery/v9/providers/core"},"build":{"name":"build","type":"\u0007","is_mandatory":true,"title":"Build version of the platform (optional)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"eol":{"name":"eol","type":"\u001basset.eol","title":"Information about the assets platform end-of-life.","is_implicit_resource":true,"provider":"go.mondoo.com/cnquery/v9/providers/core"},"family":{"name":"family","type":"\u0019\u0007","is_mandatory":true,"title":"List of platform families that this platform belongs to","provider":"go.mondoo.com/cnquery/v9/providers/core"},"fqdn":{"name":"fqdn","type":"\u0007","is_mandatory":true,"title":"Fully qualified domain name (optional)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"ids":{"name":"ids","type":"\u0019\u0007","is_mandatory":true,"title":"All identifiers for this asset","provider":"go.mondoo.com/cnquery/v9/providers/core"},"kind":{"name":"kind","type":"\u0007","is_mandatory":true,"title":"Kind of platform, for example:","desc":"api, baremetal, vm, vm-image, container, container-image, network, ...","provider":"go.mondoo.com/cnquery/v9/providers/core"},"labels":{"name":"labels","type":"\u001a\u0007\u0007","is_mandatory":true,"title":"Optional platform information","provider":"go.mondoo.com/cnquery/v9/providers/core"},"name":{"name":"name","type":"\u0007","is_mandatory":true,"title":"Human readable name of the asset","provider":"go.mondoo.com/cnquery/v9/providers/core"},"platform":{"name":"platform","type":"\u0007","is_mandatory":true,"title":"Platform for this asset (redhat, windows, k8s-pod)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"runtime":{"name":"runtime","type":"\u0007","is_mandatory":true,"title":"Runtime is the specific kind of the platform. Examples include:","desc":"docker-container, podman-container, aws-ec2-instance, ...","provider":"go.mondoo.com/cnquery/v9/providers/core"},"title":{"name":"title","type":"\u0007","is_mandatory":true,"title":"Human-readable title of the platform (e.g., \"Red Hat 8, Container\")","provider":"go.mondoo.com/cnquery/v9/providers/core"},"version":{"name":"version","type":"\u0007","is_mandatory":true,"title":"Version of the platform","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"General asset information","min_mondoo_version":"6.13.0","defaults":"name platform version","provider":"go.mondoo.com/cnquery/v9/providers/core"},"asset.eol":{"id":"asset.eol","name":"asset.eol","fields":{"date":{"name":"date","type":"\t","is_mandatory":true,"title":"End-of-Life date","provider":"go.mondoo.com/cnquery/v9/providers/core"},"docsUrl":{"name":"docsUrl","type":"\u0007","is_mandatory":true,"title":"Documentation URL","provider":"go.mondoo.com/cnquery/v9/providers/core"},"productUrl":{"name":"productUrl","type":"\u0007","is_mandatory":true,"title":"Product URL","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"Information about the assets platform end-of-life.","min_mondoo_version":"latest","defaults":"date","provider":"go.mondoo.com/cnquery/v9/providers/core"},"cpe":{"id":"cpe","name":"cpe","fields":{"edition":{"name":"edition","type":"\u0007","title":"Edition of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"language":{"name":"language","type":"\u0007","title":"Language of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"other":{"name":"other","type":"\u0007","title":"Other of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"part":{"name":"part","type":"\u0007","title":"Part of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"product":{"name":"product","type":"\u0007","title":"Product of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"swEdition":{"name":"swEdition","type":"\u0007","title":"Software edition of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"targetHw":{"name":"targetHw","type":"\u0007","title":"Target hardware of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"targetSw":{"name":"targetSw","type":"\u0007","title":"Target software of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"update":{"name":"update","type":"\u0007","title":"Update of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"uri":{"name":"uri","type":"\u0007","is_mandatory":true,"title":"URI binding of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"vendor":{"name":"vendor","type":"\u0007","title":"Vendor of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"},"version":{"name":"version","type":"\u0007","title":"Version of the CPE","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"init":{"args":[{"name":"uri","type":"\u0007"}]},"title":"Common Platform Enumeration (CPE) identifiers","min_mondoo_version":"latest","defaults":"uri","provider":"go.mondoo.com/cnquery/v9/providers/core"},"mondoo":{"id":"mondoo","name":"mondoo","fields":{"arch":{"name":"arch","type":"\u0007","title":"Architecture of this client (e.g., linux-amd64)","min_mondoo_version":"latest","provider":"go.mondoo.com/cnquery/v9/providers/core"},"build":{"name":"build","type":"\u0007","title":"Build of the client (e.g., production, development)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"capabilities":{"name":"capabilities","type":"\u0019\u0007","title":"Connection capabilities","provider":"go.mondoo.com/cnquery/v9/providers/core"},"jobEnvironment":{"name":"jobEnvironment","type":"\n","title":"Agent execution environment","provider":"go.mondoo.com/cnquery/v9/providers/core"},"version":{"name":"version","type":"\u0007","title":"Version of the client running on the asset","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"Contextual information about MQL runtime and environment","min_mondoo_version":"5.15.0","defaults":"version","provider":"go.mondoo.com/cnquery/v9/providers/core"},"parse":{"id":"parse","name":"parse","title":"Provides common parsers (json, ini, certs, etc)","min_mondoo_version":"5.15.0","provider":"go.mondoo.com/cnquery/v9/providers/core"},"regex":{"id":"regex","name":"regex","fields":{"creditCard":{"name":"creditCard","type":"\b","title":"Matches credit card numbers","provider":"go.mondoo.com/cnquery/v9/providers/core"},"email":{"name":"email","type":"\b","title":"Matches email addresses","provider":"go.mondoo.com/cnquery/v9/providers/core"},"emoji":{"name":"emoji","type":"\b","title":"Matches emojis","provider":"go.mondoo.com/cnquery/v9/providers/core"},"ipv4":{"name":"ipv4","type":"\b","title":"Matches IPv4 addresses","provider":"go.mondoo.com/cnquery/v9/providers/core"},"ipv6":{"name":"ipv6","type":"\b","title":"Matches IPv6 addresses","provider":"go.mondoo.com/cnquery/v9/providers/core"},"mac":{"name":"mac","type":"\b","title":"Matches MAC addresses","provider":"go.mondoo.com/cnquery/v9/providers/core"},"semver":{"name":"semver","type":"\b","title":"Matches semantic version numbers","provider":"go.mondoo.com/cnquery/v9/providers/core"},"url":{"name":"url","type":"\b","title":"Matches URL addresses (HTTP/HTTPS)","provider":"go.mondoo.com/cnquery/v9/providers/core"},"uuid":{"name":"uuid","type":"\b","title":"Matches hyphen-deliminated UUIDs","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"Built-in regular expression functions","min_mondoo_version":"5.15.0","provider":"go.mondoo.com/cnquery/v9/providers/core"},"time":{"id":"time","name":"time","fields":{"day":{"name":"day","type":"\t","title":"One day, used for durations","provider":"go.mondoo.com/cnquery/v9/providers/core"},"hour":{"name":"hour","type":"\t","title":"One hour, used for durations","provider":"go.mondoo.com/cnquery/v9/providers/core"},"minute":{"name":"minute","type":"\t","title":"One minute, used for durations","provider":"go.mondoo.com/cnquery/v9/providers/core"},"now":{"name":"now","type":"\t","title":"The current time on the local system","provider":"go.mondoo.com/cnquery/v9/providers/core"},"second":{"name":"second","type":"\t","title":"One second, used for durations","provider":"go.mondoo.com/cnquery/v9/providers/core"},"today":{"name":"today","type":"\t","title":"The current day starting at midnight","provider":"go.mondoo.com/cnquery/v9/providers/core"},"tomorrow":{"name":"tomorrow","type":"\t","title":"The next day starting at midnight","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"title":"Date and time functions","min_mondoo_version":"5.15.0","provider":"go.mondoo.com/cnquery/v9/providers/core"},"uuid":{"id":"uuid","name":"uuid","fields":{"urn":{"name":"urn","type":"\u0007","title":"URN returns the RFC 2141 URN form of uuid","provider":"go.mondoo.com/cnquery/v9/providers/core"},"value":{"name":"value","type":"\u0007","is_mandatory":true,"title":"Canonical string representation xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","provider":"go.mondoo.com/cnquery/v9/providers/core"},"variant":{"name":"variant","type":"\u0007","title":"Variant encoded in UUID","provider":"go.mondoo.com/cnquery/v9/providers/core"},"version":{"name":"version","type":"\u0005","title":"Version of UUID","provider":"go.mondoo.com/cnquery/v9/providers/core"}},"init":{"args":[{"name":"value","type":"\u0007"}]},"title":"UUIDs based on RFC 4122 and DCE 1.1","min_mondoo_version":"5.15.0","defaults":"value","provider":"go.mondoo.com/cnquery/v9/providers/core"}}} \ No newline at end of file diff --git a/providers/os/connection/container/image/docker.go b/providers/os/connection/container/image/docker.go index cbb35de3b3..eee866926d 100644 --- a/providers/os/connection/container/image/docker.go +++ b/providers/os/connection/container/image/docker.go @@ -67,20 +67,27 @@ func writeCompressedTarImage(img v1.Image, digest string) (*os.File, error) { } filename := f.Name() - ref, err := name.ParseReference(digest, name.WeakValidation) - if err != nil { + if err := WriteCompressedTarImageToFile(img, digest, f); err != nil { os.Remove(filename) return nil, err + + } + return f, nil +} + +func WriteCompressedTarImageToFile(img v1.Image, digest string, f *os.File) error { + ref, err := name.ParseReference(digest, name.WeakValidation) + if err != nil { + return err } err = tarball.Write(ref, img, f) if err != nil { - os.Remove(filename) - return nil, err + return err } // Rewind, to later read the complete file for uncompress f.Seek(0, io.SeekStart) - return f, nil + return nil } diff --git a/providers/os/connection/container/image/registry.go b/providers/os/connection/container/image/registry.go index 03956b627f..ce6fb9f463 100644 --- a/providers/os/connection/container/image/registry.go +++ b/providers/os/connection/container/image/registry.go @@ -6,7 +6,6 @@ package image import ( "crypto/tls" "fmt" - "io" "net" "net/http" "strings" @@ -74,14 +73,14 @@ func GetImageDescriptor(ref name.Reference, opts ...Option) (*remote.Descriptor, return remote.Get(ref, remote.WithAuth(o.auth)) } -func LoadImageFromRegistry(ref name.Reference, opts ...Option) (v1.Image, io.ReadCloser, error) { +func LoadImageFromRegistry(ref name.Reference, opts ...Option) (v1.Image, error) { o := &options{ insecure: false, } for _, option := range opts { if err := option(o); err != nil { - return nil, nil, err + return nil, err } } @@ -98,7 +97,7 @@ func LoadImageFromRegistry(ref name.Reference, opts ...Option) (v1.Image, io.Rea auth, err := kc.Resolve(ref.Context()) if err != nil { fmt.Printf("getting creds for %q: %v", ref, err) - return nil, nil, err + return nil, err } o.auth = auth } @@ -126,15 +125,7 @@ func LoadImageFromRegistry(ref name.Reference, opts ...Option) (v1.Image, io.Rea img, err := remote.Image(ref, remote.WithAuth(o.auth), remote.WithTransport(tr)) if err != nil { - return nil, nil, err - } - - // write image to disk (conmpressed, unflattened) - // Otherwise we can not later recognize it as a valid image - f, err := writeCompressedTarImage(img, ref.String()) - if err != nil { - return nil, nil, err + return nil, err } - - return img, f, nil + return img, nil } diff --git a/providers/os/connection/docker_container.go b/providers/os/connection/docker_container.go index 9966dfcbdd..e9e91a574d 100644 --- a/providers/os/connection/docker_container.go +++ b/providers/os/connection/docker_container.go @@ -7,13 +7,11 @@ import ( "context" "errors" "io" - "os" "strconv" "strings" "github.com/docker/docker/client" "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/rs/zerolog/log" "github.com/spf13/afero" "go.mondoo.com/cnquery/v10/providers-sdk/v1/inventory" @@ -166,7 +164,8 @@ func (c *DockerContainerConnection) RunCommand(command string) (*shared.Command, // this happens, when we try to run /bin/sh in a container, which does not have it if err == nil && res.ExitStatus == 126 { output := "" - b, err := io.ReadAll(res.Stdout) + var b []byte + b, err = io.ReadAll(res.Stdout) if err == nil { output = string(b) } @@ -183,34 +182,19 @@ func NewContainerRegistryImage(id uint32, conf *inventory.Config, asset *invento registryOpts := []image.Option{image.WithInsecure(conf.Insecure)} remoteOpts := auth.AuthOption(conf.Credentials) - for i := range remoteOpts { - registryOpts = append(registryOpts, remoteOpts[i]) - } + registryOpts = append(registryOpts, remoteOpts...) - var img v1.Image - var rc io.ReadCloser - loadedImage := false - if asset.Connections[0].Options != nil { - if _, ok := asset.Connections[0].Options[COMPRESSED_IMAGE]; ok { - // read image from disk - img, rc, err = image.LoadImageFromDisk(asset.Connections[0].Options[COMPRESSED_IMAGE]) - if err != nil { - return nil, err - } - loadedImage = true - } + img, err := image.LoadImageFromRegistry(ref, registryOpts...) + if err != nil { + return nil, err } - if !loadedImage { - img, rc, err = image.LoadImageFromRegistry(ref, registryOpts...) - if err != nil { - return nil, err - } - if asset.Connections[0].Options == nil { - asset.Connections[0].Options = map[string]string{} - } - osFile := rc.(*os.File) - filename := osFile.Name() - asset.Connections[0].Options[COMPRESSED_IMAGE] = filename + if asset.Connections[0].Options == nil { + asset.Connections[0].Options = map[string]string{} + } + + conn, err := NewTarConnectionForContainer(id, conf, asset, img) + if err != nil { + return nil, err } var identifier string @@ -219,10 +203,6 @@ func NewContainerRegistryImage(id uint32, conf *inventory.Config, asset *invento identifier = containerid.MondooContainerImageID(hash.String()) } - conn, err := NewWithReader(id, conf, asset, rc) - if err != nil { - return nil, err - } conn.PlatformIdentifier = identifier conn.Metadata.Name = containerid.ShortContainerImageID(hash.String()) diff --git a/providers/os/connection/tar.go b/providers/os/connection/tar.go index 2a329cdc99..4236563aad 100644 --- a/providers/os/connection/tar.go +++ b/providers/os/connection/tar.go @@ -9,6 +9,7 @@ import ( "errors" "io" "os" + "sync" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/mutate" @@ -24,17 +25,18 @@ import ( ) const ( - OPTION_FILE = "path" - FLATTENED_IMAGE = "flattened_path" - COMPRESSED_IMAGE = "compressed_path" + OPTION_FILE = "path" + FLATTENED_IMAGE = "flattened_path" ) var _ shared.Connection = (*TarConnection)(nil) type TarConnection struct { - id uint32 - asset *inventory.Asset - conf *inventory.Config + id uint32 + asset *inventory.Asset + conf *inventory.Config + fetchFn func() (string, error) + fetchOnce sync.Once Fs *provider_tar.FS CloseFN func() @@ -83,7 +85,24 @@ func (p *TarConnection) RunCommand(command string) (*shared.Command, error) { return &res, nil } +func (p *TarConnection) EnsureLoaded() { + if p.fetchFn != nil { + p.fetchOnce.Do(func() { + f, err := p.fetchFn() + if err != nil { + log.Error().Err(err).Msg("tar> could not fetch tar file") + return + } + if err := p.LoadFile(f); err != nil { + log.Error().Err(err).Msg("tar> could not load tar file") + return + } + }) + } +} + func (p *TarConnection) FileSystem() afero.Fs { + p.EnsureLoaded() return p.Fs } @@ -155,10 +174,42 @@ func (c *TarConnection) Runtime() string { return c.PlatformRuntime } +func NewTarConnectionForContainer(id uint32, conf *inventory.Config, asset *inventory.Asset, img v1.Image) (*TarConnection, error) { + f, err := cache.RandomFile() + if err != nil { + return nil, err + } + + return &TarConnection{ + id: id, + asset: asset, + Fs: provider_tar.NewFs(""), + fetchFn: func() (string, error) { + err = cache.StreamToTmpFile(mutate.Extract(img), f) + if err != nil { + os.Remove(f.Name()) + return "", err + } + log.Debug().Msg("tar> extracted image to temporary file") + asset.Connections[0].Options[FLATTENED_IMAGE] = f.Name() + return f.Name(), nil + }, + CloseFN: func() { + log.Debug().Str("tar", f.Name()).Msg("tar> remove temporary tar file on connection close") + os.Remove(f.Name()) + }, + PlatformKind: conf.Type, + PlatformRuntime: conf.Runtime, + conf: conf, + }, nil +} + +// TODO: this one is used by plain tar connection func NewTarConnection(id uint32, conf *inventory.Config, asset *inventory.Asset) (*TarConnection, error) { return NewWithClose(id, conf, asset, nil) } +// Used with docker snapshots // NewWithReader provides a tar provider from a container image stream func NewWithReader(id uint32, conf *inventory.Config, asset *inventory.Asset, rc io.ReadCloser) (*TarConnection, error) { filename := ""