diff --git a/providers/os/connection/container/cache/stream.go b/providers/os/connection/container/cache/stream.go index c48d298616..139e5d8e2c 100644 --- a/providers/os/connection/container/cache/stream.go +++ b/providers/os/connection/container/cache/stream.go @@ -5,12 +5,11 @@ package cache import ( "io" - "io/ioutil" "os" ) func RandomFile() (*os.File, error) { - return ioutil.TempFile("", "mondoo.inspection") + return os.CreateTemp("", "mondoo.inspection") } // This streams a binary stream into a file. The user of this method diff --git a/providers/os/connection/container/image/docker.go b/providers/os/connection/container/image/docker.go index 6094ba5c34..514ed8ca2f 100644 --- a/providers/os/connection/container/image/docker.go +++ b/providers/os/connection/container/image/docker.go @@ -5,12 +5,14 @@ package image import ( "io" + "os" "strings" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/daemon" - "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "go.mondoo.com/cnquery/providers/os/connection/container/cache" ) type ShaReference struct { @@ -46,5 +48,39 @@ func LoadImageFromDockerEngine(sha string, disableBuffer bool) (v1.Image, io.Rea if err != nil { return nil, nil, err } - return img, mutate.Extract(img), nil + + // write image to disk (conmpressed, unflattened) + // Otherwise we can not later recognize it as a valid image + f, err := writeCompressedTarImage(img, sha) + if err != nil { + return nil, nil, err + } + + return img, f, nil +} + +// writeCompressedTarImage writes image including the metradata unflattened to disk +func writeCompressedTarImage(img v1.Image, digest string) (*os.File, error) { + f, err := cache.RandomFile() + if err != nil { + return nil, err + } + filename := f.Name() + + ref, err := name.ParseReference(digest, name.WeakValidation) + if err != nil { + os.Remove(filename) + return nil, err + } + + err = tarball.Write(ref, img, f) + if err != nil { + os.Remove(filename) + return nil, err + } + + // Rewindo, to later read the complete file for uncompress + f.Seek(0, io.SeekStart) + + return f, nil } diff --git a/providers/os/connection/container/image/registry.go b/providers/os/connection/container/image/registry.go index 5ac08c5e25..03956b627f 100644 --- a/providers/os/connection/container/image/registry.go +++ b/providers/os/connection/container/image/registry.go @@ -16,7 +16,6 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" ) @@ -129,5 +128,13 @@ func LoadImageFromRegistry(ref name.Reference, opts ...Option) (v1.Image, io.Rea if err != nil { return nil, nil, err } - return img, mutate.Extract(img), nil + + // write image to disk (conmpressed, unflattened) + // Otherwise we can not later recognize it as a valid image + f, err := writeCompressedTarImage(img, ref.String()) + if err != nil { + return nil, nil, err + } + + return img, f, nil } diff --git a/providers/os/connection/docker_container.go b/providers/os/connection/docker_container.go index b2b8cf779f..f9d5a6736f 100644 --- a/providers/os/connection/docker_container.go +++ b/providers/os/connection/docker_container.go @@ -214,6 +214,17 @@ func NewContainerRegistryImage(id uint32, conf *inventory.Config, asset *invento conn.PlatformArchitecture = imgConfig.Architecture } + labels := map[string]string{} + labels["docker.io/digests"] = ref.String() + + manifest, err := img.Manifest() + if err == nil { + labels["mondoo.com/image-id"] = manifest.Config.Digest.String() + } + + conn.Metadata.Labels = labels + asset.Labels = labels + return conn, err } log.Debug().Str("image", conf.Host).Msg("Could not detect a valid repository url") diff --git a/providers/os/connection/tar.go b/providers/os/connection/tar.go index 9373eca028..1e59749f2a 100644 --- a/providers/os/connection/tar.go +++ b/providers/os/connection/tar.go @@ -10,6 +10,7 @@ import ( "io" "os" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/rs/zerolog/log" @@ -157,19 +158,24 @@ func NewTarConnection(id uint32, conf *inventory.Config, asset *inventory.Asset) // NewWithReader provides a tar provider from a container image stream func NewWithReader(id uint32, conf *inventory.Config, asset *inventory.Asset, rc io.ReadCloser, close func()) (*TarConnection, error) { - // we cache the flattened image locally - f, err := cache.RandomFile() - if err != nil { - return nil, err - } + filename := "" + if x, ok := rc.(*os.File); ok { + filename = x.Name() + } else { + // cache file locally + f, err := cache.RandomFile() + if err != nil { + return nil, err + } - // we return a pure tar image - filename := f.Name() + // we return a pure tar image + filename = f.Name() - err = cache.StreamToTmpFile(rc, f) - if err != nil { - os.Remove(filename) - return nil, err + err = cache.StreamToTmpFile(rc, f) + if err != nil { + os.Remove(filename) + return nil, err + } } return NewWithClose(id, &inventory.Config{ @@ -200,11 +206,18 @@ func NewWithClose(id uint32, conf *inventory.Config, asset *inventory.Asset, clo return nil, err } identifier = containerid.MondooContainerImageID(hash.String()) - // if it is a container image, we need to transform the tar first, so that all layers are flattened - c, err := NewWithReader(id, conf, asset, mutate.Extract(img), closeFn) + + // we cache the flattened image locally + c, err := newWithFlattenedImage(id, conf, asset, &img) if err != nil { return nil, err } + + // remove unflattened image file, we now have a flattened image + if closeFn != nil { + closeFn() + } + c.PlatformIdentifier = identifier return c, nil } else { @@ -233,3 +246,41 @@ func NewWithClose(id uint32, conf *inventory.Config, asset *inventory.Asset, clo return c, nil } } + +func newWithFlattenedImage(id uint32, conf *inventory.Config, asset *inventory.Asset, img *v1.Image) (*TarConnection, error) { + f, err := cache.RandomFile() + if err != nil { + return nil, err + } + imageFilename := f.Name() + err = cache.StreamToTmpFile(mutate.Extract(*img), f) + if err != nil { + os.Remove(imageFilename) + return nil, err + } + + c := &TarConnection{ + id: id, + asset: asset, + Fs: provider_tar.NewFs(imageFilename), + CloseFN: func() { + // remove temporary file on stream close + os.Remove(imageFilename) + }, + PlatformKind: conf.Backend, + PlatformRuntime: conf.Runtime, + conf: &inventory.Config{ + Options: map[string]string{ + OPTION_FILE: imageFilename, + }, + }, + } + + err = c.LoadFile(imageFilename) + if err != nil { + log.Error().Err(err).Str("tar", imageFilename).Msg("tar> could not load tar file") + return nil, err + } + + return c, nil +} diff --git a/providers/os/provider/provider.go b/providers/os/provider/provider.go index 7b16d67442..9614ab0c43 100644 --- a/providers/os/provider/provider.go +++ b/providers/os/provider/provider.go @@ -218,6 +218,12 @@ func (s *Service) Connect(req *plugin.ConnectReq, callback plugin.ProviderCallba // It is not necessary to implement this method. // If you want to do some cleanup, you can do it here. func (s *Service) Shutdown(req *plugin.ShutdownReq) (*plugin.ShutdownRes, error) { + for i := range s.runtimes { + runtime := s.runtimes[i] + if x, ok := runtime.Connection.(*connection.TarConnection); ok { + x.CloseFN() + } + } return &plugin.ShutdownRes{}, nil } diff --git a/providers/os/resources/container.go b/providers/os/resources/container.go index 3a694b33f6..668b562806 100644 --- a/providers/os/resources/container.go +++ b/providers/os/resources/container.go @@ -7,14 +7,15 @@ import ( "github.com/google/go-containerregistry/pkg/name" "go.mondoo.com/cnquery/llx" "go.mondoo.com/cnquery/providers-sdk/v1/plugin" + "go.mondoo.com/cnquery/providers/os/connection" ) func initContainerImage(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[string]*llx.RawData, plugin.Resource, error) { - raw, ok := args["reference"] - if !ok || len(args) != 1 { + if len(args) > 1 { return args, nil, nil } - reference := raw.Value.(string) + conn := runtime.Connection.(*connection.TarConnection) + reference := conn.Metadata.Labels["docker.io/digests"] ref, err := name.ParseReference(reference) if err != nil {