🧹 refactor os provider connections (#3415)

* 🧹 refactor os connections * 🧹 refactor tar connection * fix build Signed-off-by: Ivan Milchev <[email protected]> * fix test panic Signed-off-by: Ivan Milchev <[email protected]> * add tests for container connections * 🧹 simplify os connection names * 🧹 simplify docker connection * 🧹 more improvements --------- Signed-off-by: Ivan Milchev <[email protected]> Co-authored-by: Ivan Milchev <[email protected]>
mondoohq · Feb 26, 2024 · 97352b5 · 97352b5
1 parent 39df8dc
commit 97352b5
Show file tree

Hide file tree

Showing 35 changed files with 1,164 additions and 837 deletions.
diff --git a/providers/os/connection/container/image/docker.go b/providers/os/connection/container/image/docker.go
@@ -12,7 +12,6 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
-	"go.mondoo.com/cnquery/v10/providers/os/connection/container/cache"
 )
 
 type ShaReference struct {
@@ -39,40 +38,17 @@ func (r ShaReference) Scope(scope string) string {
 	return ""
 }
 
-func LoadImageFromDockerEngine(sha string, disableBuffer bool) (v1.Image, io.ReadCloser, error) {
+func LoadImageFromDockerEngine(sha string, disableBuffer bool) (v1.Image, error) {
 	opts := []daemon.Option{}
 	if disableBuffer {
 		opts = append(opts, daemon.WithUnbufferedOpener())
 	}
 	img, err := daemon.Image(&ShaReference{SHA: strings.Replace(sha, "sha256:", "", -1)}, opts...)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	// write image to disk (conmpressed, unflattened)
-	// Otherwise we can not later recognize it as a valid image
-	f, err := writeCompressedTarImage(img, sha)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return img, f, nil
-}
-
-// writeCompressedTarImage writes image including the metradata unflattened to disk
-func writeCompressedTarImage(img v1.Image, digest string) (*os.File, error) {
-	f, err := cache.RandomFile()
 	if err != nil {
 		return nil, err
 	}
-	filename := f.Name()
 
-	if err := WriteCompressedTarImageToFile(img, digest, f); err != nil {
-		os.Remove(filename)
-		return nil, err
-
-	}
-	return f, nil
+	return img, nil
 }
 
 func WriteCompressedTarImageToFile(img v1.Image, digest string, f *os.File) error {

diff --git a/providers/os/connection/container/image_connection.go b/providers/os/connection/container/image_connection.go
@@ -0,0 +1,169 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package container
+
+import (
+	"errors"
+	"os"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/rs/zerolog/log"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/inventory"
+	"go.mondoo.com/cnquery/v10/providers/os/connection/container/auth"
+	"go.mondoo.com/cnquery/v10/providers/os/connection/container/image"
+	"go.mondoo.com/cnquery/v10/providers/os/connection/tar"
+	"go.mondoo.com/cnquery/v10/providers/os/id/containerid"
+)
+
+// NewImageConnection uses a container image reference as input and creates a tar connection
+func NewImageConnection(id uint32, conf *inventory.Config, asset *inventory.Asset, img v1.Image) (*tar.Connection, error) {
+	f, err := tar.RandomFile()
+	if err != nil {
+		return nil, err
+	}
+
+	conf.Options[tar.OPTION_FILE] = f.Name()
+
+	return tar.NewConnection(id, conf, asset,
+		tar.WithFetchFn(func() (string, error) {
+			err = tar.StreamToTmpFile(mutate.Extract(img), f)
+			if err != nil {
+				_ = os.Remove(f.Name())
+				return "", err
+			}
+			log.Debug().Msg("tar> extracted image to temporary file")
+			return f.Name(), nil
+		}),
+		tar.WithCloseFn(func() {
+			log.Debug().Str("tar", f.Name()).Msg("tar> remove temporary tar file on connection close")
+			_ = os.Remove(f.Name())
+		}),
+	)
+}
+
+// NewRegistryImage loads a container image from a remote registry
+func NewRegistryImage(id uint32, conf *inventory.Config, asset *inventory.Asset) (*tar.Connection, error) {
+	ref, err := name.ParseReference(conf.Host, name.WeakValidation)
+	if err != nil {
+		return nil, errors.New("invalid container registry reference: " + conf.Host)
+	}
+	log.Debug().Str("ref", ref.Name()).Msg("found valid container registry reference")
+
+	registryOpts := []image.Option{image.WithInsecure(conf.Insecure)}
+	remoteOpts := auth.AuthOption(conf.Credentials)
+	registryOpts = append(registryOpts, remoteOpts...)
+
+	img, err := image.LoadImageFromRegistry(ref, registryOpts...)
+	if err != nil {
+		return nil, err
+	}
+	if conf.Options == nil {
+		conf.Options = map[string]string{}
+	}
+
+	conn, err := NewImageConnection(id, conf, asset, img)
+	if err != nil {
+		return nil, err
+	}
+
+	var identifier string
+	hash, err := img.Digest()
+	if err == nil {
+		identifier = containerid.MondooContainerImageID(hash.String())
+	}
+
+	conn.PlatformIdentifier = identifier
+	conn.Metadata.Name = containerid.ShortContainerImageID(hash.String())
+
+	repoName := ref.Context().Name()
+	imgDigest := hash.String()
+	containerAssetName := repoName + "@" + containerid.ShortContainerImageID(imgDigest)
+	if asset.Name == "" {
+		asset.Name = containerAssetName
+	}
+	if len(asset.PlatformIds) == 0 {
+		asset.PlatformIds = []string{identifier}
+	} else {
+		asset.PlatformIds = append(asset.PlatformIds, identifier)
+	}
+
+	// set the platform architecture using the image configuration
+	imgConfig, err := img.ConfigFile()
+	if err == nil {
+		conn.PlatformArchitecture = imgConfig.Architecture
+	}
+
+	labels := map[string]string{}
+	labels["docker.io/digests"] = ref.String()
+
+	manifest, err := img.Manifest()
+	if err == nil {
+		labels["mondoo.com/image-id"] = manifest.Config.Digest.String()
+	}
+
+	conn.Metadata.Labels = labels
+	asset.Labels = labels
+
+	return conn, err
+}
+
+func NewFromTar(id uint32, conf *inventory.Config, asset *inventory.Asset) (*tar.Connection, error) {
+	if conf == nil || len(conf.Options[tar.OPTION_FILE]) == 0 {
+		return nil, errors.New("tar provider requires a valid tar file")
+	}
+
+	if conf.Options == nil {
+		conf.Options = map[string]string{}
+	}
+
+	filename := conf.Options[tar.OPTION_FILE]
+	var identifier string
+
+	// try to determine if the tar is a container image
+	img, iErr := tarball.ImageFromPath(filename, nil)
+	if iErr != nil {
+		return nil, iErr
+	}
+
+	hash, err := img.Digest()
+	if err != nil {
+		return nil, err
+	}
+	identifier = containerid.MondooContainerImageID(hash.String())
+
+	// we need to extract the image from the tar file and create a new tar connection
+	imageFilename := ""
+
+	f, err := tar.RandomFile()
+	if err != nil {
+		return nil, err
+	}
+	imageFilename = f.Name()
+	conf.Options[tar.OPTION_FILE] = imageFilename
+
+	c, err := tar.NewConnection(id, conf, asset,
+		tar.WithFetchFn(func() (string, error) {
+			err = tar.StreamToTmpFile(mutate.Extract(img), f)
+			if err != nil {
+				_ = os.Remove(imageFilename)
+				return imageFilename, err
+			}
+			return imageFilename, nil
+		}),
+		tar.WithCloseFn(func() {
+			// remove temporary file on stream close
+			log.Debug().Str("tar", imageFilename).Msg("tar> remove temporary flattened image file on connection close")
+			_ = os.Remove(imageFilename)
+		}),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	c.PlatformIdentifier = identifier
+	return c, nil
+}