diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go index d1948e7d5e..3e0b65d275 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go @@ -2,10 +2,11 @@ package docker_kurtosis_backend import ( "context" - "github.com/sirupsen/logrus" "io" "sync" + "github.com/sirupsen/logrus" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector" @@ -23,6 +24,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/engine" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" @@ -78,11 +80,23 @@ func NewDockerKurtosisBackend( } } -func (backend *DockerKurtosisBackend) FetchImage(ctx context.Context, image string) (bool, error) { - pulledFromRemote, err := backend.dockerManager.FetchImage(ctx, image) +func (backend *DockerKurtosisBackend) FetchImage(ctx context.Context, image string, download_mode image_download_mode.ImageDownloadMode) (bool, error) { + var err error + var pulledFromRemote bool = false + + switch image_pulling := download_mode; image_pulling { + case image_download_mode.Always: + err = backend.dockerManager.FetchLatestImage(ctx, image) + case image_download_mode.Missing: + pulledFromRemote, err = backend.dockerManager.FetchImageMissing(ctx, image) + case image_download_mode.Never: + return false, stacktrace.NewError("Undefined image pulling mode: '%v'", image_pulling) + } + if err != nil { return false, stacktrace.Propagate(err, "An error occurred fetching image from kurtosis backend") } + return pulledFromRemote, nil } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go index 19e91facbe..3a25807973 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go @@ -167,7 +167,7 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( labelStrs, ).WithRestartPolicy(docker_manager.RestartOnFailure).Build() - if _, err = backend.dockerManager.FetchImage(ctx, image); err != nil { + if _, err = backend.dockerManager.FetchImageMissing(ctx, image); err != nil { logrus.Warnf("Failed to pull the latest version of API container image '%v'; you may be running an out-of-date version", image) } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 457b8477f2..87de9096d6 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -527,7 +527,7 @@ func (manager *DockerManager) CreateAndStartContainer( case Always: err = manager.FetchLatestImage(ctx, dockerImage) case Missing: - _, err = manager.FetchImage(ctx, dockerImage) + _, err = manager.FetchImageMissing(ctx, dockerImage) case Never: return "", nil, stacktrace.NewError("Undefined image pulling mode: '%v'", image_pulling) } @@ -1189,10 +1189,10 @@ func (manager *DockerManager) GetContainersByLabels(ctx context.Context, labels return result, nil } -// [FetchImage] uses the local [dockerImage] if it's available. +// [FetchImageMissing] uses the local [dockerImage] if it's available. // If unavailable, will attempt to fetch the latest image. // Returns error if local [dockerImage] is unavailable and pulling image fails. -func (manager *DockerManager) FetchImage(ctx context.Context, dockerImage string) (bool, error) { +func (manager *DockerManager) FetchImageMissing(ctx context.Context, dockerImage string) (bool, error) { // if the image name doesn't have version information we concatenate `:latest` // this behavior is similar to CreateAndStartContainer above // this allows us to be deterministic in our behaviour diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go index c997c9c845..180586128e 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go @@ -2,6 +2,8 @@ package kubernetes_kurtosis_backend import ( "context" + "io" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/user_services_functions" @@ -13,12 +15,12 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/engine" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" - "io" ) const ( @@ -117,7 +119,7 @@ func NewKubernetesKurtosisBackend( } } -func (backend *KubernetesKurtosisBackend) FetchImage(ctx context.Context, image string) (bool, error) { +func (backend *KubernetesKurtosisBackend) FetchImage(ctx context.Context, image string, download_mode image_download_mode.ImageDownloadMode) (bool, error) { logrus.Warnf("FetchImage isn't implemented for Kubernetes yet") return false, nil } diff --git a/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go index 4ff0083d70..c0454f9af0 100644 --- a/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go @@ -2,18 +2,20 @@ package metrics_reporting import ( "context" + "io" + "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/api_container" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/compute_resources" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/engine" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/stacktrace" - "io" - "time" ) // TODO CALL THE METRICS LIBRARY EVENT-REGISTRATION FUNCTIONS HERE!!!! @@ -25,8 +27,8 @@ func NewMetricsReportingKurtosisBackend(underlying backend_interface.KurtosisBac return &MetricsReportingKurtosisBackend{underlying: underlying} } -func (backend *MetricsReportingKurtosisBackend) FetchImage(ctx context.Context, image string) (bool, error) { - pulledFromRemote, err := backend.underlying.FetchImage(ctx, image) +func (backend *MetricsReportingKurtosisBackend) FetchImage(ctx context.Context, image string, download_mode image_download_mode.ImageDownloadMode) (bool, error) { + pulledFromRemote, err := backend.underlying.FetchImage(ctx, image, download_mode) if err != nil { return false, stacktrace.Propagate(err, "An error occurred pulling image '%v'", image) } diff --git a/container-engine-lib/lib/backend_interface/kurtosis_backend.go b/container-engine-lib/lib/backend_interface/kurtosis_backend.go index f2dcd28c45..49accef3d8 100644 --- a/container-engine-lib/lib/backend_interface/kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/kurtosis_backend.go @@ -2,16 +2,18 @@ package backend_interface import ( "context" + "io" + "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/api_container" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/compute_resources" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/engine" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" - "io" - "time" ) // TODO This mega-backend should really have its individual functionalities split up into @@ -29,7 +31,7 @@ type KurtosisBackend interface { // FetchImage always attempts to retrieve the latest image. // If retrieving the latest [dockerImage] fails, the local image will be used. // Returns True is it was retrieved from cloud or False if it's a local image - FetchImage(ctx context.Context, image string) (bool, error) + FetchImage(ctx context.Context, image string, download_mode image_download_mode.ImageDownloadMode) (bool, error) PruneUnusedImages(ctx context.Context) ([]string, error) diff --git a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go index 6b3e058bd4..821ab3b4d8 100644 --- a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.29.0. DO NOT EDIT. +// Code generated by mockery v2.34.0. DO NOT EDIT. package backend_interface @@ -14,6 +14,8 @@ import ( exec_result "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/exec_result" + image_download_mode "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" + io "io" logs_aggregator "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" @@ -800,23 +802,23 @@ func (_c *MockKurtosisBackend_DumpKurtosis_Call) RunAndReturn(run func(context.C return _c } -// FetchImage provides a mock function with given fields: ctx, image -func (_m *MockKurtosisBackend) FetchImage(ctx context.Context, image string) (bool, error) { - ret := _m.Called(ctx, image) +// FetchImage provides a mock function with given fields: ctx, image, download_mode +func (_m *MockKurtosisBackend) FetchImage(ctx context.Context, image string, download_mode image_download_mode.ImageDownloadMode) (bool, error) { + ret := _m.Called(ctx, image, download_mode) var r0 bool var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string) (bool, error)); ok { - return rf(ctx, image) + if rf, ok := ret.Get(0).(func(context.Context, string, image_download_mode.ImageDownloadMode) (bool, error)); ok { + return rf(ctx, image, download_mode) } - if rf, ok := ret.Get(0).(func(context.Context, string) bool); ok { - r0 = rf(ctx, image) + if rf, ok := ret.Get(0).(func(context.Context, string, image_download_mode.ImageDownloadMode) bool); ok { + r0 = rf(ctx, image, download_mode) } else { r0 = ret.Get(0).(bool) } - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, image) + if rf, ok := ret.Get(1).(func(context.Context, string, image_download_mode.ImageDownloadMode) error); ok { + r1 = rf(ctx, image, download_mode) } else { r1 = ret.Error(1) } @@ -832,13 +834,14 @@ type MockKurtosisBackend_FetchImage_Call struct { // FetchImage is a helper method to define mock.On call // - ctx context.Context // - image string -func (_e *MockKurtosisBackend_Expecter) FetchImage(ctx interface{}, image interface{}) *MockKurtosisBackend_FetchImage_Call { - return &MockKurtosisBackend_FetchImage_Call{Call: _e.mock.On("FetchImage", ctx, image)} +// - download_mode image_download_mode.ImageDownloadMode +func (_e *MockKurtosisBackend_Expecter) FetchImage(ctx interface{}, image interface{}, download_mode interface{}) *MockKurtosisBackend_FetchImage_Call { + return &MockKurtosisBackend_FetchImage_Call{Call: _e.mock.On("FetchImage", ctx, image, download_mode)} } -func (_c *MockKurtosisBackend_FetchImage_Call) Run(run func(ctx context.Context, image string)) *MockKurtosisBackend_FetchImage_Call { +func (_c *MockKurtosisBackend_FetchImage_Call) Run(run func(ctx context.Context, image string, download_mode image_download_mode.ImageDownloadMode)) *MockKurtosisBackend_FetchImage_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string)) + run(args[0].(context.Context), args[1].(string), args[2].(image_download_mode.ImageDownloadMode)) }) return _c } @@ -848,7 +851,7 @@ func (_c *MockKurtosisBackend_FetchImage_Call) Return(_a0 bool, _a1 error) *Mock return _c } -func (_c *MockKurtosisBackend_FetchImage_Call) RunAndReturn(run func(context.Context, string) (bool, error)) *MockKurtosisBackend_FetchImage_Call { +func (_c *MockKurtosisBackend_FetchImage_Call) RunAndReturn(run func(context.Context, string, image_download_mode.ImageDownloadMode) (bool, error)) *MockKurtosisBackend_FetchImage_Call { _c.Call.Return(run) return _c } @@ -2149,13 +2152,12 @@ func (_c *MockKurtosisBackend_UpdateEnclave_Call) RunAndReturn(run func(context. return _c } -type mockConstructorTestingTNewMockKurtosisBackend interface { +// NewMockKurtosisBackend creates a new instance of MockKurtosisBackend. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockKurtosisBackend(t interface { mock.TestingT Cleanup(func()) -} - -// NewMockKurtosisBackend creates a new instance of MockKurtosisBackend. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMockKurtosisBackend(t mockConstructorTestingTNewMockKurtosisBackend) *MockKurtosisBackend { +}) *MockKurtosisBackend { mock := &MockKurtosisBackend{} mock.Mock.Test(t) diff --git a/container-engine-lib/lib/backend_interface/objects/image_download_mode/image_download_mode.go b/container-engine-lib/lib/backend_interface/objects/image_download_mode/image_download_mode.go new file mode 100644 index 0000000000..5dffcb9a04 --- /dev/null +++ b/container-engine-lib/lib/backend_interface/objects/image_download_mode/image_download_mode.go @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2021 - present Kurtosis Technologies Inc. + * All Rights Reserved. + */ + +package image_download_mode + +const ( + Never ImageDownloadMode = "never" + Always = "always" + Missing = "missing" +) + +type ImageDownloadMode string diff --git a/core/server/api_container/server/startosis_engine/startosis_validator/docker_images_validator.go b/core/server/api_container/server/startosis_engine/startosis_validator/docker_images_validator.go index 2e11514a0f..9d563c9dc9 100644 --- a/core/server/api_container/server/startosis_engine/startosis_validator/docker_images_validator.go +++ b/core/server/api_container/server/startosis_engine/startosis_validator/docker_images_validator.go @@ -2,10 +2,12 @@ package startosis_validator import ( "context" + "sync" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" "github.com/sirupsen/logrus" - "sync" ) const maxNumberOfConcurrentDownloads = int64(4) @@ -61,7 +63,7 @@ func fetchImageFromBackend(ctx context.Context, wg *sync.WaitGroup, imageCurrent }() logrus.Debugf("Starting the download of image: '%s'", imageName) - imagePulledFromRemote, err := (*backend).FetchImage(ctx, imageName) + imagePulledFromRemote, err := (*backend).FetchImage(ctx, imageName, image_download_mode.Missing) if err != nil { logrus.Warnf("Container image '%s' download failed. Error was: '%s'", imageName, err.Error()) pullErrors <- startosis_errors.WrapWithValidationError(err, "Failed fetching the required image '%v'.", imageName)