feat: regular replace package dependency and replace package with no-…

…main-branch (#1481)

## Description:
regular replace package dependency and replace package with
no-main-branch

## Is this change user-facing?
YES

## References (if applicable):
Fix #770 
This is part of the [Forked cleanup package
project](https://www.notion.so/kurtosistech/Forked-Package-Cleanup-16d86c4e274547b28496f17154bf3d62)

IMPORTANT: replace with a local package has not been implemented yet,
and it will be released separately

api/golang/core/lib/enclaves/enclave_context.go

@@ -125,7 +125,7 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
 ) (chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine, context.CancelFunc, error) {
 	serializedParams, err := maybeParseYaml(runConfig.SerializedParams)
 	if err != nil {
-		return nil, nil, stacktrace.Propagate(err, "An error occured when parsing YAML args for package '%v'", serializedParams)
+		return nil, nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams)
 	}
 	executionStartedSuccessfully := false
 	ctxWithCancel, cancelCtxFunc := context.WithCancel(ctx)
@@ -193,7 +193,7 @@ func (enclaveCtx *EnclaveContext) RunStarlarkRemotePackage(
 ) (chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine, context.CancelFunc, error) {
 	serializedParams, err := maybeParseYaml(runConfig.SerializedParams)
 	if err != nil {
-		return nil, nil, stacktrace.Propagate(err, "An error occured when parsing YAML args for remote package '%v'", serializedParams)
+		return nil, nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for remote package '%v'", serializedParams)
 	}
 	executionStartedSuccessfully := false
 	ctxWithCancel, cancelCtxFunc := context.WithCancel(ctx)

api/protobuf/core/api_container_service.proto

@@ -47,7 +47,7 @@ service ApiContainerService {
   rpc ListFilesArtifactNamesAndUuids(google.protobuf.Empty) returns (ListFilesArtifactNamesAndUuidsResponse) {}
 
   rpc InspectFilesArtifactContents(InspectFilesArtifactContentsRequest) returns (InspectFilesArtifactContentsResponse) {}
-  
+
   // User services port forwarding
   rpc ConnectServices(ConnectServicesArgs) returns (ConnectServicesResponse) {};
 

cli/cli/commands/run/run.go

@@ -490,11 +490,11 @@ func executePackage(
 ) (<-chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine, context.CancelFunc, error) {
 	// we get the absolute path so that the logs make more sense
 	absolutePackagePath, err := filepath.Abs(packagePath)
-	logrus.Infof("Executing Starlark package at '%v' as the passed argument '%v' looks like a directory", absolutePackagePath, packagePath)
-
 	if err != nil {
 		return nil, nil, stacktrace.Propagate(err, "An error occurred while getting the absolute path for '%v'", packagePath)
 	}
+	logrus.Infof("Executing Starlark package at '%v' as the passed argument '%v' looks like a directory", absolutePackagePath, packagePath)
+
 	return enclaveCtx.RunStarlarkPackage(ctx, packagePath, runConfig)
 }
 

core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go

@@ -6,6 +6,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_helper"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
 	"github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages"
+	"github.com/sirupsen/logrus"
 	"go.starlark.net/starlark"
 	"go.starlark.net/starlarkstruct"
 )
@@ -36,6 +37,7 @@ func NewImportModule(
 	recursiveInterpret func(moduleId string, scriptContent string) (starlark.StringDict, *startosis_errors.InterpretationError),
 	packageContentProvider startosis_packages.PackageContentProvider,
 	moduleGlobalCache map[string]*startosis_packages.ModuleCacheEntry,
+	packageReplaceOptions map[string]string,
 ) *kurtosis_helper.KurtosisHelper {
 	return &kurtosis_helper.KurtosisHelper{
 		KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{
@@ -56,6 +58,7 @@ func NewImportModule(
 			packageContentProvider: packageContentProvider,
 			recursiveInterpret:     recursiveInterpret,
 			moduleGlobalCache:      moduleGlobalCache,
+			packageReplaceOptions:  packageReplaceOptions,
 		},
 	}
 }
@@ -64,56 +67,58 @@ type importModuleCapabilities struct {
 	packageContentProvider startosis_packages.PackageContentProvider
 	recursiveInterpret     func(moduleId string, scriptContent string) (starlark.StringDict, *startosis_errors.InterpretationError)
 	moduleGlobalCache      map[string]*startosis_packages.ModuleCacheEntry
+	packageReplaceOptions  map[string]string
 }
 
 func (builtin *importModuleCapabilities) Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
-	moduleInPackageStarlarkStr, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ModuleFileArgName)
+	moduleLocatorArgValue, err := builtin_argument.ExtractArgumentValue[starlark.String](arguments, ModuleFileArgName)
 	if err != nil {
 		return nil, explicitInterpretationError(err)
 	}
-	moduleInPackage := moduleInPackageStarlarkStr.GoString()
-	moduleInPackage, relativePathParsingInterpretationErr := builtin.packageContentProvider.GetAbsoluteLocatorForRelativeModuleLocator(locatorOfModuleInWhichThisBuiltInIsBeingCalled, moduleInPackage)
+	relativeOrAbsoluteModuleLocator := moduleLocatorArgValue.GoString()
+	absoluteModuleLocator, relativePathParsingInterpretationErr := builtin.packageContentProvider.GetAbsoluteLocatorForRelativeLocator(locatorOfModuleInWhichThisBuiltInIsBeingCalled, relativeOrAbsoluteModuleLocator, builtin.packageReplaceOptions)
 	if relativePathParsingInterpretationErr != nil {
 		return nil, relativePathParsingInterpretationErr
 	}
+	logrus.Debugf("importing module from absolute locator '%s'", absoluteModuleLocator)
 
 	var loadInProgress *startosis_packages.ModuleCacheEntry
-	cacheEntry, found := builtin.moduleGlobalCache[moduleInPackage]
+	cacheEntry, found := builtin.moduleGlobalCache[absoluteModuleLocator]
 	if found && cacheEntry == loadInProgress {
 		return nil, startosis_errors.NewInterpretationError("There's a cycle in the import_module calls")
 	}
 	if found {
 		return cacheEntry.GetModule(), cacheEntry.GetError()
 	}
 
-	builtin.moduleGlobalCache[moduleInPackage] = loadInProgress
+	builtin.moduleGlobalCache[absoluteModuleLocator] = loadInProgress
 	shouldUnsetLoadInProgress := true
 	defer func() {
 		if shouldUnsetLoadInProgress {
-			delete(builtin.moduleGlobalCache, moduleInPackage)
+			delete(builtin.moduleGlobalCache, absoluteModuleLocator)
 		}
 	}()
 
 	// Load it.
-	contents, interpretationError := builtin.packageContentProvider.GetModuleContents(moduleInPackage)
+	contents, interpretationError := builtin.packageContentProvider.GetModuleContents(absoluteModuleLocator)
 	if interpretationError != nil {
-		return nil, startosis_errors.WrapWithInterpretationError(interpretationError, "An error occurred while loading the module '%v'", moduleInPackage)
+		return nil, startosis_errors.WrapWithInterpretationError(interpretationError, "An error occurred while loading the module '%v'", absoluteModuleLocator)
 	}
 
-	globalVariables, interpretationErr := builtin.recursiveInterpret(moduleInPackage, contents)
+	globalVariables, interpretationErr := builtin.recursiveInterpret(absoluteModuleLocator, contents)
 	// the above error goes unchecked as it needs to be persisted to the cache and then returned to the parent loader
 
 	// Update the cache.
 	if interpretationErr == nil {
 		newModule := &starlarkstruct.Module{
-			Name:    moduleInPackage,
+			Name:    absoluteModuleLocator,
 			Members: globalVariables,
 		}
 		cacheEntry = startosis_packages.NewModuleCacheEntry(newModule, nil)
 	} else {
 		cacheEntry = startosis_packages.NewModuleCacheEntry(nil, interpretationErr)
 	}
-	builtin.moduleGlobalCache[moduleInPackage] = cacheEntry
+	builtin.moduleGlobalCache[absoluteModuleLocator] = cacheEntry
 
 	shouldUnsetLoadInProgress = false
 	if cacheEntry.GetError() != nil {

core/server/api_container/server/startosis_engine/builtins/read_file/read_file.go

@@ -15,7 +15,11 @@ const (
 	SrcArgName = "src"
 )
 
-func NewReadFileHelper(packageId string, packageContentProvider startosis_packages.PackageContentProvider) *kurtosis_helper.KurtosisHelper {
+func NewReadFileHelper(
+	packageId string,
+	packageContentProvider startosis_packages.PackageContentProvider,
+	packageReplaceOptions map[string]string,
+) *kurtosis_helper.KurtosisHelper {
 	return &kurtosis_helper.KurtosisHelper{
 		KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{
 			Name: ReadFileBuiltinName,
@@ -33,12 +37,14 @@ func NewReadFileHelper(packageId string, packageContentProvider startosis_packag
 
 		Capabilities: &readFileCapabilities{
 			packageContentProvider: packageContentProvider,
+			packageReplaceOptions:  packageReplaceOptions,
 		},
 	}
 }
 
 type readFileCapabilities struct {
 	packageContentProvider startosis_packages.PackageContentProvider
+	packageReplaceOptions  map[string]string
 }
 
 func (builtin *readFileCapabilities) Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
@@ -47,7 +53,7 @@ func (builtin *readFileCapabilities) Interpret(locatorOfModuleInWhichThisBuiltIn
 		return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for arg '%s'", srcValue)
 	}
 	fileToReadStr := srcValue.GoString()
-	fileToReadStr, relativePathParsingInterpretationErr := builtin.packageContentProvider.GetAbsoluteLocatorForRelativeModuleLocator(locatorOfModuleInWhichThisBuiltInIsBeingCalled, fileToReadStr)
+	fileToReadStr, relativePathParsingInterpretationErr := builtin.packageContentProvider.GetAbsoluteLocatorForRelativeLocator(locatorOfModuleInWhichThisBuiltInIsBeingCalled, fileToReadStr, builtin.packageReplaceOptions)
 	if relativePathParsingInterpretationErr != nil {
 		return nil, relativePathParsingInterpretationErr
 	}

core/server/api_container/server/startosis_engine/kurtosis_builtins.go

@@ -52,7 +52,13 @@ func Predeclared() starlark.StringDict {
 // can have an effect at both interpretation and execution time.
 //
 // Examples: add_service, exec, wait, etc.
-func KurtosisPlanInstructions(packageId string, serviceNetwork service_network.ServiceNetwork, runtimeValueStore *runtime_value_store.RuntimeValueStore, packageContentProvider startosis_packages.PackageContentProvider) []*kurtosis_plan_instruction.KurtosisPlanInstruction {
+func KurtosisPlanInstructions(
+	packageId string,
+	serviceNetwork service_network.ServiceNetwork,
+	runtimeValueStore *runtime_value_store.RuntimeValueStore,
+	packageContentProvider startosis_packages.PackageContentProvider,
+	packageReplaceOptions map[string]string,
+) []*kurtosis_plan_instruction.KurtosisPlanInstruction {
 	return []*kurtosis_plan_instruction.KurtosisPlanInstruction{
 		add_service.NewAddService(serviceNetwork, runtimeValueStore),
 		add_service.NewAddServices(serviceNetwork, runtimeValueStore),
@@ -67,7 +73,7 @@ func KurtosisPlanInstructions(packageId string, serviceNetwork service_network.S
 		tasks.NewRunShService(serviceNetwork, runtimeValueStore),
 		stop_service.NewStopService(serviceNetwork),
 		store_service_files.NewStoreServiceFiles(serviceNetwork),
-		upload_files.NewUploadFiles(packageId, serviceNetwork, packageContentProvider),
+		upload_files.NewUploadFiles(packageId, serviceNetwork, packageContentProvider, packageReplaceOptions),
 		wait.NewWait(serviceNetwork, runtimeValueStore),
 	}
 }
@@ -79,11 +85,11 @@ func KurtosisPlanInstructions(packageId string, serviceNetwork service_network.S
 // Kurtosis enclave.
 //
 // Example: read_file, import_package, etc.
-func KurtosisHelpers(packageId string, recursiveInterpret func(moduleId string, scriptContent string) (starlark.StringDict, *startosis_errors.InterpretationError), packageContentProvider startosis_packages.PackageContentProvider, packageGlobalCache map[string]*startosis_packages.ModuleCacheEntry) []*starlark.Builtin {
+func KurtosisHelpers(packageId string, recursiveInterpret func(moduleId string, scriptContent string) (starlark.StringDict, *startosis_errors.InterpretationError), packageContentProvider startosis_packages.PackageContentProvider, packageGlobalCache map[string]*startosis_packages.ModuleCacheEntry, packageReplaceOptions map[string]string) []*starlark.Builtin {
 	return []*starlark.Builtin{
-		starlark.NewBuiltin(import_module.ImportModuleBuiltinName, import_module.NewImportModule(packageId, recursiveInterpret, packageContentProvider, packageGlobalCache).CreateBuiltin()),
+		starlark.NewBuiltin(import_module.ImportModuleBuiltinName, import_module.NewImportModule(packageId, recursiveInterpret, packageContentProvider, packageGlobalCache, packageReplaceOptions).CreateBuiltin()),
 		starlark.NewBuiltin(print_builtin.PrintBuiltinName, print_builtin.GeneratePrintBuiltin()),
-		starlark.NewBuiltin(read_file.ReadFileBuiltinName, read_file.NewReadFileHelper(packageId, packageContentProvider).CreateBuiltin()),
+		starlark.NewBuiltin(read_file.ReadFileBuiltinName, read_file.NewReadFileHelper(packageId, packageContentProvider, packageReplaceOptions).CreateBuiltin()),
 	}
 }
 

core/server/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go

@@ -30,7 +30,12 @@ const (
 	readOnlyFilePerm        = 0400
 )
 
-func NewUploadFiles(packageId string, serviceNetwork service_network.ServiceNetwork, packageContentProvider startosis_packages.PackageContentProvider) *kurtosis_plan_instruction.KurtosisPlanInstruction {
+func NewUploadFiles(
+	packageId string,
+	serviceNetwork service_network.ServiceNetwork,
+	packageContentProvider startosis_packages.PackageContentProvider,
+	packageReplaceOptions map[string]string,
+) *kurtosis_plan_instruction.KurtosisPlanInstruction {
 	return &kurtosis_plan_instruction.KurtosisPlanInstruction{
 		KurtosisBaseBuiltin: &kurtosis_starlark_framework.KurtosisBaseBuiltin{
 			Name: UploadFilesBuiltinName,
@@ -58,10 +63,11 @@ func NewUploadFiles(packageId string, serviceNetwork service_network.ServiceNetw
 				serviceNetwork:         serviceNetwork,
 				packageContentProvider: packageContentProvider,
 
-				src:               "",  // populated at interpretation time
-				artifactName:      "",  // populated at interpretation time
-				archivePathOnDisk: "",  // populated at interpretation time
-				filesArtifactMd5:  nil, // populated at interpretation time
+				src:                   "",  // populated at interpretation time
+				artifactName:          "",  // populated at interpretation time
+				archivePathOnDisk:     "",  // populated at interpretation time
+				filesArtifactMd5:      nil, // populated at interpretation time
+				packageReplaceOptions: packageReplaceOptions,
 			}
 		},
 
@@ -76,10 +82,11 @@ type UploadFilesCapabilities struct {
 	serviceNetwork         service_network.ServiceNetwork
 	packageContentProvider startosis_packages.PackageContentProvider
 
-	src               string
-	artifactName      string
-	archivePathOnDisk string
-	filesArtifactMd5  []byte
+	src                   string
+	artifactName          string
+	archivePathOnDisk     string
+	filesArtifactMd5      []byte
+	packageReplaceOptions map[string]string
 }
 
 func (builtin *UploadFilesCapabilities) Interpret(locatorOfModuleInWhichThisBuiltInIsBeingCalled string, arguments *builtin_argument.ArgumentValuesSet) (starlark.Value, *startosis_errors.InterpretationError) {
@@ -102,7 +109,7 @@ func (builtin *UploadFilesCapabilities) Interpret(locatorOfModuleInWhichThisBuil
 		return nil, startosis_errors.WrapWithInterpretationError(err, "Unable to extract value for '%s' argument", SrcArgName)
 	}
 
-	absoluteLocator, interpretationErr := builtin.packageContentProvider.GetAbsoluteLocatorForRelativeModuleLocator(locatorOfModuleInWhichThisBuiltInIsBeingCalled, src.GoString())
+	absoluteLocator, interpretationErr := builtin.packageContentProvider.GetAbsoluteLocatorForRelativeLocator(locatorOfModuleInWhichThisBuiltInIsBeingCalled, src.GoString(), builtin.packageReplaceOptions)
 	if interpretationErr != nil {
 		return nil, startosis_errors.WrapWithInterpretationError(interpretationErr, "Tried to convert locator '%v' into absolute locator but failed", src.GoString())
 	}

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/add_service_framework_test.go

@@ -25,13 +25,13 @@ type addServiceTestCase struct {
 }
 
 func (suite *KurtosisPlanInstructionTestSuite) TestAddService() {
-	suite.serviceNetwork.EXPECT().ExistServiceRegistration(TestServiceName).Times(1).Return(false, nil)
+	suite.serviceNetwork.EXPECT().ExistServiceRegistration(testServiceName).Times(1).Return(false, nil)
 	suite.serviceNetwork.EXPECT().AddService(
 		mock.Anything,
-		TestServiceName,
+		testServiceName,
 		mock.MatchedBy(func(serviceConfig *service.ServiceConfig) bool {
 			expectedServiceConfig := service.NewServiceConfig(
-				TestContainerImageName,
+				testContainerImageName,
 				map[string]*port_spec.PortSpec{},
 				map[string]*port_spec.PortSpec{},
 				nil,
@@ -51,7 +51,7 @@ func (suite *KurtosisPlanInstructionTestSuite) TestAddService() {
 			return true
 		}),
 	).Times(1).Return(
-		service.NewService(service.NewServiceRegistration(TestServiceName, TestServiceUuid, TestEnclaveUuid, nil, string(TestServiceName)), nil, nil, nil, container.NewContainer(container.ContainerStatus_Running, "", nil, nil, nil)),
+		service.NewService(service.NewServiceRegistration(testServiceName, testServiceUuid, testEnclaveUuid, nil, string(testServiceName)), nil, nil, nil, container.NewContainer(container.ContainerStatus_Running, "", nil, nil, nil)),
 		nil,
 	)
 
@@ -67,8 +67,8 @@ func (t *addServiceTestCase) GetInstruction() *kurtosis_plan_instruction.Kurtosi
 }
 
 func (t *addServiceTestCase) GetStarlarkCode() string {
-	serviceConfig := fmt.Sprintf("ServiceConfig(image=%q)", TestContainerImageName)
-	return fmt.Sprintf(`%s(%s=%q, %s=%s)`, add_service.AddServiceBuiltinName, add_service.ServiceNameArgName, TestServiceName, add_service.ServiceConfigArgName, serviceConfig)
+	serviceConfig := fmt.Sprintf("ServiceConfig(image=%q)", testContainerImageName)
+	return fmt.Sprintf(`%s(%s=%q, %s=%s)`, add_service.AddServiceBuiltinName, add_service.ServiceNameArgName, testServiceName, add_service.ServiceConfigArgName, serviceConfig)
 }
 
 func (t *addServiceTestCase) GetStarlarkCodeForAssertion() string {
@@ -79,9 +79,9 @@ func (t *addServiceTestCase) Assert(interpretationResult starlark.Value, executi
 	serviceObj, ok := interpretationResult.(*kurtosis_types.Service)
 	require.True(t, ok, "interpretation result should be a dictionary")
 	require.NotNil(t, serviceObj)
-	expectedServiceObj := fmt.Sprintf(`Service\(name="%v", hostname="{{kurtosis:[0-9a-f]{32}:hostname.runtime_value}}", ip_address="{{kurtosis:[0-9a-f]{32}:ip_address.runtime_value}}", ports={}\)`, TestServiceName)
+	expectedServiceObj := fmt.Sprintf(`Service\(name="%v", hostname="{{kurtosis:[0-9a-f]{32}:hostname.runtime_value}}", ip_address="{{kurtosis:[0-9a-f]{32}:ip_address.runtime_value}}", ports={}\)`, testServiceName)
 	require.Regexp(t, expectedServiceObj, serviceObj.String())
 
-	expectedExecutionResult := fmt.Sprintf("Service '%s' added with service UUID '%s'", TestServiceName, TestServiceUuid)
+	expectedExecutionResult := fmt.Sprintf("Service '%s' added with service UUID '%s'", testServiceName, testServiceUuid)
 	require.Equal(t, expectedExecutionResult, *executionResult)
 }