feat: local replace package dependency (#1521)

## Description:
local replace package dependency

## Is this change user facing?
YES

## References (if applicable):
This is part of the [Forked cleanup package
project](https://www.notion.so/kurtosistech/Forked-Package-Cleanup-16d86c4e274547b28496f17154bf3d62)

api/golang/core/lib/enclaves/enclave_context.go

@@ -32,6 +32,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/emptypb"
 	"io"
+	"os"
 	"path"
 	"strings"
 )
@@ -41,6 +42,10 @@ type EnclaveUUID string
 const (
 	kurtosisYamlFilename    = "kurtosis.yml"
 	enforceMaxFileSizeLimit = true
+
+	osPathSeparatorString = string(os.PathSeparator)
+
+	dotRelativePathIndicatorString = "."
 )
 
 // Docs available at https://docs.kurtosis.com/sdk/#enclavecontext
@@ -136,7 +141,13 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
 	}()
 
 	starlarkResponseLineChan := make(chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine)
-	executeStartosisPackageArgs, err := enclaveCtx.assembleRunStartosisPackageArg(packageRootPath, runConfig.RelativePathToMainFile, runConfig.MainFunctionName, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId)
+
+	kurtosisYml, err := getKurtosisYaml(packageRootPath)
+	if err != nil {
+		return nil, nil, stacktrace.Propagate(err, "An error occurred getting Kurtosis yaml file from path '%s'", packageRootPath)
+	}
+
+	executeStartosisPackageArgs, err := enclaveCtx.assembleRunStartosisPackageArg(kurtosisYml, runConfig.RelativePathToMainFile, runConfig.MainFunctionName, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId)
 	if err != nil {
 		return nil, nil, stacktrace.Propagate(err, "Error preparing package '%s' for execution", packageRootPath)
 	}
@@ -146,6 +157,12 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
 		return nil, nil, stacktrace.Propagate(err, "Error uploading package '%s' prior to executing it", packageRootPath)
 	}
 
+	if len(kurtosisYml.PackageReplaceOptions) > 0 {
+		if err = enclaveCtx.uploadLocalStarlarkPackageDependencies(packageRootPath, kurtosisYml.PackageReplaceOptions); err != nil {
+			return nil, nil, stacktrace.Propagate(err, "An error occurred while uploading the local starlark package dependencies from the replace options '%+v'", kurtosisYml.PackageReplaceOptions)
+		}
+	}
+
 	stream, err := enclaveCtx.client.RunStarlarkPackage(ctxWithCancel, executeStartosisPackageArgs)
 	if err != nil {
 		return nil, nil, stacktrace.Propagate(err, "Unexpected error happened executing Starlark package '%v'", packageRootPath)
@@ -156,6 +173,18 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
 	return starlarkResponseLineChan, cancelCtxFunc, nil
 }
 
+func (enclaveCtx *EnclaveContext) uploadLocalStarlarkPackageDependencies(packageRootPath string, packageReplaceOptions map[string]string) error {
+	for dependencyPackageId, replaceOption := range packageReplaceOptions {
+		if isLocalDependencyReplace(replaceOption) {
+			localPackagePath := path.Join(packageRootPath, replaceOption)
+			if err := enclaveCtx.uploadStarlarkPackage(dependencyPackageId, localPackagePath); err != nil {
+				return stacktrace.Propagate(err, "Error uploading package '%s' prior to executing it", replaceOption)
+			}
+		}
+	}
+	return nil
+}
+
 // Docs available at https://docs.kurtosis.com/sdk/#runstarlarkpackageblockingstring-packagerootpath-string-serializedparams-boolean-dryrun---starlarkrunresult-runresult-error-error
 func (enclaveCtx *EnclaveContext) RunStarlarkPackageBlocking(
 	ctx context.Context,
@@ -493,7 +522,7 @@ func getErrFromStarlarkRunResult(result *StarlarkRunResult) error {
 }
 
 func (enclaveCtx *EnclaveContext) assembleRunStartosisPackageArg(
-	packageRootPath string,
+	kurtosisYaml *KurtosisYaml,
 	relativePathToMainFile string,
 	mainFunctionName string,
 	serializedParams string,
@@ -503,12 +532,7 @@ func (enclaveCtx *EnclaveContext) assembleRunStartosisPackageArg(
 	cloudInstanceId string,
 	cloudUserId string,
 ) (*kurtosis_core_rpc_api_bindings.RunStarlarkPackageArgs, error) {
-	kurtosisYamlFilepath := path.Join(packageRootPath, kurtosisYamlFilename)
 
-	kurtosisYaml, err := ParseKurtosisYaml(kurtosisYamlFilepath)
-	if err != nil {
-		return nil, stacktrace.Propagate(err, "There was an error parsing the '%v' at '%v'", kurtosisYamlFilename, packageRootPath)
-	}
 	return binding_constructors.NewRunStarlarkPackageArgs(kurtosisYaml.PackageName, relativePathToMainFile, mainFunctionName, serializedParams, dryRun, parallelism, experimentalFeatures, cloudInstanceId, cloudUserId), nil
 }
 
@@ -546,3 +570,20 @@ func (enclaveCtx *EnclaveContext) uploadStarlarkPackage(packageId string, packag
 	}
 	return nil
 }
+
+func getKurtosisYaml(packageRootPath string) (*KurtosisYaml, error) {
+	kurtosisYamlFilepath := path.Join(packageRootPath, kurtosisYamlFilename)
+
+	kurtosisYaml, err := ParseKurtosisYaml(kurtosisYamlFilepath)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "There was an error parsing the '%v' at '%v'", kurtosisYamlFilename, packageRootPath)
+	}
+	return kurtosisYaml, nil
+}
+
+func isLocalDependencyReplace(replace string) bool {
+	if strings.HasPrefix(replace, osPathSeparatorString) || strings.HasPrefix(replace, dotRelativePathIndicatorString) {
+		return true
+	}
+	return false
+}

api/golang/core/lib/enclaves/kurtosis_yaml.go

@@ -12,7 +12,9 @@ const (
 
 // fields are public because it's needed for YAML decoding
 type KurtosisYaml struct {
-	PackageName string `yaml:"name"`
+	PackageName           string            `yaml:"name"`
+	PackageDescription    string            `yaml:"description"`
+	PackageReplaceOptions map[string]string `yaml:"replace"`
 }
 
 func ParseKurtosisYaml(kurtosisYamlFilepath string) (*KurtosisYaml, error) {
@@ -25,7 +27,7 @@ func ParseKurtosisYaml(kurtosisYamlFilepath string) (*KurtosisYaml, error) {
 	}
 
 	var kurtosisYaml KurtosisYaml
-	err = yaml.Unmarshal(kurtosisYamlContents, &kurtosisYaml)
+	err = yaml.UnmarshalStrict(kurtosisYamlContents, &kurtosisYaml)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred while parsing the '%v' file at '%v'", kurtosisYamlFilename, kurtosisYamlFilepath)
 	}

api/typescript/src/core/lib/enclaves/enclave_context.ts

@@ -34,7 +34,7 @@ import {
     GetStarlarkRunResponse,
 } from "../../kurtosis_core_rpc_api_bindings/api_container_service_pb";
 import * as path from "path";
-import {parseKurtosisYaml} from "./kurtosis_yaml";
+import {parseKurtosisYaml, KurtosisYaml} from "./kurtosis_yaml";
 import {Readable} from "stream";
 import {readStreamContentUntilClosed, StarlarkRunResult} from "./starlark_run_blocking";
 import {ServiceIdentifiers} from "../services/service_identifiers";
@@ -44,6 +44,10 @@ export type EnclaveUUID = string;
 
 export const KURTOSIS_YAML_FILENAME = "kurtosis.yml";
 
+const OS_PATH_SEPARATOR_STRING = "/"
+
+const DOT_RELATIVE_PATH_INDICATOR_STRING = "."
+
 
 // Docs available at https://docs.kurtosis.com/sdk/#enclavecontext
 export class EnclaveContext {
@@ -142,10 +146,37 @@ export class EnclaveContext {
         packageRootPath: string,
         runConfig: StarlarkRunConfig,
     ): Promise<Result<Readable, Error>> {
-        const args = await this.assembleRunStarlarkPackageArg(packageRootPath, runConfig.relativePathToMainFile, runConfig.mainFunctionName, runConfig.serializedParams, runConfig.dryRun, runConfig.cloudInstanceId, runConfig.cloudUserId)
+        const kurtosisYmlResult = await this.getKurtosisYaml(packageRootPath)
+        if (kurtosisYmlResult.isErr()) {
+            return err(new Error(`Unexpected error while getting the Kurtosis yaml file from path '${packageRootPath}'`))
+        }
+
+        const kurtosisYaml: KurtosisYaml = kurtosisYmlResult.value
+        const packageId: string = kurtosisYaml.name
+        const packageReplaceOptions: Map<string, string> = kurtosisYaml.packageReplaceOptions
+
+        const args = await this.assembleRunStarlarkPackageArg(kurtosisYaml, runConfig.relativePathToMainFile, runConfig.mainFunctionName, runConfig.serializedParams, runConfig.dryRun, runConfig.cloudInstanceId, runConfig.cloudUserId)
         if (args.isErr()) {
             return err(new Error(`Unexpected error while assembling arguments to pass to the Starlark executor \n${args.error}`))
         }
+
+        const archiverResponse = await this.genericTgzArchiver.createTgzByteArray(packageRootPath)
+        if (archiverResponse.isErr()){
+            return err(new Error(`Unexpected error while creating the package's tgs file from '${packageRootPath}'\n${archiverResponse.error}`))
+        }
+
+        const uploadStarlarkPackageResponse = await this.backend.uploadStarlarkPackage(packageId, archiverResponse.value)
+        if (uploadStarlarkPackageResponse.isErr()){
+            return err(new Error(`Unexpected error while uploading Starlark package '${packageId}'\n${uploadStarlarkPackageResponse.error}`))
+        }
+
+        if (packageReplaceOptions !== undefined && packageReplaceOptions.size > 0) {
+            const uploadLocalStarlarkPackageDependenciesResponse = await this.uploadLocalStarlarkPackageDependencies(packageRootPath, packageReplaceOptions)
+            if (uploadLocalStarlarkPackageDependenciesResponse.isErr()) {
+                return err(new Error(`Unexpected error while uploading local Starlark package dependencies '${packageReplaceOptions}' from '${packageRootPath}' \n${uploadLocalStarlarkPackageDependenciesResponse.error}`))
+            }
+        }
+
         const packageRunResult : Result<Readable, Error> = await this.backend.runStarlarkPackage(args.value)
         if (packageRunResult.isErr()) {
             return err(new Error(`Unexpected error happened executing Starlark package \n${packageRunResult.error}`))
@@ -381,29 +412,16 @@ export class EnclaveContext {
     }
 
     private async assembleRunStarlarkPackageArg(
-        packageRootPath: string,
+        kurtosisYaml: KurtosisYaml,
         relativePathToMainFile: string,
         mainFunctionName: string,
         serializedParams: string,
         dryRun: boolean,
         cloudInstanceId: string,
         cloudUserId: string,
         ): Promise<Result<RunStarlarkPackageArgs, Error>> {
-        const kurtosisYamlFilepath = path.join(packageRootPath, KURTOSIS_YAML_FILENAME)
-
-        const resultParseKurtosisYaml = await parseKurtosisYaml(kurtosisYamlFilepath)
-        if (resultParseKurtosisYaml.isErr()) {
-            return err(resultParseKurtosisYaml.error)
-        }
-        const kurtosisYaml = resultParseKurtosisYaml.value
-
-        const archiverResponse = await this.genericTgzArchiver.createTgzByteArray(packageRootPath)
-        if (archiverResponse.isErr()){
-            return err(archiverResponse.error)
-        }
 
         const args = new RunStarlarkPackageArgs;
-        args.setLocal(archiverResponse.value)
         args.setPackageId(kurtosisYaml.name)
         args.setSerializedParams(serializedParams)
         args.setDryRun(dryRun)
@@ -413,4 +431,47 @@ export class EnclaveContext {
         args.setCloudUserId(cloudUserId)
         return ok(args)
     }
+
+    private async getKurtosisYaml(packageRootPath: string): Promise<Result<KurtosisYaml, Error>> {
+        const kurtosisYamlFilepath = path.join(packageRootPath, KURTOSIS_YAML_FILENAME)
+
+        const resultParseKurtosisYaml = await parseKurtosisYaml(kurtosisYamlFilepath)
+        if (resultParseKurtosisYaml.isErr()) {
+            return err(resultParseKurtosisYaml.error)
+        }
+        const kurtosisYaml = resultParseKurtosisYaml.value
+
+        return ok(kurtosisYaml)
+    }
+
+
+    private async uploadLocalStarlarkPackageDependencies(
+        packageRootPath: string,
+        packageReplaceOptions: Map<string, string>,
+    ): Promise<Result<null, Error>> {
+        for (const [dependencyPackageId, replaceOption] of packageReplaceOptions.entries()) {
+            if (this.isLocalDependencyReplace(replaceOption)) {
+                const localPackagePath: string = path.join(packageRootPath, replaceOption)
+
+                const archiverResponse = await this.genericTgzArchiver.createTgzByteArray(localPackagePath)
+                if (archiverResponse.isErr()){
+                    return err(archiverResponse.error)
+                }
+
+                const uploadStarlarkPackageResponse = await this.backend.uploadStarlarkPackage(dependencyPackageId, archiverResponse.value)
+                if (uploadStarlarkPackageResponse.isErr()){
+                    return err(uploadStarlarkPackageResponse.error)
+                }
+                return ok(null)
+            }
+        }
+        return ok(null)
+    }
+
+    private isLocalDependencyReplace(replace: string): boolean {
+        if (replace.startsWith(OS_PATH_SEPARATOR_STRING) || replace.startsWith(DOT_RELATIVE_PATH_INDICATOR_STRING)) {
+            return true
+        }
+        return false
+    }
 }

api/typescript/src/core/lib/enclaves/generic_api_container_client.ts

@@ -36,6 +36,7 @@ export interface GenericApiContainerClient {
     getServices(getServicesArgs: GetServicesArgs): Promise<Result<GetServicesResponse, Error>>
     execCommand(execCommandArgs: ExecCommandArgs): Promise<Result<ExecCommandResponse, Error>>
     uploadFiles(name: string, payload: Uint8Array): Promise<Result<UploadFilesArtifactResponse, Error>>
+    uploadStarlarkPackage(packageId: string, payload: Uint8Array): Promise<Result<null, Error>>
     storeWebFilesArtifact(storeWebFilesArtifactArgs: StoreWebFilesArtifactArgs): Promise<Result<StoreWebFilesArtifactResponse, Error>>
     downloadFilesArtifact(downloadFilesArtifactArgs: DownloadFilesArtifactArgs): Promise<Result<Uint8Array, Error>>
     getExistingAndHistoricalServiceIdentifiers(): Promise<Result<GetExistingAndHistoricalServiceIdentifiersResponse, Error>>

api/typescript/src/core/lib/enclaves/grpc_node_api_container_client.ts

@@ -202,6 +202,50 @@ export class GrpcNodeApiContainerClient implements GenericApiContainerClient {
         return ok(uploadFilesArtifactResponse)
     }
 
+    public async uploadStarlarkPackage(packageId: string, payload: Uint8Array): Promise<Result<null, Error>> {
+        const uploadStarlarkPackagePromise: Promise<Result<null, Error>> = new Promise((resolve, _unusedReject) => {
+            const clientStream = this.client.uploadStarlarkPackage((error: ServiceError | null, response?: google_protobuf_empty_pb.Empty) => {
+                if (error === null) {
+                    if (!response) {
+                        resolve(err(new Error("No error was encountered but the response was still falsy; this should never happen")));
+                    } else {
+                        resolve(ok(null));
+                    }
+                } else {
+                    resolve(err(error));
+                }
+            })
+
+            const constantChunkMetadata = new DataChunkMetadata()
+                .setName(packageId)
+
+            let previousChunkHash = ""
+            for (let idx = 0; idx < payload.length; idx += DATA_STREAM_CHUNK_SIZE) {
+                const currentChunk = payload.subarray(idx, Math.min(idx + DATA_STREAM_CHUNK_SIZE, payload.length))
+
+                const dataChunk = new StreamedDataChunk()
+                    .setData(currentChunk)
+                    .setPreviousChunkHash(previousChunkHash)
+                    .setMetadata(constantChunkMetadata)
+                clientStream.write(dataChunk, (streamErr: Error | null | undefined) => {
+                    if (streamErr != undefined) {
+                        resolve(err(new Error(`An error occurred sending data through the stream:\n${streamErr.message}`)))
+                    }
+                })
+                previousChunkHash = this.computeHexHash(currentChunk)
+            }
+            clientStream.end() // close the stream, no more data will be sent
+        });
+
+        const uploadStarlarkPackageResponseResult: Result<null, Error> = await uploadStarlarkPackagePromise;
+        if(uploadStarlarkPackageResponseResult.isErr()){
+            return err(uploadStarlarkPackageResponseResult.error)
+        }
+
+        const uploadStarlarkPackageResponse = uploadStarlarkPackageResponseResult.value
+        return ok(uploadStarlarkPackageResponse)
+    }
+
     public async storeWebFilesArtifact(storeWebFilesArtifactArgs: StoreWebFilesArtifactArgs): Promise<Result<StoreWebFilesArtifactResponse, Error>> {
         const storeWebFilesArtifactPromise: Promise<Result<StoreWebFilesArtifactResponse, Error>> = new Promise((resolve, _unusedReject) => {
             this.client.storeWebFilesArtifact(storeWebFilesArtifactArgs, (error: ServiceError | null, response?: StoreWebFilesArtifactResponse) => {

api/typescript/src/core/lib/enclaves/kurtosis_yaml.ts

@@ -7,7 +7,9 @@ const PACKAGES_URL = "https://docs.kurtosis.com/concepts-reference/packages";
 
 export class KurtosisYaml {
     constructor(
-        public readonly  name: string,
+        public readonly name: string,
+        public readonly description: string,
+        public readonly packageReplaceOptions: Map<string, string>,
     ){}
 }
 

cli/cli/go.mod

@@ -55,7 +55,6 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/xlab/treeprint v1.2.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	gopkg.in/segmentio/analytics-go.v3 v3.1.0
 	k8s.io/api v0.27.2
 )
 

core/server/api_container/main.go

@@ -125,14 +125,14 @@ func runMain() error {
 		return stacktrace.NewError("Kurtosis backend type is '%v' but cluster configuration parameters are null.", args.KurtosisBackendType_Kubernetes.String())
 	}
 
-	gitPackageContentProvider, err := enclaveDataDir.GetGitPackageContentProvider()
+	enclaveDb, err := enclave_db.GetOrCreateEnclaveDatabase()
 	if err != nil {
-		return stacktrace.Propagate(err, "An error occurred while creating the Git module content provider")
+		return stacktrace.Propagate(err, "An error occurred while getting the enclave db")
 	}
 
-	enclaveDb, err := enclave_db.GetOrCreateEnclaveDatabase()
+	gitPackageContentProvider, err := enclaveDataDir.GetGitPackageContentProvider(enclaveDb)
 	if err != nil {
-		return stacktrace.Propagate(err, "An error occurred while getting the enclave db")
+		return stacktrace.Propagate(err, "An error occurred while creating the Git module content provider")
 	}
 
 	// TODO Extract into own function

core/server/api_container/server/startosis_engine/startosis_interpreter.go

@@ -93,6 +93,10 @@ func (interpreter *StartosisInterpreter) InterpretAndOptimizePlan(
 	currentEnclavePlan *enclave_plan_persistence.EnclavePlan,
 ) (string, *instructions_plan.InstructionsPlan, *kurtosis_core_rpc_api_bindings.StarlarkInterpretationError) {
 
+	if interpretationErr := interpreter.moduleContentProvider.CloneReplacedPackagesIfNeeded(packageReplaceOptions); interpretationErr != nil {
+		return "", nil, interpretationErr.ToAPIType()
+	}
+
 	// run interpretation with no mask at all to generate the list of instructions as if the enclave was empty
 	enclaveComponents := enclave_structure.NewEnclaveComponents()
 	emptyPlanInstructionsMask := resolver.NewInstructionsPlanMask(0)