Merge branch 'main' into derek/docs_read_file

.github/workflows/conventional-pr-title-checker.yml

@@ -7,9 +7,7 @@ on:
       - reopened
       - edited
       - synchronize
-  push:
-    branches:
-      - gh-readonly-queue/main/**
+  merge_group:
 
 # cancel redundant builds
 concurrency:

.github/workflows/do-not-merge.yml

@@ -3,6 +3,7 @@ name: Do Not Merge
 on:
   pull_request:
     types: [synchronize, opened, reopened, labeled, unlabeled]
+  merge_group:
 
 jobs:
   do-not-merge:

.github/workflows/gofmt.yml

@@ -7,10 +7,8 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  push:
-    branches:
-      - gh-readonly-queue/main/**
   pull_request:
+  merge_group:
 
 jobs:
   gofmt:

.github/workflows/golangci-lint.yml

@@ -14,10 +14,8 @@ permissions:
   contents: read
 
 on:
-  push:
-    branches:
-      - gh-readonly-queue/main/**
   pull_request:
+  merge_group:
 
 env:
   GOLINT_VERSION: v1.53.3

.github/workflows/push-docs.yml

@@ -1,6 +1,8 @@
 name: Deploy documentation
 
-on: push
+on:
+  push:
+  merge_group:
 
 jobs:
   publish:

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## [0.84.6](https://github.com/kurtosis-tech/kurtosis/compare/0.84.5...0.84.6) (2023-10-13)
+
+
+### Features
+
+* Unused images are cleaned even without -a flag ([#1551](https://github.com/kurtosis-tech/kurtosis/issues/1551)) ([e1317aa](https://github.com/kurtosis-tech/kurtosis/commit/e1317aaa0853943d73234dde344fa5102ab41bd8)), closes [#1523](https://github.com/kurtosis-tech/kurtosis/issues/1523)
+
+## [0.84.5](https://github.com/kurtosis-tech/kurtosis/compare/0.84.4...0.84.5) (2023-10-12)
+
+
+### Features
+
+* highlight the active cluster in kurtosis cluster ls ([#1514](https://github.com/kurtosis-tech/kurtosis/issues/1514)) ([67e0111](https://github.com/kurtosis-tech/kurtosis/commit/67e0111af7483efdab14743f5e10897054db96a2))
+* local replace package dependency ([#1521](https://github.com/kurtosis-tech/kurtosis/issues/1521)) ([d5e3126](https://github.com/kurtosis-tech/kurtosis/commit/d5e3126900f1a16523a3c8ba33b25c3a7bed6e0d))
+* manage script return value ([#1546](https://github.com/kurtosis-tech/kurtosis/issues/1546)) ([a53508f](https://github.com/kurtosis-tech/kurtosis/commit/a53508f825985a26e306e10038305798f3e3ce4d))
+
+
+### Bug Fixes
+
+* run package bug ([#1539](https://github.com/kurtosis-tech/kurtosis/issues/1539)) ([1f5380a](https://github.com/kurtosis-tech/kurtosis/commit/1f5380afeb91c3dfe8b365b7752494f5444376e7)), closes [#1501](https://github.com/kurtosis-tech/kurtosis/issues/1501) [#1479](https://github.com/kurtosis-tech/kurtosis/issues/1479)
+
 ## [0.84.4](https://github.com/kurtosis-tech/kurtosis/compare/0.84.3...0.84.4) (2023-10-10)
 
 

LICENSE.md

@@ -3,7 +3,7 @@ Business Source License 1.1
 Parameters
 
 Licensor:             Kurtosis Technologies, Inc.
-Licensed Work:        Kurtosis 0.84.4
+Licensed Work:        Kurtosis 0.84.6
 The Licensed Work is (c) 2023 Kurtosis Technologies, Inc.
 Additional Use Grant: You may make use of the Licensed Work, provided that
 you may not use the Licensed Work for an Environment Orchestration Service.
@@ -12,7 +12,7 @@ you may not use the Licensed Work for an Environment Orchestration Service.
                       allows third parties (other than your employees and
                       contractors) to create distributed system environments.
 
-Change Date:          2027-10-10
+Change Date:          2027-10-13
 
 Change License:       AGPLv3 (GNU Affero General Public License Version 3)
 

api/golang/core/lib/enclaves/enclave_context.go

@@ -32,6 +32,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/emptypb"
 	"io"
+	"os"
 	"path"
 	"strings"
 )
@@ -41,6 +42,10 @@ type EnclaveUUID string
 const (
 	kurtosisYamlFilename    = "kurtosis.yml"
 	enforceMaxFileSizeLimit = true
+
+	osPathSeparatorString = string(os.PathSeparator)
+
+	dotRelativePathIndicatorString = "."
 )
 
 // Docs available at https://docs.kurtosis.com/sdk/#enclavecontext
@@ -136,7 +141,13 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
 	}()
 
 	starlarkResponseLineChan := make(chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine)
-	executeStartosisPackageArgs, err := enclaveCtx.assembleRunStartosisPackageArg(packageRootPath, runConfig.RelativePathToMainFile, runConfig.MainFunctionName, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId)
+
+	kurtosisYml, err := getKurtosisYaml(packageRootPath)
+	if err != nil {
+		return nil, nil, stacktrace.Propagate(err, "An error occurred getting Kurtosis yaml file from path '%s'", packageRootPath)
+	}
+
+	executeStartosisPackageArgs, err := enclaveCtx.assembleRunStartosisPackageArg(kurtosisYml, runConfig.RelativePathToMainFile, runConfig.MainFunctionName, serializedParams, runConfig.DryRun, runConfig.Parallelism, runConfig.ExperimentalFeatureFlags, runConfig.CloudInstanceId, runConfig.CloudUserId)
 	if err != nil {
 		return nil, nil, stacktrace.Propagate(err, "Error preparing package '%s' for execution", packageRootPath)
 	}
@@ -146,6 +157,12 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
 		return nil, nil, stacktrace.Propagate(err, "Error uploading package '%s' prior to executing it", packageRootPath)
 	}
 
+	if len(kurtosisYml.PackageReplaceOptions) > 0 {
+		if err = enclaveCtx.uploadLocalStarlarkPackageDependencies(packageRootPath, kurtosisYml.PackageReplaceOptions); err != nil {
+			return nil, nil, stacktrace.Propagate(err, "An error occurred while uploading the local starlark package dependencies from the replace options '%+v'", kurtosisYml.PackageReplaceOptions)
+		}
+	}
+
 	stream, err := enclaveCtx.client.RunStarlarkPackage(ctxWithCancel, executeStartosisPackageArgs)
 	if err != nil {
 		return nil, nil, stacktrace.Propagate(err, "Unexpected error happened executing Starlark package '%v'", packageRootPath)
@@ -156,6 +173,18 @@ func (enclaveCtx *EnclaveContext) RunStarlarkPackage(
 	return starlarkResponseLineChan, cancelCtxFunc, nil
 }
 
+func (enclaveCtx *EnclaveContext) uploadLocalStarlarkPackageDependencies(packageRootPath string, packageReplaceOptions map[string]string) error {
+	for dependencyPackageId, replaceOption := range packageReplaceOptions {
+		if isLocalDependencyReplace(replaceOption) {
+			localPackagePath := path.Join(packageRootPath, replaceOption)
+			if err := enclaveCtx.uploadStarlarkPackage(dependencyPackageId, localPackagePath); err != nil {
+				return stacktrace.Propagate(err, "Error uploading package '%s' prior to executing it", replaceOption)
+			}
+		}
+	}
+	return nil
+}
+
 // Docs available at https://docs.kurtosis.com/sdk/#runstarlarkpackageblockingstring-packagerootpath-string-serializedparams-boolean-dryrun---starlarkrunresult-runresult-error-error
 func (enclaveCtx *EnclaveContext) RunStarlarkPackageBlocking(
 	ctx context.Context,
@@ -493,7 +522,7 @@ func getErrFromStarlarkRunResult(result *StarlarkRunResult) error {
 }
 
 func (enclaveCtx *EnclaveContext) assembleRunStartosisPackageArg(
-	packageRootPath string,
+	kurtosisYaml *KurtosisYaml,
 	relativePathToMainFile string,
 	mainFunctionName string,
 	serializedParams string,
@@ -503,12 +532,7 @@ func (enclaveCtx *EnclaveContext) assembleRunStartosisPackageArg(
 	cloudInstanceId string,
 	cloudUserId string,
 ) (*kurtosis_core_rpc_api_bindings.RunStarlarkPackageArgs, error) {
-	kurtosisYamlFilepath := path.Join(packageRootPath, kurtosisYamlFilename)
 
-	kurtosisYaml, err := ParseKurtosisYaml(kurtosisYamlFilepath)
-	if err != nil {
-		return nil, stacktrace.Propagate(err, "There was an error parsing the '%v' at '%v'", kurtosisYamlFilename, packageRootPath)
-	}
 	return binding_constructors.NewRunStarlarkPackageArgs(kurtosisYaml.PackageName, relativePathToMainFile, mainFunctionName, serializedParams, dryRun, parallelism, experimentalFeatures, cloudInstanceId, cloudUserId), nil
 }
 
@@ -546,3 +570,20 @@ func (enclaveCtx *EnclaveContext) uploadStarlarkPackage(packageId string, packag
 	}
 	return nil
 }
+
+func getKurtosisYaml(packageRootPath string) (*KurtosisYaml, error) {
+	kurtosisYamlFilepath := path.Join(packageRootPath, kurtosisYamlFilename)
+
+	kurtosisYaml, err := ParseKurtosisYaml(kurtosisYamlFilepath)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "There was an error parsing the '%v' at '%v'", kurtosisYamlFilename, packageRootPath)
+	}
+	return kurtosisYaml, nil
+}
+
+func isLocalDependencyReplace(replace string) bool {
+	if strings.HasPrefix(replace, osPathSeparatorString) || strings.HasPrefix(replace, dotRelativePathIndicatorString) {
+		return true
+	}
+	return false
+}

api/golang/core/lib/enclaves/kurtosis_yaml.go

@@ -12,7 +12,9 @@ const (
 
 // fields are public because it's needed for YAML decoding
 type KurtosisYaml struct {
-	PackageName string `yaml:"name"`
+	PackageName           string            `yaml:"name"`
+	PackageDescription    string            `yaml:"description"`
+	PackageReplaceOptions map[string]string `yaml:"replace"`
 }
 
 func ParseKurtosisYaml(kurtosisYamlFilepath string) (*KurtosisYaml, error) {
@@ -25,7 +27,7 @@ func ParseKurtosisYaml(kurtosisYamlFilepath string) (*KurtosisYaml, error) {
 	}
 
 	var kurtosisYaml KurtosisYaml
-	err = yaml.Unmarshal(kurtosisYamlContents, &kurtosisYaml)
+	err = yaml.UnmarshalStrict(kurtosisYamlContents, &kurtosisYaml)
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred while parsing the '%v' file at '%v'", kurtosisYamlFilename, kurtosisYamlFilepath)
 	}

api/golang/kurtosis_version/kurtosis_version.go

@@ -9,6 +9,6 @@ const (
 	// !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!!
 	// This is necessary so that Kurt Core consumers will know if they're compatible with the currently-running
 	//  API container
-	KurtosisVersion = "0.84.4"
+	KurtosisVersion = "0.84.6"
 	// !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!!
 )

api/rust/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kurtosis-sdk"
-version = "0.84.4"
+version = "0.84.6"
 license = "BUSL-1.1"
 description = "Rust SDK for Kurtosis"
 edition = "2021"

api/typescript/package.json

@@ -1,7 +1,7 @@
 {
     "name": "kurtosis-sdk",
     "//": "NOTE: DO NOT UPDATE THIS VERSION MANUALLY - IT WILL BE UPDATED DURING THE RELEASE PROCESS!",
-    "version": "0.84.4",
+    "version": "0.84.6",
     "main": "./build/index",
     "description": "This repo contains a Typescript client for communicating with the Kurtosis Engine server, which is responsible for creating, managing and destroying Kurtosis Enclaves.",
     "types": "./build/index",

api/typescript/src/core/lib/enclaves/enclave_context.ts

@@ -34,7 +34,7 @@ import {
     GetStarlarkRunResponse,
 } from "../../kurtosis_core_rpc_api_bindings/api_container_service_pb";
 import * as path from "path";
-import {parseKurtosisYaml} from "./kurtosis_yaml";
+import {parseKurtosisYaml, KurtosisYaml} from "./kurtosis_yaml";
 import {Readable} from "stream";
 import {readStreamContentUntilClosed, StarlarkRunResult} from "./starlark_run_blocking";
 import {ServiceIdentifiers} from "../services/service_identifiers";
@@ -44,6 +44,10 @@ export type EnclaveUUID = string;
 
 export const KURTOSIS_YAML_FILENAME = "kurtosis.yml";
 
+const OS_PATH_SEPARATOR_STRING = "/"
+
+const DOT_RELATIVE_PATH_INDICATOR_STRING = "."
+
 
 // Docs available at https://docs.kurtosis.com/sdk/#enclavecontext
 export class EnclaveContext {
@@ -142,10 +146,37 @@ export class EnclaveContext {
         packageRootPath: string,
         runConfig: StarlarkRunConfig,
     ): Promise<Result<Readable, Error>> {
-        const args = await this.assembleRunStarlarkPackageArg(packageRootPath, runConfig.relativePathToMainFile, runConfig.mainFunctionName, runConfig.serializedParams, runConfig.dryRun, runConfig.cloudInstanceId, runConfig.cloudUserId)
+        const kurtosisYmlResult = await this.getKurtosisYaml(packageRootPath)
+        if (kurtosisYmlResult.isErr()) {
+            return err(new Error(`Unexpected error while getting the Kurtosis yaml file from path '${packageRootPath}'`))
+        }
+
+        const kurtosisYaml: KurtosisYaml = kurtosisYmlResult.value
+        const packageId: string = kurtosisYaml.name
+        const packageReplaceOptions: Map<string, string> = kurtosisYaml.packageReplaceOptions
+
+        const args = await this.assembleRunStarlarkPackageArg(kurtosisYaml, runConfig.relativePathToMainFile, runConfig.mainFunctionName, runConfig.serializedParams, runConfig.dryRun, runConfig.cloudInstanceId, runConfig.cloudUserId)
         if (args.isErr()) {
             return err(new Error(`Unexpected error while assembling arguments to pass to the Starlark executor \n${args.error}`))
         }
+
+        const archiverResponse = await this.genericTgzArchiver.createTgzByteArray(packageRootPath)
+        if (archiverResponse.isErr()){
+            return err(new Error(`Unexpected error while creating the package's tgs file from '${packageRootPath}'\n${archiverResponse.error}`))
+        }
+
+        const uploadStarlarkPackageResponse = await this.backend.uploadStarlarkPackage(packageId, archiverResponse.value)
+        if (uploadStarlarkPackageResponse.isErr()){
+            return err(new Error(`Unexpected error while uploading Starlark package '${packageId}'\n${uploadStarlarkPackageResponse.error}`))
+        }
+
+        if (packageReplaceOptions !== undefined && packageReplaceOptions.size > 0) {
+            const uploadLocalStarlarkPackageDependenciesResponse = await this.uploadLocalStarlarkPackageDependencies(packageRootPath, packageReplaceOptions)
+            if (uploadLocalStarlarkPackageDependenciesResponse.isErr()) {
+                return err(new Error(`Unexpected error while uploading local Starlark package dependencies '${packageReplaceOptions}' from '${packageRootPath}' \n${uploadLocalStarlarkPackageDependenciesResponse.error}`))
+            }
+        }
+
         const packageRunResult : Result<Readable, Error> = await this.backend.runStarlarkPackage(args.value)
         if (packageRunResult.isErr()) {
             return err(new Error(`Unexpected error happened executing Starlark package \n${packageRunResult.error}`))
@@ -381,29 +412,16 @@ export class EnclaveContext {
     }
 
     private async assembleRunStarlarkPackageArg(
-        packageRootPath: string,
+        kurtosisYaml: KurtosisYaml,
         relativePathToMainFile: string,
         mainFunctionName: string,
         serializedParams: string,
         dryRun: boolean,
         cloudInstanceId: string,
         cloudUserId: string,
         ): Promise<Result<RunStarlarkPackageArgs, Error>> {
-        const kurtosisYamlFilepath = path.join(packageRootPath, KURTOSIS_YAML_FILENAME)
-
-        const resultParseKurtosisYaml = await parseKurtosisYaml(kurtosisYamlFilepath)
-        if (resultParseKurtosisYaml.isErr()) {
-            return err(resultParseKurtosisYaml.error)
-        }
-        const kurtosisYaml = resultParseKurtosisYaml.value
-
-        const archiverResponse = await this.genericTgzArchiver.createTgzByteArray(packageRootPath)
-        if (archiverResponse.isErr()){
-            return err(archiverResponse.error)
-        }
 
         const args = new RunStarlarkPackageArgs;
-        args.setLocal(archiverResponse.value)
         args.setPackageId(kurtosisYaml.name)
         args.setSerializedParams(serializedParams)
         args.setDryRun(dryRun)
@@ -413,4 +431,47 @@ export class EnclaveContext {
         args.setCloudUserId(cloudUserId)
         return ok(args)
     }
+
+    private async getKurtosisYaml(packageRootPath: string): Promise<Result<KurtosisYaml, Error>> {
+        const kurtosisYamlFilepath = path.join(packageRootPath, KURTOSIS_YAML_FILENAME)
+
+        const resultParseKurtosisYaml = await parseKurtosisYaml(kurtosisYamlFilepath)
+        if (resultParseKurtosisYaml.isErr()) {
+            return err(resultParseKurtosisYaml.error)
+        }
+        const kurtosisYaml = resultParseKurtosisYaml.value
+
+        return ok(kurtosisYaml)
+    }
+
+
+    private async uploadLocalStarlarkPackageDependencies(
+        packageRootPath: string,
+        packageReplaceOptions: Map<string, string>,
+    ): Promise<Result<null, Error>> {
+        for (const [dependencyPackageId, replaceOption] of packageReplaceOptions.entries()) {
+            if (this.isLocalDependencyReplace(replaceOption)) {
+                const localPackagePath: string = path.join(packageRootPath, replaceOption)
+
+                const archiverResponse = await this.genericTgzArchiver.createTgzByteArray(localPackagePath)
+                if (archiverResponse.isErr()){
+                    return err(archiverResponse.error)
+                }
+
+                const uploadStarlarkPackageResponse = await this.backend.uploadStarlarkPackage(dependencyPackageId, archiverResponse.value)
+                if (uploadStarlarkPackageResponse.isErr()){
+                    return err(uploadStarlarkPackageResponse.error)
+                }
+                return ok(null)
+            }
+        }
+        return ok(null)
+    }
+
+    private isLocalDependencyReplace(replace: string): boolean {
+        if (replace.startsWith(OS_PATH_SEPARATOR_STRING) || replace.startsWith(DOT_RELATIVE_PATH_INDICATOR_STRING)) {
+            return true
+        }
+        return false
+    }
 }

api/typescript/src/core/lib/enclaves/generic_api_container_client.ts

@@ -36,6 +36,7 @@ export interface GenericApiContainerClient {
     getServices(getServicesArgs: GetServicesArgs): Promise<Result<GetServicesResponse, Error>>
     execCommand(execCommandArgs: ExecCommandArgs): Promise<Result<ExecCommandResponse, Error>>
     uploadFiles(name: string, payload: Uint8Array): Promise<Result<UploadFilesArtifactResponse, Error>>
+    uploadStarlarkPackage(packageId: string, payload: Uint8Array): Promise<Result<null, Error>>
     storeWebFilesArtifact(storeWebFilesArtifactArgs: StoreWebFilesArtifactArgs): Promise<Result<StoreWebFilesArtifactResponse, Error>>
     downloadFilesArtifact(downloadFilesArtifactArgs: DownloadFilesArtifactArgs): Promise<Result<Uint8Array, Error>>
     getExistingAndHistoricalServiceIdentifiers(): Promise<Result<GetExistingAndHistoricalServiceIdentifiersResponse, Error>>