Merge branch 'main' into gyani/cloud-everywhere

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [0.85.4](https://github.com/kurtosis-tech/kurtosis/compare/0.85.3...0.85.4) (2023-11-07)
+
+
+### Features
+
+* add autocomplete for cluster set command ([#1695](https://github.com/kurtosis-tech/kurtosis/issues/1695)) ([d36164d](https://github.com/kurtosis-tech/kurtosis/commit/d36164d2dd5738abe2ade548346ec7aa4d8353a3))
+* manage parameters and URL ([#1689](https://github.com/kurtosis-tech/kurtosis/issues/1689)) ([eafc056](https://github.com/kurtosis-tech/kurtosis/commit/eafc056e04643e289998a18d560d5d998c0cdffa))
+* new em ui enclave logs ([#1696](https://github.com/kurtosis-tech/kurtosis/issues/1696)) ([788c7bc](https://github.com/kurtosis-tech/kurtosis/commit/788c7bc8948f750afcdf9452e9c9f010ddd4bc9f))
+* print Made with Kurtosis at the end of a run ([#1687](https://github.com/kurtosis-tech/kurtosis/issues/1687)) ([a08b0b1](https://github.com/kurtosis-tech/kurtosis/commit/a08b0b1e77abbec4fcdff0ff47a9f9f3c1c47c80))
+
+
+### Bug Fixes
+
+* correct is_ci value in metrics from APIC ([#1697](https://github.com/kurtosis-tech/kurtosis/issues/1697)) ([9df62dd](https://github.com/kurtosis-tech/kurtosis/commit/9df62dd9eac2fa8aed456da27c06e63267c54618))
+* kurtosis run considers every nonexistent path to be a URL and fails with a suspicious error ([#1706](https://github.com/kurtosis-tech/kurtosis/issues/1706)) ([0f7809e](https://github.com/kurtosis-tech/kurtosis/commit/0f7809e0a846e77ee26e7f5f92a1a4bfa1622d2f)), closes [#1705](https://github.com/kurtosis-tech/kurtosis/issues/1705)
+* return the correct yaml parsing error ([#1691](https://github.com/kurtosis-tech/kurtosis/issues/1691)) ([c6170ec](https://github.com/kurtosis-tech/kurtosis/commit/c6170eccae1f9b346fc45b1a8289363b82667582))
+* user/instance id values were flipped ([#1698](https://github.com/kurtosis-tech/kurtosis/issues/1698)) ([901069c](https://github.com/kurtosis-tech/kurtosis/commit/901069c1d3bee4e8408848f4bc3cee63fb1be00c))
+
 ## [0.85.3](https://github.com/kurtosis-tech/kurtosis/compare/0.85.2...0.85.3) (2023-11-03)
 
 

LICENSE.md

@@ -3,7 +3,7 @@ Business Source License 1.1
 Parameters
 
 Licensor:             Kurtosis Technologies, Inc.
-Licensed Work:        Kurtosis 0.85.3
+Licensed Work:        Kurtosis 0.85.4
 The Licensed Work is (c) 2023 Kurtosis Technologies, Inc.
 Additional Use Grant: You may make use of the Licensed Work, provided that
 you may not use the Licensed Work for an Environment Orchestration Service.
@@ -12,7 +12,7 @@ you may not use the Licensed Work for an Environment Orchestration Service.
                       allows third parties (other than your employees and
                       contractors) to create distributed system environments.
 
-Change Date:          2027-11-03
+Change Date:          2027-11-07
 
 Change License:       AGPLv3 (GNU Affero General Public License Version 3)
 

api/golang/kurtosis_version/kurtosis_version.go

@@ -9,6 +9,6 @@ const (
 	// !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!!
 	// This is necessary so that Kurt Core consumers will know if they're compatible with the currently-running
 	//  API container
-	KurtosisVersion = "0.85.3"
+	KurtosisVersion = "0.85.4"
 	// !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!!
 )

api/rust/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kurtosis-sdk"
-version = "0.85.3"
+version = "0.85.4"
 license = "BUSL-1.1"
 description = "Rust SDK for Kurtosis"
 edition = "2021"

api/typescript/package.json

@@ -1,7 +1,7 @@
 {
     "name": "kurtosis-sdk",
     "//": "NOTE: DO NOT UPDATE THIS VERSION MANUALLY - IT WILL BE UPDATED DURING THE RELEASE PROCESS!",
-    "version": "0.85.3",
+    "version": "0.85.4",
     "main": "./build/index",
     "description": "This repo contains a Typescript client for communicating with the Kurtosis Engine server, which is responsible for creating, managing and destroying Kurtosis Enclaves.",
     "types": "./build/index",

api/typescript/src/kurtosis_version/kurtosis_version.ts

@@ -1,5 +1,5 @@
 // !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!!
 // This is necessary so that Kurt Core consumers (e.g. modules) will know if they're compatible with the currently-running
 //  API container
-export const KURTOSIS_VERSION: string = "0.85.3"
+export const KURTOSIS_VERSION: string = "0.85.4"
 // !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!!

cli/cli/commands/run/run.go

@@ -12,6 +12,7 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/kurtosis-tech/kurtosis/api/golang/core/lib/starlark_run_config"
@@ -104,6 +105,8 @@ const (
 
 	imageDownloadFlagKey = "image-download"
 	defaultImageDownload = "missing"
+
+	httpProtocolRegexStr = "^(http|https)://"
 )
 
 var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisCommand{
@@ -694,20 +697,8 @@ func validateSerializedArgs(serializedArgs string) error {
 
 func getArgsFromFilepathOrURL(packageArgsFile string) (string, error) {
 	var packageArgsFileBytes []byte
-	isFileURL := true
-	_, err := os.Stat(packageArgsFile)
-	if err == nil {
-		isFileURL = false
-		packageArgsFileBytes, err = os.ReadFile(packageArgsFile)
-		if err != nil {
-			return "", stacktrace.Propagate(err, "attempted to read file provided by flag '%v' with path '%v' but failed", packageArgsFileFlagKey, packageArgsFile)
-		}
-	}
-	if err != nil && !os.IsNotExist(err) {
-		return "", stacktrace.Propagate(err, "An error occurred checking for argument's file existence on '%s'", packageArgsFile)
-	}
 
-	if isFileURL {
+	if isHttpUrl(packageArgsFile) {
 		argsFileURL, parseErr := url.Parse(packageArgsFile)
 		if parseErr != nil {
 			return "", stacktrace.Propagate(parseErr, "An error occurred while parsing file args URL '%s'", argsFileURL)
@@ -722,6 +713,16 @@ func getArgsFromFilepathOrURL(packageArgsFile string) (string, error) {
 			return "", stacktrace.Propagate(readAllErr, "An error occurred reading the args file content")
 		}
 		packageArgsFileBytes = responseBodyBytes
+	} else {
+		_, err := os.Stat(packageArgsFile)
+		if err != nil {
+			return "", stacktrace.Propagate(err, "An error occurred checking for argument's file existence on '%s'", packageArgsFile)
+		}
+
+		packageArgsFileBytes, err = os.ReadFile(packageArgsFile)
+		if err != nil {
+			return "", stacktrace.Propagate(err, "attempted to read file provided by flag '%v' with path '%v' but failed", packageArgsFileFlagKey, packageArgsFile)
+		}
 	}
 
 	packageArgsFileStr := string(packageArgsFileBytes)
@@ -730,4 +731,10 @@ func getArgsFromFilepathOrURL(packageArgsFile string) (string, error) {
 	}
 
 	return packageArgsFileStr, nil
+
+}
+
+func isHttpUrl(maybeHttpUrl string) bool {
+	httpProtocolRegex := regexp.MustCompile(httpProtocolRegexStr)
+	return httpProtocolRegex.MatchString(maybeHttpUrl)
 }

cli/cli/commands/run/run_test.go

@@ -42,3 +42,35 @@ func TestValidateArgs_invalid(t *testing.T) {
 	err = validatePackageArgs(testCtx, testParsedFlags, parsedArgs)
 	require.NotNil(t, err)
 }
+
+func TestIsHttpUrl_ValidHTTP(t *testing.T) {
+	fileUrl := "http://www.mysite.com/myfile.json"
+
+	isHttpUrlResult := isHttpUrl(fileUrl)
+
+	require.True(t, isHttpUrlResult)
+}
+
+func TestIsHttpUrl_ValidHTTPS(t *testing.T) {
+	fileUrl := "https://www.mysite.com/myfile.json"
+
+	isHttpUrlResult := isHttpUrl(fileUrl)
+
+	require.True(t, isHttpUrlResult)
+}
+
+func TestIsHttpUrl_NoValidBecauseIsAbsoluteFilepath(t *testing.T) {
+	fileUrl := "/my-folder/myfile.json"
+
+	isHttpUrlResult := isHttpUrl(fileUrl)
+
+	require.False(t, isHttpUrlResult)
+}
+
+func TestIsHttpUrl_NoValidBecauseIsRelativeFilepath(t *testing.T) {
+	fileUrl := "../my-folder/myfile.json"
+
+	isHttpUrlResult := isHttpUrl(fileUrl)
+
+	require.False(t, isHttpUrlResult)
+}

enclave-manager/web/.env

@@ -1,4 +1,5 @@
 REACT_APP_KURTOSIS_DEFAULT_HOST=localhost
-REACT_APP_KURTOSIS_DEFAULT_PORT=8081
+REACT_APP_KURTOSIS_DEFAULT_EM_API_PORT=8081
 
-REACT_APP_KURTOSIS_CLOUD_URL=https://cloud.kurtosis.com:9770
+REACT_APP_KURTOSIS_CLOUD_UI_URL=https://cloud.kurtosis.com
+REACT_APP_KURTOSIS_PACKAGE_INDEXER_URL=https://cloud.kurtosis.com:9770

enclave-manager/web/.prettierignore

@@ -1,3 +1,6 @@
+Ignore generated code:
+src/client/packageIndexer/api/*.ts
+
 # Ignore artifacts:
 build
 coverage

enclave-manager/web/.prettierrc

@@ -1,4 +1,4 @@
 {
   "printWidth": 120,
-  "plugins": ["prettier-plugin-organize-imports"]
+  "plugins": ["prettier-plugin-organize-imports", "prettier-plugin-curly"]
 }

enclave-manager/web/package.json

@@ -1,7 +1,8 @@
 {
-  "name": "frontend",
-  "version": "0.2.0",
+  "name": "enclave-manager-ui",
+  "version": "0.1.0",
   "private": true,
+  "homepage": ".",
   "dependencies": {
     "@chakra-ui/icons": "^2.1.1",
     "@chakra-ui/react": "^2.8.1",
@@ -30,14 +31,17 @@
     "@types/react": "^18.0.0",
     "@types/react-dom": "^18.0.0",
     "prettier": "3.0.3",
+    "prettier-plugin-curly": "^0.1.3",
     "prettier-plugin-organize-imports": "^3.2.3",
     "typescript": "^4.4.2"
   },
   "scripts": {
+    "prebuild": "rm -rf ../../engine/server/webapp",
     "clean": "rm -rf build",
     "cleanInstall": "rm -rf node_modules; yarn install",
     "start": "react-scripts start",
     "build": "react-scripts build",
+    "postbuild": "cp -r build/ ../../engine/server/webapp",
     "prettier": "prettier . --check",
     "prettier:fix": "prettier . --write",
     "test": "react-scripts test",

enclave-manager/web/src/client/constants.ts

@@ -1,10 +1,12 @@
 import { isDefined } from "../utils";
 
-export const KURTOSIS_DEFAULT_HOST = process.env.REACT_APP_KURTOSIS_DEFAULT_HOST || "localhost";
-export const KURTOSIS_DEFAULT_PORT = isDefined(process.env.REACT_APP_KURTOSIS_DEFAULT_PORT)
-  ? parseInt(process.env.REACT_APP_KURTOSIS_DEFAULT_PORT)
+export const KURTOSIS_EM_DEFAULT_HOST = process.env.REACT_APP_KURTOSIS_DEFAULT_HOST || "localhost";
+export const KURTOSIS_DEFAULT_EM_API_PORT = isDefined(process.env.REACT_APP_KURTOSIS_DEFAULT_EM_API_PORT)
+  ? parseInt(process.env.REACT_APP_KURTOSIS_DEFAULT_EM_API_PORT)
   : 8081;
-export const KURTOSIS_DEFAULT_URL =
-  process.env.REACT_APP_KURTOSIS_DEFAULT_URL || `http://${KURTOSIS_DEFAULT_HOST}:${KURTOSIS_DEFAULT_PORT}`;
+export const KURTOSIS_EM_API_DEFAULT_URL =
+  process.env.REACT_APP_KURTOSIS_DEFAULT_URL || `http://${KURTOSIS_EM_DEFAULT_HOST}:${KURTOSIS_DEFAULT_EM_API_PORT}`;
 
-export const KURTOSIS_CLOUD_URL = process.env.REACT_APP_KURTOSIS_CLOUD_URL || "https://cloud.kurtosis.com:9770";
+export const KURTOSIS_PACKAGE_INDEXER_URL =
+  process.env.REACT_APP_KURTOSIS_PACKAGE_INDEXER_URL || "https://cloud.kurtosis.com:9770";
+export const KURTOSIS_CLOUD_HOST = process.env.REACT_APP_KURTOSIS_CLOUD_UI_URL || "https://cloud.kurtosis.com";

enclave-manager/web/src/client/enclaveManager/AuthenticatedKurtosisClient.ts

@@ -1,19 +1,24 @@
 import { createPromiseClient } from "@connectrpc/connect";
 import { createConnectTransport } from "@connectrpc/connect-web";
 import { KurtosisEnclaveManagerServer } from "enclave-manager-sdk/build/kurtosis_enclave_manager_api_connect";
-import { KURTOSIS_DEFAULT_PORT } from "../constants";
+import { KURTOSIS_CLOUD_HOST, KURTOSIS_DEFAULT_EM_API_PORT } from "../constants";
 import { KurtosisClient } from "./KurtosisClient";
 
-function constructGatewayURL(host: string): string {
-  return `https://cloud.kurtosis.com/gateway/ips/${host}/ports/${KURTOSIS_DEFAULT_PORT}`;
+function constructGatewayURL(remoteHost: string): string {
+  return `${KURTOSIS_CLOUD_HOST}/gateway/ips/${remoteHost}/ports/${KURTOSIS_DEFAULT_EM_API_PORT}`;
 }
 
 export class AuthenticatedKurtosisClient extends KurtosisClient {
-  private token: string;
+  private readonly token: string;
 
-  constructor(host: string, token: string) {
+  constructor(gatewayHost: string, token: string, parentUrl: URL, childUrl: URL) {
     super(
-      createPromiseClient(KurtosisEnclaveManagerServer, createConnectTransport({ baseUrl: constructGatewayURL(host) })),
+      createPromiseClient(
+        KurtosisEnclaveManagerServer,
+        createConnectTransport({ baseUrl: constructGatewayURL(gatewayHost) }),
+      ),
+      parentUrl,
+      childUrl,
     );
     this.token = token;
   }

enclave-manager/web/src/client/enclaveManager/KurtosisClient.ts

@@ -1,11 +1,13 @@
 import { PromiseClient } from "@connectrpc/connect";
-import { RunStarlarkPackageArgs } from "enclave-manager-sdk/build/api_container_service_pb";
+import { RunStarlarkPackageArgs, ServiceInfo } from "enclave-manager-sdk/build/api_container_service_pb";
 import {
   CreateEnclaveArgs,
   DestroyEnclaveArgs,
   EnclaveAPIContainerInfo,
   EnclaveInfo,
   EnclaveMode,
+  GetServiceLogsArgs,
+  LogLineFilter,
 } from "enclave-manager-sdk/build/engine_service_pb";
 import { KurtosisEnclaveManagerServer } from "enclave-manager-sdk/build/kurtosis_enclave_manager_api_connect";
 import {
@@ -14,18 +16,49 @@ import {
   GetStarlarkRunRequest,
   RunStarlarkPackageRequest,
 } from "enclave-manager-sdk/build/kurtosis_enclave_manager_api_pb";
-import { assertDefined, asyncResult } from "../../utils";
+import { assertDefined, asyncResult, isDefined } from "../../utils";
 import { RemoveFunctions } from "../../utils/types";
+import { EnclaveFullInfo } from "../../emui/enclaves/types";
 
 export abstract class KurtosisClient {
-  protected client: PromiseClient<typeof KurtosisEnclaveManagerServer>;
+  protected readonly client: PromiseClient<typeof KurtosisEnclaveManagerServer>;
 
-  constructor(client: PromiseClient<typeof KurtosisEnclaveManagerServer>) {
+  /* Full URL of the browser containing the EM UI covering two use cases:
+   * In local-mode this is: http://localhost:9711, http://localhost:3000 (with `yarn start` / dev mode)
+   * In authenticated mode this is: https://cloud.kurtosis.com/enclave-manager (this data/url is provided as a search param when the code loads)
+   *
+   * This URL is primarily used to generate links to the EM UI (where the hostname is included).
+   * */
+  protected readonly parentUrl: URL;
+
+  /* Full URL of the EM UI, covering two use cases:
+   * In local-mode this is the same as the `parentUrl`
+   * In authenticated mode : https://cloud.kurtosis.com/enclave-manager/gateway/ips/1-2-3-4/ports/1234/?searchparams... (this data/url is provided as a search param when the code loads)
+   *
+   * This URL is primarily used to set the React router basename so that the router is able to ignore any leading subdirectories.
+   * */
+  protected readonly childUrl: URL;
+
+  constructor(client: PromiseClient<typeof KurtosisEnclaveManagerServer>, parentUrl: URL, childUrl: URL) {
     this.client = client;
+    this.parentUrl = parentUrl;
+    this.childUrl = childUrl;
   }
 
   abstract getHeaderOptions(): { headers?: Headers };
 
+  getParentBasePathUrl() {
+    return `${this.parentUrl.origin}${this.parentUrl.pathname}`;
+  }
+
+  getChildPath() {
+    return this.childUrl.pathname;
+  }
+
+  getChildUrl() {
+    return this.childUrl;
+  }
+
   async checkHealth() {
     return asyncResult(this.client.check({}, this.getHeaderOptions()));
   }
@@ -50,7 +83,28 @@ export abstract class KurtosisClient {
         apicPort: apicInfo.grpcPortInsideEnclave,
       });
       return this.client.getServices(request, this.getHeaderOptions());
-    }, "KurtosisClient could not getServices");
+    }, `KurtosisClient could not getServices for ${enclave.name}`);
+  }
+
+  async getServiceLogs(
+    abortController: AbortController,
+    enclave: RemoveFunctions<EnclaveFullInfo>,
+    services: ServiceInfo[],
+    followLogs?: boolean,
+    numLogLines?: number,
+    returnAllLogs?: boolean,
+    conjunctiveFilters: LogLineFilter[] = [],
+  ) {
+    // Not currently using asyncResult as the return type here is an asyncIterable
+    const request = new GetServiceLogsArgs({
+      enclaveIdentifier: enclave.name,
+      serviceUuidSet: services.reduce((acc, service) => ({ ...acc, [service.serviceUuid]: true }), {}),
+      followLogs: isDefined(followLogs) ? followLogs : true,
+      conjunctiveFilters: conjunctiveFilters,
+      numLogLines: isDefined(numLogLines) ? numLogLines : 1500,
+      returnAllLogs: !!returnAllLogs,
+    });
+    return this.client.getServiceLogs(request, { ...this.getHeaderOptions(), signal: abortController.signal });
   }
 
   async getStarlarkRun(enclave: RemoveFunctions<EnclaveInfo>) {
@@ -65,7 +119,7 @@ export abstract class KurtosisClient {
         apicPort: apicInfo.grpcPortInsideEnclave,
       });
       return this.client.getStarlarkRun(request, this.getHeaderOptions());
-    }, "KurtosisClient could not getStarlarkRun");
+    }, `KurtosisClient could not getStarlarkRun for ${enclave.name}`);
   }
 
   async listFilesArtifactNamesAndUuids(enclave: RemoveFunctions<EnclaveInfo>) {
@@ -80,7 +134,7 @@ export abstract class KurtosisClient {
         apicPort: apicInfo.grpcPortInsideEnclave,
       });
       return this.client.listFilesArtifactNamesAndUuids(request, this.getHeaderOptions());
-    }, "KurtosisClient could not listFilesArtifactNamesAndUuids");
+    }, `KurtosisClient could not listFilesArtifactNamesAndUuids for ${enclave.name}`);
   }
 
   async createEnclave(