From 59ba36d6bfbff676cd56deeff5d6e0ce7fe75b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marijan=20Petri=C4=8Devi=C4=87?= Date: Sat, 25 May 2024 17:57:09 +0200 Subject: [PATCH 01/47] fix: add enclave-manager/local to workspace --- enclave-manager/local/go.mod | 15 +++++++++++++++ enclave-manager/local/go.sum | 15 +++++++++++++++ go.work | 1 + 3 files changed, 31 insertions(+) create mode 100644 enclave-manager/local/go.mod create mode 100644 enclave-manager/local/go.sum diff --git a/enclave-manager/local/go.mod b/enclave-manager/local/go.mod new file mode 100644 index 0000000000..0d00c4cf4a --- /dev/null +++ b/enclave-manager/local/go.mod @@ -0,0 +1,15 @@ +module github.com/kurtosis-tech/kurtosis/enclave-manager/local + +go 1.20 +replace ( + github.com/kurtosis-tech/kurtosis/enclave-manager => ../server + github.com/kurtosis-tech/kurtosis/contexts-config-store => ../../contexts-config-store + github.com/kurtosis-tech/kurtosis/kurtosis_version => ../../kurtosis_version +) + +require ( + github.com/kurtosis-tech/kurtosis/enclave-manager v0.0.0-20230828153722-32770ca96513 + github.com/sirupsen/logrus v1.9.3 +) + +require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/enclave-manager/local/go.sum b/enclave-manager/local/go.sum new file mode 100644 index 0000000000..21f9bfb372 --- /dev/null +++ b/enclave-manager/local/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work b/go.work index 318f912450..9a20f823bb 100644 --- a/go.work +++ b/go.work @@ -10,6 +10,7 @@ use ( ./core/launcher ./core/server ./enclave-manager/api/golang + ./enclave-manager/local ./enclave-manager/server ./engine/launcher ./engine/server From bc0e31cdb51f3375468ecf84360b20288481cb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marijan=20Petri=C4=8Devi=C4=87?= Date: Sat, 25 May 2024 18:01:43 +0200 Subject: [PATCH 02/47] feat: package kurtosis with nix --- flake.lock | 6 +++--- flake.nix | 13 ++++++++++--- package.nix | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 package.nix diff --git a/flake.lock b/flake.lock index 60e9d3d49c..9c05eae1a9 100644 --- a/flake.lock +++ b/flake.lock @@ -58,11 +58,11 @@ }, "unstable": { "locked": { - "lastModified": 1706371002, - "narHash": "sha256-dwuorKimqSYgyu8Cw6ncKhyQjUDOyuXoxDTVmAXq88s=", + "lastModified": 1716509168, + "narHash": "sha256-4zSIhSRRIoEBwjbPm3YiGtbd8HDWzFxJjw5DYSDy1n8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c002c6aa977ad22c60398daaa9be52f2203d0006", + "rev": "bfb7a882678e518398ce9a31a881538679f6f092", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index ef4d08fceb..cb00960565 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ flake-utils.url = "github:numtide/flake-utils"; }; - outputs = { nixpkgs, unstable, flake-utils, ... }: + outputs = { self, nixpkgs, unstable, flake-utils, ... }: let utils = flake-utils; in utils.lib.eachDefaultSystem (system: let @@ -17,9 +17,15 @@ inherit pkgs system; nodejs = pkgs.nodejs_20; }; - in { + in + { formatter = pkgs.nixpkgs-fmt; + packages = rec { + default = kurtosis; + kurtosis = unstable_pkgs.callPackage ./package.nix { }; + }; + devShell = pkgs.mkShell { nativeBuildInputs = with pkgs; let @@ -27,7 +33,8 @@ import ./nix-pkgs/openapi-codegen.nix { inherit pkgs; }; grpc-tools-node = import ./nix-pkgs/grpc-tools-node.nix { inherit pkgs; }; - in [ + in + [ goreleaser go_1_20 gopls diff --git a/package.nix b/package.nix new file mode 100644 index 0000000000..92aead7926 --- /dev/null +++ b/package.nix @@ -0,0 +1,54 @@ +{ lib +, buildGoModule +, writeTextFile +}: + +buildGoModule rec { + pname = "kurtosis"; + version = "0.89.11"; + + src = ./.; + + proxyVendor = true; + vendorHash = "sha256-GaEIitoRiuYxtS7cDKobFyIlraDNQjcvbRvzG3nUKFU="; + + postPatch = + let + kurtosisVersion = writeTextFile { + name = "kurtosis_verion.go"; + text = '' + package kurtosis_version + const ( + KurtosisVersion = "${version}" + ) + ''; + }; + in + '' + ln -s ${kurtosisVersion} kurtosis_version/kurtosis_version.go + ''; + + # disable checks temporarily since they connect to the internet + # namely user_support_constants_test.go + doCheck = false; + + # keep this for future reference + preCheck = '' + # some tests in commands use XDG home related environment variables + export HOME=/tmp + ''; + + postInstall = '' + mv $out/bin/cli $out/bin/kurtosis + mv $out/bin/files_artifacts_expander $out/bin/files-artifacts-expander + mv $out/bin/api_container $out/bin/api-container + + ''; + + meta = with lib; { + description = "A platform for launching an ephemeral Ethereum backend"; + mainProgram = "kurtosis"; + homepage = "https://github.com/kurtosis-tech/kurtosis"; + license = licenses.asl20; + }; +} From 41bd2dc103c0aa00504af884e8dc4d13f586be30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marijan=20Petri=C4=8Devi=C4=87?= Date: Sat, 25 May 2024 18:02:00 +0200 Subject: [PATCH 03/47] feat: ignore nix related files --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 21b3b83910..7c9666618d 100644 --- a/.gitignore +++ b/.gitignore @@ -131,4 +131,8 @@ lerna-debug.log # contains several libraries that get shipped yarn.lock package.json -package-lock.json \ No newline at end of file +package-lock.json + +# Nix +# ignores the default result symlink created when building with nix +result From 744e330693ca37ed4ecd29c1ae86d7a8eff019fc Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:17:47 -0400 Subject: [PATCH 04/47] use podman socket --- .../backend_creator/backend_creator.go | 5 +++-- .../docker/docker_kurtosis_backend/consts/consts.go | 7 +++++-- .../implementations/traefik/consts.go | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go index 86d787d599..234cedff40 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go @@ -3,11 +3,12 @@ package backend_creator import ( "context" "fmt" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "net" "os" "path" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" + "github.com/docker/docker/client" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" @@ -27,7 +28,7 @@ import ( const ( unixSocketPrefix = "unix://" - systemDaemonSocket = "/var/run/docker.sock" + systemDaemonSocket = "/var/run/podman/podman.sock" userOwnDaemonSocket = "/.docker/run/docker.sock" noTempDirPrefix = "" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go index a13632c54e..2da615c2bd 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go @@ -22,14 +22,17 @@ const ( EngineTransportProtocol = port_spec.TransportProtocol_TCP // This needs to be bind-mounted into the engine & API containers so they can manipulate Docker - DockerSocketFilepath = "/var/run/docker.sock" + DockerSocketFilepath = "/var/run/podman/podman.sock" // The host engine config directory to mount and its local mapping HostEngineConfigDirToMount = "/root/engine_config" EngineConfigLocalDir = "/run/engine" //The Docker network name where all the containers in the engine and logs service context will be added - NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" + // NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" + // HttpApplicationProtocol = "http" + // The Docker network name where all the containers in the engine and logs service context will be added + NameOfNetworkToStartEngineAndLogServiceContainersIn = "podman" HttpApplicationProtocol = "http" GitHubAuthStorageDirPath = "/kurtosis-data/github-auth/" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go index 7df8087c81..bf2b63fc56 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -28,7 +28,7 @@ entryPoints: providers: docker: - endpoint: "unix:///var/run/docker.sock" + endpoint: "unix:///var/run/podman/podman.sock" exposedByDefault: false network: "{{ .NetworkId }}" ` From 70fa36ae0f167e3d27badcfe5fac440dbd14f248 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:19:29 -0400 Subject: [PATCH 05/47] turn off log collector --- .../start_user_services.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go index 371937bbf6..ce2759b592 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions/start_user_services.go @@ -669,15 +669,15 @@ func createStartServiceOperation( } } - if logsCollectorAddress == "" { - return nil, stacktrace.NewError("Expected to have a logs collector server address value to send the user service logs, but it is empty") - } + // if logsCollectorAddress == "" { + // return nil, stacktrace.NewError("Expected to have a logs collector server address value to send the user service logs, but it is empty") + // } - // The container will be configured to send the logs to the Fluentbit logs collector server - fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( - logsCollectorAddress, - logsCollectorLabels, - ) + // // The container will be configured to send the logs to the Fluentbit logs collector server + // fluentdLoggingDriverCnfg := docker_manager.NewFluentdLoggingDriver( + // logsCollectorAddress, + // logsCollectorLabels, + // ) createAndStartArgsBuilder := docker_manager.NewCreateAndStartContainerArgsBuilder( containerImageName, @@ -703,8 +703,8 @@ func createStartServiceOperation( tiniEnabled, ).WithVolumeMounts( volumeMounts, - ).WithLoggingDriver( - fluentdLoggingDriverCnfg, + // ).WithLoggingDriver( + // fluentdLoggingDriverCnfg, ).WithRestartPolicy( restartPolicy, ).WithUser( From df42701d46c537543825085da082de4ce45c51e0 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:21:15 -0400 Subject: [PATCH 06/47] fix volume inconsistency match --- .../docker/docker_manager/docker_manager.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index bd4e771c04..7949a88621 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -481,6 +481,21 @@ func (manager *DockerManager) GetVolumesByLabels(ctx context.Context, labels map result := []*volume.Volume{} if resp.Volumes != nil { + // Podman API inconsistency - filter out the union matches that podman returns while docker only returns the intersect matches when filtering by label + for _, vol := range resp.Volumes { + allLabelsMatch := true + + for label, val := range labels { + if volValue, exists := vol.Labels[label]; !exists || volValue != val { + allLabelsMatch = false + break + } + } + + if allLabelsMatch { + result = append(result, vol) + } + } result = resp.Volumes } From 68c13874d27d88c51d05b91e2145b0ecd0c382f1 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:22:36 -0400 Subject: [PATCH 07/47] retrieve network id --- .../docker/docker_manager/docker_manager.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 7949a88621..73fee107ba 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -819,8 +819,17 @@ func (manager *DockerManager) GetContainerIps(ctx context.Context, containerId s return nil, stacktrace.Propagate(err, "An error occurred inspecting container with ID '%v'", containerId) } allNetworkInfo := resp.NetworkSettings.Networks - for _, networkInfo := range allNetworkInfo { - containerIps[networkInfo.NetworkID] = networkInfo.IPAddress + // for _, networkInfo := range allNetworkInfo { + // containerIps[networkInfo.NetworkID] = networkInfo.IPAddress + // } + for networkKey, networkInfo := range allNetworkInfo { + // podman does not return the networkID properly and as such we need to make sure we get it. + network, err := manager.dockerClient.NetworkInspect(ctx, networkInfo.NetworkID, types.NetworkInspectOptions{}) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred inspecting network: '%v'", networkKey) + } + + containerIps[network.ID] = networkInfo.IPAddress } return containerIps, nil } From e062bf17e5c20ae0598a58c641ef23d3243e0b7b Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:23:09 -0400 Subject: [PATCH 08/47] change host ip --- .../lib/backend_impls/docker/docker_manager/docker_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 73fee107ba..1779f8eedc 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -1762,7 +1762,7 @@ func (manager *DockerManager) getContainerHostConfig( portMap[containerPort] = []nat.PortBinding{ // Leaving this struct empty will cause Docker to automatically choose an interface IP & port on the host machine { - HostIP: "", + HostIP: "0.0.0.0", HostPort: "", }, } From 15aa10a93f5cfe70e305e43cc9bcf233163e4599 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 15:24:47 -0400 Subject: [PATCH 09/47] add xdg fallback --- .../host_machine_directories.go | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cli/cli/helpers/host_machine_directories/host_machine_directories.go b/cli/cli/helpers/host_machine_directories/host_machine_directories.go index d78a53dbb9..3d210728be 100644 --- a/cli/cli/helpers/host_machine_directories/host_machine_directories.go +++ b/cli/cli/helpers/host_machine_directories/host_machine_directories.go @@ -1,9 +1,13 @@ package host_machine_directories import ( + "io/ioutil" + "path" + "path/filepath" + "github.com/adrg/xdg" "github.com/kurtosis-tech/stacktrace" - "path" + "github.com/sirupsen/logrus" ) const ( @@ -107,9 +111,19 @@ func GetLastPesteredUserAboutOldVersionsFilepath() (string, error) { func GetKurtosisCliLogsFileDirPath(fileName string) (string, error) { xdgRelDirPath := getRelativeFilePathForKurtosisCliLogs() - kurtosisCliLogFilePath, err := xdg.DataFile(path.Join(xdgRelDirPath, fileName)) - if err != nil { - return "", stacktrace.Propagate(err, "An error occurred getting the kurtosis cli logs file path using '%v'", kurtosisCliLogFilePath) + // kurtosisCliLogFilePath, err := xdg.DataFile(path.Join(xdgRelDirPath, fileName)) + // if err != nil { + // return "", stacktrace.Propagate(err, "An error occurred getting the kurtosis cli logs file path using '%v'", kurtosisCliLogFilePath) + // } + kurtosisCliLogFilePath, errXdg := xdg.DataFile(path.Join(xdgRelDirPath, fileName)) + if errXdg != nil { + // Fallback to temp folder if XDG fails to find a suitable location. For instance XDG will fail when testing inside Nix sandbox. + logrus.WithError(errXdg).Warnf("Couldn't create kurtosis cli logs file path in the user space '%v'. Trying on temp folder.", kurtosisCliLogFilePath) + kurtosisCliLogDir, errTemp := ioutil.TempDir("", applicationDirname) + if errTemp != nil { + return "", stacktrace.Propagate(errTemp, "An error occurred creating kurtosis cli logs file path using '%v'", kurtosisCliLogDir) + } + kurtosisCliLogFilePath = filepath.Join(kurtosisCliLogDir, fileName) } return kurtosisCliLogFilePath, nil } From 555fcb3b326d460fe8b7c563f7684a736af4b75c Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 16:00:23 -0400 Subject: [PATCH 10/47] use local goreleaser --- cli/cli/scripts/build.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cli/cli/scripts/build.sh b/cli/cli/scripts/build.sh index 0f8401c209..026a0de506 100755 --- a/cli/cli/scripts/build.sh +++ b/cli/cli/scripts/build.sh @@ -97,10 +97,11 @@ fi exit 1 fi # Executing goreleaser v1.26.2 without needing to install it - if ! curl -sfL https://goreleaser.com/static/run | VERSION=v1.26.2 DISTRIBUTION=oss bash -s -- ${goreleaser_verb_and_flags}; then - echo "Error: Couldn't build the CLI binary for the current OS/arch" >&2 - exit 1 - fi +# if ! curl -sfL https://goreleaser.com/static/run | VERSION=v1.26.2 DISTRIBUTION=oss bash -s -- ${goreleaser_verb_and_flags}; then + if ! GORELEASER_CURRENT_TAG=$(cat $root_dirpath/version.txt) goreleaser ${goreleaser_verb_and_flags}; then + echo "Error: Couldn't build the CLI binary for the current OS/arch" >&2 + exit 1 + fi ) # Final verification From 3e20f4c2012a0f6100bd3e01db15cbca6e769606 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 16:17:05 -0400 Subject: [PATCH 11/47] Revert "use podman socket" This reverts commit 744e330693ca37ed4ecd29c1ae86d7a8eff019fc. --- .../backend_creator/backend_creator.go | 5 ++--- .../docker/docker_kurtosis_backend/consts/consts.go | 7 ++----- .../implementations/traefik/consts.go | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go index 234cedff40..86d787d599 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go @@ -3,12 +3,11 @@ package backend_creator import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "net" "os" "path" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" - "github.com/docker/docker/client" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" @@ -28,7 +27,7 @@ import ( const ( unixSocketPrefix = "unix://" - systemDaemonSocket = "/var/run/podman/podman.sock" + systemDaemonSocket = "/var/run/docker.sock" userOwnDaemonSocket = "/.docker/run/docker.sock" noTempDirPrefix = "" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go index 2da615c2bd..a13632c54e 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go @@ -22,17 +22,14 @@ const ( EngineTransportProtocol = port_spec.TransportProtocol_TCP // This needs to be bind-mounted into the engine & API containers so they can manipulate Docker - DockerSocketFilepath = "/var/run/podman/podman.sock" + DockerSocketFilepath = "/var/run/docker.sock" // The host engine config directory to mount and its local mapping HostEngineConfigDirToMount = "/root/engine_config" EngineConfigLocalDir = "/run/engine" //The Docker network name where all the containers in the engine and logs service context will be added - // NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" - // HttpApplicationProtocol = "http" - // The Docker network name where all the containers in the engine and logs service context will be added - NameOfNetworkToStartEngineAndLogServiceContainersIn = "podman" + NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" HttpApplicationProtocol = "http" GitHubAuthStorageDirPath = "/kurtosis-data/github-auth/" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go index bf2b63fc56..7df8087c81 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -28,7 +28,7 @@ entryPoints: providers: docker: - endpoint: "unix:///var/run/podman/podman.sock" + endpoint: "unix:///var/run/docker.sock" exposedByDefault: false network: "{{ .NetworkId }}" ` From f54369b7346ad650f587fb3a1285b044df217ebc Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 17:21:27 -0400 Subject: [PATCH 12/47] fix get engine logs and components --- .../shared_helpers/shared_helpers.go | 39 ++++++++++--------- .../docker/docker_manager/docker_manager.go | 26 +++++++++++++ 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers/shared_helpers.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers/shared_helpers.go index 18c3e533e2..ba9c236e66 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers/shared_helpers.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers/shared_helpers.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" "io" "net" "os" @@ -14,6 +13,8 @@ import ( "strings" "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" + "github.com/docker/docker/pkg/stdcopy" "github.com/docker/go-connections/nat" "github.com/gammazero/workerpool" @@ -417,24 +418,24 @@ func GetEngineAndLogsComponentsNetwork( ctx context.Context, dockerManager *docker_manager.DockerManager, ) (*types.Network, error) { - matchingNetworks, err := dockerManager.GetNetworksByName(ctx, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn) - if err != nil { - return nil, stacktrace.Propagate( - err, - "An error occurred getting networks matching the network we want to start the engine in, '%v'", - consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, - ) - } - numMatchingNetworks := len(matchingNetworks) - if numMatchingNetworks == 0 && numMatchingNetworks > 1 { - return nil, stacktrace.NewError( - "Expected exactly one network matching the name of the network that we want to start the engine in, '%v', but got %v", - consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, - numMatchingNetworks, - ) - } - targetNetwork := matchingNetworks[0] - return targetNetwork, nil + // matchingNetworks, err := dockerManager.GetNetworksByName(ctx, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn) + // if err != nil { + // return nil, stacktrace.Propagate( + // err, + // "An error occurred getting networks matching the network we want to start the engine in, '%v'", + // consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, + // ) + // } + // numMatchingNetworks := len(matchingNetworks) + // if numMatchingNetworks == 0 && numMatchingNetworks > 1 { + // return nil, stacktrace.NewError( + // "Expected exactly one network matching the name of the network that we want to start the engine in, '%v', but got %v", + // consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, + // numMatchingNetworks, + // ) + // } + // targetNetwork := matchingNetworks[0] + return dockerManager.GetDefaultNetwork(ctx) } func DumpContainers(ctx context.Context, dockerManager *docker_manager.DockerManager, containers []*types.Container, outputDirpath string) error { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 1779f8eedc..1e5909af96 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -413,6 +413,32 @@ func (manager *DockerManager) RemoveNetwork(context context.Context, networkId s return nil } +func (manager *DockerManager) GetDefaultNetwork(ctx context.Context) (*docker_manager_types.Network, error) { + + matchingNetworks, err := manager.GetNetworksByName(ctx, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn) + if err != nil { + return nil, stacktrace.Propagate( + err, + "An error occurred getting networks matching the network we want to start the engine in, '%v'", + consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, + ) + } + numMatchingNetworks := len(matchingNetworks) + if numMatchingNetworks > 1 { + return nil, stacktrace.NewError( + "Expected exactly one network matching the name of the network that we want to start the engine in, '%v', but got %v", + consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, + numMatchingNetworks, + ) + } + + if numMatchingNetworks == 0 { + return nil, stacktrace.NewError(fmt.Sprintf("No matching network found with the configured name: %v", consts.NameOfNetworkToStartEngineAndLogServiceContainersIn)) + } + + return matchingNetworks[0], nil +} + /* CreateVolume Creates a Docker volume identified by the given name. From e48fc234ea263a89aa4958d7ffbd6bd591008a7e Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 12 Jul 2024 17:32:46 -0400 Subject: [PATCH 13/47] bridge -> podman --- .../docker/docker_kurtosis_backend/consts/consts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go index a13632c54e..dc5816377b 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go @@ -29,7 +29,7 @@ const ( EngineConfigLocalDir = "/run/engine" //The Docker network name where all the containers in the engine and logs service context will be added - NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" + NameOfNetworkToStartEngineAndLogServiceContainersIn = "podman" HttpApplicationProtocol = "http" GitHubAuthStorageDirPath = "/kurtosis-data/github-auth/" From 9789f1cee6f4cd135d6b4acfee3d348a90c8ccda Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 25 Jul 2024 11:46:58 -0400 Subject: [PATCH 14/47] disable security opts --- .../docker/docker_manager/docker_manager.go | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 1e5909af96..cd43767885 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -624,7 +624,8 @@ func (manager *DockerManager) CreateAndStartContainer( } containerHostConfigPtr, err := manager.getContainerHostConfig( args.addedCapabilities, - args.securityOpts, + //args.securityOpts, + args.networkMode, args.bindMounts, args.volumeMounts, @@ -1823,6 +1824,7 @@ func (manager *DockerManager) getContainerHostConfig( securityOptStr := string(securityOpt) securityOptsSlice = append(securityOptsSlice, securityOptStr) } + securityOptsSlice = append(securityOptsSlice, "") extraHosts := []string{} if needsToAccessDockerHostMachine { @@ -1925,21 +1927,25 @@ func (manager *DockerManager) getContainerHostConfig( Privileged: false, PublishAllPorts: false, ReadonlyRootfs: false, - SecurityOpt: securityOptsSlice, - StorageOpt: nil, - Tmpfs: nil, - UTSMode: "", - UsernsMode: "", - ShmSize: 0, - Sysctls: nil, - Runtime: "", - ConsoleSize: [2]uint{}, - Isolation: "", - Resources: resources, - Mounts: nil, - MaskedPaths: nil, - ReadonlyPaths: nil, - Init: &useInit, + //SecurityOpt: securityOptsSlice, + SecurityOpt: []string{ + "label=disable", // Disables SELinux + "apparmor:unconfined", // Disables AppArmor + }, + StorageOpt: nil, + Tmpfs: nil, + UTSMode: "", + UsernsMode: "", + ShmSize: 0, + Sysctls: nil, + Runtime: "", + ConsoleSize: [2]uint{}, + Isolation: "", + Resources: resources, + Mounts: nil, + MaskedPaths: nil, + ReadonlyPaths: nil, + Init: &useInit, } return containerHostConfigPtr, nil } From 4edfe23b263e121279b5cd10b771e85ff67be6c4 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 25 Jul 2024 11:47:08 -0400 Subject: [PATCH 15/47] use podman for image building --- scripts/docker-image-builder.sh | 72 ++++++++++++++++----------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/scripts/docker-image-builder.sh b/scripts/docker-image-builder.sh index 7c470f322f..ca47784529 100755 --- a/scripts/docker-image-builder.sh +++ b/scripts/docker-image-builder.sh @@ -70,33 +70,33 @@ done ## Start by making sure the builder and the context do not already exist. If that's the case remove them kurtosis_docker_builder="kurtosis-docker-builder" docker_buildx_context='kurtosis-docker-builder-context' -if docker buildx inspect "${kurtosis_docker_builder}" &>/dev/null; then - echo "Removing docker buildx builder ${kurtosis_docker_builder} as it seems to already exist" - if ! docker buildx rm ${kurtosis_docker_builder} &>/dev/null; then - echo "Failed removing docker buildx builder ${kurtosis_docker_builder}. Try removing it manually with 'docker buildx rm ${kurtosis_docker_builder}' before re-running this script" - exit 1 - fi -fi -if docker context inspect "${docker_buildx_context}" &>/dev/null; then - echo "Removing docker context ${docker_buildx_context} as it seems to already exist" - if ! docker context rm ${docker_buildx_context} &>/dev/null; then - echo "Failed removing docker context ${docker_buildx_context}. Try removing it manually with 'docker context rm ${docker_buildx_context}' before re-running this script" - exit 1 - fi -fi - -## Create Docker context and buildx builder -if ! docker context create "${docker_buildx_context}" &>/dev/null; then - echo "Error: Docker context creation for buildx failed" >&2 - exit 1 -fi -if ! docker buildx create --use --name "${kurtosis_docker_builder}" "${docker_buildx_context}" &>/dev/null; then - echo "Error: Docker context switch for buildx failed" >&2d - exit 1 -fi +#if docker buildx inspect "${kurtosis_docker_builder}" &>/dev/null; then +# echo "Removing docker buildx builder ${kurtosis_docker_builder} as it seems to already exist" +# if ! docker buildx rm ${kurtosis_docker_builder} &>/dev/null; then +# echo "Failed removing docker buildx builder ${kurtosis_docker_builder}. Try removing it manually with 'docker buildx rm ${kurtosis_docker_builder}' before re-running this script" +# exit 1 +# fi +#fi +#if docker context inspect "${docker_buildx_context}" &>/dev/null; then +# echo "Removing docker context ${docker_buildx_context} as it seems to already exist" +# if ! docker context rm ${docker_buildx_context} &>/dev/null; then +# echo "Failed removing docker context ${docker_buildx_context}. Try removing it manually with 'docker context rm ${docker_buildx_context}' before re-running this script" +# exit 1 +# fi +#fi +# +### Create Docker context and buildx builder +#if ! docker context create "${docker_buildx_context}" &>/dev/null; then +# echo "Error: Docker context creation for buildx failed" >&2 +# exit 1 +#fi +#if ! docker buildx create --use --name "${kurtosis_docker_builder}" "${docker_buildx_context}" &>/dev/null; then +# echo "Error: Docker context switch for buildx failed" >&2d +# exit 1 +#fi ## Actually build the Docker image -docker_buildx_cmd="docker buildx build ${push_flag} --platform ${buildx_platform_arg} ${image_tags_concatenated} -f ${dockerfile_filepath} ${dockerfile_dirpath}" +docker_buildx_cmd="sudo podman buildx build ${push_flag} --platform ${buildx_platform_arg} ${image_tags_concatenated} -f ${dockerfile_filepath} ${dockerfile_dirpath}" echo "Running the following docker buildx command:" echo "${docker_buildx_cmd}" if ! eval "${docker_buildx_cmd}"; then @@ -104,14 +104,14 @@ if ! eval "${docker_buildx_cmd}"; then exit 1 fi -# Cleanup context and buildx runner -echo "Cleaning up remaining resources" -if ! docker buildx rm "${kurtosis_docker_builder}" &>/dev/null; then - echo "Warn: Failed removing the buildx builder '${kurtosis_docker_builder}'. Try manually removing it with 'docker buildx rm ${kurtosis_docker_builder}'" >&2 - exit 1 -fi -if ! docker context rm "${docker_buildx_context}" &>/dev/null; then - echo "Warn: Failed removing the buildx context '${docker_buildx_context}'. Try manually removing it with 'docker context rm ${docker_buildx_context}'" >&2 - exit 1 -fi -echo "Successfully built docker image" +## Cleanup context and buildx runner +#echo "Cleaning up remaining resources" +#if ! docker buildx rm "${kurtosis_docker_builder}" &>/dev/null; then +# echo "Warn: Failed removing the buildx builder '${kurtosis_docker_builder}'. Try manually removing it with 'docker buildx rm ${kurtosis_docker_builder}'" >&2 +# exit 1 +#fi +#if ! docker context rm "${docker_buildx_context}" &>/dev/null; then +# echo "Warn: Failed removing the buildx context '${docker_buildx_context}'. Try manually removing it with 'docker context rm ${docker_buildx_context}'" >&2 +# exit 1 +#fi +#echo "Successfully built docker image" From aaca583056888cb9196f9c8e93a683a3b11d721a Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 25 Jul 2024 12:08:34 -0400 Subject: [PATCH 16/47] fix --- .../lib/backend_impls/docker/docker_manager/docker_manager.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index cd43767885..0293d194f8 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -624,8 +624,7 @@ func (manager *DockerManager) CreateAndStartContainer( } containerHostConfigPtr, err := manager.getContainerHostConfig( args.addedCapabilities, - //args.securityOpts, - + args.securityOpts, args.networkMode, args.bindMounts, args.volumeMounts, From 900a5315fb5bbbcc8706ad9a2ff6d243ac6a46ed Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 31 Jul 2024 16:29:21 -0400 Subject: [PATCH 17/47] wait and list networks before connecting --- .../reverse_proxy_functions/network_reverse_proxy.go | 5 +++++ .../docker/docker_manager/docker_manager.go | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go index 109e4c1c20..539e91eb9c 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go @@ -2,7 +2,9 @@ package reverse_proxy_functions import ( "context" + "github.com/sirupsen/logrus" "net" + "time" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" @@ -34,6 +36,9 @@ func ConnectReverseProxyToNetwork(ctx context.Context, dockerManager *docker_man return nil } + logrus.Infof("Waiting a few second before connecting reverse proxy to network...") + time.Sleep(10 * time.Second) + logrus.Infof("Okay lets try and connect now.") if err = dockerManager.ConnectContainerToNetwork(ctx, networkId, maybeReverseProxyContainerId, autoAssignIpAddressToReverseProxy, emptyAliasForReverseProxy); err != nil { return stacktrace.Propagate(err, "An error occurred while connecting the reverse proxy with container id '%v' to the enclave network '%v'", maybeReverseProxyContainerId, networkId) } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 0293d194f8..4e38b5afe7 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -1222,6 +1222,15 @@ func (manager *DockerManager) ConnectContainerToNetwork(ctx context.Context, net config := getEndpointSettingsForIpAddress(staticIpAddressStr, alias) + logrus.Infof("Listing all networks right before attempting to connect to network...") + networkResources, err := manager.dockerClient.NetworkList(ctx, types.NetworkListOptions{}) + for _, networkResource := range networkResources { + logrus.Infof("Information about network '%v' that exists: %v", networkResource.Name, networkResource) + if networkResource.ID == networkId { + logrus.Infof("The network we are trying to connect to exists right before connection.") + } + } + err = manager.dockerClient.NetworkConnect( ctx, networkId, From 572f4b859e5ea74fe300140c8f0ca67d7324d81d Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 1 Aug 2024 12:17:02 -0400 Subject: [PATCH 18/47] improve err logs for storing pkg contents --- .../git_package_content_provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go index af1c8b1f5d..d3a1303ac0 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go @@ -230,7 +230,7 @@ func (provider *GitPackageContentProvider) StorePackageContents(packageId string tempFile, err := os.CreateTemp(defaultTmpDir, temporaryArchiveFilePattern) if err != nil { - return "", startosis_errors.NewInterpretationError("An error occurred while creating temporary file to write compressed '%v' to", packageId) + return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred while creating temporary file to write compressed '%v' to temporary directory '%v' with temporary archive file pattern '%v'.", packageId, defaultTmpDir, temporaryArchiveFilePattern) } defer os.Remove(tempFile.Name()) From aba722acf18fd6e1640a8067bef7d4f2c253f802 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 23 Aug 2024 17:06:13 -0400 Subject: [PATCH 19/47] start dependency list --- .../builtins/import_module/import_module.go | 2 +- .../startosis_engine/plan_yaml/plan_yaml.go | 22 +++++++++++++++---- .../startosis_engine/startosis_interpreter.go | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go b/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go index 7e2351748f..143c9ce354 100644 --- a/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go +++ b/core/server/api_container/server/startosis_engine/builtins/import_module/import_module.go @@ -19,7 +19,7 @@ const ( /* NewImportModule returns a sequential (not parallel) implementation of an equivalent or `load` in Starlark - This function returns a starlarkstruct.Module object that can then me used to get variables and call functions from the loaded module. + This function returns a starlarkstruct.Module object that can then be used to get variables and call functions from the loaded module. How does the returned function work? 1. The function first checks whether a module is currently loading. If so then there's cycle and it errors immediately, diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go index 3677b2004c..e7f6167719 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go @@ -9,10 +9,11 @@ const ( ) type privatePlanYaml struct { - PackageId string `yaml:"packageId,omitempty"` - Services []*Service `yaml:"services,omitempty"` - FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` - Tasks []*Task `yaml:"tasks,omitempty"` + PackageId string `yaml:"packageId,omitempty"` + Services []*Service `yaml:"services,omitempty"` + FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` + Tasks []*Task `yaml:"tasks,omitempty"` + PackageDependencyGraph PackageDependencyGraph `yaml:"packagDependencyGraph,omitempty"` } // Service represents a service in the system. @@ -110,3 +111,16 @@ type Task struct { // TaskType represents the type of task (either python or shell) type TaskType string + +type Package struct { + PackageId string + ContainerImages []string +} + +type PackageDependencyGraph struct { + RootPackageId string + + PackageIndex map[string]Package + + PackageGraph map[string][]Package +} diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter.go b/core/server/api_container/server/startosis_engine/startosis_interpreter.go index 8660df5ce4..a737262db0 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter.go @@ -358,6 +358,7 @@ func (interpreter *StartosisInterpreter) buildBindings( packageReplaceOptions map[string]string, ) (*starlark.StringDict, *startosis_errors.InterpretationError) { recursiveInterpretForModuleLoading := func(moduleId string, serializedStartosis string) (starlark.StringDict, *startosis_errors.InterpretationError) { + logrus.Infof("PACKAGE DEPENDENCY: %v", moduleId) result, err := interpreter.interpretInternal(packageId, moduleId, serializedStartosis, instructionPlan, moduleGlobalCache, packageReplaceOptions) if err != nil { return nil, err From 507ef1329ba03973b07c6b1d2e96bfa7c3d132e9 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 29 Aug 2024 22:55:57 -0400 Subject: [PATCH 20/47] add images and package dependencies to plan yaml --- .../instructions_plan/instructions_plan.go | 9 +++++++ .../startosis_engine/plan_yaml/plan_yaml.go | 11 +++++---- .../plan_yaml/plan_yaml_generator.go | 24 +++++++++++++++---- .../startosis_engine/startosis_interpreter.go | 1 + 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go index 38e42655d4..1ec0e1e309 100644 --- a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go +++ b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go @@ -21,6 +21,9 @@ type InstructionsPlan struct { scheduledInstructionsIndex map[ScheduledInstructionUuid]*ScheduledInstruction instructionsSequence []ScheduledInstructionUuid + + // list of package names that this instructions plan relies on + packageDependencies []string } func NewInstructionsPlan() *InstructionsPlan { @@ -28,6 +31,7 @@ func NewInstructionsPlan() *InstructionsPlan { indexOfFirstInstruction: 0, scheduledInstructionsIndex: map[ScheduledInstructionUuid]*ScheduledInstruction{}, instructionsSequence: []ScheduledInstructionUuid{}, + packageDependencies: []string{}, } } @@ -88,9 +92,14 @@ func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYaml) (string return "", startosis_errors.WrapWithInterpretationError(err, "An error occurred updating the plan with instruction: %v.", instructionUuid) } } + planYaml.AddPackageDependencies(plan.packageDependencies) return planYaml.GenerateYaml() } +func (plan *InstructionsPlan) AddPackageDependency(packageDependency string) { + plan.packageDependencies = append(plan.packageDependencies, packageDependency) +} + func (plan *InstructionsPlan) Size() int { return len(plan.instructionsSequence) } diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go index e7f6167719..df7b2d17dc 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go @@ -9,11 +9,12 @@ const ( ) type privatePlanYaml struct { - PackageId string `yaml:"packageId,omitempty"` - Services []*Service `yaml:"services,omitempty"` - FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` - Tasks []*Task `yaml:"tasks,omitempty"` - PackageDependencyGraph PackageDependencyGraph `yaml:"packagDependencyGraph,omitempty"` + PackageId string `yaml:"packageId,omitempty"` + Services []*Service `yaml:"services,omitempty"` + FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` + Tasks []*Task `yaml:"tasks,omitempty"` + Images []string `yaml:"images,omitempty"` + PackageDependencies []string `yaml:"packageDependencies,omitempty"` } // Service represents a service in the system. diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index 36fa1ea211..cdc2ac5ce8 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -29,15 +29,19 @@ type PlanYaml struct { futureReferenceIndex map[string]string filesArtifactIndex map[string]*FilesArtifact latestUuid int + images []string + packageDependencies []string } func CreateEmptyPlan(packageId string) *PlanYaml { return &PlanYaml{ privatePlanYaml: &privatePlanYaml{ - PackageId: packageId, - Services: []*Service{}, - Tasks: []*Task{}, - FilesArtifacts: []*FilesArtifact{}, + PackageId: packageId, + Services: []*Service{}, + Tasks: []*Task{}, + FilesArtifacts: []*FilesArtifact{}, + Images: []string{}, + PackageDependencies: []string{}, }, futureReferenceIndex: map[string]string{}, filesArtifactIndex: map[string]*FilesArtifact{}, @@ -84,6 +88,8 @@ func (planYaml *PlanYaml) AddService( imageYaml := &ImageSpec{} //nolint:exhaustruct imageYaml.ImageName = serviceConfig.GetContainerImageName() + planYaml.addImage(imageYaml.ImageName) + imageYaml.BuildContextLocator = imageBuildContextLocator imageYaml.TargetStage = imageTargetStage imageYaml.Registry = imageRegistryAddress @@ -168,6 +174,7 @@ func (planYaml *PlanYaml) AddRunSh( taskYaml.RunCmd = []string{planYaml.swapFutureReference(runCommand)} taskYaml.Image = serviceConfig.GetContainerImageName() + planYaml.addImage(taskYaml.Image) var envVars []*EnvironmentVariable for key, val := range serviceConfig.GetEnvVars() { @@ -243,6 +250,7 @@ func (planYaml *PlanYaml) AddRunPython( taskYaml.RunCmd = []string{planYaml.swapFutureReference(runCommand)} taskYaml.Image = serviceConfig.GetContainerImageName() + planYaml.addImage(taskYaml.Image) var envVars []*EnvironmentVariable for key, val := range serviceConfig.GetEnvVars() { @@ -375,6 +383,10 @@ func (planYaml *PlanYaml) RemoveService(serviceName string) { } } +func (planYaml *PlanYaml) AddPackageDependencies(packageDependency []string) { + planYaml.privatePlanYaml.PackageDependencies = append(planYaml.privatePlanYaml.PackageDependencies, packageDependency...) +} + // getFileMountsFromFilesArtifacts turns filesArtifactExpansions into FileMount's // file mounts have two cases: // 1. the referenced files artifact already exists in the planYaml, in which case add the referenced files artifact to the proper filepath as a file mount @@ -433,6 +445,10 @@ func (planYaml *PlanYaml) addTaskYaml(task *Task) { planYaml.privatePlanYaml.Tasks = append(planYaml.privatePlanYaml.Tasks, task) } +func (planYaml *PlanYaml) addImage(img string) { + planYaml.privatePlanYaml.Images = append(planYaml.privatePlanYaml.Images, img) +} + // yaml future reference format: {{ kurtosis.. Date: Thu, 29 Aug 2024 22:57:32 -0400 Subject: [PATCH 21/47] hook up to cli --- cli/cli/commands/run/run.go | 64 ++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) mode change 100644 => 100755 cli/cli/commands/run/run.go diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go old mode 100644 new mode 100755 index e271c849ae..d61e4fd51d --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -55,6 +55,9 @@ const ( dryRunFlagKey = "dry-run" defaultDryRun = "false" + dependenciesFlagKey = "dependencies" + defaultDependencies = "false" + fullUuidsFlagKey = "full-uuids" fullUuidFlagKeyDefault = "false" @@ -128,6 +131,12 @@ var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisC Type: flags.FlagType_Bool, Default: defaultDryRun, }, + { + Key: dependenciesFlagKey, + Usage: "If true, a yaml will be returned with a list of images and packages that this run depends on.", + Type: flags.FlagType_Bool, + Default: defaultDependencies, + }, { Key: enclaveIdentifierFlagKey, Usage: "The enclave identifier of the enclave in which the script or package will be ran. " + @@ -262,6 +271,11 @@ func run( return stacktrace.Propagate(err, "Expected a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dryRunFlagKey) } + dependencies, err := flags.GetBool(dependenciesFlagKey) + if err != nil { + return stacktrace.Propagate(err, "Expectew a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dependencies) + } + parallelism, err := flags.GetUint32(parallelismFlagKey) if err != nil { return stacktrace.Propagate(err, "Expected a integer flag with key '%v' but none was found; this is an error in Kurtosis!", parallelismFlagKey) @@ -366,6 +380,17 @@ func run( defer output_printers.PrintEnclaveName(enclaveCtx.GetEnclaveName()) } + isRemotePackage := strings.HasPrefix(starlarkScriptOrPackagePath, githubDomainPrefix) + + if dependencies { + packageDependencyYamlStr, err := getPackageDependencyYamlStr(ctx, enclaveCtx, starlarkScriptOrPackagePath, isRemotePackage, packageArgs) + if err != nil { + return stacktrace.Propagate(err, "An error occurred getting package dependencies") + } + fmt.Printf("%v\n", packageDependencyYamlStr) + return nil + } + var responseLineChan <-chan *kurtosis_core_rpc_api_bindings.StarlarkRunResponseLine var cancelFunc context.CancelFunc var errRunningKurtosis error @@ -375,7 +400,6 @@ func run( connect = kurtosis_core_rpc_api_bindings.Connect_NO_CONNECT } - isRemotePackage := strings.HasPrefix(starlarkScriptOrPackagePath, githubDomainPrefix) if isRemotePackage { responseLineChan, cancelFunc, errRunningKurtosis = executeRemotePackage(ctx, enclaveCtx, starlarkScriptOrPackagePath, starlarkRunConfig) } else { @@ -588,6 +612,44 @@ func getOrCreateEnclaveContext( return enclaveContext, isNewEnclaveFlagWhenCreated, nil } +type PackageDependency struct { + Packages []string `yaml:"packages,omitempty"` + Images []string `yaml:"images,omitempty"` +} + +func getPackageDependencyYamlStr( + ctx context.Context, + enclaveCtx *enclaves.EnclaveContext, + starlarkScriptOrPackageId string, + isRemote bool, + packageArgs string, +) (string, error) { + var packageYaml *kurtosis_core_rpc_api_bindings.PlanYaml + var err error + if isRemote { + packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) + } else { + packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) + } + if err != nil { + return "", stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") + } + //var dependencies map[string][]string + //if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), dependencies); err != nil { + // return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) + //} + //pd := PackageDependency{ + // Packages: dependencies["packageDependencies"], + // Images: dependencies["images"], + //} + //packageDependencyYamlBytes, err := yaml.Marshal(&pd) + //if err != nil { + // return "", stacktrace.Propagate(err, "") + //} + //return string(packageDependencyYamlBytes), nil + return packageYaml.PlanYaml, nil +} + // validatePackageArgs just validates the args is a valid JSON or YAML string func validatePackageArgs(_ context.Context, _ *flags.ParsedFlags, args *args.ParsedArgs) error { serializedArgs, err := args.GetNonGreedyArg(inputArgsArgKey) From 8df95e7ae0f767ebab404b1e287f78f7d416e462 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 29 Aug 2024 22:58:00 -0400 Subject: [PATCH 22/47] add get starlark plan yaml to api --- .../core/lib/enclaves/enclave_context.go | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) mode change 100644 => 100755 api/golang/core/lib/enclaves/enclave_context.go diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go old mode 100644 new mode 100755 index 9849652114..bb54691c61 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -531,6 +531,31 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis return response, nil } +func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { + response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{ + PackageId: packageId, + SerializedParams: &serializedParams, + RelativePathToMainFile: nil, + MainFunctionName: nil, + }) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while getting the starlark package plan yaml run.") + } + return response, nil +} + +func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { + response, err := enclaveCtx.client.GetStarlarkScriptPlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkScriptPlanYamlArgs{ + SerializedScript: serializedScript, + SerializedParams: &serializedParams, + MainFunctionName: nil, + }) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while getting the last starlark script plan yaml run.") + } + return response, nil +} + // ==================================================================================================== // // Private helper methods From d0bba73c93ab207005192c1fffb04ae959777ef8 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 30 Aug 2024 10:54:06 -0400 Subject: [PATCH 23/47] update tests --- .../startosis_interpreter_plan_yaml_test.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go index 73838db876..9618d10827 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go @@ -147,6 +147,8 @@ services: filesArtifacts: - uuid: "2" name: hi-file +images: +- kurtosistech/example-datastore-server ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -212,6 +214,8 @@ tasks: envVar: - key: HELLO value: Hello! +images: +- badouralix/curl-jq ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -288,6 +292,8 @@ tasks: - requests pythonArgs: - something +images: +- python:3.11-alpine ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -362,6 +368,8 @@ tasks: serviceName: db acceptableCodes: - 0 +images: +- postgres:latest ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -428,6 +436,8 @@ tasks: filesArtifacts: - uuid: "1" name: bye-file +images: +- badouralix/curl-jq ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -490,6 +500,8 @@ services: filesArtifacts: - uuid: "2" name: hi-file +images: +- kurtosistech/example-datastore-server ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -550,6 +562,8 @@ services: filesArtifacts: - uuid: "2" name: hi-file +images: +- kurtosistech/example-datastore-server ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -610,6 +624,8 @@ tasks: filesArtifacts: - uuid: "1" name: dockerfile +images: +- badouralix/curl-jq ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -673,6 +689,8 @@ filesArtifacts: name: bye-file files: - bye.txt +images: +- postgres:latest ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -717,6 +735,8 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRemoveService() { filesArtifacts: - uuid: "2" name: hi-file +images: +- postgres:latest ` require.Equal(suite.T(), expectedYaml, planYaml) } @@ -813,6 +833,10 @@ tasks: command: - echo {{ kurtosis.4.code }} {{ kurtosis.4.output }} image: badouralix/curl-jq +images: +- postgres:latest +- badouralix/curl-jq +- badouralix/curl-jq ` require.Equal(suite.T(), expectedYaml, planYaml) } From 3831674792484c1088b284dfcf477c501c0fe87f Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 30 Aug 2024 11:06:43 -0400 Subject: [PATCH 24/47] make imgs unique and sort --- .../instructions_plan/instructions_plan.go | 6 +++--- .../plan_yaml/plan_yaml_generator.go | 21 ++++++++++++++----- .../startosis_interpreter_plan_yaml_test.go | 3 +-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go index 1ec0e1e309..4b2a871d33 100644 --- a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go +++ b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go @@ -23,7 +23,7 @@ type InstructionsPlan struct { instructionsSequence []ScheduledInstructionUuid // list of package names that this instructions plan relies on - packageDependencies []string + packageDependencies map[string]bool } func NewInstructionsPlan() *InstructionsPlan { @@ -31,7 +31,7 @@ func NewInstructionsPlan() *InstructionsPlan { indexOfFirstInstruction: 0, scheduledInstructionsIndex: map[ScheduledInstructionUuid]*ScheduledInstruction{}, instructionsSequence: []ScheduledInstructionUuid{}, - packageDependencies: []string{}, + packageDependencies: map[string]bool{}, } } @@ -97,7 +97,7 @@ func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYaml) (string } func (plan *InstructionsPlan) AddPackageDependency(packageDependency string) { - plan.packageDependencies = append(plan.packageDependencies, packageDependency) + plan.packageDependencies[packageDependency] = true } func (plan *InstructionsPlan) Size() int { diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index cdc2ac5ce8..f2aaa3c708 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -29,8 +29,7 @@ type PlanYaml struct { futureReferenceIndex map[string]string filesArtifactIndex map[string]*FilesArtifact latestUuid int - images []string - packageDependencies []string + imageSet map[string]bool } func CreateEmptyPlan(packageId string) *PlanYaml { @@ -43,6 +42,7 @@ func CreateEmptyPlan(packageId string) *PlanYaml { Images: []string{}, PackageDependencies: []string{}, }, + imageSet: map[string]bool{}, futureReferenceIndex: map[string]string{}, filesArtifactIndex: map[string]*FilesArtifact{}, latestUuid: 0, @@ -50,6 +50,7 @@ func CreateEmptyPlan(packageId string) *PlanYaml { } func (planYaml *PlanYaml) GenerateYaml() (string, error) { + planYaml.AddImages() yamlBytes, err := yaml.Marshal(planYaml.privatePlanYaml) if err != nil { return "", stacktrace.Propagate(err, "An error occurred generating plan yaml.") @@ -383,8 +384,18 @@ func (planYaml *PlanYaml) RemoveService(serviceName string) { } } -func (planYaml *PlanYaml) AddPackageDependencies(packageDependency []string) { - planYaml.privatePlanYaml.PackageDependencies = append(planYaml.privatePlanYaml.PackageDependencies, packageDependency...) +func (planYaml *PlanYaml) AddPackageDependencies(packageDependency map[string]bool) { + for dependency := range packageDependency { + planYaml.privatePlanYaml.PackageDependencies = append(planYaml.privatePlanYaml.PackageDependencies, dependency) + } + slices.Sort(planYaml.privatePlanYaml.PackageDependencies) +} + +func (planYaml *PlanYaml) AddImages() { + for img := range planYaml.imageSet { + planYaml.privatePlanYaml.Images = append(planYaml.privatePlanYaml.Images, img) + } + slices.Sort(planYaml.privatePlanYaml.Images) } // getFileMountsFromFilesArtifacts turns filesArtifactExpansions into FileMount's @@ -446,7 +457,7 @@ func (planYaml *PlanYaml) addTaskYaml(task *Task) { } func (planYaml *PlanYaml) addImage(img string) { - planYaml.privatePlanYaml.Images = append(planYaml.privatePlanYaml.Images, img) + planYaml.imageSet[img] = true } // yaml future reference format: {{ kurtosis.. Date: Fri, 30 Aug 2024 11:29:18 -0400 Subject: [PATCH 25/47] refactor plan yaml --- cli/cli/commands/run/run.go | 28 +++++------ .../instructions_plan/instructions_plan.go | 2 +- .../add_service/add_service.go | 2 +- .../add_service/add_services.go | 2 +- .../kurtosis_instruction/exec/exec.go | 2 +- .../get_files_artifact/get_files_artifact.go | 2 +- .../get_service/get_service.go | 2 +- .../get_services/get_services.go | 2 +- .../kurtosis_instruction.go | 2 +- .../kurtosis_print/kurtosis_print.go | 2 +- .../mock_kurtosis_instruction.go | 12 ++--- .../remove_service/remove_service.go | 2 +- .../render_templates/render_templates.go | 2 +- .../kurtosis_instruction/request/request.go | 2 +- .../set_service/set_service.go | 2 +- .../start_service/start_service.go | 2 +- .../stop_service/stop_service.go | 2 +- .../store_service_files.go | 2 +- .../kurtosis_instruction/tasks/run_python.go | 2 +- .../kurtosis_instruction/tasks/run_sh.go | 2 +- .../upload_files/upload_files.go | 2 +- .../kurtosis_instruction/verify/verify.go | 2 +- .../kurtosis_instruction/wait/wait.go | 2 +- .../kurtosis_plan_instruction_capabilities.go | 2 +- .../kurtosis_plan_instruction_internal.go | 2 +- .../startosis_engine/plan_yaml/plan_yaml.go | 2 +- .../plan_yaml/plan_yaml_generator.go | 50 +++++++++---------- 27 files changed, 69 insertions(+), 69 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index d61e4fd51d..ce6358ce9e 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/user_support_constants" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml" "io" "net/http" "net/url" @@ -634,20 +635,19 @@ func getPackageDependencyYamlStr( if err != nil { return "", stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") } - //var dependencies map[string][]string - //if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), dependencies); err != nil { - // return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) - //} - //pd := PackageDependency{ - // Packages: dependencies["packageDependencies"], - // Images: dependencies["images"], - //} - //packageDependencyYamlBytes, err := yaml.Marshal(&pd) - //if err != nil { - // return "", stacktrace.Propagate(err, "") - //} - //return string(packageDependencyYamlBytes), nil - return packageYaml.PlanYaml, nil + var planYaml plan_yaml.PlanYaml + if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), &planYaml); err != nil { + return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) + } + planYaml.Tasks = nil + planYaml.FilesArtifacts = nil + planYaml.Services = nil + packageDependencyYamlBytes, err := yaml.Marshal(&planYaml) + if err != nil { + return "", stacktrace.Propagate(err, "") + } + return string(packageDependencyYamlBytes), nil + //return packageYaml.PlanYaml, nil } // validatePackageArgs just validates the args is a valid JSON or YAML string diff --git a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go index 4b2a871d33..4b5f8b2dc6 100644 --- a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go +++ b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go @@ -81,7 +81,7 @@ func (plan *InstructionsPlan) GeneratePlan() ([]*ScheduledInstruction, *startosi } // GenerateYaml takes in an existing planYaml (usually empty) and returns a yaml string containing the effects of the plan -func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYaml) (string, error) { +func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator) (string, error) { for _, instructionUuid := range plan.instructionsSequence { instruction, found := plan.scheduledInstructionsIndex[instructionUuid] if !found { diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go index 903b30c358..b92b70528c 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go @@ -286,7 +286,7 @@ func (builtin *AddServiceCapabilities) FillPersistableAttributes(builder *enclav ) } -func (builtin *AddServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *AddServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { var buildContextLocator string var targetStage string var registryAddress string diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go index c221488f85..6e1ec87f8a 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go @@ -370,7 +370,7 @@ func (builtin *AddServicesCapabilities) allServicesReadinessCheck( return failedServiceChecksRegularMap } -func (builtin *AddServicesCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *AddServicesCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // TOOD: Implement logrus.Warn("ADD SERVICES NOT IMPLEMENTED YET FOR UPDATING PLAN YAML.") return nil diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/exec/exec.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/exec/exec.go index 14f9ef0821..bcea5246b1 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/exec/exec.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/exec/exec.go @@ -214,7 +214,7 @@ func (builtin *ExecCapabilities) FillPersistableAttributes(builder *enclave_plan builder.SetType(ExecBuiltinName) } -func (builtin *ExecCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *ExecCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { err := planYaml.AddExec(string(builtin.serviceName), builtin.description, builtin.returnValue, builtin.cmdList, builtin.acceptableCodes) if err != nil { return stacktrace.Propagate(err, "An error occurred updating plan with exec.") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_files_artifact/get_files_artifact.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_files_artifact/get_files_artifact.go index da3b565027..60122ee167 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_files_artifact/get_files_artifact.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_files_artifact/get_files_artifact.go @@ -93,7 +93,7 @@ func (builtin *GetFilesArtifactCapabilities) FillPersistableAttributes(builder * builder.SetType(GetFilesArtifactBuiltinName).AddFilesArtifact(builtin.artifactName, nil) } -func (builtin *GetFilesArtifactCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *GetFilesArtifactCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { // get files artifact does not affect the planYaml return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_service/get_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_service/get_service.go index 17b5aa7a2a..98ce8d9fd1 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_service/get_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_service/get_service.go @@ -103,7 +103,7 @@ func (builtin *GetServiceCapabilities) FillPersistableAttributes(builder *enclav builder.SetType(GetServiceBuiltinName).AddServiceName(builtin.serviceName) } -func (builtin *GetServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *GetServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { // get service does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go index 05d2667a24..991cbdb907 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/get_services/get_services.go @@ -92,7 +92,7 @@ func (builtin *GetServicesCapabilities) FillPersistableAttributes(builder *encla builder.SetType(GetServicesBuiltinName) } -func (builtin *GetServicesCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *GetServicesCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { // get services does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_instruction.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_instruction.go index 97c2fd1749..cf959b6c98 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_instruction.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_instruction.go @@ -36,5 +36,5 @@ type KurtosisInstruction interface { GetPersistableAttributes() *enclave_plan_persistence.EnclavePlanInstructionBuilder // UpdatePlan updates the plan with the effects of running this instruction. - UpdatePlan(plan *plan_yaml.PlanYaml) error + UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_print/kurtosis_print.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_print/kurtosis_print.go index c111b0a098..6608f7e68c 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_print/kurtosis_print.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/kurtosis_print/kurtosis_print.go @@ -106,7 +106,7 @@ func (builtin *PrintCapabilities) FillPersistableAttributes(builder *enclave_pla builder.SetType(PrintBuiltinName) } -func (builitin *PrintCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builitin *PrintCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // print does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/mock_instruction/mock_kurtosis_instruction.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/mock_instruction/mock_kurtosis_instruction.go index 6d22f66a9c..a81b33fe94 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/mock_instruction/mock_kurtosis_instruction.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/mock_instruction/mock_kurtosis_instruction.go @@ -347,11 +347,11 @@ func (_c *MockKurtosisInstruction_TryResolveWith_Call) RunAndReturn(run func(*en } // UpdatePlan provides a mock function with given fields: plan -func (_m *MockKurtosisInstruction) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (_m *MockKurtosisInstruction) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { ret := _m.Called(plan) var r0 error - if rf, ok := ret.Get(0).(func(*plan_yaml.PlanYaml) error); ok { + if rf, ok := ret.Get(0).(func(*plan_yaml.PlanYamlGenerator) error); ok { r0 = rf(plan) } else { r0 = ret.Error(0) @@ -366,14 +366,14 @@ type MockKurtosisInstruction_UpdatePlan_Call struct { } // UpdatePlan is a helper method to define mock.On call -// - plan *plan_yaml.PlanYaml +// - plan *plan_yaml.PlanYamlGenerator func (_e *MockKurtosisInstruction_Expecter) UpdatePlan(plan interface{}) *MockKurtosisInstruction_UpdatePlan_Call { return &MockKurtosisInstruction_UpdatePlan_Call{Call: _e.mock.On("UpdatePlan", plan)} } -func (_c *MockKurtosisInstruction_UpdatePlan_Call) Run(run func(plan *plan_yaml.PlanYaml)) *MockKurtosisInstruction_UpdatePlan_Call { +func (_c *MockKurtosisInstruction_UpdatePlan_Call) Run(run func(plan *plan_yaml.PlanYamlGenerator)) *MockKurtosisInstruction_UpdatePlan_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*plan_yaml.PlanYaml)) + run(args[0].(*plan_yaml.PlanYamlGenerator)) }) return _c } @@ -383,7 +383,7 @@ func (_c *MockKurtosisInstruction_UpdatePlan_Call) Return(_a0 error) *MockKurtos return _c } -func (_c *MockKurtosisInstruction_UpdatePlan_Call) RunAndReturn(run func(*plan_yaml.PlanYaml) error) *MockKurtosisInstruction_UpdatePlan_Call { +func (_c *MockKurtosisInstruction_UpdatePlan_Call) RunAndReturn(run func(*plan_yaml.PlanYamlGenerator) error) *MockKurtosisInstruction_UpdatePlan_Call { _c.Call.Return(run) return _c } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service/remove_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service/remove_service.go index d905da7e81..d4358a62b8 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service/remove_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/remove_service/remove_service.go @@ -114,7 +114,7 @@ func (builtin *RemoveServiceCapabilities) FillPersistableAttributes(builder *enc ) } -func (builtin *RemoveServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RemoveServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { plan.RemoveService(string(builtin.serviceName)) return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates/render_templates.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates/render_templates.go index 40e92fbf56..e2a05e08a6 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates/render_templates.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/render_templates/render_templates.go @@ -174,7 +174,7 @@ func (builtin *RenderTemplatesCapabilities) FillPersistableAttributes(builder *e ) } -func (builtin *RenderTemplatesCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RenderTemplatesCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { filepaths := []string{} for filepath := range builtin.templatesAndDataByDestRelFilepath { filepaths = append(filepaths, filepath) diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/request/request.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/request/request.go index 2a893d2672..538ec28afa 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/request/request.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/request/request.go @@ -209,7 +209,7 @@ func (builtin *RequestCapabilities) FillPersistableAttributes(builder *enclave_p builder.SetType(RequestBuiltinName) } -func (builtin *RequestCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RequestCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // TODO: Implement logrus.Warn("REQUEST NOT IMPLEMENTED YET FOR UPDATING PLAN") return nil diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go index 03f3d75094..d741ec199d 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/set_service/set_service.go @@ -187,7 +187,7 @@ func (builtin *SetServiceCapabilities) FillPersistableAttributes(builder *enclav builder.SetType(SetServiceBuiltinName).AddServiceName(builtin.serviceName) } -func (builtin *SetServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYaml) error { +func (builtin *SetServiceCapabilities) UpdatePlan(planYaml *plan_yaml.PlanYamlGenerator) error { // update service does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/start_service/start_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/start_service/start_service.go index 2856274444..05822d05bc 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/start_service/start_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/start_service/start_service.go @@ -107,7 +107,7 @@ func (builtin *StartServiceCapabilities) FillPersistableAttributes(builder *encl ) } -func (builtin *StartServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *StartServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // start services doesn't affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/stop_service/stop_service.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/stop_service/stop_service.go index 82fc1a86c7..6e7cb18898 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/stop_service/stop_service.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/stop_service/stop_service.go @@ -107,7 +107,7 @@ func (builtin *StopServiceCapabilities) FillPersistableAttributes(builder *encla ) } -func (builtin *StopServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *StopServiceCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // stop service does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files/store_service_files.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files/store_service_files.go index ab17ad1b5b..a5cbdc4de5 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files/store_service_files.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/store_service_files/store_service_files.go @@ -180,7 +180,7 @@ func (builtin *StoreServiceFilesCapabilities) FillPersistableAttributes(builder ) } -func (builtin *StoreServiceFilesCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *StoreServiceFilesCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { err := plan.AddStoreServiceFiles(builtin.artifactName, builtin.src) if err != nil { return stacktrace.Propagate(err, "An error occurred updating plan with store service files") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go index fd1e820452..86e1f66f85 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go @@ -373,7 +373,7 @@ func (builtin *RunPythonCapabilities) FillPersistableAttributes(builder *enclave builder.SetType(RunPythonBuiltinName) } -func (builtin *RunPythonCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RunPythonCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { err := plan.AddRunPython(builtin.run, builtin.description, builtin.returnValue, builtin.serviceConfig, builtin.storeSpecList, builtin.pythonArguments, builtin.packages) if err != nil { return stacktrace.Propagate(err, "An error occurred updating plan with run python") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go index afe8de2ee2..a1e0b1cf4b 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go @@ -329,7 +329,7 @@ func (builtin *RunShCapabilities) FillPersistableAttributes(builder *enclave_pla builder.SetType(RunShBuiltinName) } -func (builtin *RunShCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *RunShCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { err := plan.AddRunSh(builtin.run, builtin.description, builtin.returnValue, builtin.serviceConfig, builtin.storeSpecList) if err != nil { return stacktrace.Propagate(err, "An error occurred adding run sh task to the plan") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go index b1b02397fc..80e6e2572b 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/upload_files/upload_files.go @@ -219,7 +219,7 @@ func (builtin *UploadFilesCapabilities) FillPersistableAttributes(builder *encla ) } -func (builtin *UploadFilesCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *UploadFilesCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { err := plan.AddUploadFiles(builtin.artifactName, builtin.src) if err != nil { return stacktrace.Propagate(err, "An error occurred updating plan with upload files.") diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/verify/verify.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/verify/verify.go index 842f5d0ddf..b5944d85ea 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/verify/verify.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/verify/verify.go @@ -161,7 +161,7 @@ func (builtin *VerifyCapabilities) FillPersistableAttributes(builder *enclave_pl builder.SetType(VerifyBuiltinName) } -func (builtin *VerifyCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *VerifyCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // verify does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_instruction/wait/wait.go b/core/server/api_container/server/startosis_engine/kurtosis_instruction/wait/wait.go index ee89af066e..6526cf6860 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_instruction/wait/wait.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_instruction/wait/wait.go @@ -294,7 +294,7 @@ func (builtin *WaitCapabilities) FillPersistableAttributes(builder *enclave_plan builder.SetType(WaitBuiltinName) } -func (builtin *WaitCapabilities) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *WaitCapabilities) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { // wait does not affect the plan return nil } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_capabilities.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_capabilities.go index 5037f15fad..47a2b32853 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_capabilities.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_capabilities.go @@ -28,5 +28,5 @@ type KurtosisPlanInstructionCapabilities interface { Description() string // UpdatePlan applies the effect of this instruction capabilities onto the yaml representation of the instruction plan. - UpdatePlan(plan *plan_yaml.PlanYaml) error + UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error } diff --git a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_internal.go b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_internal.go index 968181cb55..41aa54aeda 100644 --- a/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_internal.go +++ b/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/kurtosis_plan_instruction/kurtosis_plan_instruction_internal.go @@ -98,7 +98,7 @@ func (builtin *kurtosisPlanInstructionInternal) GetPersistableAttributes() *encl return enclavePlaneInstructionBuilder.SetStarlarkCode(builtin.String()) } -func (builtin *kurtosisPlanInstructionInternal) UpdatePlan(plan *plan_yaml.PlanYaml) error { +func (builtin *kurtosisPlanInstructionInternal) UpdatePlan(plan *plan_yaml.PlanYamlGenerator) error { return builtin.capabilities.UpdatePlan(plan) } diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go index df7b2d17dc..fe04379788 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml.go @@ -8,7 +8,7 @@ const ( exec TaskType = "exec" ) -type privatePlanYaml struct { +type PlanYaml struct { PackageId string `yaml:"packageId,omitempty"` Services []*Service `yaml:"services,omitempty"` FilesArtifacts []*FilesArtifact `yaml:"filesArtifacts,omitempty"` diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index f2aaa3c708..e7098c83db 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -22,9 +22,9 @@ const ( outputFutureRefType = "output" ) -// PlanYaml is a yaml representation of the effect of an Instructions Plan or sequence of instructions on the state of the Enclave. -type PlanYaml struct { - privatePlanYaml *privatePlanYaml +// PlanYamlGenerator is a yaml representation of the effect of an Instructions Plan or sequence of instructions on the state of the Enclave. +type PlanYamlGenerator struct { + privatePlanYaml *PlanYaml futureReferenceIndex map[string]string filesArtifactIndex map[string]*FilesArtifact @@ -32,9 +32,9 @@ type PlanYaml struct { imageSet map[string]bool } -func CreateEmptyPlan(packageId string) *PlanYaml { - return &PlanYaml{ - privatePlanYaml: &privatePlanYaml{ +func CreateEmptyPlan(packageId string) *PlanYamlGenerator { + return &PlanYamlGenerator{ + privatePlanYaml: &PlanYaml{ PackageId: packageId, Services: []*Service{}, Tasks: []*Task{}, @@ -49,7 +49,7 @@ func CreateEmptyPlan(packageId string) *PlanYaml { } } -func (planYaml *PlanYaml) GenerateYaml() (string, error) { +func (planYaml *PlanYamlGenerator) GenerateYaml() (string, error) { planYaml.AddImages() yamlBytes, err := yaml.Marshal(planYaml.privatePlanYaml) if err != nil { @@ -58,7 +58,7 @@ func (planYaml *PlanYaml) GenerateYaml() (string, error) { return string(yamlBytes), nil } -func (planYaml *PlanYaml) AddService( +func (planYaml *PlanYamlGenerator) AddService( serviceName service.ServiceName, serviceInfo *kurtosis_types.Service, serviceConfig *service.ServiceConfig, @@ -138,7 +138,7 @@ func (planYaml *PlanYaml) AddService( return nil } -func (planYaml *PlanYaml) AddRunSh( +func (planYaml *PlanYamlGenerator) AddRunSh( runCommand string, description string, returnValue *starlarkstruct.Struct, @@ -213,7 +213,7 @@ func (planYaml *PlanYaml) AddRunSh( return nil } -func (planYaml *PlanYaml) AddRunPython( +func (planYaml *PlanYamlGenerator) AddRunPython( runCommand string, description string, returnValue *starlarkstruct.Struct, @@ -293,7 +293,7 @@ func (planYaml *PlanYaml) AddRunPython( return nil } -func (planYaml *PlanYaml) AddExec( +func (planYaml *PlanYamlGenerator) AddExec( serviceName string, description string, returnValue *starlark.Dict, @@ -345,7 +345,7 @@ func (planYaml *PlanYaml) AddExec( return nil } -func (planYaml *PlanYaml) AddRenderTemplates(filesArtifactName string, filepaths []string) error { +func (planYaml *PlanYamlGenerator) AddRenderTemplates(filesArtifactName string, filepaths []string) error { uuid := planYaml.generateUuid() filesArtifactYaml := &FilesArtifact{} //nolint exhaustruct filesArtifactYaml.Uuid = uuid @@ -355,7 +355,7 @@ func (planYaml *PlanYaml) AddRenderTemplates(filesArtifactName string, filepaths return nil } -func (planYaml *PlanYaml) AddUploadFiles(filesArtifactName, locator string) error { +func (planYaml *PlanYamlGenerator) AddUploadFiles(filesArtifactName, locator string) error { uuid := planYaml.generateUuid() filesArtifactYaml := &FilesArtifact{} //nolint exhauststruct filesArtifactYaml.Uuid = uuid @@ -365,7 +365,7 @@ func (planYaml *PlanYaml) AddUploadFiles(filesArtifactName, locator string) erro return nil } -func (planYaml *PlanYaml) AddStoreServiceFiles(filesArtifactName, locator string) error { +func (planYaml *PlanYamlGenerator) AddStoreServiceFiles(filesArtifactName, locator string) error { uuid := planYaml.generateUuid() filesArtifactYaml := &FilesArtifact{} //nolint exhaustruct filesArtifactYaml.Uuid = uuid @@ -375,7 +375,7 @@ func (planYaml *PlanYaml) AddStoreServiceFiles(filesArtifactName, locator string return nil } -func (planYaml *PlanYaml) RemoveService(serviceName string) { +func (planYaml *PlanYamlGenerator) RemoveService(serviceName string) { for idx, service := range planYaml.privatePlanYaml.Services { if service.Name == serviceName { planYaml.privatePlanYaml.Services = slices.Delete(planYaml.privatePlanYaml.Services, idx, idx+1) @@ -384,14 +384,14 @@ func (planYaml *PlanYaml) RemoveService(serviceName string) { } } -func (planYaml *PlanYaml) AddPackageDependencies(packageDependency map[string]bool) { +func (planYaml *PlanYamlGenerator) AddPackageDependencies(packageDependency map[string]bool) { for dependency := range packageDependency { planYaml.privatePlanYaml.PackageDependencies = append(planYaml.privatePlanYaml.PackageDependencies, dependency) } slices.Sort(planYaml.privatePlanYaml.PackageDependencies) } -func (planYaml *PlanYaml) AddImages() { +func (planYaml *PlanYamlGenerator) AddImages() { for img := range planYaml.imageSet { planYaml.privatePlanYaml.Images = append(planYaml.privatePlanYaml.Images, img) } @@ -406,7 +406,7 @@ func (planYaml *PlanYaml) AddImages() { // - create new files artifact // - add the files artifact to the plan // - add it to as a file mount accordingly -func (planYaml *PlanYaml) getFileMountsFromFilesArtifacts(filesArtifactExpansion *service_directory.FilesArtifactsExpansion) []*FileMount { +func (planYaml *PlanYamlGenerator) getFileMountsFromFilesArtifacts(filesArtifactExpansion *service_directory.FilesArtifactsExpansion) []*FileMount { var fileMounts []*FileMount if filesArtifactExpansion == nil { return fileMounts @@ -443,30 +443,30 @@ func (planYaml *PlanYaml) getFileMountsFromFilesArtifacts(filesArtifactExpansion return fileMounts } -func (planYaml *PlanYaml) addServiceYaml(service *Service) { +func (planYaml *PlanYamlGenerator) addServiceYaml(service *Service) { planYaml.privatePlanYaml.Services = append(planYaml.privatePlanYaml.Services, service) } -func (planYaml *PlanYaml) addFilesArtifactYaml(filesArtifact *FilesArtifact) { +func (planYaml *PlanYamlGenerator) addFilesArtifactYaml(filesArtifact *FilesArtifact) { planYaml.filesArtifactIndex[filesArtifact.Name] = filesArtifact planYaml.privatePlanYaml.FilesArtifacts = append(planYaml.privatePlanYaml.FilesArtifacts, filesArtifact) } -func (planYaml *PlanYaml) addTaskYaml(task *Task) { +func (planYaml *PlanYamlGenerator) addTaskYaml(task *Task) { planYaml.privatePlanYaml.Tasks = append(planYaml.privatePlanYaml.Tasks, task) } -func (planYaml *PlanYaml) addImage(img string) { +func (planYaml *PlanYamlGenerator) addImage(img string) { planYaml.imageSet[img] = true } // yaml future reference format: {{ kurtosis.. Date: Fri, 30 Aug 2024 11:37:34 -0400 Subject: [PATCH 26/47] only add module prefix and dont add parent package --- .../server/startosis_engine/startosis_interpreter.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter.go b/core/server/api_container/server/startosis_engine/startosis_interpreter.go index 9f1bd3fba9..b26fef893a 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter.go @@ -350,6 +350,10 @@ func (interpreter *StartosisInterpreter) interpretInternal( return globalVariables, nil } +func getModulePrefix(moduleId string) string { + return strings.Join(strings.SplitN(moduleId, "/", 4)[:3], "/") +} + func (interpreter *StartosisInterpreter) buildBindings( packageId string, thread *starlark.Thread, @@ -357,9 +361,12 @@ func (interpreter *StartosisInterpreter) buildBindings( moduleGlobalCache map[string]*startosis_packages.ModuleCacheEntry, packageReplaceOptions map[string]string, ) (*starlark.StringDict, *startosis_errors.InterpretationError) { + packagePrefix := getModulePrefix(packageId) recursiveInterpretForModuleLoading := func(moduleId string, serializedStartosis string) (starlark.StringDict, *startosis_errors.InterpretationError) { - logrus.Infof("PACKAGE DEPENDENCY: %v", moduleId) - instructionPlan.AddPackageDependency(moduleId) + modulePrefix := getModulePrefix(moduleId) + if modulePrefix != packagePrefix { + instructionPlan.AddPackageDependency(modulePrefix) + } result, err := interpreter.interpretInternal(packageId, moduleId, serializedStartosis, instructionPlan, moduleGlobalCache, packageReplaceOptions) if err != nil { return nil, err From c1cd1fc37b6e7e75671ec7fe8fee91333e3d9b09 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 30 Aug 2024 12:20:36 -0400 Subject: [PATCH 27/47] maybe parse yaml --- .../core/lib/enclaves/enclave_context.go | 8 ++++++ cli/cli/commands/run/run.go | 27 +++++++++---------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go index bb54691c61..589dbd0055 100755 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -532,6 +532,10 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis } func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { + serializedParams, err := maybeParseYaml(serializedParams) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) + } response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{ PackageId: packageId, SerializedParams: &serializedParams, @@ -545,6 +549,10 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context } func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { + serializedParams, err := maybeParseYaml(serializedParams) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) + } response, err := enclaveCtx.client.GetStarlarkScriptPlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkScriptPlanYamlArgs{ SerializedScript: serializedScript, SerializedParams: &serializedParams, diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index ce6358ce9e..8f0529ec4c 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/user_support_constants" - "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/plan_yaml" "io" "net/http" "net/url" @@ -635,19 +634,19 @@ func getPackageDependencyYamlStr( if err != nil { return "", stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") } - var planYaml plan_yaml.PlanYaml - if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), &planYaml); err != nil { - return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) - } - planYaml.Tasks = nil - planYaml.FilesArtifacts = nil - planYaml.Services = nil - packageDependencyYamlBytes, err := yaml.Marshal(&planYaml) - if err != nil { - return "", stacktrace.Propagate(err, "") - } - return string(packageDependencyYamlBytes), nil - //return packageYaml.PlanYaml, nil + //var planYaml plan_yaml.PlanYaml + //if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), &planYaml); err != nil { + // return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) + //} + //planYaml.Tasks = nil + //planYaml.FilesArtifacts = nil + //planYaml.Services = nil + //packageDependencyYamlBytes, err := yaml.Marshal(&planYaml) + //if err != nil { + // return "", stacktrace.Propagate(err, "") + //} + //return string(packageDependencyYamlBytes), nil + return packageYaml.PlanYaml, nil } // validatePackageArgs just validates the args is a valid JSON or YAML string From 554b754abc75b7998b3fb510ca0288a573864403 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 11 Sep 2024 11:05:45 -0400 Subject: [PATCH 28/47] pull dependencies locally --- .../api_container_service.pb.go | 73 ++++--- .../core/lib/enclaves/enclave_context.go | 6 +- api/protobuf/core/api_container_service.proto | 6 + api/rust/src/api_container_api.rs | 6 + .../api_container_service_pb.d.ts | 22 +++ .../api_container_service_pb.js | 100 +++++++++- .../connect/api_container_service_pb.d.ts | 14 ++ .../connect/api_container_service_pb.js | 2 + cli/cli/commands/run/run.go | 178 ++++++++++++++---- .../server/api_container_service.go | 6 +- .../instructions_plan/instructions_plan.go | 4 +- .../plan_yaml/plan_yaml_generator.go | 17 +- .../startosis_engine/startosis_interpreter.go | 14 +- .../startosis_interpreter_plan_yaml_test.go | 22 +-- .../startosis_interpreter_test.go | 11 ++ 15 files changed, 398 insertions(+), 83 deletions(-) diff --git a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go index ebb3f041d2..af0a975ee5 100644 --- a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go +++ b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go @@ -3381,6 +3381,8 @@ type StarlarkScriptPlanYamlArgs struct { SerializedParams *string `protobuf:"bytes,2,opt,name=serialized_params,json=serializedParams,proto3,oneof" json:"serialized_params,omitempty"` // The name of the main function, the default value is "run" MainFunctionName *string `protobuf:"bytes,5,opt,name=main_function_name,json=mainFunctionName,proto3,oneof" json:"main_function_name,omitempty"` + // Whether to return only the dependencies of the starlark script + DependenciesOnly *bool `protobuf:"varint,6,opt,name=dependencies_only,json=dependenciesOnly,proto3,oneof" json:"dependencies_only,omitempty"` } func (x *StarlarkScriptPlanYamlArgs) Reset() { @@ -3436,6 +3438,13 @@ func (x *StarlarkScriptPlanYamlArgs) GetMainFunctionName() string { return "" } +func (x *StarlarkScriptPlanYamlArgs) GetDependenciesOnly() bool { + if x != nil && x.DependenciesOnly != nil { + return *x.DependenciesOnly + } + return false +} + type StarlarkPackagePlanYamlArgs struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3449,6 +3458,8 @@ type StarlarkPackagePlanYamlArgs struct { RelativePathToMainFile *string `protobuf:"bytes,3,opt,name=relative_path_to_main_file,json=relativePathToMainFile,proto3,oneof" json:"relative_path_to_main_file,omitempty"` // The name of the main function, the default value is "run" MainFunctionName *string `protobuf:"bytes,4,opt,name=main_function_name,json=mainFunctionName,proto3,oneof" json:"main_function_name,omitempty"` + // Whether to return only the dependencies of the Starlark package + DependenciesOnly *bool `protobuf:"varint,5,opt,name=dependencies_only,json=dependenciesOnly,proto3,oneof" json:"dependencies_only,omitempty"` } func (x *StarlarkPackagePlanYamlArgs) Reset() { @@ -3511,6 +3522,13 @@ func (x *StarlarkPackagePlanYamlArgs) GetMainFunctionName() string { return "" } +func (x *StarlarkPackagePlanYamlArgs) GetDependenciesOnly() bool { + if x != nil && x.DependenciesOnly != nil { + return *x.DependenciesOnly + } + return false +} + var File_api_container_service_proto protoreflect.FileDescriptor var file_api_container_service_proto_rawDesc = []byte{ @@ -4085,7 +4103,7 @@ var file_api_container_service_proto_rawDesc = []byte{ 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x27, 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, - 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0xdb, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x61, 0x72, + 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0xa3, 0x02, 0x0a, 0x1a, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, @@ -4096,29 +4114,38 @@ var file_api_container_service_proto_rawDesc = []byte{ 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, - 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x15, - 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xae, 0x02, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, - 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, - 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, - 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x61, - 0x67, 0x65, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, - 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, - 0x00, 0x52, 0x10, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3f, 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, - 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, - 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x16, 0x72, 0x65, - 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x54, 0x6f, 0x4d, 0x61, 0x69, 0x6e, - 0x46, 0x69, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, + 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x30, 0x0a, 0x11, 0x64, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x48, 0x02, 0x52, 0x10, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, + 0x69, 0x65, 0x73, 0x4f, 0x6e, 0x6c, 0x79, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, - 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x42, - 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x2a, 0x36, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x64, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x22, 0xf6, 0x02, + 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, + 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1d, 0x0a, + 0x0a, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x11, + 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x10, 0x73, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3f, + 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, + 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x01, 0x52, 0x16, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, + 0x74, 0x68, 0x54, 0x6f, 0x4d, 0x61, 0x69, 0x6e, 0x46, 0x69, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, + 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x10, 0x6d, + 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, + 0x01, 0x01, 0x12, 0x30, 0x0a, 0x11, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, + 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x48, 0x03, 0x52, + 0x10, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x4f, 0x6e, 0x6c, + 0x79, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, + 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x72, + 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, + 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, + 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, + 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x2a, 0x36, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x2a, 0x2c, diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go index 589dbd0055..eb5707a8be 100755 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -531,7 +531,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis return response, nil } -func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { +func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string, dependenciesOnly bool) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { serializedParams, err := maybeParseYaml(serializedParams) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) @@ -541,6 +541,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context SerializedParams: &serializedParams, RelativePathToMainFile: nil, MainFunctionName: nil, + DependenciesOnly: &dependenciesOnly, }) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred while getting the starlark package plan yaml run.") @@ -548,7 +549,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context return response, nil } -func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { +func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string, dependenciesOnly bool) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { serializedParams, err := maybeParseYaml(serializedParams) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) @@ -557,6 +558,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, SerializedScript: serializedScript, SerializedParams: &serializedParams, MainFunctionName: nil, + DependenciesOnly: &dependenciesOnly, }) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred while getting the last starlark script plan yaml run.") diff --git a/api/protobuf/core/api_container_service.proto b/api/protobuf/core/api_container_service.proto index cc4d0aacaa..3bded04cad 100644 --- a/api/protobuf/core/api_container_service.proto +++ b/api/protobuf/core/api_container_service.proto @@ -598,6 +598,9 @@ message StarlarkScriptPlanYamlArgs { // The name of the main function, the default value is "run" optional string main_function_name = 5; + + // Whether to return only the dependencies of the starlark script + optional bool dependencies_only = 6; } message StarlarkPackagePlanYamlArgs { @@ -612,4 +615,7 @@ message StarlarkPackagePlanYamlArgs { // The name of the main function, the default value is "run" optional string main_function_name = 4; + + // Whether to return only the dependencies of the Starlark package + optional bool dependencies_only = 5; } diff --git a/api/rust/src/api_container_api.rs b/api/rust/src/api_container_api.rs index a3de2c65bc..c81e2200ad 100644 --- a/api/rust/src/api_container_api.rs +++ b/api/rust/src/api_container_api.rs @@ -709,6 +709,9 @@ pub struct StarlarkScriptPlanYamlArgs { /// The name of the main function, the default value is "run" #[prost(string, optional, tag = "5")] pub main_function_name: ::core::option::Option<::prost::alloc::string::String>, + /// Whether to return only the dependencies of the starlark script + #[prost(bool, optional, tag = "6")] + pub dependencies_only: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -727,6 +730,9 @@ pub struct StarlarkPackagePlanYamlArgs { /// The name of the main function, the default value is "run" #[prost(string, optional, tag = "4")] pub main_function_name: ::core::option::Option<::prost::alloc::string::String>, + /// Whether to return only the dependencies of the Starlark package + #[prost(bool, optional, tag = "5")] + pub dependencies_only: ::core::option::Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts index 1d23b22991..2fb6b04a0f 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts @@ -1511,6 +1511,11 @@ export class StarlarkScriptPlanYamlArgs extends jspb.Message { hasMainFunctionName(): boolean; clearMainFunctionName(): StarlarkScriptPlanYamlArgs; + getDependenciesOnly(): boolean; + setDependenciesOnly(value: boolean): StarlarkScriptPlanYamlArgs; + hasDependenciesOnly(): boolean; + clearDependenciesOnly(): StarlarkScriptPlanYamlArgs; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): StarlarkScriptPlanYamlArgs.AsObject; static toObject(includeInstance: boolean, msg: StarlarkScriptPlanYamlArgs): StarlarkScriptPlanYamlArgs.AsObject; @@ -1524,6 +1529,7 @@ export namespace StarlarkScriptPlanYamlArgs { serializedScript: string, serializedParams?: string, mainFunctionName?: string, + dependenciesOnly?: boolean, } export enum SerializedParamsCase { @@ -1535,6 +1541,11 @@ export namespace StarlarkScriptPlanYamlArgs { _MAIN_FUNCTION_NAME_NOT_SET = 0, MAIN_FUNCTION_NAME = 5, } + + export enum DependenciesOnlyCase { + _DEPENDENCIES_ONLY_NOT_SET = 0, + DEPENDENCIES_ONLY = 6, + } } export class StarlarkPackagePlanYamlArgs extends jspb.Message { @@ -1556,6 +1567,11 @@ export class StarlarkPackagePlanYamlArgs extends jspb.Message { hasMainFunctionName(): boolean; clearMainFunctionName(): StarlarkPackagePlanYamlArgs; + getDependenciesOnly(): boolean; + setDependenciesOnly(value: boolean): StarlarkPackagePlanYamlArgs; + hasDependenciesOnly(): boolean; + clearDependenciesOnly(): StarlarkPackagePlanYamlArgs; + serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): StarlarkPackagePlanYamlArgs.AsObject; static toObject(includeInstance: boolean, msg: StarlarkPackagePlanYamlArgs): StarlarkPackagePlanYamlArgs.AsObject; @@ -1570,6 +1586,7 @@ export namespace StarlarkPackagePlanYamlArgs { serializedParams?: string, relativePathToMainFile?: string, mainFunctionName?: string, + dependenciesOnly?: boolean, } export enum SerializedParamsCase { @@ -1586,6 +1603,11 @@ export namespace StarlarkPackagePlanYamlArgs { _MAIN_FUNCTION_NAME_NOT_SET = 0, MAIN_FUNCTION_NAME = 4, } + + export enum DependenciesOnlyCase { + _DEPENDENCIES_ONLY_NOT_SET = 0, + DEPENDENCIES_ONLY = 5, + } } export enum ServiceStatus { diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js index d0c4855650..714c868f8a 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js @@ -10882,7 +10882,8 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.toObject = function(includeIn var f, obj = { serializedScript: jspb.Message.getFieldWithDefault(msg, 1, ""), serializedParams: jspb.Message.getFieldWithDefault(msg, 2, ""), - mainFunctionName: jspb.Message.getFieldWithDefault(msg, 5, "") + mainFunctionName: jspb.Message.getFieldWithDefault(msg, 5, ""), + dependenciesOnly: jspb.Message.getBooleanFieldWithDefault(msg, 6, false) }; if (includeInstance) { @@ -10931,6 +10932,10 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.deserializeBinaryFromReader = var value = /** @type {string} */ (reader.readString()); msg.setMainFunctionName(value); break; + case 6: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setDependenciesOnly(value); + break; default: reader.skipField(); break; @@ -10981,6 +10986,13 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.serializeBinaryToWriter = fun f ); } + f = /** @type {boolean} */ (jspb.Message.getField(message, 6)); + if (f != null) { + writer.writeBool( + 6, + f + ); + } }; @@ -11074,6 +11086,42 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.hasMainFunctionName }; +/** + * optional bool dependencies_only = 6; + * @return {boolean} + */ +proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.getDependenciesOnly = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 6, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.api_container_api.StarlarkScriptPlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.setDependenciesOnly = function(value) { + return jspb.Message.setField(this, 6, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.api_container_api.StarlarkScriptPlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.clearDependenciesOnly = function() { + return jspb.Message.setField(this, 6, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.hasDependenciesOnly = function() { + return jspb.Message.getField(this, 6) != null; +}; + + @@ -11109,7 +11157,8 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.toObject = function(includeI packageId: jspb.Message.getFieldWithDefault(msg, 1, ""), serializedParams: jspb.Message.getFieldWithDefault(msg, 2, ""), relativePathToMainFile: jspb.Message.getFieldWithDefault(msg, 3, ""), - mainFunctionName: jspb.Message.getFieldWithDefault(msg, 4, "") + mainFunctionName: jspb.Message.getFieldWithDefault(msg, 4, ""), + dependenciesOnly: jspb.Message.getBooleanFieldWithDefault(msg, 5, false) }; if (includeInstance) { @@ -11162,6 +11211,10 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.deserializeBinaryFromReader var value = /** @type {string} */ (reader.readString()); msg.setMainFunctionName(value); break; + case 5: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setDependenciesOnly(value); + break; default: reader.skipField(); break; @@ -11219,6 +11272,13 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.serializeBinaryToWriter = fu f ); } + f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeBool( + 5, + f + ); + } }; @@ -11348,6 +11408,42 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasMainFunctionNam }; +/** + * optional bool dependencies_only = 5; + * @return {boolean} + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getDependenciesOnly = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setDependenciesOnly = function(value) { + return jspb.Message.setField(this, 5, value); +}; + + +/** + * Clears the field making it undefined. + * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearDependenciesOnly = function() { + return jspb.Message.setField(this, 5, undefined); +}; + + +/** + * Returns whether this field is set. + * @return {boolean} + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasDependenciesOnly = function() { + return jspb.Message.getField(this, 5) != null; +}; + + /** * @enum {number} */ diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts index de9778ca61..b297f0233d 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts @@ -1882,6 +1882,13 @@ export declare class StarlarkScriptPlanYamlArgs extends Message); static readonly runtime: typeof proto3; @@ -1928,6 +1935,13 @@ export declare class StarlarkPackagePlanYamlArgs extends Message); static readonly runtime: typeof proto3; diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js index 164d68d7d5..e0812b1b9d 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js @@ -659,6 +659,7 @@ export const StarlarkScriptPlanYamlArgs = proto3.makeMessageType( { no: 1, name: "serialized_script", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 2, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 5, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 6, name: "dependencies_only", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ], ); @@ -672,6 +673,7 @@ export const StarlarkPackagePlanYamlArgs = proto3.makeMessageType( { no: 2, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 3, name: "relative_path_to_main_file", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 4, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 5, name: "dependencies_only", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ], ); diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 8f0529ec4c..8abfea3c01 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -4,8 +4,16 @@ import ( "context" "encoding/json" "fmt" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/starlark_run_config" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/configs" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/user_support_constants" + "gopkg.in/yaml.v2" "io" + "k8s.io/utils/strings/slices" "net/http" "net/url" "os" @@ -15,10 +23,6 @@ import ( "regexp" "strings" - "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/starlark_run_config" - "gopkg.in/yaml.v2" - "k8s.io/utils/strings/slices" - "github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings" "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/enclaves" "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/services" @@ -55,8 +59,10 @@ const ( dryRunFlagKey = "dry-run" defaultDryRun = "false" - dependenciesFlagKey = "dependencies" - defaultDependencies = "false" + dependenciesFlagKey = "dependencies" + defaultDependencies = "false" + pullDependenciesFlagKey = "pull" + defaultPullDependencies = "false" fullUuidsFlagKey = "full-uuids" fullUuidFlagKeyDefault = "false" @@ -137,6 +143,12 @@ var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisC Type: flags.FlagType_Bool, Default: defaultDependencies, }, + { + Key: pullDependenciesFlagKey, + Usage: "If true, and the dependencies flag is passed, attempts to pull all images and packages that the run depends on locally. kurtosis.yml is updated with the replace directives pointing to locally pulled packages.", + Type: flags.FlagType_Bool, + Default: defaultPullDependencies, + }, { Key: enclaveIdentifierFlagKey, Usage: "The enclave identifier of the enclave in which the script or package will be ran. " + @@ -273,7 +285,12 @@ func run( dependencies, err := flags.GetBool(dependenciesFlagKey) if err != nil { - return stacktrace.Propagate(err, "Expectew a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dependencies) + return stacktrace.Propagate(err, "Expected a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dependenciesFlagKey) + } + + pullDependencies, err := flags.GetBool(pullDependenciesFlagKey) + if err != nil { + return stacktrace.Propagate(err, "Expected a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", pullDependenciesFlagKey) } parallelism, err := flags.GetUint32(parallelismFlagKey) @@ -383,11 +400,38 @@ func run( isRemotePackage := strings.HasPrefix(starlarkScriptOrPackagePath, githubDomainPrefix) if dependencies { - packageDependencyYamlStr, err := getPackageDependencyYamlStr(ctx, enclaveCtx, starlarkScriptOrPackagePath, isRemotePackage, packageArgs) + dependencyYaml, err := getPackageDependencyYaml(ctx, enclaveCtx, starlarkScriptOrPackagePath, isRemotePackage, packageArgs) if err != nil { return stacktrace.Propagate(err, "An error occurred getting package dependencies") } - fmt.Printf("%v\n", packageDependencyYamlStr) + fmt.Printf("%v\n", dependencyYaml.PlanYaml) + + if pullDependencies { + type PackageDependencies struct { + Images []string `yaml:"images"` + Packages []string `yaml:"packageDependencies"` + } + var pkgDeps PackageDependencies + err := yaml.Unmarshal([]byte(dependencyYaml.PlanYaml), &pkgDeps) + if err != nil { + return stacktrace.Propagate(err, "An error occurred unmarhsaling dependency yaml string") + } + + err = pullImagesLocally(ctx, pkgDeps.Images) + if err != nil { + return err + } + + packageNamesToLocalFilepaths, err := pullPackagesLocally(pkgDeps.Packages) + if err != nil { + return err + } + + err = updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths) + if err != nil { + return err + } + } return nil } @@ -612,41 +656,24 @@ func getOrCreateEnclaveContext( return enclaveContext, isNewEnclaveFlagWhenCreated, nil } -type PackageDependency struct { - Packages []string `yaml:"packages,omitempty"` - Images []string `yaml:"images,omitempty"` -} - -func getPackageDependencyYamlStr( +func getPackageDependencyYaml( ctx context.Context, enclaveCtx *enclaves.EnclaveContext, starlarkScriptOrPackageId string, isRemote bool, packageArgs string, -) (string, error) { +) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { var packageYaml *kurtosis_core_rpc_api_bindings.PlanYaml var err error if isRemote { - packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) + packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs, true) } else { - packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) - } - if err != nil { - return "", stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") - } - //var planYaml plan_yaml.PlanYaml - //if err = yaml.Unmarshal([]byte(packageYaml.PlanYaml), &planYaml); err != nil { - // return "", stacktrace.Propagate(err, "An error occurred unmarshaling plan yaml string: %v.", packageYaml.PlanYaml) - //} - //planYaml.Tasks = nil - //planYaml.FilesArtifacts = nil - //planYaml.Services = nil - //packageDependencyYamlBytes, err := yaml.Marshal(&planYaml) - //if err != nil { - // return "", stacktrace.Propagate(err, "") - //} - //return string(packageDependencyYamlBytes), nil - return packageYaml.PlanYaml, nil + packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, starlarkScriptOrPackageId, packageArgs, true) + } + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") + } + return packageYaml, nil } // validatePackageArgs just validates the args is a valid JSON or YAML string @@ -795,3 +822,84 @@ func isHttpUrl(maybeHttpUrl string) bool { httpProtocolRegex := regexp.MustCompile(httpProtocolRegexStr) return httpProtocolRegex.MatchString(maybeHttpUrl) } + +func pullImagesLocally(ctx context.Context, images []string) error { + kurtosisBackend, err := backend_creator.GetDockerKurtosisBackend(backend_creator.NoAPIContainerModeArgs, configs.NoRemoteBackendConfig) + if err != nil { + return stacktrace.Propagate(err, "An error occurred retrieving Docker Kurtosis Backend") + } + for _, img := range images { + _, _, err := kurtosisBackend.FetchImage(ctx, img, nil, image_download_mode.ImageDownloadMode_Always) + if err != nil { + return stacktrace.Propagate(err, "An error occurred pulling '%v' locally.", img) + } + } + return nil +} + +func pullPackagesLocally(packageDependencies []string) (map[string]string, error) { + var localPackagesToRelativeFilepaths map[string]string + wd, err := os.Getwd() + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting current working directory.") + } + parentCwd := filepath.Dir(wd) + relParentCwd, err := filepath.Rel(wd, parentCwd) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting rel path between '%v' and '%v'.", wd, relParentCwd) + } + logrus.Infof("current wid: %v", wd) + logrus.Infof("rel parent wd: %v", relParentCwd) + for _, dep := range packageDependencies { + logrus.Infof("Pulling package: %v", dep) + // assume dep doesn't have http:// prefix or .git suffix + repoUrl := fmt.Sprintf("%s%s%s", "http://", dep, ".git") + logrus.Infof("repo url: %v", repoUrl) + _, err := git.PlainClone(relParentCwd, false, &git.CloneOptions{ + URL: repoUrl, + Auth: nil, + RemoteName: "", + ReferenceName: "", + SingleBranch: false, + Mirror: false, + NoCheckout: false, + Depth: 0, + RecurseSubmodules: 0, + ShallowSubmodules: false, + Progress: nil, + Tags: 0, + InsecureSkipTLS: false, + CABundle: nil, + ProxyOptions: transport.ProxyOptions{}, + Shared: false, + }) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred cloning package locally.") + } + parts := strings.Split(dep, "/") + packageName := parts[len(parts)-1] + localPackagesToRelativeFilepaths[dep] = fmt.Sprintf("%s/%s", relParentCwd, packageName) + } + return localPackagesToRelativeFilepaths, nil +} + +func updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths map[string]string) error { + file, err := os.OpenFile("kurtosis.yml", os.O_APPEND, 0644) + if err != nil { + return stacktrace.Propagate(err, "An error occurred opening kurtosis.yml file.") + } + defer file.Close() + + _, err = file.Write([]byte("replace:")) + if err != nil { + return stacktrace.Propagate(err, "An error occurred writing 'replace:' to kurtosis.yml") + } + + for packageName, localFilepath := range packageNamesToLocalFilepaths { + _, err := file.Write([]byte(fmt.Sprintf("\t %s: %s", packageName, localFilepath))) + if err != nil { + return stacktrace.Propagate(err, "An error occurred writing replace directive for '%v: %v'", packageName, localFilepath) + } + } + return nil +} diff --git a/core/server/api_container/server/api_container_service.go b/core/server/api_container/server/api_container_service.go index eabdf4a60d..ce2ab9db51 100644 --- a/core/server/api_container/server/api_container_service.go +++ b/core/server/api_container/server/api_container_service.go @@ -663,6 +663,7 @@ func (apicService *ApiContainerService) GetStarlarkPackagePlanYaml(ctx context.C serializedParams := args.GetSerializedParams() requestedRelativePathToMainFile := args.GetRelativePathToMainFile() mainFuncName := args.GetMainFunctionName() + dependenciesOnly := args.GetDependenciesOnly() var scriptWithRunFunction string var interpretationError *startosis_errors.InterpretationError @@ -691,7 +692,7 @@ func (apicService *ApiContainerService) GetStarlarkPackagePlanYaml(ctx context.C interpretationError = startosis_errors.NewInterpretationError(apiInterpretationError.GetErrorMessage()) return nil, stacktrace.Propagate(interpretationError, "An interpretation error occurred interpreting package for retrieving plan yaml for package: %v", packageIdFromArgs) } - planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageIdFromArgs)) + planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageIdFromArgs), dependenciesOnly) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred generating plan yaml for package: %v", packageIdFromArgs) } @@ -707,6 +708,7 @@ func (apicService *ApiContainerService) GetStarlarkScriptPlanYaml(ctx context.Co serializedStarlarkScript := args.GetSerializedScript() serializedParams := args.GetSerializedParams() mainFuncName := args.GetMainFunctionName() + dependenciesOnly := args.GetDependenciesOnly() noPackageReplaceOptions := map[string]string{} _, instructionsPlan, apiInterpretationError := apicService.startosisInterpreter.Interpret( @@ -724,7 +726,7 @@ func (apicService *ApiContainerService) GetStarlarkScriptPlanYaml(ctx context.Co if apiInterpretationError != nil { return nil, startosis_errors.NewInterpretationError(apiInterpretationError.GetErrorMessage()) } - planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), dependenciesOnly) if err != nil { return nil, err } diff --git a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go index 4b5f8b2dc6..81dd265ad7 100644 --- a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go +++ b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go @@ -81,7 +81,7 @@ func (plan *InstructionsPlan) GeneratePlan() ([]*ScheduledInstruction, *startosi } // GenerateYaml takes in an existing planYaml (usually empty) and returns a yaml string containing the effects of the plan -func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator) (string, error) { +func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator, dependenciesOnly bool) (string, error) { for _, instructionUuid := range plan.instructionsSequence { instruction, found := plan.scheduledInstructionsIndex[instructionUuid] if !found { @@ -93,7 +93,7 @@ func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator } } planYaml.AddPackageDependencies(plan.packageDependencies) - return planYaml.GenerateYaml() + return planYaml.GenerateYaml(dependenciesOnly) } func (plan *InstructionsPlan) AddPackageDependency(packageDependency string) { diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index e7098c83db..b7891f684b 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -49,9 +49,22 @@ func CreateEmptyPlan(packageId string) *PlanYamlGenerator { } } -func (planYaml *PlanYamlGenerator) GenerateYaml() (string, error) { +func (planYaml *PlanYamlGenerator) GenerateYaml(dependenciesOnly bool) (string, error) { planYaml.AddImages() - yamlBytes, err := yaml.Marshal(planYaml.privatePlanYaml) + + var planYamlCopy PlanYaml + planYamlCopy = *planYaml.privatePlanYaml + if dependenciesOnly { + planYamlCopy = PlanYaml{ + PackageId: "", + Services: nil, + FilesArtifacts: nil, + Tasks: nil, + Images: planYaml.privatePlanYaml.Images, + PackageDependencies: planYaml.privatePlanYaml.PackageDependencies, + } + } + yamlBytes, err := yaml.Marshal(planYamlCopy) if err != nil { return "", stacktrace.Propagate(err, "An error occurred generating plan yaml.") } diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter.go b/core/server/api_container/server/startosis_engine/startosis_interpreter.go index b26fef893a..48132f77e0 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter.go @@ -350,10 +350,6 @@ func (interpreter *StartosisInterpreter) interpretInternal( return globalVariables, nil } -func getModulePrefix(moduleId string) string { - return strings.Join(strings.SplitN(moduleId, "/", 4)[:3], "/") -} - func (interpreter *StartosisInterpreter) buildBindings( packageId string, thread *starlark.Thread, @@ -395,6 +391,16 @@ func (interpreter *StartosisInterpreter) buildBindings( return &predeclared, nil } +const ( + moduleIdSeperator = "/" + numModuleIdSeparators = 4 + prefixNum = 3 +) + +func getModulePrefix(moduleId string) string { + return strings.Join(strings.SplitN(moduleId, moduleIdSeperator, numModuleIdSeparators)[:prefixNum], moduleIdSeperator) +} + func findFirstEqualInstructionPastIndex(currentEnclaveInstructionsList []*enclave_plan_persistence.EnclavePlanInstruction, naiveInstructionsList []*instructions_plan.ScheduledInstruction, minIndex int) int { if len(naiveInstructionsList) == 0 { return -1 // no result as the naiveInstructionsList is empty diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go index bccdfe5d59..05d59ef308 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go @@ -115,7 +115,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithFilesArtifa require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := @@ -185,7 +185,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRunShWithFilesArtifacts() require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -260,7 +260,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRunPython() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -336,7 +336,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestExec() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -414,7 +414,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRenderTemplate() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -481,7 +481,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithImageBuildS require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId), false) require.NoError(suite.T(), err) expectedYaml := `packageId: ` + packageId + ` @@ -544,7 +544,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithImageSpec() require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId), false) require.NoError(suite.T(), err) expectedYaml := `packageId: ` + packageId + ` @@ -603,7 +603,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestUploadFiles() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId), false) require.NoError(suite.T(), err) expectedYaml := `packageId: ` + packageId + ` @@ -665,7 +665,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestStoreServiceFiles() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -728,7 +728,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRemoveService() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -789,7 +789,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestFutureReferencesAreSwappe require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 4, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go b/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go index d2bea6d789..2b95563878 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter_test.go @@ -1081,6 +1081,17 @@ def run(plan): require.Nil(suite.T(), interpretationError) } +func (suite *StartosisInterpreterTestSuite) TestGetModuleIdPrefix() { + githubModuleId := "github.com/some-person/some-pkg/main.star" + expectedGithubModuleId := "github.com/some-person/some-pkg" + + require.Equal(suite.T(), expectedGithubModuleId, getModulePrefix(githubModuleId)) + + moduleId := "./some-person/some-pkg/main.star" + expectedModuleId := "./some-person/some-pkg" + require.Equal(suite.T(), expectedModuleId, getModulePrefix(moduleId)) +} + // ##################################################################################################################### // // TEST HELPERS From 18c2591796f2d2f3a58342ee6cf0862c2eb890a3 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 12 Sep 2024 10:10:07 -0400 Subject: [PATCH 29/47] fix cloning pkgs --- cli/cli/commands/run/run.go | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 8abfea3c01..23502f47c5 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -3,6 +3,7 @@ package run import ( "context" "encoding/json" + "errors" "fmt" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/transport" @@ -838,7 +839,8 @@ func pullImagesLocally(ctx context.Context, images []string) error { } func pullPackagesLocally(packageDependencies []string) (map[string]string, error) { - var localPackagesToRelativeFilepaths map[string]string + localPackagesToRelativeFilepaths := map[string]string{} + wd, err := os.Getwd() if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting current working directory.") @@ -848,14 +850,17 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting rel path between '%v' and '%v'.", wd, relParentCwd) } - logrus.Infof("current wid: %v", wd) - logrus.Infof("rel parent wd: %v", relParentCwd) - for _, dep := range packageDependencies { - logrus.Infof("Pulling package: %v", dep) - // assume dep doesn't have http:// prefix or .git suffix - repoUrl := fmt.Sprintf("%s%s%s", "http://", dep, ".git") + + for _, dependency := range packageDependencies { + packageIdParts := strings.Split(dependency, "/") + packageName := packageIdParts[len(packageIdParts)-1] + logrus.Infof("Pulling package: %v", dependency) + + // assume dependency doesn't have http:// prefix or .git suffix + repoUrl := fmt.Sprintf("%s%s%s", "http://", dependency, ".git") + localPackagePath := fmt.Sprintf("%s/%s", parentCwd, packageName) logrus.Infof("repo url: %v", repoUrl) - _, err := git.PlainClone(relParentCwd, false, &git.CloneOptions{ + _, err := git.PlainClone(localPackagePath, false, &git.CloneOptions{ URL: repoUrl, Auth: nil, RemoteName: "", @@ -873,33 +878,33 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error ProxyOptions: transport.ProxyOptions{}, Shared: false, }) - if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred cloning package locally.") + if err != nil && !errors.Is(err, git.ErrRepositoryAlreadyExists) { + return nil, stacktrace.Propagate(err, "An error occurred cloning package '%s' to '%s'.", dependency, localPackagePath) } - parts := strings.Split(dep, "/") - packageName := parts[len(parts)-1] - localPackagesToRelativeFilepaths[dep] = fmt.Sprintf("%s/%s", relParentCwd, packageName) + localPackagesToRelativeFilepaths[dependency] = fmt.Sprintf("%s/%s", relParentCwd, packageName) } + return localPackagesToRelativeFilepaths, nil } func updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths map[string]string) error { - file, err := os.OpenFile("kurtosis.yml", os.O_APPEND, 0644) + file, err := os.OpenFile("kurtosis.yml", os.O_WRONLY|os.O_APPEND, 0644) if err != nil { return stacktrace.Propagate(err, "An error occurred opening kurtosis.yml file.") } defer file.Close() - _, err = file.Write([]byte("replace:")) + _, err = file.Write([]byte("replace:\n")) if err != nil { return stacktrace.Propagate(err, "An error occurred writing 'replace:' to kurtosis.yml") } for packageName, localFilepath := range packageNamesToLocalFilepaths { - _, err := file.Write([]byte(fmt.Sprintf("\t %s: %s", packageName, localFilepath))) + _, err := file.Write([]byte(fmt.Sprintf(" %s: %s\n", packageName, localFilepath))) if err != nil { return stacktrace.Propagate(err, "An error occurred writing replace directive for '%v: %v'", packageName, localFilepath) } } + return nil } From 7e56e5b37ff6cbfc96db8d247fd5b7c509e5eeab Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 12 Sep 2024 10:39:23 -0400 Subject: [PATCH 30/47] replace magic vlues --- cli/cli/commands/run/run.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 23502f47c5..d9d6fb2b13 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -94,7 +94,8 @@ const ( kurtosisBackendCtxKey = "kurtosis-backend" engineClientCtxKey = "engine-client" - kurtosisYMLFilePath = "kurtosis.yml" + kurtosisYMLFilePath = "kurtosis.yml" + kurtosisYMLFilePerms = 0644 portMappingSeparatorForLogs = ", " @@ -888,7 +889,7 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error } func updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths map[string]string) error { - file, err := os.OpenFile("kurtosis.yml", os.O_WRONLY|os.O_APPEND, 0644) + file, err := os.OpenFile(kurtosisYMLFilePath, os.O_WRONLY|os.O_APPEND, kurtosisYMLFilePerms) if err != nil { return stacktrace.Propagate(err, "An error occurred opening kurtosis.yml file.") } From 24bbebbd5da5e4071a11dac2387785bc1dff8f8f Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Thu, 12 Sep 2024 10:44:17 -0400 Subject: [PATCH 31/47] exhaust struct --- cli/cli/commands/run/run.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index d9d6fb2b13..15273af4bb 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -876,8 +876,12 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error Tags: 0, InsecureSkipTLS: false, CABundle: nil, - ProxyOptions: transport.ProxyOptions{}, - Shared: false, + ProxyOptions: transport.ProxyOptions{ + URL: "", + Username: "", + Password: "", + }, + Shared: false, }) if err != nil && !errors.Is(err, git.ErrRepositoryAlreadyExists) { return nil, stacktrace.Propagate(err, "An error occurred cloning package '%s' to '%s'.", dependency, localPackagePath) From f5d4014502ea3af8cd58e998c9be9e6a13783634 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 13 Nov 2024 21:44:34 -0800 Subject: [PATCH 32/47] start addressing comments --- .../core/lib/enclaves/enclave_context.go | 6 +- cli/cli/commands/run/run.go | 57 ++++++++++++------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go index eb5707a8be..03e25f8d50 100755 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -534,7 +534,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string, dependenciesOnly bool) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { serializedParams, err := maybeParseYaml(serializedParams) if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) + return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%s':\n%s", packageId, serializedParams) } response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{ PackageId: packageId, @@ -544,7 +544,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context DependenciesOnly: &dependenciesOnly, }) if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred while getting the starlark package plan yaml run.") + return nil, stacktrace.Propagate(err, "An error occurred while getting the Starlark package plan yaml.") } return response, nil } @@ -561,7 +561,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, DependenciesOnly: &dependenciesOnly, }) if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred while getting the last starlark script plan yaml run.") + return nil, stacktrace.Propagate(err, "An error occurred while getting the starlark script plan yaml.") } return response, nil } diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 15273af4bb..8c551d32bd 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -60,10 +60,10 @@ const ( dryRunFlagKey = "dry-run" defaultDryRun = "false" - dependenciesFlagKey = "dependencies" - defaultDependencies = "false" - pullDependenciesFlagKey = "pull" - defaultPullDependencies = "false" + dependenciesFlagKey = "dependencies" + dependenciesFlagDefault = "false" + pullDependenciesFlagKey = "pull" + pullDependenciesFlagDefault = "false" fullUuidsFlagKey = "full-uuids" fullUuidFlagKeyDefault = "false" @@ -117,7 +117,8 @@ const ( nonBlockingModeFlagKey = "non-blocking-tasks" defaultBlockingMode = "false" - httpProtocolRegexStr = "^(http|https)://" + httpProtocolRegexStr = "^(http|https)://" + shouldCloneNormalRepo = false ) var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisCommand{ @@ -141,15 +142,15 @@ var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisC }, { Key: dependenciesFlagKey, - Usage: "If true, a yaml will be returned with a list of images and packages that this run depends on.", + Usage: "If true, a yaml will be output (to stdout) with a list of images and packages that this run depends on.", Type: flags.FlagType_Bool, - Default: defaultDependencies, + Default: dependenciesFlagDefault, }, { Key: pullDependenciesFlagKey, - Usage: "If true, and the dependencies flag is passed, attempts to pull all images and packages that the run depends on locally. kurtosis.yml is updated with the replace directives pointing to locally pulled packages.", + Usage: fmt.Sprintf("If true, and the %s flag is passed, attempts to pull all images and packages that the run depends on locally. %s is updated with replace directives pointing to locally pulled packages. If a replace directive already exists an error is thrown. Note: this currently only works on the Docker backend.", dependenciesFlagKey, kurtosisYMLFilePath), Type: flags.FlagType_Bool, - Default: defaultPullDependencies, + Default: pullDependenciesFlagDefault, }, { Key: enclaveIdentifierFlagKey, @@ -285,7 +286,7 @@ func run( return stacktrace.Propagate(err, "Expected a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dryRunFlagKey) } - dependencies, err := flags.GetBool(dependenciesFlagKey) + isDependenciesOnly, err := flags.GetBool(dependenciesFlagKey) if err != nil { return stacktrace.Propagate(err, "Expected a boolean flag with key '%v' but none was found; this is an error in Kurtosis!", dependenciesFlagKey) } @@ -401,10 +402,10 @@ func run( isRemotePackage := strings.HasPrefix(starlarkScriptOrPackagePath, githubDomainPrefix) - if dependencies { + if isDependenciesOnly { dependencyYaml, err := getPackageDependencyYaml(ctx, enclaveCtx, starlarkScriptOrPackagePath, isRemotePackage, packageArgs) if err != nil { - return stacktrace.Propagate(err, "An error occurred getting package dependencies") + return stacktrace.Propagate(err, "An error occurred getting package dependencies.") } fmt.Printf("%v\n", dependencyYaml.PlanYaml) @@ -416,9 +417,10 @@ func run( var pkgDeps PackageDependencies err := yaml.Unmarshal([]byte(dependencyYaml.PlanYaml), &pkgDeps) if err != nil { - return stacktrace.Propagate(err, "An error occurred unmarhsaling dependency yaml string") + return stacktrace.Propagate(err, "An error occurred unmarshalling dependency yaml string") } + // errors below already wrapped w propagate err = pullImagesLocally(ctx, pkgDeps.Images) if err != nil { return err @@ -842,26 +844,37 @@ func pullImagesLocally(ctx context.Context, images []string) error { func pullPackagesLocally(packageDependencies []string) (map[string]string, error) { localPackagesToRelativeFilepaths := map[string]string{} - wd, err := os.Getwd() + workingDirectory, err := os.Getwd() if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting current working directory.") } - parentCwd := filepath.Dir(wd) - relParentCwd, err := filepath.Rel(wd, parentCwd) + parentCwd := filepath.Dir(workingDirectory) + relParentCwd, err := filepath.Rel(workingDirectory, parentCwd) if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred getting rel path between '%v' and '%v'.", wd, relParentCwd) + return nil, stacktrace.Propagate(err, "An error occurred getting rel path between '%v' and '%v'.", workingDirectory, relParentCwd) } + if _, err := os.Stat(fmt.Sprintf("%s/%s", parentCwd, kurtosisYMLFilePath)); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, stacktrace.Propagate(err, "%s does not exist in current working directory. Make sure you are running this at the root of the package with the %s.", kurtosisYMLFilePath, kurtosisYMLFilePath) + } else { + return nil, stacktrace.Propagate(err, "An error occurred checking if %s exists.", kurtosisYMLFilePath) + } + } for _, dependency := range packageDependencies { packageIdParts := strings.Split(dependency, "/") packageName := packageIdParts[len(packageIdParts)-1] logrus.Infof("Pulling package: %v", dependency) - // assume dependency doesn't have http:// prefix or .git suffix - repoUrl := fmt.Sprintf("%s%s%s", "http://", dependency, ".git") + var repoUrl string + if !strings.HasPrefix("http://", dependency) { + repoUrl = "http://" + dependency + } + if !strings.HasSuffix(".git", dependency) { + repoUrl += ".git" + } localPackagePath := fmt.Sprintf("%s/%s", parentCwd, packageName) - logrus.Infof("repo url: %v", repoUrl) - _, err := git.PlainClone(localPackagePath, false, &git.CloneOptions{ + _, err := git.PlainClone(localPackagePath, shouldCloneNormalRepo, &git.CloneOptions{ URL: repoUrl, Auth: nil, RemoteName: "", @@ -895,7 +908,7 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error func updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths map[string]string) error { file, err := os.OpenFile(kurtosisYMLFilePath, os.O_WRONLY|os.O_APPEND, kurtosisYMLFilePerms) if err != nil { - return stacktrace.Propagate(err, "An error occurred opening kurtosis.yml file.") + return stacktrace.Propagate(err, "An error occurred opening %s file.", kurtosisYMLFilePath) } defer file.Close() From db66969a6850f84fa61f5feccd829f693b376000 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Fri, 15 Nov 2024 16:20:58 -0500 Subject: [PATCH 33/47] remove dependencies only flag --- api/golang/core/lib/enclaves/enclave_context.go | 6 ++---- .../api_container/server/api_container_service.go | 6 ++---- .../instructions_plan/instructions_plan.go | 4 ++-- .../plan_yaml/plan_yaml_generator.go | 14 ++------------ 4 files changed, 8 insertions(+), 22 deletions(-) diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go index 03e25f8d50..2807daa39b 100755 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -531,7 +531,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis return response, nil } -func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string, dependenciesOnly bool) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { +func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { serializedParams, err := maybeParseYaml(serializedParams) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%s':\n%s", packageId, serializedParams) @@ -541,7 +541,6 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context SerializedParams: &serializedParams, RelativePathToMainFile: nil, MainFunctionName: nil, - DependenciesOnly: &dependenciesOnly, }) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred while getting the Starlark package plan yaml.") @@ -549,7 +548,7 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context return response, nil } -func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string, dependenciesOnly bool) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { +func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, serializedScript string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { serializedParams, err := maybeParseYaml(serializedParams) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%v'", serializedParams) @@ -558,7 +557,6 @@ func (enclaveCtx *EnclaveContext) GetStarlarkScriptPlanYaml(ctx context.Context, SerializedScript: serializedScript, SerializedParams: &serializedParams, MainFunctionName: nil, - DependenciesOnly: &dependenciesOnly, }) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred while getting the starlark script plan yaml.") diff --git a/core/server/api_container/server/api_container_service.go b/core/server/api_container/server/api_container_service.go index ce2ab9db51..eabdf4a60d 100644 --- a/core/server/api_container/server/api_container_service.go +++ b/core/server/api_container/server/api_container_service.go @@ -663,7 +663,6 @@ func (apicService *ApiContainerService) GetStarlarkPackagePlanYaml(ctx context.C serializedParams := args.GetSerializedParams() requestedRelativePathToMainFile := args.GetRelativePathToMainFile() mainFuncName := args.GetMainFunctionName() - dependenciesOnly := args.GetDependenciesOnly() var scriptWithRunFunction string var interpretationError *startosis_errors.InterpretationError @@ -692,7 +691,7 @@ func (apicService *ApiContainerService) GetStarlarkPackagePlanYaml(ctx context.C interpretationError = startosis_errors.NewInterpretationError(apiInterpretationError.GetErrorMessage()) return nil, stacktrace.Propagate(interpretationError, "An interpretation error occurred interpreting package for retrieving plan yaml for package: %v", packageIdFromArgs) } - planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageIdFromArgs), dependenciesOnly) + planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageIdFromArgs)) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred generating plan yaml for package: %v", packageIdFromArgs) } @@ -708,7 +707,6 @@ func (apicService *ApiContainerService) GetStarlarkScriptPlanYaml(ctx context.Co serializedStarlarkScript := args.GetSerializedScript() serializedParams := args.GetSerializedParams() mainFuncName := args.GetMainFunctionName() - dependenciesOnly := args.GetDependenciesOnly() noPackageReplaceOptions := map[string]string{} _, instructionsPlan, apiInterpretationError := apicService.startosisInterpreter.Interpret( @@ -726,7 +724,7 @@ func (apicService *ApiContainerService) GetStarlarkScriptPlanYaml(ctx context.Co if apiInterpretationError != nil { return nil, startosis_errors.NewInterpretationError(apiInterpretationError.GetErrorMessage()) } - planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), dependenciesOnly) + planYamlStr, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) if err != nil { return nil, err } diff --git a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go index 81dd265ad7..4b5f8b2dc6 100644 --- a/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go +++ b/core/server/api_container/server/startosis_engine/instructions_plan/instructions_plan.go @@ -81,7 +81,7 @@ func (plan *InstructionsPlan) GeneratePlan() ([]*ScheduledInstruction, *startosi } // GenerateYaml takes in an existing planYaml (usually empty) and returns a yaml string containing the effects of the plan -func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator, dependenciesOnly bool) (string, error) { +func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator) (string, error) { for _, instructionUuid := range plan.instructionsSequence { instruction, found := plan.scheduledInstructionsIndex[instructionUuid] if !found { @@ -93,7 +93,7 @@ func (plan *InstructionsPlan) GenerateYaml(planYaml *plan_yaml.PlanYamlGenerator } } planYaml.AddPackageDependencies(plan.packageDependencies) - return planYaml.GenerateYaml(dependenciesOnly) + return planYaml.GenerateYaml() } func (plan *InstructionsPlan) AddPackageDependency(packageDependency string) { diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index b7891f684b..434d98f37c 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -22,7 +22,7 @@ const ( outputFutureRefType = "output" ) -// PlanYamlGenerator is a yaml representation of the effect of an Instructions Plan or sequence of instructions on the state of the Enclave. +// PlanYamlGenerator generates a PlanYaml representing the effect of an Instructions Plan or sequence of instructions on the state of the Enclave. type PlanYamlGenerator struct { privatePlanYaml *PlanYaml @@ -49,21 +49,11 @@ func CreateEmptyPlan(packageId string) *PlanYamlGenerator { } } -func (planYaml *PlanYamlGenerator) GenerateYaml(dependenciesOnly bool) (string, error) { +func (planYaml *PlanYamlGenerator) GenerateYaml() (string, error) { planYaml.AddImages() var planYamlCopy PlanYaml planYamlCopy = *planYaml.privatePlanYaml - if dependenciesOnly { - planYamlCopy = PlanYaml{ - PackageId: "", - Services: nil, - FilesArtifacts: nil, - Tasks: nil, - Images: planYaml.privatePlanYaml.Images, - PackageDependencies: planYaml.privatePlanYaml.PackageDependencies, - } - } yamlBytes, err := yaml.Marshal(planYamlCopy) if err != nil { return "", stacktrace.Propagate(err, "An error occurred generating plan yaml.") From ec6e9af282fce2c28b99fd965acbda46bc632e00 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 19:38:43 -0500 Subject: [PATCH 34/47] rename test file --- ...s_interpreter_plan_yaml_generator_test.go} | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) rename core/server/api_container/server/startosis_engine/{startosis_interpreter_plan_yaml_test.go => startosis_interpreter_plan_yaml_generator_test.go} (94%) diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_generator_test.go similarity index 94% rename from core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go rename to core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_generator_test.go index 05d59ef308..2e568f6420 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_test.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter_plan_yaml_generator_test.go @@ -24,7 +24,7 @@ const ( mockApicVersion = "1234" ) -type StartosisIntepreterPlanYamlTestSuite struct { +type StartosisIntepreterPlanYamlGeneratorTestSuite struct { suite.Suite serviceNetwork *service_network.MockServiceNetwork packageContentProvider *mock_package_content_provider.MockPackageContentProvider @@ -34,7 +34,7 @@ type StartosisIntepreterPlanYamlTestSuite struct { interpreter *StartosisInterpreter } -func (suite *StartosisIntepreterPlanYamlTestSuite) SetupTest() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) SetupTest() { // mock package content provider suite.packageContentProvider = mock_package_content_provider.NewMockPackageContentProvider() enclaveDb := getEnclaveDBForTest(suite.T()) @@ -72,15 +72,15 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) SetupTest() { suite.interpreter = NewStartosisInterpreter(suite.serviceNetwork, suite.packageContentProvider, suite.runtimeValueStore, nil, "", suite.interpretationTimeValueStore) } -func TestRunStartosisIntepreterPlanYamlTestSuite(t *testing.T) { - suite.Run(t, new(StartosisIntepreterPlanYamlTestSuite)) +func TestRunStartosisIntepreterPlanYamlGeneratorTestSuite(t *testing.T) { + suite.Run(t, new(StartosisIntepreterPlanYamlGeneratorTestSuite)) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TearDownTest() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TearDownTest() { suite.packageContentProvider.RemoveAll() } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithFilesArtifact() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestAddServiceWithFilesArtifact() { script := `def run(plan, hi_files_artifact): service_name = "serviceA" config = ServiceConfig( @@ -115,7 +115,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithFilesArtifa require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) require.NoError(suite.T(), err) expectedYaml := @@ -153,7 +153,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestRunShWithFilesArtifacts() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestRunShWithFilesArtifacts() { script := `def run(plan, hi_files_artifact): plan.run_sh( run="echo bye > /bye.txt", @@ -185,7 +185,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRunShWithFilesArtifacts() require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -220,7 +220,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestRunPython() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestRunPython() { script := `def run(plan, hi_files_artifact): plan.run_python( run = """ @@ -260,7 +260,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRunPython() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -298,7 +298,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestExec() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestExec() { script := `def run(plan, hi_files_artifact): plan.add_service( name="db", @@ -336,7 +336,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestExec() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -374,7 +374,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestRenderTemplate() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestRenderTemplate() { script := `def run(plan, args): bye_files_artifact = plan.render_templates( name="bye-file", @@ -414,7 +414,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRenderTemplate() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -442,7 +442,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithImageBuildSpec() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestAddServiceWithImageBuildSpec() { dockerfileModulePath := "github.com/kurtosis-tech/plan-yaml-prac/server/Dockerfile" serverModulePath := "github.com/kurtosis-tech/plan-yaml-prac/server" dockerfileContents := `RUN ["something"]` @@ -481,7 +481,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithImageBuildS require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId)) require.NoError(suite.T(), err) expectedYaml := `packageId: ` + packageId + ` @@ -506,7 +506,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithImageSpec() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestAddServiceWithImageSpec() { dockerfileModulePath := "github.com/kurtosis-tech/plan-yaml-prac/server/Dockerfile" serverModulePath := "github.com/kurtosis-tech/plan-yaml-prac/server" dockerfileContents := `RUN ["something"]` @@ -544,7 +544,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestAddServiceWithImageSpec() require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 1, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId)) require.NoError(suite.T(), err) expectedYaml := `packageId: ` + packageId + ` @@ -568,7 +568,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestUploadFiles() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestUploadFiles() { dockerfileModulePath := "github.com/kurtosis-tech/plan-yaml-prac/server/Dockerfile" serverModulePath := "github.com/kurtosis-tech/plan-yaml-prac/server" dockerfileContents := `RUN ["something"]` @@ -603,7 +603,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestUploadFiles() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(packageId)) require.NoError(suite.T(), err) expectedYaml := `packageId: ` + packageId + ` @@ -630,7 +630,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestStoreServiceFiles() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestStoreServiceFiles() { script := `def run(plan, hi_files_artifact): plan.add_service( name="db", @@ -665,7 +665,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestStoreServiceFiles() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -695,7 +695,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestRemoveService() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestRemoveService() { script := `def run(plan, hi_files_artifact): plan.add_service( name="db", @@ -728,7 +728,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestRemoveService() { require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 2, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT @@ -741,7 +741,7 @@ images: require.Equal(suite.T(), expectedYaml, planYaml) } -func (suite *StartosisIntepreterPlanYamlTestSuite) TestFutureReferencesAreSwapped() { +func (suite *StartosisIntepreterPlanYamlGeneratorTestSuite) TestFutureReferencesAreSwapped() { script := `def run(plan, hi_files_artifact): service = plan.add_service( name="db", @@ -789,7 +789,7 @@ func (suite *StartosisIntepreterPlanYamlTestSuite) TestFutureReferencesAreSwappe require.Nil(suite.T(), interpretationError) require.Equal(suite.T(), 4, instructionsPlan.Size()) - planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript), false) + planYaml, err := instructionsPlan.GenerateYaml(plan_yaml.CreateEmptyPlan(startosis_constants.PackageIdPlaceholderForStandaloneScript)) require.NoError(suite.T(), err) expectedYaml := `packageId: DEFAULT_PACKAGE_ID_FOR_SCRIPT From 512186093d09af8115baa5b87fad2f8fd4c83329 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 19:55:54 -0500 Subject: [PATCH 35/47] typo --- cli/cli/kurtosis_package/kurtosis_package.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cli/kurtosis_package/kurtosis_package.go b/cli/cli/kurtosis_package/kurtosis_package.go index aad6888721..e4ec209f18 100644 --- a/cli/cli/kurtosis_package/kurtosis_package.go +++ b/cli/cli/kurtosis_package/kurtosis_package.go @@ -26,7 +26,7 @@ func InitializeKurtosisPackage(packageDirpath string, packageName string, isExec // validate package name _, err := shared_utils.ParseGitURL(packageName) if err != nil { - return stacktrace.Propagate(err, "An erro occurred validating package name '%v', invalid GitHub URL", packageName) + return stacktrace.Propagate(err, "An error occurred validating package name '%v', invalid GitHub URL", packageName) } logrus.Debugf("Initializaing the '%s' Kurtosis package...", packageName) From 294c325a41604cb09ec2733972de67a239d6ca10 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 20:25:06 -0500 Subject: [PATCH 36/47] remove add images --- .../plan_yaml/plan_yaml_generator.go | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go index 434d98f37c..114b85432e 100644 --- a/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go +++ b/core/server/api_container/server/startosis_engine/plan_yaml/plan_yaml_generator.go @@ -30,6 +30,7 @@ type PlanYamlGenerator struct { filesArtifactIndex map[string]*FilesArtifact latestUuid int imageSet map[string]bool + packageDependencySet map[string]bool } func CreateEmptyPlan(packageId string) *PlanYamlGenerator { @@ -43,6 +44,7 @@ func CreateEmptyPlan(packageId string) *PlanYamlGenerator { PackageDependencies: []string{}, }, imageSet: map[string]bool{}, + packageDependencySet: map[string]bool{}, futureReferenceIndex: map[string]string{}, filesArtifactIndex: map[string]*FilesArtifact{}, latestUuid: 0, @@ -50,11 +52,10 @@ func CreateEmptyPlan(packageId string) *PlanYamlGenerator { } func (planYaml *PlanYamlGenerator) GenerateYaml() (string, error) { - planYaml.AddImages() + planYaml.privatePlanYaml.Images = convertStrMapSetToSortedStrList(planYaml.imageSet) + planYaml.privatePlanYaml.PackageDependencies = convertStrMapSetToSortedStrList(planYaml.packageDependencySet) - var planYamlCopy PlanYaml - planYamlCopy = *planYaml.privatePlanYaml - yamlBytes, err := yaml.Marshal(planYamlCopy) + yamlBytes, err := yaml.Marshal(*planYaml.privatePlanYaml) if err != nil { return "", stacktrace.Propagate(err, "An error occurred generating plan yaml.") } @@ -389,9 +390,8 @@ func (planYaml *PlanYamlGenerator) RemoveService(serviceName string) { func (planYaml *PlanYamlGenerator) AddPackageDependencies(packageDependency map[string]bool) { for dependency := range packageDependency { - planYaml.privatePlanYaml.PackageDependencies = append(planYaml.privatePlanYaml.PackageDependencies, dependency) + planYaml.packageDependencySet[dependency] = true } - slices.Sort(planYaml.privatePlanYaml.PackageDependencies) } func (planYaml *PlanYamlGenerator) AddImages() { @@ -481,3 +481,12 @@ func (planYaml *PlanYamlGenerator) generateUuid() string { planYaml.latestUuid++ return strconv.Itoa(planYaml.latestUuid) } + +func convertStrMapSetToSortedStrList(mapSet map[string]bool) []string { + l := make([]string, 0, len(mapSet)) + for v := range mapSet { + l = append(l, v) + } + slices.Sort(l) + return l +} From 6b489ca26c0bcc779b990a770166aead63787bf1 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 20:43:09 -0500 Subject: [PATCH 37/47] clarify get module prefix --- .../server/startosis_engine/startosis_interpreter.go | 9 +++++---- .../git_package_content_provider.go | 12 ++++++------ .../git_package_content_provider/locators.go | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/core/server/api_container/server/startosis_engine/startosis_interpreter.go b/core/server/api_container/server/startosis_engine/startosis_interpreter.go index 48132f77e0..99b3e9a5a7 100644 --- a/core/server/api_container/server/startosis_engine/startosis_interpreter.go +++ b/core/server/api_container/server/startosis_engine/startosis_interpreter.go @@ -20,6 +20,7 @@ import ( "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_constants" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors" "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages" + "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider" "github.com/sirupsen/logrus" "go.starlark.net/resolve" "go.starlark.net/starlark" @@ -392,13 +393,13 @@ func (interpreter *StartosisInterpreter) buildBindings( } const ( - moduleIdSeperator = "/" - numModuleIdSeparators = 4 - prefixNum = 3 + numModIdSeparators = 3 ) +// gets the prefix of a module id +// eg. "github.com/kurtosis-tech/postgres-package/main.star" returns "github.com/kurtosis-tech/postgres-package" func getModulePrefix(moduleId string) string { - return strings.Join(strings.SplitN(moduleId, moduleIdSeperator, numModuleIdSeparators)[:prefixNum], moduleIdSeperator) + return strings.Join(strings.SplitN(moduleId, git_package_content_provider.OsPathSeparatorString, numModIdSeparators+1)[:numModIdSeparators], git_package_content_provider.OsPathSeparatorString) } func findFirstEqualInstructionPastIndex(currentEnclaveInstructionsList []*enclave_plan_persistence.EnclavePlanInstruction, naiveInstructionsList []*instructions_plan.ScheduledInstruction, minIndex int) int { diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go index af1c8b1f5d..83704d7f1f 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/git_package_content_provider.go @@ -44,7 +44,7 @@ const ( filePathToKurtosisOrComposeYamlNotFound = "" replaceCountPackageDirWithGithubConstant = 1 - osPathSeparatorString = string(os.PathSeparator) + OsPathSeparatorString = string(os.PathSeparator) onlyOneReplace = 1 @@ -549,8 +549,8 @@ func validatePackageNameMatchesKurtosisYamlLocation(kurtosisYaml *yaml_parser.Ku packageNameFromAbsPackagePath := strings.Replace(absPathToKurtosisYmlInThePackage, packageDir, shared_utils.GithubDomainPrefix, replaceCountPackageDirWithGithubConstant) packageName := kurtosisYaml.GetPackageName() - if strings.HasSuffix(packageName, osPathSeparatorString) { - return startosis_errors.NewInterpretationError("Kurtosis package name cannot have trailing %q; package name: %v and kurtosis.yml is found at: %v", osPathSeparatorString, packageName, packageNameFromAbsPackagePath) + if strings.HasSuffix(packageName, OsPathSeparatorString) { + return startosis_errors.NewInterpretationError("Kurtosis package name cannot have trailing %q; package name: %v and kurtosis.yml is found at: %v", OsPathSeparatorString, packageName, packageNameFromAbsPackagePath) } // re-using ParseGitURL with packageName found from kurtosis.yml as it already does some validations @@ -592,8 +592,8 @@ func getKurtosisOrComposeYamlPathForFileUrlInternal(absPathToFile string, packag return filePathToKurtosisOrComposeYamlNotFound, startosis_errors.NewInterpretationError("Absolute path to file: %v must start with following prefix %v", absPathToFile, packagesDir) } - removeTrailingPathSeparator := strings.Trim(beginSearchForKurtosisYamlFromRepo, osPathSeparatorString) - dirs := strings.Split(removeTrailingPathSeparator, osPathSeparatorString) + removeTrailingPathSeparator := strings.Trim(beginSearchForKurtosisYamlFromRepo, OsPathSeparatorString) + dirs := strings.Split(removeTrailingPathSeparator, OsPathSeparatorString) logrus.Debugf("Found directories: %v", dirs) var validYamlFilenames []string @@ -619,7 +619,7 @@ func getKurtosisOrComposeYamlPathForFileUrlInternal(absPathToFile string, packag } func isLocalDependencyReplace(replace string) bool { - if strings.HasPrefix(replace, osPathSeparatorString) || strings.HasPrefix(replace, dotRelativePathIndicatorString) { + if strings.HasPrefix(replace, OsPathSeparatorString) || strings.HasPrefix(replace, dotRelativePathIndicatorString) { return true } return false diff --git a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go index c5472eb361..b3a034ec54 100644 --- a/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go +++ b/core/server/api_container/server/startosis_engine/startosis_packages/git_package_content_provider/locators.go @@ -13,7 +13,7 @@ const ( ) func isLocalLocator(locator string) bool { - if strings.HasPrefix(locator, osPathSeparatorString) || strings.HasPrefix(locator, dotRelativePathIndicatorString) { + if strings.HasPrefix(locator, OsPathSeparatorString) || strings.HasPrefix(locator, dotRelativePathIndicatorString) { return true } return false From 807de9b443d0a9c82c5391aedef3225691d22212 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 21:19:08 -0500 Subject: [PATCH 38/47] finish addressing comments --- cli/cli/commands/run/run.go | 84 ++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 25 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 8c551d32bd..1c46202335 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -8,6 +8,7 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/starlark_run_config" + "github.com/kurtosis-tech/kurtosis/cli/cli/out" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/configs" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" @@ -117,8 +118,9 @@ const ( nonBlockingModeFlagKey = "non-blocking-tasks" defaultBlockingMode = "false" - httpProtocolRegexStr = "^(http|https)://" - shouldCloneNormalRepo = false + httpProtocolRegexStr = "^(http|https)://" + shouldCloneNormalRepo = false + packageReplaceKeyInKurtosisYml = "replace:" ) var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisCommand{ @@ -407,19 +409,26 @@ func run( if err != nil { return stacktrace.Propagate(err, "An error occurred getting package dependencies.") } - fmt.Printf("%v\n", dependencyYaml.PlanYaml) - if pullDependencies { - type PackageDependencies struct { - Images []string `yaml:"images"` - Packages []string `yaml:"packageDependencies"` - } - var pkgDeps PackageDependencies - err := yaml.Unmarshal([]byte(dependencyYaml.PlanYaml), &pkgDeps) - if err != nil { - return stacktrace.Propagate(err, "An error occurred unmarshalling dependency yaml string") - } + type PackageDependencies struct { + Images []string `yaml:"images"` + Packages []string `yaml:"packageDependencies"` + } + var pkgDeps PackageDependencies + err = yaml.Unmarshal([]byte(dependencyYaml.PlanYaml), &pkgDeps) + if err != nil { + return stacktrace.Propagate(err, "An error occurred unmarshalling dependency yaml string") + } + out.PrintOutLn("Images:") + for _, imageStr := range pkgDeps.Images { + out.PrintOutLn(fmt.Sprintf(" %s", imageStr)) + } + out.PrintOutLn("Packages:") + for _, packageStr := range pkgDeps.Packages { + out.PrintOutLn(fmt.Sprintf(" %s", packageStr)) + } + if pullDependencies { // errors below already wrapped w propagate err = pullImagesLocally(ctx, pkgDeps.Images) if err != nil { @@ -670,9 +679,20 @@ func getPackageDependencyYaml( var packageYaml *kurtosis_core_rpc_api_bindings.PlanYaml var err error if isRemote { - packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs, true) + packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) } else { - packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, starlarkScriptOrPackageId, packageArgs, true) + fileOrDir, err := os.Stat(starlarkScriptOrPackageId) + if err != nil { + return nil, stacktrace.Propagate(err, "There was an error reading file or package from disk at '%v'", starlarkScriptOrPackageId) + } + + if isStandaloneScript(fileOrDir, kurtosisYMLFilePath) { + scriptContentBytes, err := os.ReadFile(starlarkScriptOrPackageId) + if err != nil { + return nil, stacktrace.Propagate(err, "Unable to read content of Starlark script file '%s'", starlarkScriptOrPackageId) + } + packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, string(scriptContentBytes), packageArgs) + } } if err != nil { return nil, stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") @@ -704,7 +724,6 @@ func parseVerbosityFlag(flags *flags.ParsedFlags) (command_args_run.Verbosity, e // Get the image download flag is present, and parse it to a valid ImageDownload value func parseImageDownloadFlag(flags *flags.ParsedFlags) (*kurtosis_core_rpc_api_bindings.ImageDownloadMode, error) { - imageDownloadStr, err := flags.GetString(imageDownloadFlagKey) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting the image-download using flag key '%s'", imageDownloadFlagKey) @@ -848,6 +867,13 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting current working directory.") } + // ensure a kurtosis yml exists here so that packages get cloned to the right place (one dir above/nested in the same dir as the package) + file, err := os.OpenFile(kurtosisYMLFilePath, os.O_WRONLY|os.O_APPEND, kurtosisYMLFilePerms) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred opening '%s' file. Make sure this command is being run from within the directory of a kurtosis package.", kurtosisYMLFilePath) + } + defer file.Close() + parentCwd := filepath.Dir(workingDirectory) relParentCwd, err := filepath.Rel(workingDirectory, parentCwd) if err != nil { @@ -874,6 +900,7 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error repoUrl += ".git" } localPackagePath := fmt.Sprintf("%s/%s", parentCwd, packageName) + // find the kurtosis.yml _, err := git.PlainClone(localPackagePath, shouldCloneNormalRepo, &git.CloneOptions{ URL: repoUrl, Auth: nil, @@ -908,21 +935,28 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error func updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths map[string]string) error { file, err := os.OpenFile(kurtosisYMLFilePath, os.O_WRONLY|os.O_APPEND, kurtosisYMLFilePerms) if err != nil { - return stacktrace.Propagate(err, "An error occurred opening %s file.", kurtosisYMLFilePath) + return stacktrace.Propagate(err, "An error occurred opening '%s' file.", kurtosisYMLFilePath) } defer file.Close() - _, err = file.Write([]byte("replace:\n")) + // assume kurtosis.yml is a small file so okay to read into memory + kurtosisYmlBytes, err := os.ReadFile(kurtosisYMLFilePath) if err != nil { - return stacktrace.Propagate(err, "An error occurred writing 'replace:' to kurtosis.yml") + return stacktrace.Propagate(err, "An error occurred reading '%s' file.", kurtosisYMLFilePath) } - + replaceDirectiveStr := fmt.Sprintf("%s:\n", packageReplaceKeyInKurtosisYml) for packageName, localFilepath := range packageNamesToLocalFilepaths { - _, err := file.Write([]byte(fmt.Sprintf(" %s: %s\n", packageName, localFilepath))) - if err != nil { - return stacktrace.Propagate(err, "An error occurred writing replace directive for '%v: %v'", packageName, localFilepath) - } + // TODO: this assumes the users kurtosis yml is indented by two spaces which might always not be true and this could break a users kurtosis.yml + // TODO: find a way to handle other indentation levels + replaceDirectiveStr += fmt.Sprintf(" %s: %s\n", packageName, localFilepath) + } + if strings.Contains(string(kurtosisYmlBytes), packageReplaceKeyInKurtosisYml) { + logrus.Infof("A replace directive was already detected in '%s' so we will avoid overwriting it. Update the replace directive with the following:\n%s", kurtosisYMLFilePath, replaceDirectiveStr) + return nil + } + _, err = file.Write([]byte(replaceDirectiveStr)) + if err != nil { + return stacktrace.Propagate(err, "An error occurred writing '%s' to kurtosis.yml", replaceDirectiveStr) } - return nil } From 0a0c10b595ed155ec4550a20b204c6d0c73bc868 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 21:31:23 -0500 Subject: [PATCH 39/47] account for official imgs --- .../lib/backend_impls/docker/docker_manager/docker_auth.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth.go index e956126d92..8e0d47f19c 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth.go @@ -143,6 +143,11 @@ func GetAuthFromDockerConfig(repo string) (*registry.AuthConfig, error) { return nil, err } + // if repo string doesn't contain a repo prefix assume its an official docker library image + if !strings.Contains(repo, "/") { + repo = "library/" + repo + } + registryHost := dockerregistry.ConvertToHostname(repo) if !strings.Contains(registryHost, ".") || registryHost == "docker.io" || registryHost == "registry-1.docker.io" { From f7e5d9d53f8a37b3b73732db19070fc15d8fe6ef Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 21:34:10 -0500 Subject: [PATCH 40/47] regen proto to remove dependencies only --- .../api_container_service.pb.go | 73 ++++--------- api/protobuf/core/api_container_service.proto | 6 -- api/rust/src/api_container_api.rs | 6 -- .../api_container_service_pb.d.ts | 22 ---- .../api_container_service_pb.js | 100 +----------------- .../connect/api_container_service_pb.d.ts | 14 --- .../connect/api_container_service_pb.js | 2 - 7 files changed, 25 insertions(+), 198 deletions(-) diff --git a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go index af0a975ee5..ebb3f041d2 100644 --- a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go +++ b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go @@ -3381,8 +3381,6 @@ type StarlarkScriptPlanYamlArgs struct { SerializedParams *string `protobuf:"bytes,2,opt,name=serialized_params,json=serializedParams,proto3,oneof" json:"serialized_params,omitempty"` // The name of the main function, the default value is "run" MainFunctionName *string `protobuf:"bytes,5,opt,name=main_function_name,json=mainFunctionName,proto3,oneof" json:"main_function_name,omitempty"` - // Whether to return only the dependencies of the starlark script - DependenciesOnly *bool `protobuf:"varint,6,opt,name=dependencies_only,json=dependenciesOnly,proto3,oneof" json:"dependencies_only,omitempty"` } func (x *StarlarkScriptPlanYamlArgs) Reset() { @@ -3438,13 +3436,6 @@ func (x *StarlarkScriptPlanYamlArgs) GetMainFunctionName() string { return "" } -func (x *StarlarkScriptPlanYamlArgs) GetDependenciesOnly() bool { - if x != nil && x.DependenciesOnly != nil { - return *x.DependenciesOnly - } - return false -} - type StarlarkPackagePlanYamlArgs struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3458,8 +3449,6 @@ type StarlarkPackagePlanYamlArgs struct { RelativePathToMainFile *string `protobuf:"bytes,3,opt,name=relative_path_to_main_file,json=relativePathToMainFile,proto3,oneof" json:"relative_path_to_main_file,omitempty"` // The name of the main function, the default value is "run" MainFunctionName *string `protobuf:"bytes,4,opt,name=main_function_name,json=mainFunctionName,proto3,oneof" json:"main_function_name,omitempty"` - // Whether to return only the dependencies of the Starlark package - DependenciesOnly *bool `protobuf:"varint,5,opt,name=dependencies_only,json=dependenciesOnly,proto3,oneof" json:"dependencies_only,omitempty"` } func (x *StarlarkPackagePlanYamlArgs) Reset() { @@ -3522,13 +3511,6 @@ func (x *StarlarkPackagePlanYamlArgs) GetMainFunctionName() string { return "" } -func (x *StarlarkPackagePlanYamlArgs) GetDependenciesOnly() bool { - if x != nil && x.DependenciesOnly != nil { - return *x.DependenciesOnly - } - return false -} - var File_api_container_service_proto protoreflect.FileDescriptor var file_api_container_service_proto_rawDesc = []byte{ @@ -4103,7 +4085,7 @@ var file_api_container_service_proto_rawDesc = []byte{ 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x27, 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x79, 0x61, 0x6d, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, - 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0xa3, 0x02, 0x0a, 0x1a, 0x53, 0x74, 0x61, 0x72, + 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0xdb, 0x01, 0x0a, 0x1a, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, @@ -4114,38 +4096,29 @@ var file_api_container_service_proto_rawDesc = []byte{ 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x30, 0x0a, 0x11, 0x64, 0x65, 0x70, 0x65, - 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x08, 0x48, 0x02, 0x52, 0x10, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, - 0x69, 0x65, 0x73, 0x4f, 0x6e, 0x6c, 0x79, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, + 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, + 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x15, + 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xae, 0x02, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, + 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, + 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x61, + 0x67, 0x65, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, + 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x10, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3f, 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, + 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x16, 0x72, 0x65, + 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x54, 0x6f, 0x4d, 0x61, 0x69, 0x6e, + 0x46, 0x69, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x64, 0x65, 0x70, 0x65, - 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x22, 0xf6, 0x02, - 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, - 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1d, 0x0a, - 0x0a, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x11, - 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x10, 0x73, 0x65, 0x72, 0x69, 0x61, - 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3f, - 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, - 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x01, 0x52, 0x16, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, - 0x74, 0x68, 0x54, 0x6f, 0x4d, 0x61, 0x69, 0x6e, 0x46, 0x69, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, - 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x10, 0x6d, - 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, - 0x01, 0x01, 0x12, 0x30, 0x0a, 0x11, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, - 0x65, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x48, 0x03, 0x52, - 0x10, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x4f, 0x6e, 0x6c, - 0x79, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, - 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x72, - 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, - 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, - 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, - 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x2a, 0x36, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, + 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x42, + 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x2a, 0x36, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x2a, 0x2c, diff --git a/api/protobuf/core/api_container_service.proto b/api/protobuf/core/api_container_service.proto index 3bded04cad..cc4d0aacaa 100644 --- a/api/protobuf/core/api_container_service.proto +++ b/api/protobuf/core/api_container_service.proto @@ -598,9 +598,6 @@ message StarlarkScriptPlanYamlArgs { // The name of the main function, the default value is "run" optional string main_function_name = 5; - - // Whether to return only the dependencies of the starlark script - optional bool dependencies_only = 6; } message StarlarkPackagePlanYamlArgs { @@ -615,7 +612,4 @@ message StarlarkPackagePlanYamlArgs { // The name of the main function, the default value is "run" optional string main_function_name = 4; - - // Whether to return only the dependencies of the Starlark package - optional bool dependencies_only = 5; } diff --git a/api/rust/src/api_container_api.rs b/api/rust/src/api_container_api.rs index c81e2200ad..a3de2c65bc 100644 --- a/api/rust/src/api_container_api.rs +++ b/api/rust/src/api_container_api.rs @@ -709,9 +709,6 @@ pub struct StarlarkScriptPlanYamlArgs { /// The name of the main function, the default value is "run" #[prost(string, optional, tag = "5")] pub main_function_name: ::core::option::Option<::prost::alloc::string::String>, - /// Whether to return only the dependencies of the starlark script - #[prost(bool, optional, tag = "6")] - pub dependencies_only: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -730,9 +727,6 @@ pub struct StarlarkPackagePlanYamlArgs { /// The name of the main function, the default value is "run" #[prost(string, optional, tag = "4")] pub main_function_name: ::core::option::Option<::prost::alloc::string::String>, - /// Whether to return only the dependencies of the Starlark package - #[prost(bool, optional, tag = "5")] - pub dependencies_only: ::core::option::Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts index 2fb6b04a0f..1d23b22991 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts @@ -1511,11 +1511,6 @@ export class StarlarkScriptPlanYamlArgs extends jspb.Message { hasMainFunctionName(): boolean; clearMainFunctionName(): StarlarkScriptPlanYamlArgs; - getDependenciesOnly(): boolean; - setDependenciesOnly(value: boolean): StarlarkScriptPlanYamlArgs; - hasDependenciesOnly(): boolean; - clearDependenciesOnly(): StarlarkScriptPlanYamlArgs; - serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): StarlarkScriptPlanYamlArgs.AsObject; static toObject(includeInstance: boolean, msg: StarlarkScriptPlanYamlArgs): StarlarkScriptPlanYamlArgs.AsObject; @@ -1529,7 +1524,6 @@ export namespace StarlarkScriptPlanYamlArgs { serializedScript: string, serializedParams?: string, mainFunctionName?: string, - dependenciesOnly?: boolean, } export enum SerializedParamsCase { @@ -1541,11 +1535,6 @@ export namespace StarlarkScriptPlanYamlArgs { _MAIN_FUNCTION_NAME_NOT_SET = 0, MAIN_FUNCTION_NAME = 5, } - - export enum DependenciesOnlyCase { - _DEPENDENCIES_ONLY_NOT_SET = 0, - DEPENDENCIES_ONLY = 6, - } } export class StarlarkPackagePlanYamlArgs extends jspb.Message { @@ -1567,11 +1556,6 @@ export class StarlarkPackagePlanYamlArgs extends jspb.Message { hasMainFunctionName(): boolean; clearMainFunctionName(): StarlarkPackagePlanYamlArgs; - getDependenciesOnly(): boolean; - setDependenciesOnly(value: boolean): StarlarkPackagePlanYamlArgs; - hasDependenciesOnly(): boolean; - clearDependenciesOnly(): StarlarkPackagePlanYamlArgs; - serializeBinary(): Uint8Array; toObject(includeInstance?: boolean): StarlarkPackagePlanYamlArgs.AsObject; static toObject(includeInstance: boolean, msg: StarlarkPackagePlanYamlArgs): StarlarkPackagePlanYamlArgs.AsObject; @@ -1586,7 +1570,6 @@ export namespace StarlarkPackagePlanYamlArgs { serializedParams?: string, relativePathToMainFile?: string, mainFunctionName?: string, - dependenciesOnly?: boolean, } export enum SerializedParamsCase { @@ -1603,11 +1586,6 @@ export namespace StarlarkPackagePlanYamlArgs { _MAIN_FUNCTION_NAME_NOT_SET = 0, MAIN_FUNCTION_NAME = 4, } - - export enum DependenciesOnlyCase { - _DEPENDENCIES_ONLY_NOT_SET = 0, - DEPENDENCIES_ONLY = 5, - } } export enum ServiceStatus { diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js index 714c868f8a..d0c4855650 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js @@ -10882,8 +10882,7 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.toObject = function(includeIn var f, obj = { serializedScript: jspb.Message.getFieldWithDefault(msg, 1, ""), serializedParams: jspb.Message.getFieldWithDefault(msg, 2, ""), - mainFunctionName: jspb.Message.getFieldWithDefault(msg, 5, ""), - dependenciesOnly: jspb.Message.getBooleanFieldWithDefault(msg, 6, false) + mainFunctionName: jspb.Message.getFieldWithDefault(msg, 5, "") }; if (includeInstance) { @@ -10932,10 +10931,6 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.deserializeBinaryFromReader = var value = /** @type {string} */ (reader.readString()); msg.setMainFunctionName(value); break; - case 6: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setDependenciesOnly(value); - break; default: reader.skipField(); break; @@ -10986,13 +10981,6 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.serializeBinaryToWriter = fun f ); } - f = /** @type {boolean} */ (jspb.Message.getField(message, 6)); - if (f != null) { - writer.writeBool( - 6, - f - ); - } }; @@ -11086,42 +11074,6 @@ proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.hasMainFunctionName }; -/** - * optional bool dependencies_only = 6; - * @return {boolean} - */ -proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.getDependenciesOnly = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 6, false)); -}; - - -/** - * @param {boolean} value - * @return {!proto.api_container_api.StarlarkScriptPlanYamlArgs} returns this - */ -proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.setDependenciesOnly = function(value) { - return jspb.Message.setField(this, 6, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.api_container_api.StarlarkScriptPlanYamlArgs} returns this - */ -proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.clearDependenciesOnly = function() { - return jspb.Message.setField(this, 6, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.api_container_api.StarlarkScriptPlanYamlArgs.prototype.hasDependenciesOnly = function() { - return jspb.Message.getField(this, 6) != null; -}; - - @@ -11157,8 +11109,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.toObject = function(includeI packageId: jspb.Message.getFieldWithDefault(msg, 1, ""), serializedParams: jspb.Message.getFieldWithDefault(msg, 2, ""), relativePathToMainFile: jspb.Message.getFieldWithDefault(msg, 3, ""), - mainFunctionName: jspb.Message.getFieldWithDefault(msg, 4, ""), - dependenciesOnly: jspb.Message.getBooleanFieldWithDefault(msg, 5, false) + mainFunctionName: jspb.Message.getFieldWithDefault(msg, 4, "") }; if (includeInstance) { @@ -11211,10 +11162,6 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.deserializeBinaryFromReader var value = /** @type {string} */ (reader.readString()); msg.setMainFunctionName(value); break; - case 5: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setDependenciesOnly(value); - break; default: reader.skipField(); break; @@ -11272,13 +11219,6 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.serializeBinaryToWriter = fu f ); } - f = /** @type {boolean} */ (jspb.Message.getField(message, 5)); - if (f != null) { - writer.writeBool( - 5, - f - ); - } }; @@ -11408,42 +11348,6 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasMainFunctionNam }; -/** - * optional bool dependencies_only = 5; - * @return {boolean} - */ -proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getDependenciesOnly = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 5, false)); -}; - - -/** - * @param {boolean} value - * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this - */ -proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setDependenciesOnly = function(value) { - return jspb.Message.setField(this, 5, value); -}; - - -/** - * Clears the field making it undefined. - * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this - */ -proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearDependenciesOnly = function() { - return jspb.Message.setField(this, 5, undefined); -}; - - -/** - * Returns whether this field is set. - * @return {boolean} - */ -proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasDependenciesOnly = function() { - return jspb.Message.getField(this, 5) != null; -}; - - /** * @enum {number} */ diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts index b297f0233d..de9778ca61 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts @@ -1882,13 +1882,6 @@ export declare class StarlarkScriptPlanYamlArgs extends Message); static readonly runtime: typeof proto3; @@ -1935,13 +1928,6 @@ export declare class StarlarkPackagePlanYamlArgs extends Message); static readonly runtime: typeof proto3; diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js index e0812b1b9d..164d68d7d5 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.js @@ -659,7 +659,6 @@ export const StarlarkScriptPlanYamlArgs = proto3.makeMessageType( { no: 1, name: "serialized_script", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 2, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 5, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, - { no: 6, name: "dependencies_only", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ], ); @@ -673,7 +672,6 @@ export const StarlarkPackagePlanYamlArgs = proto3.makeMessageType( { no: 2, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 3, name: "relative_path_to_main_file", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 4, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, - { no: 5, name: "dependencies_only", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ], ); From 3de186fccd8b492fe335beb51e65546ccd514db3 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 21:34:35 -0500 Subject: [PATCH 41/47] change location of err handle --- cli/cli/commands/run/run.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 1c46202335..c2caf223d1 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -680,6 +680,9 @@ func getPackageDependencyYaml( var err error if isRemote { packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") + } } else { fileOrDir, err := os.Stat(starlarkScriptOrPackageId) if err != nil { @@ -694,9 +697,6 @@ func getPackageDependencyYaml( packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, string(scriptContentBytes), packageArgs) } } - if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") - } return packageYaml, nil } From a938a4084e1a98de81dfbd2dd58a60e2fbc233f4 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 21:53:58 -0500 Subject: [PATCH 42/47] use work dir for kurt yml check --- cli/cli/commands/run/run.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index c2caf223d1..d01811a0de 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -868,6 +868,14 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error return nil, stacktrace.Propagate(err, "An error occurred getting current working directory.") } // ensure a kurtosis yml exists here so that packages get cloned to the right place (one dir above/nested in the same dir as the package) + if _, err := os.Stat(fmt.Sprintf("%s/%s", workingDirectory, kurtosisYMLFilePath)); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, stacktrace.Propagate(err, "'%s' does not exist in current working directory. Make sure you are running this at the root of the package with the %s.", kurtosisYMLFilePath, kurtosisYMLFilePath) + } else { + return nil, stacktrace.Propagate(err, "An error occurred checking if %s exists.", kurtosisYMLFilePath) + } + } + file, err := os.OpenFile(kurtosisYMLFilePath, os.O_WRONLY|os.O_APPEND, kurtosisYMLFilePerms) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred opening '%s' file. Make sure this command is being run from within the directory of a kurtosis package.", kurtosisYMLFilePath) @@ -879,14 +887,6 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting rel path between '%v' and '%v'.", workingDirectory, relParentCwd) } - - if _, err := os.Stat(fmt.Sprintf("%s/%s", parentCwd, kurtosisYMLFilePath)); err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil, stacktrace.Propagate(err, "%s does not exist in current working directory. Make sure you are running this at the root of the package with the %s.", kurtosisYMLFilePath, kurtosisYMLFilePath) - } else { - return nil, stacktrace.Propagate(err, "An error occurred checking if %s exists.", kurtosisYMLFilePath) - } - } for _, dependency := range packageDependencies { packageIdParts := strings.Split(dependency, "/") packageName := packageIdParts[len(packageIdParts)-1] @@ -900,7 +900,6 @@ func pullPackagesLocally(packageDependencies []string) (map[string]string, error repoUrl += ".git" } localPackagePath := fmt.Sprintf("%s/%s", parentCwd, packageName) - // find the kurtosis.yml _, err := git.PlainClone(localPackagePath, shouldCloneNormalRepo, &git.CloneOptions{ URL: repoUrl, Auth: nil, From 57ab266485d3e5c8e6486d2c9b0724487364e9e0 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 21:57:10 -0500 Subject: [PATCH 43/47] remove extra colon --- cli/cli/commands/run/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index d01811a0de..e84f4046ae 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -943,7 +943,7 @@ func updateKurtosisYamlWithReplaceDirectives(packageNamesToLocalFilepaths map[st if err != nil { return stacktrace.Propagate(err, "An error occurred reading '%s' file.", kurtosisYMLFilePath) } - replaceDirectiveStr := fmt.Sprintf("%s:\n", packageReplaceKeyInKurtosisYml) + replaceDirectiveStr := fmt.Sprintf("%s\n", packageReplaceKeyInKurtosisYml) for packageName, localFilepath := range packageNamesToLocalFilepaths { // TODO: this assumes the users kurtosis yml is indented by two spaces which might always not be true and this could break a users kurtosis.yml // TODO: find a way to handle other indentation levels From c2ce7f45feae0aa733998413bfb6044741795aac Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Tue, 19 Nov 2024 22:42:40 -0500 Subject: [PATCH 44/47] lint --- cli/cli/commands/run/run.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index e84f4046ae..920bdc2a71 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -695,6 +695,9 @@ func getPackageDependencyYaml( return nil, stacktrace.Propagate(err, "Unable to read content of Starlark script file '%s'", starlarkScriptOrPackageId) } packageYaml, err = enclaveCtx.GetStarlarkScriptPlanYaml(ctx, string(scriptContentBytes), packageArgs) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") + } } } return packageYaml, nil From dcb43e181eb46a3381ad22cf0495bd1e33a50e31 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Wed, 20 Nov 2024 10:39:04 -0500 Subject: [PATCH 45/47] change docker auth err to warn logs --- .../backend_impls/docker/docker_manager/docker_manager.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index cff367eb32..c75e39862b 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -2283,13 +2283,15 @@ func pullImage(dockerClient *client.Client, imageName string, registrySpec *imag // Try to obtain the auth configuration from the docker config file authConfig, err := GetAuthFromDockerConfig(imageName) if err != nil { - logrus.Errorf("An error occurred while getting auth config for image: %s: %s", imageName, err.Error()) + logrus.Warnf("An error occurred while getting auth config for image: %s: %s", imageName, err.Error()) + logrus.Warnf("Falling back to pulling image with no auth config.") } if authConfig != nil { authFromConfig, err := registry.EncodeAuthConfig(*authConfig) if err != nil { - logrus.Errorf("An error occurred while encoding auth config for image: %s: %s", imageName, err.Error()) + logrus.Warnf("An error occurred while encoding auth config for image: %s: %s", imageName, err.Error()) + logrus.Warnf("Falling back to pulling image with no auth config.") } else { imagePullOptions.RegistryAuth = authFromConfig } From 18de25eb323facdb7b7076e8d44b158a446c2aa3 Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Mon, 25 Nov 2024 00:45:43 -0500 Subject: [PATCH 46/47] pull local package --- .../api_container_service.pb.go | 331 +++++++++--------- .../core/lib/enclaves/enclave_context.go | 38 +- api/protobuf/core/api_container_service.proto | 9 +- api/rust/src/api_container_api.rs | 9 +- .../api_container_service_pb.d.ts | 10 +- .../api_container_service_pb.js | 76 ++-- .../connect/api_container_service_pb.d.ts | 13 +- .../connect/api_container_service_pb.js | 7 +- cli/cli/commands/run/run.go | 16 +- .../server/api_container_service.go | 2 +- 10 files changed, 310 insertions(+), 201 deletions(-) diff --git a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go index ebb3f041d2..94897f9c7a 100644 --- a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go +++ b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go @@ -3442,13 +3442,15 @@ type StarlarkPackagePlanYamlArgs struct { unknownFields protoimpl.UnknownFields PackageId string `protobuf:"bytes,1,opt,name=package_id,json=packageId,proto3" json:"package_id,omitempty"` + // whether or not this is package yaml should be pulled from on disk package or cloned + IsRemote bool `protobuf:"varint,2,opt,name=is_remote,json=isRemote,proto3" json:"is_remote,omitempty"` // Serialized parameters data for the Starlark package main function // This should be a valid JSON string - SerializedParams *string `protobuf:"bytes,2,opt,name=serialized_params,json=serializedParams,proto3,oneof" json:"serialized_params,omitempty"` + SerializedParams *string `protobuf:"bytes,3,opt,name=serialized_params,json=serializedParams,proto3,oneof" json:"serialized_params,omitempty"` // The relative main file filepath, the default value is the "main.star" file in the root of a package - RelativePathToMainFile *string `protobuf:"bytes,3,opt,name=relative_path_to_main_file,json=relativePathToMainFile,proto3,oneof" json:"relative_path_to_main_file,omitempty"` + RelativePathToMainFile *string `protobuf:"bytes,4,opt,name=relative_path_to_main_file,json=relativePathToMainFile,proto3,oneof" json:"relative_path_to_main_file,omitempty"` // The name of the main function, the default value is "run" - MainFunctionName *string `protobuf:"bytes,4,opt,name=main_function_name,json=mainFunctionName,proto3,oneof" json:"main_function_name,omitempty"` + MainFunctionName *string `protobuf:"bytes,5,opt,name=main_function_name,json=mainFunctionName,proto3,oneof" json:"main_function_name,omitempty"` } func (x *StarlarkPackagePlanYamlArgs) Reset() { @@ -3490,6 +3492,13 @@ func (x *StarlarkPackagePlanYamlArgs) GetPackageId() string { return "" } +func (x *StarlarkPackagePlanYamlArgs) GetIsRemote() bool { + if x != nil { + return x.IsRemote + } + return false +} + func (x *StarlarkPackagePlanYamlArgs) GetSerializedParams() string { if x != nil && x.SerializedParams != nil { return *x.SerializedParams @@ -4099,177 +4108,179 @@ var file_api_container_service_proto_rawDesc = []byte{ 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xae, 0x02, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xcb, 0x02, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x61, - 0x67, 0x65, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, - 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, - 0x00, 0x52, 0x10, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, - 0x61, 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3f, 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, - 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, - 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x16, 0x72, 0x65, - 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x54, 0x6f, 0x4d, 0x61, 0x69, 0x6e, - 0x46, 0x69, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, - 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x42, 0x1d, 0x0a, 0x1b, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, - 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x42, - 0x15, 0x0a, 0x13, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x2a, 0x36, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, - 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x2a, 0x2c, - 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4d, - 0x6f, 0x64, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x61, 0x6c, 0x77, 0x61, 0x79, 0x73, 0x10, 0x00, 0x12, - 0x0b, 0x0a, 0x07, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x2a, 0x26, 0x0a, 0x07, - 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x4e, 0x45, - 0x43, 0x54, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x4f, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, - 0x43, 0x54, 0x10, 0x01, 0x2a, 0x32, 0x0a, 0x13, 0x4b, 0x75, 0x72, 0x74, 0x6f, 0x73, 0x69, 0x73, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x46, 0x6c, 0x61, 0x67, 0x12, 0x1b, 0x0a, 0x17, 0x4e, - 0x4f, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x52, 0x55, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x5f, 0x43, - 0x41, 0x43, 0x48, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x2a, 0x26, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x09, 0x0a, 0x05, 0x4e, 0x45, 0x56, - 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x01, - 0x32, 0xa6, 0x10, 0x0a, 0x13, 0x41, 0x70, 0x69, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6d, 0x0a, 0x11, 0x52, 0x75, 0x6e, 0x53, - 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, 0x12, 0x28, 0x2e, + 0x67, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x52, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x12, 0x30, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x10, + 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x88, 0x01, 0x01, 0x12, 0x3f, 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, + 0x70, 0x61, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x16, 0x72, 0x65, 0x6c, 0x61, 0x74, + 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x54, 0x6f, 0x4d, 0x61, 0x69, 0x6e, 0x46, 0x69, 0x6c, + 0x65, 0x88, 0x01, 0x01, 0x12, 0x31, 0x0a, 0x12, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x48, 0x02, 0x52, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x4e, 0x61, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x73, 0x65, 0x72, 0x69, + 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x1d, 0x0a, + 0x1b, 0x5f, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, + 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x42, 0x15, 0x0a, 0x13, + 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x2a, 0x36, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x4f, 0x50, 0x50, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0b, + 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x2a, 0x2c, 0x0a, 0x11, 0x49, + 0x6d, 0x61, 0x67, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, + 0x12, 0x0a, 0x0a, 0x06, 0x61, 0x6c, 0x77, 0x61, 0x79, 0x73, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, + 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x2a, 0x26, 0x0a, 0x07, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x10, + 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x4f, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x10, + 0x01, 0x2a, 0x32, 0x0a, 0x13, 0x4b, 0x75, 0x72, 0x74, 0x6f, 0x73, 0x69, 0x73, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x46, 0x6c, 0x61, 0x67, 0x12, 0x1b, 0x0a, 0x17, 0x4e, 0x4f, 0x5f, 0x49, + 0x4e, 0x53, 0x54, 0x52, 0x55, 0x43, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x5f, 0x43, 0x41, 0x43, 0x48, + 0x49, 0x4e, 0x47, 0x10, 0x00, 0x2a, 0x26, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x09, 0x0a, 0x05, 0x4e, 0x45, 0x56, 0x45, 0x52, 0x10, + 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x01, 0x32, 0xa6, 0x10, + 0x0a, 0x13, 0x41, 0x70, 0x69, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6d, 0x0a, 0x11, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, 0x72, + 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, 0x12, 0x28, 0x2e, 0x61, 0x70, 0x69, + 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x52, + 0x75, 0x6e, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x41, 0x72, 0x67, 0x73, 0x1a, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, + 0x6b, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4c, 0x69, 0x6e, 0x65, + 0x22, 0x00, 0x30, 0x01, 0x12, 0x59, 0x0a, 0x15, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x74, + 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, - 0x69, 0x2e, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x72, - 0x6c, 0x61, 0x72, 0x6b, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4c, - 0x69, 0x6e, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x59, 0x0a, 0x15, 0x55, 0x70, 0x6c, 0x6f, 0x61, - 0x64, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, - 0x12, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74, - 0x61, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, - 0x28, 0x01, 0x12, 0x6f, 0x0a, 0x12, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, - 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x75, 0x6e, - 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x41, - 0x72, 0x67, 0x73, 0x1a, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, - 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x22, - 0x00, 0x30, 0x01, 0x12, 0x5b, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x12, 0x22, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x26, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x8d, 0x01, 0x0a, 0x2a, 0x47, 0x65, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, - 0x41, 0x6e, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x45, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x45, - 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x41, 0x6e, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, - 0x69, 0x63, 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, - 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x5b, 0x0a, 0x0b, 0x45, 0x78, 0x65, 0x63, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, + 0x69, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x43, 0x68, + 0x75, 0x6e, 0x6b, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x28, 0x01, 0x12, + 0x6f, 0x0a, 0x12, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, + 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x52, 0x75, 0x6e, 0x53, 0x74, 0x61, + 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x41, 0x72, 0x67, 0x73, + 0x1a, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x52, 0x75, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x4c, 0x69, 0x6e, 0x65, 0x22, 0x00, 0x30, 0x01, + 0x12, 0x5b, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x22, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, - 0x61, 0x70, 0x69, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x41, + 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x26, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x43, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x79, 0x0a, - 0x22, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x45, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, - 0x69, 0x74, 0x79, 0x12, 0x39, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x48, - 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x41, 0x76, - 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x7b, 0x0a, 0x23, 0x57, 0x61, 0x69, 0x74, - 0x46, 0x6f, 0x72, 0x48, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, - 0x3a, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, - 0x61, 0x70, 0x69, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x48, 0x74, 0x74, 0x70, 0x50, - 0x6f, 0x73, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, + 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8d, 0x01, + 0x0a, 0x2a, 0x47, 0x65, 0x74, 0x45, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x41, 0x6e, 0x64, + 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x73, 0x12, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x45, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x78, 0x69, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x41, 0x6e, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, + 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, + 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5b, 0x0a, + 0x0b, 0x45, 0x78, 0x65, 0x63, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x22, 0x2e, 0x61, + 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, + 0x2e, 0x45, 0x78, 0x65, 0x63, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x41, 0x72, 0x67, 0x73, + 0x1a, 0x26, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x79, 0x0a, 0x22, 0x57, 0x61, + 0x69, 0x74, 0x46, 0x6f, 0x72, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x64, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, + 0x12, 0x39, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x48, 0x74, 0x74, 0x70, + 0x47, 0x65, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x13, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x46, - 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x24, 0x2e, 0x61, + 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x7b, 0x0a, 0x23, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, + 0x48, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x3a, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, - 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x43, 0x68, 0x75, - 0x6e, 0x6b, 0x1a, 0x2e, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x69, 0x6c, - 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x6f, 0x0a, 0x15, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, - 0x61, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, - 0x2c, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, - 0x61, 0x70, 0x69, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x69, 0x6c, 0x65, - 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x24, 0x2e, - 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, - 0x69, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x43, 0x68, - 0x75, 0x6e, 0x6b, 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x15, 0x53, 0x74, 0x6f, 0x72, 0x65, - 0x57, 0x65, 0x62, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x12, 0x2c, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x57, 0x65, 0x62, 0x46, 0x69, 0x6c, - 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x30, - 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, - 0x70, 0x69, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x57, 0x65, 0x62, 0x46, 0x69, 0x6c, 0x65, 0x73, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x91, 0x01, 0x0a, 0x1d, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65, - 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x46, 0x69, - 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x38, 0x2e, 0x61, 0x70, 0x69, - 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, - 0x74, 0x6f, 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x75, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, - 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, - 0x41, 0x6e, 0x64, 0x55, 0x75, 0x69, 0x64, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x2e, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x48, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, + 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, + 0x6c, 0x69, 0x74, 0x79, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x1a, 0x39, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x41, 0x6e, 0x64, 0x55, 0x75, - 0x69, 0x64, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x91, 0x01, - 0x0a, 0x1c, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x36, - 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, - 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, - 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x37, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, - 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, - 0x63, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x67, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x2a, 0x2e, 0x61, + 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x13, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x69, 0x6c, 0x65, + 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x5f, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, + 0x2e, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, + 0x61, 0x70, 0x69, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x28, 0x01, 0x12, 0x6f, 0x0a, 0x15, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x46, + 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x2c, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, - 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x47, 0x65, - 0x74, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x52, 0x75, 0x6e, 0x12, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, - 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x72, - 0x6c, 0x61, 0x72, 0x6b, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x69, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, - 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x2d, - 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, - 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1b, 0x2e, - 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, - 0x69, 0x2e, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0x00, 0x12, 0x6b, 0x0a, 0x1a, - 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, - 0x67, 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x2e, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x24, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, - 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x6c, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x65, 0x64, 0x44, 0x61, 0x74, 0x61, 0x43, 0x68, 0x75, 0x6e, 0x6b, + 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x15, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x57, 0x65, 0x62, + 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x2c, 0x2e, + 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, + 0x69, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x57, 0x65, 0x62, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x30, 0x2e, 0x61, 0x70, + 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, + 0x53, 0x74, 0x6f, 0x72, 0x65, 0x57, 0x65, 0x62, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x91, 0x01, 0x0a, 0x1d, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x34, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x38, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x6f, 0x72, + 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x46, 0x72, + 0x6f, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x75, 0x0a, 0x1e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x41, 0x6e, 0x64, + 0x55, 0x75, 0x69, 0x64, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x39, 0x2e, + 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, + 0x69, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x41, 0x6e, 0x64, 0x55, 0x75, 0x69, 0x64, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x91, 0x01, 0x0a, 0x1c, 0x49, + 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, + 0x61, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x36, 0x2e, 0x61, 0x70, + 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, + 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x37, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x49, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x74, 0x46, + 0x69, 0x6c, 0x65, 0x73, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x67, + 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x12, 0x26, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x2a, 0x2e, 0x61, 0x70, 0x69, 0x5f, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x53, 0x74, + 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x52, 0x75, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x1a, 0x29, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, + 0x6b, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x69, + 0x0a, 0x19, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x2d, 0x2e, 0x61, 0x70, + 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, + 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x53, 0x63, 0x72, 0x69, 0x70, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x50, - 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0x00, 0x42, 0x52, 0x5a, 0x50, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x72, 0x74, 0x6f, 0x73, 0x69, 0x73, - 0x2d, 0x74, 0x65, 0x63, 0x68, 0x2f, 0x6b, 0x75, 0x72, 0x74, 0x6f, 0x73, 0x69, 0x73, 0x2f, 0x61, - 0x70, 0x69, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x6b, - 0x75, 0x72, 0x74, 0x6f, 0x73, 0x69, 0x73, 0x5f, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x70, 0x63, - 0x5f, 0x61, 0x70, 0x69, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x22, 0x00, 0x12, 0x6b, 0x0a, 0x1a, 0x47, 0x65, 0x74, + 0x53, 0x74, 0x61, 0x72, 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, + 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x12, 0x2e, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x53, 0x74, 0x61, 0x72, + 0x6c, 0x61, 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, + 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1b, 0x2e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x69, 0x2e, 0x50, 0x6c, 0x61, 0x6e, + 0x59, 0x61, 0x6d, 0x6c, 0x22, 0x00, 0x42, 0x52, 0x5a, 0x50, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x72, 0x74, 0x6f, 0x73, 0x69, 0x73, 0x2d, 0x74, 0x65, + 0x63, 0x68, 0x2f, 0x6b, 0x75, 0x72, 0x74, 0x6f, 0x73, 0x69, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, + 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x6b, 0x75, 0x72, 0x74, + 0x6f, 0x73, 0x69, 0x73, 0x5f, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x72, 0x70, 0x63, 0x5f, 0x61, 0x70, + 0x69, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go index 2807daa39b..4044592084 100755 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -531,13 +531,49 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRun(ctx context.Context) (*kurtosis return response, nil } -func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { +func (enclaveCtx *EnclaveContext) GetStarlarkRemotePackagePlanYaml(ctx context.Context, packageId string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { serializedParams, err := maybeParseYaml(serializedParams) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%s':\n%s", packageId, serializedParams) } response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{ PackageId: packageId, + IsRemote: true, + SerializedParams: &serializedParams, + RelativePathToMainFile: nil, + MainFunctionName: nil, + }) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while getting the Starlark package plan yaml.") + } + return response, nil +} + +func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context, packageRootPath string, serializedParams string) (*kurtosis_core_rpc_api_bindings.PlanYaml, error) { + packageName, packageReplaceOptions, err := getPackageNameAndReplaceOptions(packageRootPath) + if err != nil { + return nil, err + } + + serializedParams, err = maybeParseYaml(serializedParams) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred when parsing YAML args for package '%s':\n%s", packageName, serializedParams) + } + + err = enclaveCtx.uploadStarlarkPackage(packageName, packageRootPath) + if err != nil { + return nil, stacktrace.Propagate(err, "Error uploading package '%s' prior to executing it", packageRootPath) + } + + if len(packageReplaceOptions) > 0 { + if err = enclaveCtx.uploadLocalStarlarkPackageDependencies(packageRootPath, packageReplaceOptions); err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while uploading the local starlark package dependencies from the replace options '%+v'", packageReplaceOptions) + } + } + + response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{ + PackageId: packageName, + IsRemote: false, SerializedParams: &serializedParams, RelativePathToMainFile: nil, MainFunctionName: nil, diff --git a/api/protobuf/core/api_container_service.proto b/api/protobuf/core/api_container_service.proto index cc4d0aacaa..1bf461ee5a 100644 --- a/api/protobuf/core/api_container_service.proto +++ b/api/protobuf/core/api_container_service.proto @@ -603,13 +603,16 @@ message StarlarkScriptPlanYamlArgs { message StarlarkPackagePlanYamlArgs { string package_id = 1; + // whether or not this is package yaml should be pulled from on disk package or cloned + bool is_remote = 2; + // Serialized parameters data for the Starlark package main function // This should be a valid JSON string - optional string serialized_params = 2; + optional string serialized_params = 3; // The relative main file filepath, the default value is the "main.star" file in the root of a package - optional string relative_path_to_main_file = 3; + optional string relative_path_to_main_file = 4; // The name of the main function, the default value is "run" - optional string main_function_name = 4; + optional string main_function_name = 5; } diff --git a/api/rust/src/api_container_api.rs b/api/rust/src/api_container_api.rs index a3de2c65bc..78a08d6fb0 100644 --- a/api/rust/src/api_container_api.rs +++ b/api/rust/src/api_container_api.rs @@ -715,17 +715,20 @@ pub struct StarlarkScriptPlanYamlArgs { pub struct StarlarkPackagePlanYamlArgs { #[prost(string, tag = "1")] pub package_id: ::prost::alloc::string::String, + /// whether or not this is package yaml should be pulled from on disk package or cloned + #[prost(bool, tag = "2")] + pub is_remote: bool, /// Serialized parameters data for the Starlark package main function /// This should be a valid JSON string - #[prost(string, optional, tag = "2")] + #[prost(string, optional, tag = "3")] pub serialized_params: ::core::option::Option<::prost::alloc::string::String>, /// The relative main file filepath, the default value is the "main.star" file in the root of a package - #[prost(string, optional, tag = "3")] + #[prost(string, optional, tag = "4")] pub relative_path_to_main_file: ::core::option::Option< ::prost::alloc::string::String, >, /// The name of the main function, the default value is "run" - #[prost(string, optional, tag = "4")] + #[prost(string, optional, tag = "5")] pub main_function_name: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts index 1d23b22991..4b7b2e9f0f 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts @@ -1541,6 +1541,9 @@ export class StarlarkPackagePlanYamlArgs extends jspb.Message { getPackageId(): string; setPackageId(value: string): StarlarkPackagePlanYamlArgs; + getIsRemote(): boolean; + setIsRemote(value: boolean): StarlarkPackagePlanYamlArgs; + getSerializedParams(): string; setSerializedParams(value: string): StarlarkPackagePlanYamlArgs; hasSerializedParams(): boolean; @@ -1567,6 +1570,7 @@ export class StarlarkPackagePlanYamlArgs extends jspb.Message { export namespace StarlarkPackagePlanYamlArgs { export type AsObject = { packageId: string, + isRemote: boolean, serializedParams?: string, relativePathToMainFile?: string, mainFunctionName?: string, @@ -1574,17 +1578,17 @@ export namespace StarlarkPackagePlanYamlArgs { export enum SerializedParamsCase { _SERIALIZED_PARAMS_NOT_SET = 0, - SERIALIZED_PARAMS = 2, + SERIALIZED_PARAMS = 3, } export enum RelativePathToMainFileCase { _RELATIVE_PATH_TO_MAIN_FILE_NOT_SET = 0, - RELATIVE_PATH_TO_MAIN_FILE = 3, + RELATIVE_PATH_TO_MAIN_FILE = 4, } export enum MainFunctionNameCase { _MAIN_FUNCTION_NAME_NOT_SET = 0, - MAIN_FUNCTION_NAME = 4, + MAIN_FUNCTION_NAME = 5, } } diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js index d0c4855650..08c88d1815 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js @@ -11107,9 +11107,10 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.toObject = functio proto.api_container_api.StarlarkPackagePlanYamlArgs.toObject = function(includeInstance, msg) { var f, obj = { packageId: jspb.Message.getFieldWithDefault(msg, 1, ""), - serializedParams: jspb.Message.getFieldWithDefault(msg, 2, ""), - relativePathToMainFile: jspb.Message.getFieldWithDefault(msg, 3, ""), - mainFunctionName: jspb.Message.getFieldWithDefault(msg, 4, "") + isRemote: jspb.Message.getBooleanFieldWithDefault(msg, 2, false), + serializedParams: jspb.Message.getFieldWithDefault(msg, 3, ""), + relativePathToMainFile: jspb.Message.getFieldWithDefault(msg, 4, ""), + mainFunctionName: jspb.Message.getFieldWithDefault(msg, 5, "") }; if (includeInstance) { @@ -11151,14 +11152,18 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.deserializeBinaryFromReader msg.setPackageId(value); break; case 2: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setIsRemote(value); + break; + case 3: var value = /** @type {string} */ (reader.readString()); msg.setSerializedParams(value); break; - case 3: + case 4: var value = /** @type {string} */ (reader.readString()); msg.setRelativePathToMainFile(value); break; - case 4: + case 5: var value = /** @type {string} */ (reader.readString()); msg.setMainFunctionName(value); break; @@ -11198,9 +11203,9 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.serializeBinaryToWriter = fu f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 2)); - if (f != null) { - writer.writeString( + f = message.getIsRemote(); + if (f) { + writer.writeBool( 2, f ); @@ -11219,6 +11224,13 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.serializeBinaryToWriter = fu f ); } + f = /** @type {string} */ (jspb.Message.getField(message, 5)); + if (f != null) { + writer.writeString( + 5, + f + ); + } }; @@ -11241,11 +11253,29 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setPackageId = fun /** - * optional string serialized_params = 2; + * optional bool is_remote = 2; + * @return {boolean} + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getIsRemote = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setIsRemote = function(value) { + return jspb.Message.setProto3BooleanField(this, 2, value); +}; + + +/** + * optional string serialized_params = 3; * @return {string} */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getSerializedParams = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); }; @@ -11254,7 +11284,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getSerializedParam * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setSerializedParams = function(value) { - return jspb.Message.setField(this, 2, value); + return jspb.Message.setField(this, 3, value); }; @@ -11263,7 +11293,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setSerializedParam * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearSerializedParams = function() { - return jspb.Message.setField(this, 2, undefined); + return jspb.Message.setField(this, 3, undefined); }; @@ -11272,16 +11302,16 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearSerializedPar * @return {boolean} */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasSerializedParams = function() { - return jspb.Message.getField(this, 2) != null; + return jspb.Message.getField(this, 3) != null; }; /** - * optional string relative_path_to_main_file = 3; + * optional string relative_path_to_main_file = 4; * @return {string} */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getRelativePathToMainFile = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 4, "")); }; @@ -11290,7 +11320,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getRelativePathToM * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setRelativePathToMainFile = function(value) { - return jspb.Message.setField(this, 3, value); + return jspb.Message.setField(this, 4, value); }; @@ -11299,7 +11329,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setRelativePathToM * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearRelativePathToMainFile = function() { - return jspb.Message.setField(this, 3, undefined); + return jspb.Message.setField(this, 4, undefined); }; @@ -11308,16 +11338,16 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearRelativePathT * @return {boolean} */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasRelativePathToMainFile = function() { - return jspb.Message.getField(this, 3) != null; + return jspb.Message.getField(this, 4) != null; }; /** - * optional string main_function_name = 4; + * optional string main_function_name = 5; * @return {string} */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getMainFunctionName = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 4, "")); + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 5, "")); }; @@ -11326,7 +11356,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getMainFunctionNam * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setMainFunctionName = function(value) { - return jspb.Message.setField(this, 4, value); + return jspb.Message.setField(this, 5, value); }; @@ -11335,7 +11365,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setMainFunctionNam * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearMainFunctionName = function() { - return jspb.Message.setField(this, 4, undefined); + return jspb.Message.setField(this, 5, undefined); }; @@ -11344,7 +11374,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearMainFunctionN * @return {boolean} */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasMainFunctionName = function() { - return jspb.Message.getField(this, 4) != null; + return jspb.Message.getField(this, 5) != null; }; diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts index de9778ca61..9c63892538 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts @@ -1906,25 +1906,32 @@ export declare class StarlarkPackagePlanYamlArgs extends Message [ { no: 1, name: "package_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, - { no: 2, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, - { no: 3, name: "relative_path_to_main_file", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, - { no: 4, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 2, name: "is_remote", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 3, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 4, name: "relative_path_to_main_file", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 5, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, ], ); diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 920bdc2a71..35f647830f 100755 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -679,7 +679,7 @@ func getPackageDependencyYaml( var packageYaml *kurtosis_core_rpc_api_bindings.PlanYaml var err error if isRemote { - packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) + packageYaml, err = enclaveCtx.GetStarlarkRemotePackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") } @@ -698,6 +698,20 @@ func getPackageDependencyYaml( if err != nil { return nil, stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") } + } else { + // if the path is a file with `kurtosis.yml` at the end it's a module dir + // we remove the `kurtosis.yml` to get just the Dir containing the module + if isKurtosisYMLFileInPackageDir(fileOrDir, kurtosisYMLFilePath) { + starlarkScriptOrPackageId = path.Dir(starlarkScriptOrPackageId) + } + // we pass the sanitized path and look for a Kurtosis YML within it to get the package name + if err != nil { + return nil, stacktrace.Propagate(err, "Tried parsing Kurtosis YML at '%v' to get package name but failed", starlarkScriptOrPackageId) + } + packageYaml, err = enclaveCtx.GetStarlarkPackagePlanYaml(ctx, starlarkScriptOrPackageId, packageArgs) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred retrieving plan yaml for provided package.") + } } } return packageYaml, nil diff --git a/core/server/api_container/server/api_container_service.go b/core/server/api_container/server/api_container_service.go index eabdf4a60d..656346d21b 100644 --- a/core/server/api_container/server/api_container_service.go +++ b/core/server/api_container/server/api_container_service.go @@ -670,7 +670,7 @@ func (apicService *ApiContainerService) GetStarlarkPackagePlanYaml(ctx context.C var detectedPackageReplaceOptions map[string]string var actualRelativePathToMainFile string scriptWithRunFunction, actualRelativePathToMainFile, detectedPackageId, detectedPackageReplaceOptions, interpretationError = - apicService.runStarlarkPackageSetup(packageIdFromArgs, true, nil, requestedRelativePathToMainFile) + apicService.runStarlarkPackageSetup(packageIdFromArgs, args.IsRemote, nil, requestedRelativePathToMainFile) if interpretationError != nil { return nil, stacktrace.Propagate(interpretationError, "An interpretation error occurred setting up the package for retrieving plan yaml for package: %v", packageIdFromArgs) } From 9820014654e8a9cdf01d6f5cd3d526e2fee5e2de Mon Sep 17 00:00:00 2001 From: Tedi Mitiku Date: Mon, 25 Nov 2024 00:51:46 -0500 Subject: [PATCH 47/47] undo breaking change --- .../api_container_service.pb.go | 32 ++++----- .../core/lib/enclaves/enclave_context.go | 4 +- api/protobuf/core/api_container_service.proto | 8 +-- api/rust/src/api_container_api.rs | 8 +-- .../api_container_service_pb.d.ts | 10 +-- .../api_container_service_pb.js | 70 +++++++++---------- .../connect/api_container_service_pb.d.ts | 14 ++-- .../connect/api_container_service_pb.js | 4 +- 8 files changed, 75 insertions(+), 75 deletions(-) diff --git a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go index 94897f9c7a..06d347f927 100644 --- a/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go +++ b/api/golang/core/kurtosis_core_rpc_api_bindings/api_container_service.pb.go @@ -3442,11 +3442,11 @@ type StarlarkPackagePlanYamlArgs struct { unknownFields protoimpl.UnknownFields PackageId string `protobuf:"bytes,1,opt,name=package_id,json=packageId,proto3" json:"package_id,omitempty"` - // whether or not this is package yaml should be pulled from on disk package or cloned - IsRemote bool `protobuf:"varint,2,opt,name=is_remote,json=isRemote,proto3" json:"is_remote,omitempty"` // Serialized parameters data for the Starlark package main function // This should be a valid JSON string - SerializedParams *string `protobuf:"bytes,3,opt,name=serialized_params,json=serializedParams,proto3,oneof" json:"serialized_params,omitempty"` + SerializedParams *string `protobuf:"bytes,2,opt,name=serialized_params,json=serializedParams,proto3,oneof" json:"serialized_params,omitempty"` + // whether or not this is package yaml should be pulled from on disk package or cloned + IsRemote bool `protobuf:"varint,3,opt,name=is_remote,json=isRemote,proto3" json:"is_remote,omitempty"` // The relative main file filepath, the default value is the "main.star" file in the root of a package RelativePathToMainFile *string `protobuf:"bytes,4,opt,name=relative_path_to_main_file,json=relativePathToMainFile,proto3,oneof" json:"relative_path_to_main_file,omitempty"` // The name of the main function, the default value is "run" @@ -3492,13 +3492,6 @@ func (x *StarlarkPackagePlanYamlArgs) GetPackageId() string { return "" } -func (x *StarlarkPackagePlanYamlArgs) GetIsRemote() bool { - if x != nil { - return x.IsRemote - } - return false -} - func (x *StarlarkPackagePlanYamlArgs) GetSerializedParams() string { if x != nil && x.SerializedParams != nil { return *x.SerializedParams @@ -3506,6 +3499,13 @@ func (x *StarlarkPackagePlanYamlArgs) GetSerializedParams() string { return "" } +func (x *StarlarkPackagePlanYamlArgs) GetIsRemote() bool { + if x != nil { + return x.IsRemote + } + return false +} + func (x *StarlarkPackagePlanYamlArgs) GetRelativePathToMainFile() string { if x != nil && x.RelativePathToMainFile != nil { return *x.RelativePathToMainFile @@ -4112,12 +4112,12 @@ var file_api_container_service_proto_rawDesc = []byte{ 0x72, 0x6b, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x6c, 0x61, 0x6e, 0x59, 0x61, 0x6d, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x61, 0x63, 0x6b, 0x61, - 0x67, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x52, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x12, 0x30, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x5f, - 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x10, - 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, - 0x88, 0x01, 0x01, 0x12, 0x3f, 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, + 0x67, 0x65, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x11, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, + 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x10, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x50, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x12, 0x3f, 0x0a, 0x1a, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x74, 0x6f, 0x5f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x16, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x54, 0x6f, 0x4d, 0x61, 0x69, 0x6e, 0x46, 0x69, 0x6c, diff --git a/api/golang/core/lib/enclaves/enclave_context.go b/api/golang/core/lib/enclaves/enclave_context.go index 4044592084..a0b3d50ed5 100755 --- a/api/golang/core/lib/enclaves/enclave_context.go +++ b/api/golang/core/lib/enclaves/enclave_context.go @@ -538,8 +538,8 @@ func (enclaveCtx *EnclaveContext) GetStarlarkRemotePackagePlanYaml(ctx context.C } response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{ PackageId: packageId, - IsRemote: true, SerializedParams: &serializedParams, + IsRemote: true, RelativePathToMainFile: nil, MainFunctionName: nil, }) @@ -573,8 +573,8 @@ func (enclaveCtx *EnclaveContext) GetStarlarkPackagePlanYaml(ctx context.Context response, err := enclaveCtx.client.GetStarlarkPackagePlanYaml(ctx, &kurtosis_core_rpc_api_bindings.StarlarkPackagePlanYamlArgs{ PackageId: packageName, - IsRemote: false, SerializedParams: &serializedParams, + IsRemote: false, RelativePathToMainFile: nil, MainFunctionName: nil, }) diff --git a/api/protobuf/core/api_container_service.proto b/api/protobuf/core/api_container_service.proto index 1bf461ee5a..c29772ede3 100644 --- a/api/protobuf/core/api_container_service.proto +++ b/api/protobuf/core/api_container_service.proto @@ -603,12 +603,12 @@ message StarlarkScriptPlanYamlArgs { message StarlarkPackagePlanYamlArgs { string package_id = 1; - // whether or not this is package yaml should be pulled from on disk package or cloned - bool is_remote = 2; - // Serialized parameters data for the Starlark package main function // This should be a valid JSON string - optional string serialized_params = 3; + optional string serialized_params = 2; + + // whether or not this is package yaml should be pulled from on disk package or cloned + bool is_remote = 3; // The relative main file filepath, the default value is the "main.star" file in the root of a package optional string relative_path_to_main_file = 4; diff --git a/api/rust/src/api_container_api.rs b/api/rust/src/api_container_api.rs index 78a08d6fb0..451c0e5848 100644 --- a/api/rust/src/api_container_api.rs +++ b/api/rust/src/api_container_api.rs @@ -715,13 +715,13 @@ pub struct StarlarkScriptPlanYamlArgs { pub struct StarlarkPackagePlanYamlArgs { #[prost(string, tag = "1")] pub package_id: ::prost::alloc::string::String, - /// whether or not this is package yaml should be pulled from on disk package or cloned - #[prost(bool, tag = "2")] - pub is_remote: bool, /// Serialized parameters data for the Starlark package main function /// This should be a valid JSON string - #[prost(string, optional, tag = "3")] + #[prost(string, optional, tag = "2")] pub serialized_params: ::core::option::Option<::prost::alloc::string::String>, + /// whether or not this is package yaml should be pulled from on disk package or cloned + #[prost(bool, tag = "3")] + pub is_remote: bool, /// The relative main file filepath, the default value is the "main.star" file in the root of a package #[prost(string, optional, tag = "4")] pub relative_path_to_main_file: ::core::option::Option< diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts index 4b7b2e9f0f..bd338a52f8 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.d.ts @@ -1541,14 +1541,14 @@ export class StarlarkPackagePlanYamlArgs extends jspb.Message { getPackageId(): string; setPackageId(value: string): StarlarkPackagePlanYamlArgs; - getIsRemote(): boolean; - setIsRemote(value: boolean): StarlarkPackagePlanYamlArgs; - getSerializedParams(): string; setSerializedParams(value: string): StarlarkPackagePlanYamlArgs; hasSerializedParams(): boolean; clearSerializedParams(): StarlarkPackagePlanYamlArgs; + getIsRemote(): boolean; + setIsRemote(value: boolean): StarlarkPackagePlanYamlArgs; + getRelativePathToMainFile(): string; setRelativePathToMainFile(value: string): StarlarkPackagePlanYamlArgs; hasRelativePathToMainFile(): boolean; @@ -1570,15 +1570,15 @@ export class StarlarkPackagePlanYamlArgs extends jspb.Message { export namespace StarlarkPackagePlanYamlArgs { export type AsObject = { packageId: string, - isRemote: boolean, serializedParams?: string, + isRemote: boolean, relativePathToMainFile?: string, mainFunctionName?: string, } export enum SerializedParamsCase { _SERIALIZED_PARAMS_NOT_SET = 0, - SERIALIZED_PARAMS = 3, + SERIALIZED_PARAMS = 2, } export enum RelativePathToMainFileCase { diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js index 08c88d1815..fcf85dd28d 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/api_container_service_pb.js @@ -11107,8 +11107,8 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.toObject = functio proto.api_container_api.StarlarkPackagePlanYamlArgs.toObject = function(includeInstance, msg) { var f, obj = { packageId: jspb.Message.getFieldWithDefault(msg, 1, ""), - isRemote: jspb.Message.getBooleanFieldWithDefault(msg, 2, false), - serializedParams: jspb.Message.getFieldWithDefault(msg, 3, ""), + serializedParams: jspb.Message.getFieldWithDefault(msg, 2, ""), + isRemote: jspb.Message.getBooleanFieldWithDefault(msg, 3, false), relativePathToMainFile: jspb.Message.getFieldWithDefault(msg, 4, ""), mainFunctionName: jspb.Message.getFieldWithDefault(msg, 5, "") }; @@ -11152,13 +11152,13 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.deserializeBinaryFromReader msg.setPackageId(value); break; case 2: - var value = /** @type {boolean} */ (reader.readBool()); - msg.setIsRemote(value); - break; - case 3: var value = /** @type {string} */ (reader.readString()); msg.setSerializedParams(value); break; + case 3: + var value = /** @type {boolean} */ (reader.readBool()); + msg.setIsRemote(value); + break; case 4: var value = /** @type {string} */ (reader.readString()); msg.setRelativePathToMainFile(value); @@ -11203,16 +11203,16 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.serializeBinaryToWriter = fu f ); } - f = message.getIsRemote(); - if (f) { - writer.writeBool( + f = /** @type {string} */ (jspb.Message.getField(message, 2)); + if (f != null) { + writer.writeString( 2, f ); } - f = /** @type {string} */ (jspb.Message.getField(message, 3)); - if (f != null) { - writer.writeString( + f = message.getIsRemote(); + if (f) { + writer.writeBool( 3, f ); @@ -11253,29 +11253,11 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setPackageId = fun /** - * optional bool is_remote = 2; - * @return {boolean} - */ -proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getIsRemote = function() { - return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 2, false)); -}; - - -/** - * @param {boolean} value - * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this - */ -proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setIsRemote = function(value) { - return jspb.Message.setProto3BooleanField(this, 2, value); -}; - - -/** - * optional string serialized_params = 3; + * optional string serialized_params = 2; * @return {string} */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getSerializedParams = function() { - return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 3, "")); + return /** @type {string} */ (jspb.Message.getFieldWithDefault(this, 2, "")); }; @@ -11284,7 +11266,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getSerializedParam * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setSerializedParams = function(value) { - return jspb.Message.setField(this, 3, value); + return jspb.Message.setField(this, 2, value); }; @@ -11293,7 +11275,7 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setSerializedParam * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearSerializedParams = function() { - return jspb.Message.setField(this, 3, undefined); + return jspb.Message.setField(this, 2, undefined); }; @@ -11302,7 +11284,25 @@ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.clearSerializedPar * @return {boolean} */ proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.hasSerializedParams = function() { - return jspb.Message.getField(this, 3) != null; + return jspb.Message.getField(this, 2) != null; +}; + + +/** + * optional bool is_remote = 3; + * @return {boolean} + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.getIsRemote = function() { + return /** @type {boolean} */ (jspb.Message.getBooleanFieldWithDefault(this, 3, false)); +}; + + +/** + * @param {boolean} value + * @return {!proto.api_container_api.StarlarkPackagePlanYamlArgs} returns this + */ +proto.api_container_api.StarlarkPackagePlanYamlArgs.prototype.setIsRemote = function(value) { + return jspb.Message.setProto3BooleanField(this, 3, value); }; diff --git a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts index 9c63892538..ed997492ac 100644 --- a/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts +++ b/api/typescript/src/core/kurtosis_core_rpc_api_bindings/connect/api_container_service_pb.d.ts @@ -1907,19 +1907,19 @@ export declare class StarlarkPackagePlanYamlArgs extends Message [ { no: 1, name: "package_id", kind: "scalar", T: 9 /* ScalarType.STRING */ }, - { no: 2, name: "is_remote", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, - { no: 3, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 2, name: "serialized_params", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 3, name: "is_remote", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, { no: 4, name: "relative_path_to_main_file", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 5, name: "main_function_name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, ],