diff --git a/.circleci/config.yml b/.circleci/config.yml index 792840db5b..9ff8fbf35c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -765,9 +765,9 @@ jobs: false fi - run: - name: "Verify Kurtosis cleaned up all its volumes (except for the log storage and github auth storage volumes)" + name: "Verify Kurtosis cleaned up all its volumes (except for the log storage, github auth storage, docker auth config volumes)" command: | - if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l ) -eq 2 ]; then + if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l ) -eq 3 ]; then docker volume ls false fi @@ -1004,9 +1004,9 @@ jobs: false fi - run: - name: "Verify Kurtosis cleaned up all its volumes (except for the log storage and github auth storage volumes)" + name: "Verify Kurtosis cleaned up all its volumes (except for the log storage, github auth storage, docker auth config volumes)" command: | - if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l) -eq 2 ]; then + if ! [ $(docker volume ls | grep -v kurtosis-logs-collector-vol | grep -v kurtosis-logs-db-vol | tail -n+2 | wc -l) -eq 3 ]; then docker volume ls false fi diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 887b46711e..8223f18610 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -9,16 +9,11 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'kurtosis-tech/kurtosis' steps: - - uses: google-github-actions/release-please-action@v3 + - uses: googleapis/release-please-action@d1a8f221d7723166f48a584aebba00ef3f6febec with: # We use the RELEASER_TOKEN so that the GitHub Actions # can run on the PR created # https://github.com/kurtosis-tech/kurtosis/issues/688 token: "${{ secrets.RELEASER_TOKEN }}" - release-type: simple - package-name: kurtosis - bump-minor-pre-major: false - bump-patch-for-minor-pre-major: false - # Our CI, Docker Images, Kurtosis-SDK bumps all depend on - # non v tags - include-v-in-tag: false + config-file: release-please-config.json + manifest-file: .release-please-manifest.json diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000000..efe9bfbbb1 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "1.4.2" +} diff --git a/CHANGELOG.md b/CHANGELOG.md index ec91c95fc0..cb97568dac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [1.4.2](https://github.com/kurtosis-tech/kurtosis/compare/v1.4.1...1.4.2) (2024-11-10) + + +### Bug Fixes + +* don't include component name in release please tags ([#2590](https://github.com/kurtosis-tech/kurtosis/issues/2590)) ([28582cc](https://github.com/kurtosis-tech/kurtosis/commit/28582cce6b41903b1d683126ed0f75c6503d45ff)) +* make releases work again without v in tags ([#2586](https://github.com/kurtosis-tech/kurtosis/issues/2586)) ([a87f945](https://github.com/kurtosis-tech/kurtosis/commit/a87f945ca9014a632ccde85631b364dad3304fbb)) +* remove closeChannelWhenEmpty busy loop ([#2594](https://github.com/kurtosis-tech/kurtosis/issues/2594)) ([dbb2193](https://github.com/kurtosis-tech/kurtosis/commit/dbb2193c3215ee86ea5a54a19efecc0985b4511b)) +* wrong config structure for release please ([#2588](https://github.com/kurtosis-tech/kurtosis/issues/2588)) ([e7263c3](https://github.com/kurtosis-tech/kurtosis/commit/e7263c3558b27390439a0e3456955306a06a7cb0)) + +## [1.4.1](https://github.com/kurtosis-tech/kurtosis/compare/1.4.0...v1.4.1) (2024-11-01) + + +### Bug Fixes + +* add rafael as maintainer ([#2583](https://github.com/kurtosis-tech/kurtosis/issues/2583)) ([ab5eb9e](https://github.com/kurtosis-tech/kurtosis/commit/ab5eb9efdbccfae28ee56ee96350daef7412c6ae)) +* docker auth panic ([b36ddc9](https://github.com/kurtosis-tech/kurtosis/commit/b36ddc9d6c3b2baa8fe03a3f27ec5f3cae68c043)) + +## [1.4.0](https://github.com/kurtosis-tech/kurtosis/compare/1.3.1...1.4.0) (2024-10-29) + + +### Features + +* support auth from docker config ([#2560](https://github.com/kurtosis-tech/kurtosis/issues/2560)) ([dab4470](https://github.com/kurtosis-tech/kurtosis/commit/dab4470ec18f7324b22b780de8d923ceef965428)) + ## [1.3.1](https://github.com/kurtosis-tech/kurtosis/compare/1.3.0...1.3.1) (2024-10-15) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index c590a9f1c2..55466ec816 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -9,6 +9,7 @@ This document lists the individuals who are responsible for maintaining and cont - [Dan Park](https://github.com/chunha-park) - BD/Product Marketing @KurtosisTech - [Barnabas Busa](https://github.com/barnabasbusa) - Dev Ops @Ethereum Foundation - [Parithosh Jayanthi](https://github.com/parithosh) - Dev Ops @Ethereum Foundation +- [Rafael Matias](https://github.com/skylenet) - Dev Ops @Ethereum Foundation - [John Hilliard](https://github.com/praetoriansentry) - Dev Tools @Polygon Labs - [Léo Vincent](https://github.com/leovct) - Dev Tools @Polygon Labs - [Kevin Today](https://github.com/mieubrisse) - Original Contributor diff --git a/api/golang/kurtosis_version/kurtosis_version.go b/api/golang/kurtosis_version/kurtosis_version.go index 62ab02bf8b..f6a95336e3 100644 --- a/api/golang/kurtosis_version/kurtosis_version.go +++ b/api/golang/kurtosis_version/kurtosis_version.go @@ -9,6 +9,6 @@ const ( // !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!! // This is necessary so that Kurt Core consumers will know if they're compatible with the currently-running // API container - KurtosisVersion = "1.3.1" + KurtosisVersion = "1.4.2" // !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!! ) diff --git a/api/rust/Cargo.toml b/api/rust/Cargo.toml index 6ac2a12472..26949ea416 100644 --- a/api/rust/Cargo.toml +++ b/api/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kurtosis-sdk" -version = "1.3.1" +version = "1.4.2" license = "BUSL-1.1" description = "Rust SDK for Kurtosis" edition = "2021" diff --git a/api/typescript/package.json b/api/typescript/package.json index 6fcddede22..7b4f679327 100644 --- a/api/typescript/package.json +++ b/api/typescript/package.json @@ -1,7 +1,7 @@ { "name": "kurtosis-sdk", "//": "NOTE: DO NOT UPDATE THIS VERSION MANUALLY - IT WILL BE UPDATED DURING THE RELEASE PROCESS!", - "version": "1.3.1", + "version": "1.4.2", "main": "./build/index", "description": "This repo contains a Typescript client for communicating with the Kurtosis Engine server, which is responsible for creating, managing and destroying Kurtosis Enclaves.", "types": "./build/index", diff --git a/api/typescript/src/kurtosis_version/kurtosis_version.ts b/api/typescript/src/kurtosis_version/kurtosis_version.ts index 886999da84..02710525d6 100644 --- a/api/typescript/src/kurtosis_version/kurtosis_version.ts +++ b/api/typescript/src/kurtosis_version/kurtosis_version.ts @@ -1,5 +1,5 @@ // !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!! // This is necessary so that Kurt Core consumers (e.g. modules) will know if they're compatible with the currently-running // API container -export const KURTOSIS_VERSION: string = "1.3.1" +export const KURTOSIS_VERSION: string = "1.4.2" // !!!!!!!!!!! DO NOT UPDATE! WILL BE MANUALLY UPDATED DURING THE RELEASE PROCESS !!!!!!!!!!!!!!!!!!!!!! diff --git a/cli/cli/commands/web/web.go b/cli/cli/commands/web/web.go index 3a4aafdfb9..2fb08b0419 100644 --- a/cli/cli/commands/web/web.go +++ b/cli/cli/commands/web/web.go @@ -7,9 +7,9 @@ import ( "github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/flags" "github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts" "github.com/kurtosis-tech/kurtosis/cli/cli/helpers/multi_os_command_executor" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/user_support_constants" "github.com/kurtosis-tech/kurtosis/contexts-config-store/store" "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" ) var WebCmd = &lowlevel.LowlevelKurtosisCommand{ @@ -35,9 +35,7 @@ func run(_ context.Context, _ *flags.ParsedFlags, _ *args.ParsedArgs) error { return stacktrace.Propagate(err, "tried fetching the current Kurtosis context but failed, we can't switch clusters without this information. This is a bug in Kurtosis") } if store.IsRemote(currentKurtosisContext) { - if err := multi_os_command_executor.OpenFile(user_support_constants.KurtosisCloudLink); err != nil { - return stacktrace.Propagate(err, "An error occurred while opening the Kurtosis Cloud Web UI") - } + logrus.Warn("Kurtosis Cloud has been deprecated. Switch to a local kurtosis context to use the local Kurtosis web UI instead.") } if err := multi_os_command_executor.OpenFile(webUILink); err != nil { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go index 86d787d599..3a473b1c0d 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go @@ -3,11 +3,12 @@ package backend_creator import ( "context" "fmt" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" "net" "os" "path" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db" + "github.com/docker/docker/client" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go index a13632c54e..8bf4bbefae 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts/consts.go @@ -32,7 +32,8 @@ const ( NameOfNetworkToStartEngineAndLogServiceContainersIn = "bridge" HttpApplicationProtocol = "http" - GitHubAuthStorageDirPath = "/kurtosis-data/github-auth/" + GitHubAuthStorageDirPath = "/kurtosis-data/github-auth/" + DockerConfigStorageDirPath = "/root/.docker/" EmptyApplicationURL = "" ) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go index d663de40a9..d2af3c2e81 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go @@ -618,3 +618,22 @@ func (backend *DockerKurtosisBackend) getGitHubAuthStorageVolume(ctx context.Con volume := foundVolumes[0] return volume.Name, nil } + +// Guaranteed to either return a Docker config storage volume name or throw an error +func (backend *DockerKurtosisBackend) getDockerConfigStorageVolume(ctx context.Context) (string, error) { + volumeSearchLabels := map[string]string{ + docker_label_key.VolumeTypeDockerLabelKey.GetString(): label_value_consts.DockerConfigStorageVolumeTypeDockerLabelValue.GetString(), + } + foundVolumes, err := backend.dockerManager.GetVolumesByLabels(ctx, volumeSearchLabels) + if err != nil { + return "", stacktrace.Propagate(err, "An error occurred getting Docker config storage volumes matching labels '%+v'", volumeSearchLabels) + } + if len(foundVolumes) > 1 { + return "", stacktrace.NewError("Found multiple Docker config storage volumes. This should never happen") + } + if len(foundVolumes) == 0 { + return "", stacktrace.NewError("No Docker config storage volume found.") + } + volume := foundVolumes[0] + return volume.Name, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go index a99b180b01..757b79f4ca 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go @@ -3,10 +3,11 @@ package docker_kurtosis_backend import ( "context" "encoding/json" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" "net" "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_registry_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" "github.com/docker/go-connections/nat" @@ -81,6 +82,11 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( return nil, stacktrace.Propagate(err, "An error occurred getting the GitHub auth storage volume name.") } + dockerConfigStorageVolumeName, err := backend.getDockerConfigStorageVolume(ctx) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the Docker config storage volume name.") + } + // Get the Docker network ID where we'll start the new API container enclaveNetwork, err := backend.getEnclaveNetworkByEnclaveUuid(ctx, enclaveUuid) if err != nil { @@ -191,8 +197,9 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( } volumeMounts := map[string]string{ - enclaveDataVolumeName: enclaveDataVolumeDirpath, - githubAuthStorageVolumeName: consts.GitHubAuthStorageDirPath, + enclaveDataVolumeName: enclaveDataVolumeDirpath, + githubAuthStorageVolumeName: consts.GitHubAuthStorageDirPath, + dockerConfigStorageVolumeName: consts.DockerConfigStorageDirPath, } labelStrs := map[string]string{} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 4b46974fbc..3096451b91 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -3,9 +3,11 @@ package engine_functions import ( "context" "fmt" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/github_auth_storage_creator" "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/docker_config_storage_creator" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/github_auth_storage_creator" + "github.com/docker/go-connections/nat" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions" @@ -249,14 +251,34 @@ func CreateEngine( return nil, stacktrace.Propagate(err, "An error occurred creating GitHub auth storage.") } + // Configure Docker Config by writing the provided config files to a volume that's accessible by the engine + dockerConfigStorageVolObjAttrs, err := objAttrsProvider.ForDockerConfigStorageVolume() + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred retrieving object attributes for GitHub auth storage.") + } + dockerConfigStorageVolNameStr := dockerConfigStorageVolObjAttrs.GetName().GetString() + dockerConfigStorageVolLabelStrs := map[string]string{} + for labelKey, labelValue := range dockerConfigStorageVolObjAttrs.GetLabels() { + dockerConfigStorageVolLabelStrs[labelKey.GetString()] = labelValue.GetString() + } + // This volume is created idempotently (like logs storage volume) and just write the token to the file everytime the engine starts + if err = dockerManager.CreateVolume(ctx, dockerConfigStorageVolNameStr, dockerConfigStorageVolLabelStrs); err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating Docker config storage volume.") + } + err = docker_config_storage_creator.CreateDockerConfigStorage(ctx, targetNetworkId, dockerConfigStorageVolNameStr, consts.DockerConfigStorageDirPath, dockerManager) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating Docker config storage.") + } + bindMounts := map[string]string{ // Necessary so that the engine server can interact with the Docker engine consts.DockerSocketFilepath: consts.DockerSocketFilepath, } volumeMounts := map[string]string{ - logsStorageVolNameStr: logsStorageDirPath, - githubAuthStorageVolNameStr: consts.GitHubAuthStorageDirPath, + logsStorageVolNameStr: logsStorageDirPath, + githubAuthStorageVolNameStr: consts.GitHubAuthStorageDirPath, + dockerConfigStorageVolNameStr: consts.DockerConfigStorageDirPath, } if serverArgs.OnBastionHost { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/docker_config_storage_creator/docker_config_storage_creator.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/docker_config_storage_creator/docker_config_storage_creator.go new file mode 100644 index 0000000000..e5a22cfa36 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/docker_config_storage_creator/docker_config_storage_creator.go @@ -0,0 +1,178 @@ +package docker_config_storage_creator + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "time" + + "github.com/docker/docker/api/types/registry" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" +) + +const ( + // We use this image and version because we already are using this in other projects so there is a high probability + // that the image is in the local machine's cache + creatorContainerImage = "alpine:3.17" + creatorContainerName = "kurtosis-docker-config-storage-creator" + + shBinaryFilepath = "/bin/sh" + shCmdFlag = "-c" + printfCmdName = "printf" + + creationSuccessExitCode = 0 + + creationCmdMaxRetries = 2 + creationCmdDelayInRetries = 200 * time.Millisecond + + configFilePath = "config.json" + + sleepSeconds = 1800 +) + +func CreateDockerConfigStorage( + ctx context.Context, + targetNetworkId string, + volumeName string, + storageDirPath string, + dockerManager *docker_manager.DockerManager, +) error { + entrypointArgs := []string{ + shBinaryFilepath, + shCmdFlag, + fmt.Sprintf("sleep %v", sleepSeconds), + } + + volumeMounts := map[string]string{ + volumeName: storageDirPath, + } + + createAndStartArgs := docker_manager.NewCreateAndStartContainerArgsBuilder( + creatorContainerImage, + creatorContainerName, + targetNetworkId, + ).WithEntrypointArgs( + entrypointArgs, + ).WithVolumeMounts( + volumeMounts, + ).Build() + + containerId, _, err := dockerManager.CreateAndStartContainer(ctx, createAndStartArgs) + if err != nil { + return stacktrace.Propagate(err, "An error occurred starting the Docker Config Storage Creator container with these args '%+v'", createAndStartArgs) + } + //The killing step has to be executed always in the success and also in the failed case + defer func() { + if err = dockerManager.RemoveContainer(context.Background(), containerId); err != nil { + logrus.Errorf( + "Launching the Docker Config Creator container with container ID '%v' didn't complete successfully so we "+ + "tried to remove the container we started, but doing so exited with an error:\n%v", + containerId, + err) + logrus.Errorf("ACTION REQUIRED: You'll need to manually remove the container with ID '%v'!!!!!!", containerId) + } + }() + + if err := storeConfigInVolume( + ctx, + dockerManager, + containerId, + creationCmdMaxRetries, + creationCmdDelayInRetries, + storageDirPath, + ); err != nil { + return stacktrace.Propagate(err, "An error occurred creating Docker config storage in volume.") + } + + return nil +} + +func storeConfigInVolume( + ctx context.Context, + dockerManager *docker_manager.DockerManager, + containerId string, + maxRetries uint, + timeBetweenRetries time.Duration, + storageDirPath string, +) error { + // Get all the registries from the Docker config + registries, err := docker_manager.GetAllRegistriesFromDockerConfig() + if err != nil { + return stacktrace.NewError("An error occurred getting all registries from Docker config: %v", err) + } + + cfg := struct { + Auths map[string]registry.AuthConfig `json:"auths"` + }{ + Auths: make(map[string]registry.AuthConfig), + } + + // Add the auths for each registry + for _, registry := range registries { + creds, err := docker_manager.GetAuthFromDockerConfig(registry) + if err != nil { + logrus.Warnf("An error occurred getting auth for registry '%v' from Docker config: %v", registry, err) + } + // creds can be nil if the registry doesn't have auth + if err != nil && creds != nil { + cfg.Auths[registry] = *creds + } + } + + cfgJsonStr, err := json.Marshal(cfg) + if err != nil { + return stacktrace.NewError("An error occurred marshalling the Docker config into JSON: %v", err) + } + + // Write the config.json to the volume + commandStr := fmt.Sprintf( + "%v '%v' > %v", + printfCmdName, + string(cfgJsonStr), + fmt.Sprintf("%s/%s", storageDirPath, configFilePath), + ) + + execCmd := []string{ + shBinaryFilepath, + shCmdFlag, + commandStr, + } + for i := uint(0); i < maxRetries; i++ { + outputBuffer := &bytes.Buffer{} + exitCode, err := dockerManager.RunExecCommand(ctx, containerId, execCmd, outputBuffer) + if err == nil { + if exitCode == creationSuccessExitCode { + logrus.Debugf("The Docker config file was successfully added into the volume.") + return nil + } + logrus.Debugf( + "Docker config storage creation command '%v' returned without a Docker error, but exited with non-%v exit code '%v' and logs:\n%v", + commandStr, + creationSuccessExitCode, + exitCode, + outputBuffer.String(), + ) + } else { + logrus.Debugf( + "Docker config storage creation command '%v' experienced a Docker error:\n%v", + commandStr, + err, + ) + } + + // Tiny optimization to not sleep if we're not going to run the loop again + if i < maxRetries { + time.Sleep(timeBetweenRetries) + } + } + + return stacktrace.NewError( + "The Docker config storage creation didn't return success (as measured by the command '%v') even after retrying %v times with %v between retries", + commandStr, + maxRetries, + timeBetweenRetries, + ) +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth.go new file mode 100644 index 0000000000..e956126d92 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth.go @@ -0,0 +1,201 @@ +package docker_manager + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "github.com/sirupsen/logrus" + "os" + "os/exec" + "strings" + + "github.com/docker/docker/api/types/registry" + dockerregistry "github.com/docker/docker/registry" + "github.com/kurtosis-tech/stacktrace" +) + +const ( + ENV_DOCKER_CONFIG string = "DOCKER_CONFIG" +) + +// RegistryAuthConfig holds authentication configuration for a container registry +type RegistryAuthConfig struct { + Auths map[string]registry.AuthConfig `json:"auths"` + CredHelpers map[string]string `json:"credHelpers"` + CredsStore string `json:"credsStore"` +} + +// loadDockerAuth loads the authentication configuration from the config.json file located in $DOCKER_CONFIG or ~/.docker +func loadDockerAuth() (RegistryAuthConfig, error) { + configFilePath := os.Getenv(ENV_DOCKER_CONFIG) + if configFilePath == "" { + configFilePath = os.Getenv("HOME") + "/.docker/config.json" + } else { + configFilePath = configFilePath + "/config.json" + } + + file, err := os.ReadFile(configFilePath) + if errors.Is(err, os.ErrNotExist) { + // If the auth config doesn't exist, return an empty auth config + logrus.Debugf("No docker config found at '%s'. Returning empty registry auth config.", configFilePath) + return emptyRegistryAuthConfig(), nil + } else if err != nil { + return emptyRegistryAuthConfig(), stacktrace.Propagate(err, "error reading Docker config file at '%s'", configFilePath) + } + + var authConfig RegistryAuthConfig + if err := json.Unmarshal(file, &authConfig); err != nil { + return emptyRegistryAuthConfig(), stacktrace.Propagate(err, "error unmarshalling Docker config file at '%s'", configFilePath) + } + + return authConfig, nil +} + +func emptyRegistryAuthConfig() RegistryAuthConfig { + return RegistryAuthConfig{ + Auths: map[string]registry.AuthConfig{}, + CredHelpers: map[string]string{}, + CredsStore: "", + } +} + +// getRegistriesFromCredsStore fetches all registries from a Docker credential helper (credStore) +func getRegistriesFromCredsStore(credHelper string) ([]string, error) { + credHelperCmd := "docker-credential-" + credHelper + + cmd := exec.Command(credHelperCmd, "list") + + var out bytes.Buffer + cmd.Stdout = &out + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, stacktrace.Propagate(err, "error executing credential helper '%s': %s", cmd.String(), stderr.String()) + } + // Output will look like this: {"https://index.docker.io/v1/":"username"} + var result map[string]string + outStr := out.String() + err := json.Unmarshal([]byte(outStr), &result) + if err != nil { + return nil, stacktrace.Propagate(err, "error unmarshaling credential helper list output '%s': %s", cmd.String(), outStr) + } + + registries := []string{} + for k := range result { + registries = append(registries, k) + } + return registries, nil +} + +// getCredentialsFromStore fetches credentials from a Docker credential helper (credStore) +func getCredentialsFromStore(credHelper string, registryURL string) (*registry.AuthConfig, error) { + // Prepare the helper command (docker-credential-) + credHelperCmd := "docker-credential-" + credHelper + + // Execute the credential helper to get credentials for the registry + cmd := exec.Command(credHelperCmd, "get") + cmd.Stdin = strings.NewReader(registryURL) + + var out bytes.Buffer + cmd.Stdout = &out + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, stacktrace.Propagate(err, "error executing credential helper '%s' for '%s': %s", cmd.String(), registryURL, stderr.String()) + } + + // Parse the output (it should return JSON containing "Username", "Secret" and "ServerURL") + creds := struct { + Username string `json:"Username"` + Secret string `json:"Secret"` + ServerURL string `json:"ServerURL"` + }{ + Username: "", + Secret: "", + ServerURL: "", + } + + if err := json.Unmarshal(out.Bytes(), &creds); err != nil { + return nil, stacktrace.Propagate(err, "error parsing credentials from store") + } + + return ®istry.AuthConfig{ + Username: creds.Username, + Password: creds.Secret, + ServerAddress: creds.ServerURL, + Auth: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", creds.Username, creds.Secret))), + Email: "", + IdentityToken: "", + RegistryToken: "", + }, nil +} + +// GetAuthFromDockerConfig retrieves the auth configuration for a given repository +// by checking the Docker config.json file and Docker credential helpers. +// Returns nil if no credentials were found. +func GetAuthFromDockerConfig(repo string) (*registry.AuthConfig, error) { + authConfig, err := loadDockerAuth() + if err != nil { + return nil, err + } + + registryHost := dockerregistry.ConvertToHostname(repo) + + if !strings.Contains(registryHost, ".") || registryHost == "docker.io" || registryHost == "registry-1.docker.io" { + registryHost = "https://index.docker.io/v1/" + } + + // Check if the URL contains "://", meaning it already has a protocol + if !strings.Contains(registryHost, "://") { + registryHost = "https://" + registryHost + } + + // 1. Check if there is a credHelper for this specific registry + if credHelper, exists := authConfig.CredHelpers[registryHost]; exists { + return getCredentialsFromStore(credHelper, registryHost) + } + + // 2. Check if there is a default credStore for all registries + if authConfig.CredsStore != "" { + return getCredentialsFromStore(authConfig.CredsStore, registryHost) + } + + // 3. Fallback to credentials in "auths" if no credStore is available + if auth, exists := authConfig.Auths[registryHost]; exists { + return &auth, nil + } + + // Return no AuthConfig if no credentials were found + return nil, nil +} + +// GetAllRegistriesFromDockerConfig retrieves all registries from the Docker config.json file +func GetAllRegistriesFromDockerConfig() ([]string, error) { + authConfig, err := loadDockerAuth() + if err != nil { + return nil, err + } + + var registries []string + for registry := range authConfig.Auths { + registries = append(registries, registry) + } + + for registry := range authConfig.CredHelpers { + registries = append(registries, registry) + } + + if authConfig.CredsStore != "" { + r, err := getRegistriesFromCredsStore(authConfig.CredsStore) + if err != nil { + return nil, err + } + registries = append(registries, r...) + } + + return registries, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth_test.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth_test.go new file mode 100644 index 0000000000..6e11673768 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_auth_test.go @@ -0,0 +1,103 @@ +package docker_manager + +import ( + "encoding/base64" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +// writeStaticConfig writes a static Docker config.json file to a temporary directory +func writeStaticConfig(t *testing.T, configContent string) string { + tmpDir, err := os.MkdirTemp("", "docker-config") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + + configPath := tmpDir + "/config.json" + err = os.WriteFile(configPath, []byte(configContent), 0600) + if err != nil { + t.Fatalf("Failed to write config.json: %v", err) + } + + // Set the DOCKER_CONFIG environment variable to the temp directory + os.Setenv(ENV_DOCKER_CONFIG, tmpDir) + return tmpDir +} + +func TestGetAuthWithNoAuthSetReturnsNilAndNoError(t *testing.T) { + authConfig, err := GetAuthFromDockerConfig("my-repo/my-image:latest") + assert.NoError(t, err) + assert.Nil(t, authConfig, "Auth config should be nil") +} + +func TestGetAuthConfigForRepoPlain(t *testing.T) { + expectedUser := "user" + expectedPassword := "password" + + encodedAuth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", expectedUser, expectedPassword))) + + cfg := fmt.Sprintf(` + { + "auths": { + "https://index.docker.io/v1/": { + "auth": "%s" + } + } + }`, encodedAuth) + + tmpDir := writeStaticConfig(t, cfg) + defer os.RemoveAll(tmpDir) + + // Test 1: Retrieve auth config for Docker Hub using docker.io domain + authConfig, err := GetAuthFromDockerConfig("docker.io/my-repo/my-image:latest") + assert.NoError(t, err) + assert.Equal(t, encodedAuth, authConfig.Auth, "Auth for Docker Hub should match") + + // Test 2: Retrieve auth config for Docker Hub using no domain + authConfig, err = GetAuthFromDockerConfig("my-repo/my-image:latest") + assert.NoError(t, err) + assert.Equal(t, encodedAuth, authConfig.Auth, "Auth for Docker Hub should match when using no host prefix") + + // Test 3: Retrieve auth config for Docker Hub using full domain and https:// prefix + authConfig, err = GetAuthFromDockerConfig("https://registry-1.docker.io/my-repo/my-image:latest") + assert.NoError(t, err) + assert.Equal(t, encodedAuth, authConfig.Auth, "Auth for Docker Hub should match when using no host prefix") + +} + +func TestGetAuthConfigForRepoOSX(t *testing.T) { + t.Skip("Skipping test that requires macOS keychain") + + cfg := `{ + "auths": { + "https://index.docker.io/v1/": {} + }, + "credsStore": "osxkeychain" + }` + tmpDir := writeStaticConfig(t, cfg) + defer os.RemoveAll(tmpDir) + + authConfig, err := GetAuthFromDockerConfig("my-repo/my-image:latest") + assert.NoError(t, err) + assert.NotNil(t, authConfig, "Auth config should not be nil") +} + +func TestGetAuthConfigForRepoUnix(t *testing.T) { + t.Skip("Skipping test that requires unix `pass` password manager") + + cfg := `{ + "auths": { + "https://index.docker.io/v1/": {} + }, + "credsStore": "pass" + }` + tmpDir := writeStaticConfig(t, cfg) + defer os.RemoveAll(tmpDir) + + authConfig, err := GetAuthFromDockerConfig("my-repo/my-image:latest") + assert.NoError(t, err) + assert.NotNil(t, authConfig, "Auth config should not be nil") +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index b76e691028..cff367eb32 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -2279,6 +2279,23 @@ func pullImage(dockerClient *client.Client, imageName string, registrySpec *imag PrivilegeFunc: nil, Platform: platform, } + + // Try to obtain the auth configuration from the docker config file + authConfig, err := GetAuthFromDockerConfig(imageName) + if err != nil { + logrus.Errorf("An error occurred while getting auth config for image: %s: %s", imageName, err.Error()) + } + + if authConfig != nil { + authFromConfig, err := registry.EncodeAuthConfig(*authConfig) + if err != nil { + logrus.Errorf("An error occurred while encoding auth config for image: %s: %s", imageName, err.Error()) + } else { + imagePullOptions.RegistryAuth = authFromConfig + } + } + + // If the registry spec is defined, use that for authentication if registrySpec != nil { authConfig := registry.AuthConfig{ Username: registrySpec.GetUsername(), @@ -2296,6 +2313,7 @@ func pullImage(dockerClient *client.Client, imageName string, registrySpec *imag imagePullOptions.RegistryAuth = encodedAuthConfig } + out, err := dockerClient.ImagePull(pullImageCtx, imageName, imagePullOptions) if err != nil { return stacktrace.Propagate(err, "Tried pulling image '%v' with platform '%v' but failed", imageName, platform), false diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go index c30e9d1a46..9e48d3ccdb 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go @@ -28,6 +28,7 @@ const ( logsStorageVolumeTypeLabelValueStr = "kurtosis-logs-storage" logsCollectorVolumeTypeLabelValueStr = "logs-collector-data" githubAuthStorageVolumeTypeLabelValueStr = "github-auth-storage" + dockerConfigStorageVolumeTypeLabelValueStr = "docker-config-storage" ) // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -54,3 +55,4 @@ var PersistentDirectoryVolumeTypeDockerLabelValue = docker_label_value.MustCreat var LogsStorageVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsStorageVolumeTypeLabelValueStr) var LogsCollectorVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsCollectorVolumeTypeLabelValueStr) var GitHubAuthStorageVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(githubAuthStorageVolumeTypeLabelValueStr) +var DockerConfigStorageVolumeTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(dockerConfigStorageVolumeTypeLabelValueStr) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go index 5c9af7fada..20da73b2be 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go @@ -17,12 +17,13 @@ import ( ) const ( - engineServerNamePrefix = "kurtosis-engine" - logsAggregatorName = "kurtosis-logs-aggregator" - logsStorageVolumeName = "kurtosis-logs-storage" - githubAuthStorageVolumeName = "kurtosis-github-auth-storage" - engineRESTAPIPortStr = "engine-rest-api" - reverseProxyNamePrefix = "kurtosis-reverse-proxy" + engineServerNamePrefix = "kurtosis-engine" + logsAggregatorName = "kurtosis-logs-aggregator" + logsStorageVolumeName = "kurtosis-logs-storage" + githubAuthStorageVolumeName = "kurtosis-github-auth-storage" + dockerConfigStorageVolumeName = "kurtosis-docker-config-storage" + engineRESTAPIPortStr = "engine-rest-api" + reverseProxyNamePrefix = "kurtosis-reverse-proxy" ) type DockerObjectAttributesProvider interface { @@ -38,6 +39,7 @@ type DockerObjectAttributesProvider interface { ForLogsStorageVolume() (DockerObjectAttributes, error) ForReverseProxy(engineGuid engine.EngineGUID) (DockerObjectAttributes, error) ForGitHubAuthStorageVolume() (DockerObjectAttributes, error) + ForDockerConfigStorageVolume() (DockerObjectAttributes, error) } func GetDockerObjectAttributesProvider() DockerObjectAttributesProvider { @@ -172,6 +174,23 @@ func (provider *dockerObjectAttributesProviderImpl) ForGitHubAuthStorageVolume() return objectAttributes, nil } +func (provider *dockerObjectAttributesProviderImpl) ForDockerConfigStorageVolume() (DockerObjectAttributes, error) { + name, err := docker_object_name.CreateNewDockerObjectName(dockerConfigStorageVolumeName) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating a Docker object name object from string '%v'", dockerConfigStorageVolumeName) + } + + labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{ + docker_label_key.VolumeTypeDockerLabelKey: label_value_consts.DockerConfigStorageVolumeTypeDockerLabelValue, + } + + objectAttributes, err := newDockerObjectAttributesImpl(name, labels) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while creating the ObjectAttributesImpl with the name '%s' and labels '%+v'", name, labels) + } + return objectAttributes, nil +} + func (provider *dockerObjectAttributesProviderImpl) ForReverseProxy(engineGuid engine.EngineGUID) (DockerObjectAttributes, error) { nameStr := strings.Join( diff --git a/container-engine-lib/lib/user_support_constants/user_support_constants.go b/container-engine-lib/lib/user_support_constants/user_support_constants.go index a50b590632..166b69f601 100644 --- a/container-engine-lib/lib/user_support_constants/user_support_constants.go +++ b/container-engine-lib/lib/user_support_constants/user_support_constants.go @@ -31,7 +31,6 @@ const ( FeedbackEmail = "feedback@" + OldDomain FeedbackEmailLink = "mailto:" + FeedbackEmail KurtosisTechTwitterProfileLink = "https://twitter.com/KurtosisTech" - KurtosisCloudLink = "https://cloud." + Domain // If you add new URLs above, make sure to add them to the urlsToValidateInTest below!!! // WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING @@ -56,6 +55,5 @@ var urlsToValidateInTest = []string{ KurtosisDiscordUrl, KurtosisOnBoardCalendlyUrl, HowImportWorksLink, - KurtosisCloudLink, KurtosisTechTwitterProfileLink, } diff --git a/docs/yarn.lock b/docs/yarn.lock index 9e965240f9..d281791bc2 100644 --- a/docs/yarn.lock +++ b/docs/yarn.lock @@ -4791,9 +4791,9 @@ http-parser-js@>=0.5.1: integrity sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q== http-proxy-middleware@^2.0.3: - version "2.0.6" - resolved "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz" - integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw== + version "2.0.7" + resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6" + integrity sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA== dependencies: "@types/http-proxy" "^1.17.8" http-proxy "^1.18.1" diff --git a/enclave-manager/web/lerna.json b/enclave-manager/web/lerna.json index ab22a760ef..667dcf2264 100644 --- a/enclave-manager/web/lerna.json +++ b/enclave-manager/web/lerna.json @@ -1,6 +1,6 @@ { "packages": ["packages/*"], - "version": "1.3.1", + "version": "1.4.2", "npmClient": "yarn", "$schema": "node_modules/lerna/schemas/lerna-schema.json", "useNx": false, diff --git a/enclave-manager/web/packages/app/package.json b/enclave-manager/web/packages/app/package.json index 1083b44cc7..fc75f9d28b 100644 --- a/enclave-manager/web/packages/app/package.json +++ b/enclave-manager/web/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@kurtosis/emui-app", - "version": "1.3.1", + "version": "1.4.2", "private": true, "homepage": ".", "dependencies": { @@ -10,7 +10,7 @@ "html-react-parser": "^4.2.2", "js-cookie": "^3.0.5", "kurtosis-cloud-indexer-sdk": "^0.0.31", - "kurtosis-ui-components": "1.3.1", + "kurtosis-ui-components": "1.4.2", "react-error-boundary": "^4.0.11", "react-hook-form": "^7.47.0", "react-mentions": "^4.4.10", diff --git a/enclave-manager/web/packages/components/package.json b/enclave-manager/web/packages/components/package.json index a6b9127e1d..3fbde91304 100644 --- a/enclave-manager/web/packages/components/package.json +++ b/enclave-manager/web/packages/components/package.json @@ -1,6 +1,6 @@ { "name": "kurtosis-ui-components", - "version": "1.3.1", + "version": "1.4.2", "private": false, "main": "build/index", "description": "This repo contains components used by Kurtosis UI applications.", diff --git a/enclave-manager/web/yarn.lock b/enclave-manager/web/yarn.lock index 3b1a6620df..4b41887a25 100644 --- a/enclave-manager/web/yarn.lock +++ b/enclave-manager/web/yarn.lock @@ -9002,9 +9002,9 @@ http-proxy-agent@^5.0.0: debug "4" http-proxy-middleware@^2.0.3: - version "2.0.6" - resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz#e1a4dd6979572c7ab5a4e4b55095d1f32a74963f" - integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw== + version "2.0.7" + resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6" + integrity sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA== dependencies: "@types/http-proxy" "^1.17.8" http-proxy "^1.18.1" diff --git a/engine/frontend/package-lock.json b/engine/frontend/package-lock.json index 908183d4fd..99db8646dc 100644 --- a/engine/frontend/package-lock.json +++ b/engine/frontend/package-lock.json @@ -60,7 +60,7 @@ }, "../../api/typescript": { "name": "kurtosis-sdk", - "version": "1.2.0", + "version": "1.4.2", "license": "Apache-2.0", "dependencies": { "@bufbuild/connect": "^0.12.0", @@ -8468,9 +8468,9 @@ "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" }, "node_modules/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", "engines": { "node": ">= 0.6" } @@ -8553,9 +8553,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -10644,16 +10644,16 @@ } }, "node_modules/express": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz", - "integrity": "sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -11969,9 +11969,9 @@ } }, "node_modules/http-proxy-middleware": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz", - "integrity": "sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", + "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", "dependencies": { "@types/http-proxy": "^1.17.8", "http-proxy": "^1.18.1", @@ -30536,9 +30536,9 @@ "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" }, "cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==" }, "cookie-signature": { "version": "1.0.6", @@ -30598,9 +30598,9 @@ } }, "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "requires": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -32103,16 +32103,16 @@ } }, "express": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz", - "integrity": "sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -33068,9 +33068,9 @@ } }, "http-proxy-middleware": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz", - "integrity": "sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", + "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", "requires": { "@types/http-proxy": "^1.17.8", "http-proxy": "^1.18.1", diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index 67bd58b6bb..1afdd99f30 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -90,14 +90,13 @@ func (client *persistentVolumeLogsDatabaseClient) StreamUserServiceLogs( // this go routine handles the stream cancellation go func() { - //wait for stream go routine to end + // wait for stream go routine to end wgSenders.Wait() - // send all buffered log lines + // flush should send remainder of logs in the buffer to the channel to be read logLineSender.Flush() - // wait until the channel has been fully read/empty before closing it - closeChannelWhenEmpty(logsByKurtosisUserServiceUuidChan) + close(logsByKurtosisUserServiceUuidChan) close(streamErrChan) //then cancel the context @@ -174,12 +173,3 @@ func (client *persistentVolumeLogsDatabaseClient) streamServiceLogLines( shouldReturnAllLogs, numLogLines) } - -func closeChannelWhenEmpty(logsChan chan map[service.ServiceUUID][]logline.LogLine) { - for { - if len(logsChan) == 0 { - close(logsChan) - return - } - } -} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go index d245581bde..6e3d77c2a6 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go @@ -721,11 +721,13 @@ func executeStreamCallAndGetReceivedServiceLogLines( case <-time.Tick(testTimeOut): return nil, stacktrace.NewError("Receiving stream logs in the test has reached the '%v' time out", testTimeOut) case streamErr, isChanOpen := <-errChan: - if !isChanOpen { + if !isChanOpen && len(userServiceLogsByUuidChan) == 0 { shouldReceiveStream = false break } - return nil, stacktrace.Propagate(streamErr, "Receiving streaming error.") + if isChanOpen && streamErr != nil { + return nil, stacktrace.Propagate(streamErr, "Receiving streaming error.") + } case userServiceLogsByUuid, isChanOpen := <-userServiceLogsByUuidChan: if !isChanOpen { shouldReceiveStream = false diff --git a/engine/server/engine/main.go b/engine/server/engine/main.go index 4670968c54..936bf24aa9 100644 --- a/engine/server/engine/main.go +++ b/engine/server/engine/main.go @@ -8,18 +8,6 @@ package main import ( "context" "fmt" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout" - "io/fs" - "math" - "net" - "net/http" - "os" - "path" - "path/filepath" - "runtime" - "strings" - "time" - "github.com/kurtosis-tech/kurtosis/api/golang/core/kurtosis_core_rpc_api_bindings" "github.com/kurtosis-tech/kurtosis/api/golang/engine/kurtosis_engine_rpc_api_bindings/kurtosis_engine_rpc_api_bindingsconnect" enclaveApi "github.com/kurtosis-tech/kurtosis/api/golang/http_rest/server/core_rest_api" @@ -39,6 +27,7 @@ import ( "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy" @@ -57,6 +46,17 @@ import ( echomiddleware "github.com/labstack/echo/v4/middleware" "github.com/rs/cors" "github.com/sirupsen/logrus" + "io/fs" + "math" + "net" + "net/http" + _ "net/http/pprof" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "time" ) const ( @@ -73,6 +73,7 @@ const ( functionPathSeparator = "." emptyFunctionName = "" webappPortAddr = ":9711" + pprofPath = "/debug/pprof/" remoteBackendConfigFilename = "remote_backend_config.json" pathToStaticFolder = "/run/webapp" @@ -213,8 +214,9 @@ func runMain() error { } logrus.Debugf("Created environment js file with content: \n%s", envJsFileContent) + handler := http.NewServeMux() fileServer := http.FileServer(http.Dir(pathToStaticFolder)) - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { path, err := filepath.Abs(r.URL.Path) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -236,6 +238,7 @@ func runMain() error { w.Header().Add("Cache-Control", "no-store") fileServer.ServeHTTP(w, r) }) + handler.Handle(pprofPath, http.HandlerFunc(http.DefaultServeMux.ServeHTTP)) err := http.ListenAndServe(webappPortAddr, handler) if err != nil { diff --git a/engine/server/engine/server/engine_connect_server_service.go b/engine/server/engine/server/engine_connect_server_service.go index ea2d266e9e..c19df02394 100644 --- a/engine/server/engine/server/engine_connect_server_service.go +++ b/engine/server/engine/server/engine_connect_server_service.go @@ -366,8 +366,10 @@ func (service *EngineConnectServerService) GetServiceLogs(ctx context.Context, c logrus.Debug("Exiting the stream because an error from the logs database client was received through the error chan.") return stacktrace.Propagate(err, "An error occurred streaming user service logs.") } - logrus.Debug("Exiting the stream loop after receiving a close signal from the error chan") - return nil + if len(serviceLogsByServiceUuidChan) == 0 { + logrus.Debug("Exiting the stream loop after receiving a close signal from the error chan") + return nil + } } } } diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000000..323c98d0db --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "release-type": "simple", + "bump-minor-pre-major": false, + "bump-patch-for-minor-pre-major": false, + "include-v-in-tag": false, + "include-component-in-tag": false, + "packages": { + ".": { + "package-name": "kurtosis" + } + } +} diff --git a/version.txt b/version.txt index 3a3cd8cc8b..9df886c42a 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.3.1 +1.4.2