From 3e2c856878c0c0af5820aadf3ccc92bda4b54b8a Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Thu, 30 Nov 2023 22:24:29 -0500 Subject: [PATCH 01/35] Reverse proxy and its Traefik implementation. --- .../docker_kurtosis_backend.go | 38 +++++++ .../engine_functions/create_engine.go | 24 +++- .../engine_functions/stop_engines.go | 6 + .../reverse_proxy_functions/consts.go | 6 + .../create_reverse_proxy.go | 84 ++++++++++++++ .../destroy_reverse_proxy.go | 36 ++++++ .../get_reverse_proxy.go | 21 ++++ .../implementations/traefik/consts.go | 40 +++++++ .../implementations/traefik/traefik_config.go | 37 ++++++ .../traefik_container_config_provider.go | 73 ++++++++++++ ...aefik_container_config_provider_factory.go | 6 + .../traefik_reverse_proxy_container.go | 68 +++++++++++ .../reverse_proxy_container.go | 18 +++ .../reverse_proxy_functions/shared_helpers.go | 106 ++++++++++++++++++ .../label_value_consts/label_value_consts.go | 2 + .../label_value_consts_test.go | 2 + .../object_attributes_provider.go | 19 ++++ .../objects/reverse_proxy/reverse_proxy.go | 38 +++++++ 18 files changed, 623 insertions(+), 1 deletion(-) create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/create_reverse_proxy.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/destroy_reverse_proxy.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/get_reverse_proxy.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_config.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/reverse_proxy_container.go create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go create mode 100644 container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go index a82360c4f6..506b25d8e0 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go @@ -12,6 +12,8 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions/implementations/fluentbit" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik" user_service_functions "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" @@ -27,6 +29,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/free_ip_addr_tracker" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/service_registration" @@ -471,6 +474,41 @@ func (backend *DockerKurtosisBackend) DestroyLogsCollectorForEnclave(ctx context return nil } +func (backend *DockerKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + reverseProxyContainer := traefik.NewTraefikReverseProxyContainer() //Declaring the implementation + + reverseProxy, _, err := reverse_proxy_functions.CreateReverseProxy( + ctx, + reverseProxyContainer, + backend.dockerManager, + backend.objAttrsProvider, + ) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating the logs aggregator using the logs aggregator container '%+v'.", reverseProxyContainer) + } + return reverseProxy, nil +} + +func (backend *DockerKurtosisBackend) GetReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + maybeReverseProxy, err := reverse_proxy_functions.GetReverseProxy( + ctx, + backend.dockerManager, + ) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the logs aggregator") + } + + return maybeReverseProxy, nil +} + +func (backend *DockerKurtosisBackend) DestroyReverseProxy(ctx context.Context) error { + if err := reverse_proxy_functions.DestroyReverseProxy(ctx, backend.dockerManager); err != nil { + return stacktrace.Propagate(err, "An error occurred destroying the logs aggregator") + } + + return nil +} + func (backend *DockerKurtosisBackend) GetAvailableCPUAndMemory(ctx context.Context) (compute_resources.MemoryInMegaBytes, compute_resources.CpuMilliCores, bool, error) { availableMemory, availableCpu, err := backend.dockerManager.GetAvailableCPUAndMemory(ctx) if err != nil { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 4cc40f92c7..58c52ce28f 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -3,10 +3,14 @@ package engine_functions import ( "context" "fmt" + "time" + "github.com/docker/go-connections/nat" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" @@ -17,7 +21,6 @@ import ( "github.com/kurtosis-tech/kurtosis/engine/launcher/args" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" - "time" ) const ( @@ -109,6 +112,24 @@ func CreateEngine( }() logrus.Infof("Centralized logs components started.") + reverseProxyContainer := traefik.NewTraefikReverseProxyContainer() + _, removeReverseProxyFunc, err := reverse_proxy_functions.CreateReverseProxy( + ctx, + reverseProxyContainer, + dockerManager, + objAttrsProvider) + if err != nil { + return nil, stacktrace.Propagate(err, + "An error occurred attempting to create reverse proxy for engine with GUID '%v' in Docker network with network id '%v'.", engineGuidStr, targetNetworkId) + } + shouldRemoveReverseProxy := true + defer func() { + if shouldRemoveReverseProxy { + removeReverseProxyFunc() + } + }() + logrus.Infof("Reverse proxy started.") + enclaveManagerUIPortSpec, err := port_spec.NewPortSpec(uint16(enclaveManagerUIPort), consts.EngineTransportProtocol, consts.HttpApplicationProtocol, defaultWait) if err != nil { return nil, stacktrace.Propagate( @@ -248,6 +269,7 @@ func CreateEngine( } shouldRemoveLogsAggregator = false + shouldRemoveReverseProxy = false shouldKillEngineContainer = false return result, nil } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go index 0ba2fef9b0..9ffbe75852 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go @@ -3,6 +3,7 @@ package engine_functions import ( "context" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_operation_parallelizer" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/engine" @@ -71,6 +72,11 @@ func StopEngines( if err := logs_aggregator_functions.DestroyLogsAggregator(ctx, dockerManager); err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred removing the logging components.") } + + // Stop reverse proxy + if err := reverse_proxy_functions.DestroyReverseProxy(ctx, dockerManager); err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred removing the reverse proxy.") + } return successfulGuids, erroredGuids, nil } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go new file mode 100644 index 0000000000..50c3144bab --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go @@ -0,0 +1,6 @@ +package reverse_proxy_functions + +const ( + defaultReverseProxyHttpPortNum = uint16(9720) + defaultReverseProxyDashboardPortNum = uint16(9721) +) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/create_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/create_reverse_proxy.go new file mode 100644 index 0000000000..c432263b40 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/create_reverse_proxy.go @@ -0,0 +1,84 @@ +package reverse_proxy_functions + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" +) + +const ( + defaultContainerStatusForNewReverseProxyContainer = types.ContainerStatus_Running +) + +// Create reverse proxy idempotently, if existing reverse proxy is found, then it is returned +func CreateReverseProxy( + ctx context.Context, + reverseProxyContainer ReverseProxyContainer, + dockerManager *docker_manager.DockerManager, + objAttrsProvider object_attributes_provider.DockerObjectAttributesProvider, +) ( + *reverse_proxy.ReverseProxy, + func(), + error, +) { + _, found, err := getReverseProxyContainer(ctx, dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred getting reverse proxy container.") + } + if found { + logrus.Debugf("Found existing reverse proxy; cannot start a new one.") + reverseProxyObj, _, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred getting existing reverse proxy.") + } + return reverseProxyObj, nil, nil + } + + reverseProxyNetwork, err := shared_helpers.GetEngineAndLogsComponentsNetwork(ctx, dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred getting the reverse proxy network.") + } + targetNetworkId := reverseProxyNetwork.GetId() + + containerId, containerLabels, removeReverseProxyContainerFunc, err := reverseProxyContainer.CreateAndStart( + ctx, + defaultReverseProxyHttpPortNum, + defaultReverseProxyDashboardPortNum, + targetNetworkId, + objAttrsProvider, + dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate( + err, + "An error occurred creating the reverse proxy container in Docker network with ID '%v'", + targetNetworkId, + ) + } + shouldRemoveReverseProxyContainer := true + defer func() { + if shouldRemoveReverseProxyContainer { + removeReverseProxyContainerFunc() + } + }() + + reverseProxy, err := getReverseProxyObjectFromContainerInfo( + ctx, + containerId, + defaultContainerStatusForNewReverseProxyContainer, + dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred getting reverse proxy object using container ID '%v', labels '%+v', status '%v'.", containerId, containerLabels, defaultContainerStatusForNewReverseProxyContainer) + } + + removeReverseProxyFunc := func() { + removeReverseProxyContainerFunc() + } + + shouldRemoveReverseProxyContainer = false + return reverseProxy, removeReverseProxyFunc, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/destroy_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/destroy_reverse_proxy.go new file mode 100644 index 0000000000..267605d743 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/destroy_reverse_proxy.go @@ -0,0 +1,36 @@ +package reverse_proxy_functions + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" + "time" +) + +const ( + stopReverseProxyContainerTimeout = 2 * time.Second +) + +// Destroys reverse proxy idempotently, returns nil if no reverse proxy reverse proxy container was found +func DestroyReverseProxy(ctx context.Context, dockerManager *docker_manager.DockerManager) error { + _, maybeReverseProxyContainerId, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + logrus.Warnf("Attempted to destroy reverse proxy but no reverse proxy container was found.") + return nil + } + + if maybeReverseProxyContainerId == "" { + return nil + } + + if err := dockerManager.StopContainer(ctx, maybeReverseProxyContainerId, stopReverseProxyContainerTimeout); err != nil { + return stacktrace.Propagate(err, "An error occurred stopping the reverse proxy container with ID '%v'", maybeReverseProxyContainerId) + } + + if err := dockerManager.RemoveContainer(ctx, maybeReverseProxyContainerId); err != nil { + return stacktrace.Propagate(err, "An error occurred removing the reverse proxy container with ID '%v'", maybeReverseProxyContainerId) + } + + return nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/get_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/get_reverse_proxy.go new file mode 100644 index 0000000000..a38df8fd71 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/get_reverse_proxy.go @@ -0,0 +1,21 @@ +package reverse_proxy_functions + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" + "github.com/kurtosis-tech/stacktrace" +) + +func GetReverseProxy( + ctx context.Context, + dockerManager *docker_manager.DockerManager, +) (*reverse_proxy.ReverseProxy, error) { + + maybeReverseProxyObject, _, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the reverse proxy") + } + + return maybeReverseProxyObject, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go new file mode 100644 index 0000000000..397a6e0eba --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -0,0 +1,40 @@ +package traefik + +import ( + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" +) + +const ( + ////////////////////////--TRAEFIK CONTAINER CONFIGURATION SECTION--///////////////////////////// + containerImage = "traefik:2.10.6" + + configDirpath = "/etc/traefik/" + configFilepath = configDirpath + "traefik.yml" + binaryFilepath = "/usr/local/bin/traefik" + ////////////////////////--FINISH TRAEFIK CONTAINER CONFIGURATION SECTION--///////////////////////////// + + ////////////////////////--TRAEFIK CONFIGURATION SECTION--///////////////////////////// + traefikNetworkid = consts.NameOfNetworkToStartEngineAndLogServiceContainersIn + + configFileTemplateName = "traefikConfigFileTemplate" + + configFileTemplate = ` +api: + dashboard: true + insecure: true + disabledashboardad: true + +entryPoints: + web: + address: ":{{ .WebAddress }}" + traefik: + address: ":{{ .TraefikAddress }}" + +providers: + docker: + endpoint: "unix:///var/run/docker.sock" + exposedByDefault: false + network: "{{ .NetworkId }}" +` + ////////////////////////--FINISH--TRAEFIK CONFIGURATION SECTION--///////////////////////////// +) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_config.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_config.go new file mode 100644 index 0000000000..329ae9b741 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_config.go @@ -0,0 +1,37 @@ +package traefik + +import ( + "bytes" + "text/template" + + "github.com/kurtosis-tech/stacktrace" +) + +type TraefikConfig struct { + WebAddress uint16 + TraefikAddress uint16 + NetworkId string +} + +func newDefaultTraefikConfig(httpPort uint16, dashboardPort uint16) *TraefikConfig { + return &TraefikConfig{ + WebAddress: httpPort, + TraefikAddress: dashboardPort, + NetworkId: traefikNetworkid, + } +} + +func (cfg *TraefikConfig) getConfigFileContent() (string, error) { + cfgFileTemplate, err := template.New(configFileTemplateName).Parse(configFileTemplate) + if err != nil { + return "", stacktrace.Propagate(err, "An error occurred parsing Traefik's config template.") + } + + templateStrBuffer := &bytes.Buffer{} + if err := cfgFileTemplate.Execute(templateStrBuffer, cfg); err != nil { + return "", stacktrace.Propagate(err, "An error occurred executing Traefik's config file template.") + } + templateStr := templateStrBuffer.String() + + return templateStr, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go new file mode 100644 index 0000000000..a49c88d6fd --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go @@ -0,0 +1,73 @@ +package traefik + +import ( + "fmt" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/stacktrace" +) + +const ( + shBinaryFilepath = "/bin/sh" + printfCmdName = "printf" + mkdirCmdName = "mkdir" + shCmdFlag = "-c" +) + +type traefikContainerConfigProvider struct { + config *TraefikConfig +} + +func newTraefikContainerConfigProvider(config *TraefikConfig) *traefikContainerConfigProvider { + return &traefikContainerConfigProvider{config: config} +} + +func (traefik *traefikContainerConfigProvider) GetContainerArgs( + containerName string, + containerLabels map[string]string, + networkId string, +) (*docker_manager.CreateAndStartContainerArgs, error) { + + traefikConfigContentStr, err := traefik.config.getConfigFileContent() + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the traefik configuration content") + } + + // Create cmd to + // 1. create config file in appropriate location in the traefik container + // 2. start traefik with the config file + overrideCmd := []string{ + shCmdFlag, + fmt.Sprintf( + "%v '%v' && %v '%v' > %v && %v", + mkdirCmdName, + configDirpath, + printfCmdName, + traefikConfigContentStr, + configFilepath, + binaryFilepath, + ), + } + + // Traefik should ALWAYS be running + // Thus, instruct docker to restart the container if it exits with non-zero status code for whatever reason + restartPolicy := docker_manager.RestartPolicy(docker_manager.RestartAlways) + + createAndStartArgs := docker_manager.NewCreateAndStartContainerArgsBuilder( + containerImage, + containerName, + networkId, + ).WithLabels( + containerLabels, + ).WithEntrypointArgs( + []string{ + shBinaryFilepath, + }, + ).WithCmdArgs( + overrideCmd, + ).WithRestartPolicy( + restartPolicy, + ).Build() + + return createAndStartArgs, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go new file mode 100644 index 0000000000..20729330dc --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go @@ -0,0 +1,6 @@ +package traefik + +func createTraefikContainerConfigProvider(httpPort uint16, dashboardPort uint16) *traefikContainerConfigProvider { + config := newDefaultTraefikConfig(httpPort, dashboardPort) + return newTraefikContainerConfigProvider(config) +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go new file mode 100644 index 0000000000..912b4991da --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go @@ -0,0 +1,68 @@ +package traefik + +import ( + "context" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" +) + +type traefikReverseProxyContainer struct{} + +func NewTraefikReverseProxyContainer() *traefikReverseProxyContainer { + return &traefikReverseProxyContainer{} +} + +func (traefikContainer *traefikReverseProxyContainer) CreateAndStart( + ctx context.Context, + httpPort uint16, + dashboardPort uint16, + targetNetworkId string, + objAttrsProvider object_attributes_provider.DockerObjectAttributesProvider, + dockerManager *docker_manager.DockerManager, +) (string, map[string]string, func(), error) { + traefikContainerConfigProviderObj := createTraefikContainerConfigProvider(httpPort, dashboardPort) + + reverseProxyAttrs, err := objAttrsProvider.ForReverseProxy() + if err != nil { + return "", nil, nil, stacktrace.Propagate(err, "An error occurred getting the reverse proxy container attributes.") + } + containerName := reverseProxyAttrs.GetName().GetString() + containerLabelStrs := map[string]string{} + for labelKey, labelValue := range reverseProxyAttrs.GetLabels() { + containerLabelStrs[labelKey.GetString()] = labelValue.GetString() + } + + createAndStartArgs, err := traefikContainerConfigProviderObj.GetContainerArgs(containerName, containerLabelStrs, targetNetworkId) + if err != nil { + return "", nil, nil, err + } + + containerId, _, err := dockerManager.CreateAndStartContainer(ctx, createAndStartArgs) + if err != nil { + return "", nil, nil, stacktrace.Propagate(err, "An error occurred starting the reverse proxy container with these args '%+v'", createAndStartArgs) + } + removeContainerFunc := func() { + removeCtx := context.Background() + + if err := dockerManager.RemoveContainer(removeCtx, containerId); err != nil { + logrus.Errorf( + "Launching the reverse proxy server with container ID '%v' didn't complete successfully so we "+ + "tried to remove the container we started, but doing so exited with an error:\n%v", + containerId, + err) + logrus.Errorf("ACTION REQUIRED: You'll need to manually remove the reverse proxy server with Docker container ID '%v'!!!!!!", containerId) + } + } + shouldRemoveReverseProxyContainer := true + defer func() { + if shouldRemoveReverseProxyContainer { + removeContainerFunc() + } + }() + + shouldRemoveReverseProxyContainer = false + return containerId, containerLabelStrs, removeContainerFunc, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/reverse_proxy_container.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/reverse_proxy_container.go new file mode 100644 index 0000000000..5de87e2534 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/reverse_proxy_container.go @@ -0,0 +1,18 @@ +package reverse_proxy_functions + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider" +) + +type ReverseProxyContainer interface { + CreateAndStart( + ctx context.Context, + httpPort uint16, + dashboardPort uint16, + targetNetworkId string, + objAttrsProvider object_attributes_provider.DockerObjectAttributesProvider, + dockerManager *docker_manager.DockerManager, + ) (string, map[string]string, func(), error) +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go new file mode 100644 index 0000000000..96c03a6ae1 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go @@ -0,0 +1,106 @@ +package reverse_proxy_functions + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" + "net" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" + "github.com/kurtosis-tech/stacktrace" +) + +const ( + shouldShowStoppedReverseProxyContainers = true +) + +// Returns nil [ReverseProxy] object if no container is found +func getReverseProxyObjectAndContainerId( + ctx context.Context, + dockerManager *docker_manager.DockerManager, +) (*reverse_proxy.ReverseProxy, string, error) { + reverseProxyContainer, found, err := getReverseProxyContainer(ctx, dockerManager) + if err != nil { + return nil, "", stacktrace.Propagate(err, "An error occurred getting all reverse proxy containers") + } + if !found { + return nil, "", nil + } + + reverseProxyContainerID := reverseProxyContainer.GetId() + + reverseProxyObject, err := getReverseProxyObjectFromContainerInfo( + ctx, + reverseProxyContainerID, + reverseProxyContainer.GetStatus(), + dockerManager, + ) + if err != nil { + return nil, "", stacktrace.Propagate(err, "An error occurred getting the reverse proxy object using container ID '%v', labels '%+v' and the status '%v'", reverseProxyContainer.GetId(), reverseProxyContainer.GetLabels(), reverseProxyContainer.GetStatus()) + } + + return reverseProxyObject, reverseProxyContainerID, nil +} + +// Returns nil [Container] object and false if no reverse proxy container is found +func getReverseProxyContainer(ctx context.Context, dockerManager *docker_manager.DockerManager) (*types.Container, bool, error) { + reverseProxyContainerSearchLabels := map[string]string{ + docker_label_key.AppIDDockerLabelKey.GetString(): label_value_consts.AppIDDockerLabelValue.GetString(), + docker_label_key.ContainerTypeDockerLabelKey.GetString(): label_value_consts.ReverseProxyTypeDockerLabelValue.GetString(), + } + + matchingReverseProxyContainers, err := dockerManager.GetContainersByLabels(ctx, reverseProxyContainerSearchLabels, shouldShowStoppedReverseProxyContainers) + if err != nil { + return nil, false, stacktrace.Propagate(err, "An error occurred fetching the reverse proxy container using labels: %+v", reverseProxyContainerSearchLabels) + } + + if len(matchingReverseProxyContainers) == 0 { + return nil, false, nil + } + if len(matchingReverseProxyContainers) > 1 { + return nil, false, stacktrace.NewError("Found more than one reverse proxy Docker container'; this is a bug in Kurtosis") + } + return matchingReverseProxyContainers[0], true, nil +} + +func getReverseProxyObjectFromContainerInfo( + ctx context.Context, + containerId string, + containerStatus types.ContainerStatus, + dockerManager *docker_manager.DockerManager, +) (*reverse_proxy.ReverseProxy, error) { + var privateIpAddr net.IP + + isContainerRunning, found := consts.IsContainerRunningDeterminer[containerStatus] + if !found { + // This should never happen because we enforce completeness in a unit test + return nil, stacktrace.NewError("No is-running designation found for reverse proxy container status '%v'; this is a bug in Kurtosis!", containerStatus.String()) + } + + var reverseProxyStatus container.ContainerStatus + if isContainerRunning { + reverseProxyStatus = container.ContainerStatus_Running + + privateIpAddrStr, err := dockerManager.GetContainerIP(ctx, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, containerId) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the private IP address of container '%v' in network '%v'", containerId, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn) + } + privateIpAddr = net.ParseIP(privateIpAddrStr) + if privateIpAddr == nil { + return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", privateIpAddrStr) + } + } else { + reverseProxyStatus = container.ContainerStatus_Stopped + } + + reverseProxyObj := reverse_proxy.NewReverseProxy( + reverseProxyStatus, + privateIpAddr, + ) + + return reverseProxyObj, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go index f44294d616..9b830cf813 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go @@ -15,6 +15,7 @@ const ( engineContainerTypeLabelValueStr = "kurtosis-engine" logsAggregatorContainerTypeLabelValueStr = "kurtosis-logs-aggregator" logsCollectorContainerTypeLabelValueStr = "kurtosis-logs-collector" + reverseProxyContainerTypeLabelValueStr = "kurtosis-reverse-proxy" // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!! apiContainerContainerTypeLabelValueStr = "api-container" @@ -38,6 +39,7 @@ var AppIDDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(app var EngineContainerTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(engineContainerTypeLabelValueStr) var LogsAggregatorTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsAggregatorContainerTypeLabelValueStr) var LogsCollectorTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsCollectorContainerTypeLabelValueStr) +var ReverseProxyTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(reverseProxyContainerTypeLabelValueStr) // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts_test.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts_test.go index b4d1df6886..79cddc9d5f 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts_test.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts_test.go @@ -11,6 +11,7 @@ var labelValueStrsToEnsure = map[string]string{ engineContainerTypeLabelValueStr: "kurtosis-engine", logsCollectorContainerTypeLabelValueStr: "kurtosis-logs-collector", logsAggregatorContainerTypeLabelValueStr: "kurtosis-logs-aggregator", + reverseProxyContainerTypeLabelValueStr: "kurtosis-reverse-proxy", } var labelValuesToEnsure = map[*docker_label_value.DockerLabelValue]string{ @@ -18,6 +19,7 @@ var labelValuesToEnsure = map[*docker_label_value.DockerLabelValue]string{ EngineContainerTypeDockerLabelValue: "kurtosis-engine", LogsAggregatorTypeDockerLabelValue: "kurtosis-logs-aggregator", LogsCollectorTypeDockerLabelValue: "kurtosis-logs-collector", + ReverseProxyTypeDockerLabelValue: "kurtosis-reverse-proxy", } // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTANT !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go index a68ee396f2..52fcbc54d6 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go @@ -17,6 +17,7 @@ const ( engineServerNamePrefix = "kurtosis-engine" logsAggregatorName = "kurtosis-logs-aggregator" logsStorageVolumeName = "kurtosis-logs-storage" + reverseProxyName = "kurtosis-reverse-proxy" ) type DockerObjectAttributesProvider interface { @@ -28,6 +29,7 @@ type DockerObjectAttributesProvider interface { ForEnclave(enclaveUuid enclave.EnclaveUUID) (DockerEnclaveObjectAttributesProvider, error) ForLogsAggregator() (DockerObjectAttributes, error) ForLogsStorageVolume() (DockerObjectAttributes, error) + ForReverseProxy() (DockerObjectAttributes, error) } func GetDockerObjectAttributesProvider() DockerObjectAttributesProvider { @@ -134,3 +136,20 @@ func (provider *dockerObjectAttributesProviderImpl) ForLogsStorageVolume() (Dock } return objectAttributes, nil } + +func (provider *dockerObjectAttributesProviderImpl) ForReverseProxy() (DockerObjectAttributes, error) { + name, err := docker_object_name.CreateNewDockerObjectName(reverseProxyName) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating a Docker object name object from string '%v'", reverseProxyName) + } + + labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{ + docker_label_key.ContainerTypeDockerLabelKey: label_value_consts.ReverseProxyTypeDockerLabelValue, + } + + objectAttributes, err := newDockerObjectAttributesImpl(name, labels) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while creating the ObjectAttributesImpl with the name '%s' and labels '%+v'", name, labels) + } + return objectAttributes, nil +} diff --git a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go new file mode 100644 index 0000000000..b5d2f2aae4 --- /dev/null +++ b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go @@ -0,0 +1,38 @@ +package reverse_proxy + +import ( + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" + "net" +) + +// This component is responsible for routing http traffic to the services +type ReverseProxy struct { + status container.ContainerStatus + + // This will be nil if the container is not running + maybePrivateIpAddr net.IP + + // PortNum that container will listen for logs on + logsListeningPortNum uint16 +} + +func NewReverseProxy( + status container.ContainerStatus, + maybePrivateIpAddr net.IP) *ReverseProxy { + return &ReverseProxy{ + status: status, + maybePrivateIpAddr: maybePrivateIpAddr} +} + +func (reverseProxy *ReverseProxy) GetStatus() container.ContainerStatus { + return reverseProxy.status +} + +func (reverseProxy *ReverseProxy) GetMaybePrivateIpAddr() net.IP { + return reverseProxy.maybePrivateIpAddr +} + +// Returns port number that logs aggregator listens for logs on +func (reverseProxy *ReverseProxy) GetListeningPortNum() uint16 { + return reverseProxy.logsListeningPortNum +} From eb6594c3b736ec9b2d34e605ad7ad3db53a6116d Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Fri, 1 Dec 2023 11:34:04 -0500 Subject: [PATCH 02/35] Bind docker socket and publish http/dashboard ports. --- .../implementations/traefik/consts.go | 16 ++--- .../implementations/traefik/traefik_config.go | 37 ------------ .../traefik_container_config_provider.go | 59 ++++++++++++++++++- ...aefik_container_config_provider_factory.go | 8 ++- .../traefik_reverse_proxy_container.go | 4 +- .../reverse_proxy_functions/shared_helpers.go | 2 + .../objects/reverse_proxy/consts.go | 5 ++ .../objects/reverse_proxy/reverse_proxy.go | 25 +++++--- .../reverse_proxy/reverse_proxy_config.go | 37 ++++++++++++ 9 files changed, 130 insertions(+), 63 deletions(-) delete mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_config.go create mode 100644 container-engine-lib/lib/backend_interface/objects/reverse_proxy/consts.go create mode 100644 container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy_config.go diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go index 397a6e0eba..78f8571cf8 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -1,9 +1,5 @@ package traefik -import ( - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" -) - const ( ////////////////////////--TRAEFIK CONTAINER CONFIGURATION SECTION--///////////////////////////// containerImage = "traefik:2.10.6" @@ -14,22 +10,18 @@ const ( ////////////////////////--FINISH TRAEFIK CONTAINER CONFIGURATION SECTION--///////////////////////////// ////////////////////////--TRAEFIK CONFIGURATION SECTION--///////////////////////////// - traefikNetworkid = consts.NameOfNetworkToStartEngineAndLogServiceContainersIn - - configFileTemplateName = "traefikConfigFileTemplate" - configFileTemplate = ` api: dashboard: true insecure: true disabledashboardad: true - + entryPoints: web: - address: ":{{ .WebAddress }}" + address: ":{{ .HttpPort }}" traefik: - address: ":{{ .TraefikAddress }}" - + address: ":{{ .DashboardPort }}" + providers: docker: endpoint: "unix:///var/run/docker.sock" diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_config.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_config.go deleted file mode 100644 index 329ae9b741..0000000000 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_config.go +++ /dev/null @@ -1,37 +0,0 @@ -package traefik - -import ( - "bytes" - "text/template" - - "github.com/kurtosis-tech/stacktrace" -) - -type TraefikConfig struct { - WebAddress uint16 - TraefikAddress uint16 - NetworkId string -} - -func newDefaultTraefikConfig(httpPort uint16, dashboardPort uint16) *TraefikConfig { - return &TraefikConfig{ - WebAddress: httpPort, - TraefikAddress: dashboardPort, - NetworkId: traefikNetworkid, - } -} - -func (cfg *TraefikConfig) getConfigFileContent() (string, error) { - cfgFileTemplate, err := template.New(configFileTemplateName).Parse(configFileTemplate) - if err != nil { - return "", stacktrace.Propagate(err, "An error occurred parsing Traefik's config template.") - } - - templateStrBuffer := &bytes.Buffer{} - if err := cfgFileTemplate.Execute(templateStrBuffer, cfg); err != nil { - return "", stacktrace.Propagate(err, "An error occurred executing Traefik's config file template.") - } - templateStr := templateStrBuffer.String() - - return templateStr, nil -} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go index a49c88d6fd..4246c22f4c 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go @@ -3,7 +3,12 @@ package traefik import ( "fmt" + "github.com/docker/go-connections/nat" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/stacktrace" ) @@ -15,20 +20,27 @@ const ( ) type traefikContainerConfigProvider struct { - config *TraefikConfig + config *reverse_proxy.ReverseProxyConfig } -func newTraefikContainerConfigProvider(config *TraefikConfig) *traefikContainerConfigProvider { +func newTraefikContainerConfigProvider(config *reverse_proxy.ReverseProxyConfig) *traefikContainerConfigProvider { return &traefikContainerConfigProvider{config: config} } func (traefik *traefikContainerConfigProvider) GetContainerArgs( containerName string, containerLabels map[string]string, + httpPort uint16, + dashboardPort uint16, networkId string, ) (*docker_manager.CreateAndStartContainerArgs, error) { - traefikConfigContentStr, err := traefik.config.getConfigFileContent() + bindMounts := map[string]string{ + // Necessary so that the engine server can interact with the Docker engine + consts.DockerSocketFilepath: consts.DockerSocketFilepath, + } + + traefikConfigContentStr, err := traefik.config.GetConfigFileContent(configFileTemplate) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting the traefik configuration content") } @@ -53,12 +65,53 @@ func (traefik *traefikContainerConfigProvider) GetContainerArgs( // Thus, instruct docker to restart the container if it exits with non-zero status code for whatever reason restartPolicy := docker_manager.RestartPolicy(docker_manager.RestartAlways) + defaultWait, err := port_spec.CreateWaitWithDefaultValues() + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating a wait with default values") + } + + // Publish HTTP and Dashboard ports + privateHttpPortSpec, err := port_spec.NewPortSpec(httpPort, port_spec.TransportProtocol_TCP, consts.HttpApplicationProtocol, defaultWait) + if err != nil { + return nil, stacktrace.Propagate( + err, + "An error occurred creating Traefik private http port spec object using number '%v' and protocol '%v'", + httpPort, + consts.EngineTransportProtocol.String(), + ) + } + privateHttpDockerPort, err := shared_helpers.TransformPortSpecToDockerPort(privateHttpPortSpec) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred transforming the private http port spec to a Docker port") + } + privateDashboardPortSpec, err := port_spec.NewPortSpec(dashboardPort, port_spec.TransportProtocol_TCP, consts.HttpApplicationProtocol, defaultWait) + if err != nil { + return nil, stacktrace.Propagate( + err, + "An error occurred creating Traefik private dashboard port spec object using number '%v' and protocol '%v'", + dashboardPort, + consts.EngineTransportProtocol.String(), + ) + } + privateDashboardDockerPort, err := shared_helpers.TransformPortSpecToDockerPort(privateDashboardPortSpec) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred transforming the private http port spec to a Docker port") + } + usedPorts := map[nat.Port]docker_manager.PortPublishSpec{ + privateHttpDockerPort: docker_manager.NewManualPublishingSpec(httpPort), + privateDashboardDockerPort: docker_manager.NewManualPublishingSpec(dashboardPort), + } + createAndStartArgs := docker_manager.NewCreateAndStartContainerArgsBuilder( containerImage, containerName, networkId, ).WithLabels( containerLabels, + ).WithBindMounts( + bindMounts, + ).WithUsedPorts( + usedPorts, ).WithEntrypointArgs( []string{ shBinaryFilepath, diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go index 20729330dc..2b0b30f25e 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go @@ -1,6 +1,10 @@ package traefik -func createTraefikContainerConfigProvider(httpPort uint16, dashboardPort uint16) *traefikContainerConfigProvider { - config := newDefaultTraefikConfig(httpPort, dashboardPort) +import ( + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" +) + +func createTraefikContainerConfigProvider(httpPort uint16, dashboardPort uint16, networkId string) *traefikContainerConfigProvider { + config := reverse_proxy.NewDefaultReverseProxyConfig(httpPort, dashboardPort, networkId) return newTraefikContainerConfigProvider(config) } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go index 912b4991da..ebe3f1ac48 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go @@ -23,7 +23,7 @@ func (traefikContainer *traefikReverseProxyContainer) CreateAndStart( objAttrsProvider object_attributes_provider.DockerObjectAttributesProvider, dockerManager *docker_manager.DockerManager, ) (string, map[string]string, func(), error) { - traefikContainerConfigProviderObj := createTraefikContainerConfigProvider(httpPort, dashboardPort) + traefikContainerConfigProviderObj := createTraefikContainerConfigProvider(httpPort, dashboardPort, targetNetworkId) reverseProxyAttrs, err := objAttrsProvider.ForReverseProxy() if err != nil { @@ -35,7 +35,7 @@ func (traefikContainer *traefikReverseProxyContainer) CreateAndStart( containerLabelStrs[labelKey.GetString()] = labelValue.GetString() } - createAndStartArgs, err := traefikContainerConfigProviderObj.GetContainerArgs(containerName, containerLabelStrs, targetNetworkId) + createAndStartArgs, err := traefikContainerConfigProviderObj.GetContainerArgs(containerName, containerLabelStrs, httpPort, dashboardPort, targetNetworkId) if err != nil { return "", nil, nil, err } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go index 96c03a6ae1..079e30e071 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go @@ -100,6 +100,8 @@ func getReverseProxyObjectFromContainerInfo( reverseProxyObj := reverse_proxy.NewReverseProxy( reverseProxyStatus, privateIpAddr, + defaultReverseProxyHttpPortNum, + defaultReverseProxyDashboardPortNum, ) return reverseProxyObj, nil diff --git a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/consts.go b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/consts.go new file mode 100644 index 0000000000..0044efe43e --- /dev/null +++ b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/consts.go @@ -0,0 +1,5 @@ +package reverse_proxy + +const ( + configFileTemplateName = "configFileTemplate" +) diff --git a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go index b5d2f2aae4..ae2821313c 100644 --- a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go +++ b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go @@ -12,16 +12,24 @@ type ReverseProxy struct { // This will be nil if the container is not running maybePrivateIpAddr net.IP - // PortNum that container will listen for logs on - logsListeningPortNum uint16 + // HTTP port + httpPort uint16 + + // Dashboard port + dashboardPort uint16 } func NewReverseProxy( status container.ContainerStatus, - maybePrivateIpAddr net.IP) *ReverseProxy { + maybePrivateIpAddr net.IP, + httpPort uint16, + dashboardPort uint16) *ReverseProxy { return &ReverseProxy{ status: status, - maybePrivateIpAddr: maybePrivateIpAddr} + maybePrivateIpAddr: maybePrivateIpAddr, + httpPort: httpPort, + dashboardPort: dashboardPort, + } } func (reverseProxy *ReverseProxy) GetStatus() container.ContainerStatus { @@ -32,7 +40,10 @@ func (reverseProxy *ReverseProxy) GetMaybePrivateIpAddr() net.IP { return reverseProxy.maybePrivateIpAddr } -// Returns port number that logs aggregator listens for logs on -func (reverseProxy *ReverseProxy) GetListeningPortNum() uint16 { - return reverseProxy.logsListeningPortNum +func (reverseProxy *ReverseProxy) GetHttpPort() uint16 { + return reverseProxy.httpPort +} + +func (reverseProxy *ReverseProxy) GetDashboardPort() uint16 { + return reverseProxy.dashboardPort } diff --git a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy_config.go b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy_config.go new file mode 100644 index 0000000000..7035ba9013 --- /dev/null +++ b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy_config.go @@ -0,0 +1,37 @@ +package reverse_proxy + +import ( + "bytes" + "text/template" + + "github.com/kurtosis-tech/stacktrace" +) + +type ReverseProxyConfig struct { + HttpPort uint16 + DashboardPort uint16 + NetworkId string +} + +func NewDefaultReverseProxyConfig(httpPort uint16, dashboardPort uint16, networkId string) *ReverseProxyConfig { + return &ReverseProxyConfig{ + HttpPort: httpPort, + DashboardPort: dashboardPort, + NetworkId: networkId, + } +} + +func (cfg *ReverseProxyConfig) GetConfigFileContent(configFileTemplate string) (string, error) { + cfgFileTemplate, err := template.New(configFileTemplateName).Parse(configFileTemplate) + if err != nil { + return "", stacktrace.Propagate(err, "An error occurred parsing the reverse proxy's config template.") + } + + templateStrBuffer := &bytes.Buffer{} + if err := cfgFileTemplate.Execute(templateStrBuffer, cfg); err != nil { + return "", stacktrace.Propagate(err, "An error occurred executing the reverse proxy's config file template.") + } + templateStr := templateStrBuffer.String() + + return templateStr, nil +} From 1537d3beb5b95143ebabee3639014140d3f34bb7 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Fri, 1 Dec 2023 13:39:58 -0500 Subject: [PATCH 03/35] Update docker resources cleanup check to take the reverse proxy container into consideration. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1fc9f2ea17..a6f1e2979e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -878,9 +878,9 @@ jobs: equal: [ "docker", << parameters.cli-cluster-backend >> ] steps: - run: - name: "Verify only the engine container and logs aggregator remains after the clean" + name: "Verify only the engine, logs aggregator and reverse proxy remain after the clean" command: | - if ! [ "$(docker container ls -a | tail -n+2 | wc -l)" -eq 2 ]; then + if ! [ "$(docker container ls -a | tail -n+2 | wc -l)" -eq 3 ]; then docker container ls -a false fi From e9982ed4e50af906e482e6d5d1e9e31b4d6858d9 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Fri, 1 Dec 2023 22:19:06 -0500 Subject: [PATCH 04/35] Connect and disconnect the reverse proxy to/from the enclave network. --- .../docker_kurtosis_backend.go | 16 ++ ...cker_kurtosis_backend_enclave_functions.go | 41 ++++- .../network_reverse_proxy.go | 106 ++++++++++++ .../kubernetes_kurtosis_backend.go | 18 ++ .../metrics_reporting_kurtosis_backend.go | 13 ++ .../lib/backend_interface/kurtosis_backend.go | 8 + .../mock_kurtosis_backend.go | 154 +++++++++++++++++- 7 files changed, 352 insertions(+), 4 deletions(-) create mode 100644 container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go index 506b25d8e0..0642430ee6 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go @@ -509,6 +509,22 @@ func (backend *DockerKurtosisBackend) DestroyReverseProxy(ctx context.Context) e return nil } +func (backend *DockerKurtosisBackend) ConnectReverseProxyToNetwork(ctx context.Context, networkId string) error { + if err := reverse_proxy_functions.ConnectReverseProxyToNetwork(ctx, backend.dockerManager, networkId); err != nil { + return stacktrace.Propagate(err, "An error occurred connecting the reverse proxy to the network with ID '%v'", networkId) + } + + return nil +} + +func (backend *DockerKurtosisBackend) DisconnectReverseProxyFromNetwork(ctx context.Context, networkId string) error { + if err := reverse_proxy_functions.DisconnectReverseProxyFromNetwork(ctx, backend.dockerManager, networkId); err != nil { + return stacktrace.Propagate(err, "An error occurred disconnecting the reverse proxy from the network with ID '%v'", networkId) + } + + return nil +} + func (backend *DockerKurtosisBackend) GetAvailableCPUAndMemory(ctx context.Context) (compute_resources.MemoryInMegaBytes, compute_resources.CpuMilliCores, bool, error) { availableMemory, availableCpu, err := backend.dockerManager.GetAvailableCPUAndMemory(ctx) if err != nil { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go index 54aabf1a93..f676005123 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go @@ -3,8 +3,12 @@ package docker_kurtosis_backend import ( "context" "encoding/json" + "strings" + "time" + "github.com/docker/docker/api/types/volume" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" @@ -14,8 +18,6 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" - "strings" - "time" ) const ( @@ -162,9 +164,23 @@ func (backend *DockerKurtosisBackend) CreateEnclave(ctx context.Context, enclave } }() + if err := backend.ConnectReverseProxyToNetwork(ctx, networkId); err != nil { + return nil, stacktrace.Propagate(err, "An error occurred connecting the reverse proxy to the enclave network with ID '%v'", networkId) + } + shouldDisconnectReverseProxyFromNetwork := true + defer func() { + if shouldDisconnectReverseProxyFromNetwork { + err = backend.DisconnectReverseProxyFromNetwork(ctx, networkId) + if err != nil { + logrus.Errorf("Couldn't disconnect the reverse proxy from the enclave network with ID '%v'", networkId) + } + } + }() + shouldDeleteLogsCollector = false shouldDeleteNetwork = false shouldDeleteVolume = false + shouldDisconnectReverseProxyFromNetwork = false return newEnclave, nil } @@ -374,15 +390,34 @@ func (backend *DockerKurtosisBackend) DestroyEnclaves( erroredEnclaveUuids[enclaveUuid] = volumeRemovalErr } + // Disconnect the reverse proxy from the networks + networksToDisconnect := map[enclave.EnclaveUUID]string{} + for enclaveUuid := range successfulVolumeRemovalEnclaveUuids { + networkInfo, found := matchingNetworkInfo[enclaveUuid] + if !found { + return nil, nil, stacktrace.NewError("Would have attempted to disconnect enclave network '%v' that didn't match the filters", enclaveUuid) + } + networksToDisconnect[enclaveUuid] = networkInfo.dockerNetwork.GetId() + } + + successfulDisconnectReverseProxyFromNetworkEnclaveUuids, erroredDisconnectReverseProxyFromNetworkEnclaveUuids, err := reverse_proxy_functions.DisconnectReverseProxyFromEnclaveNetworks(ctx, backend.dockerManager, networksToDisconnect) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred disconnecting the reverse proxy from the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids) + } + for enclaveUuid, networkDisconnectErr := range erroredDisconnectReverseProxyFromNetworkEnclaveUuids { + erroredEnclaveUuids[enclaveUuid] = networkDisconnectErr + } + // Remove the networks networksToDestroy := map[enclave.EnclaveUUID]string{} - for enclaveUuid := range successfulVolumeRemovalEnclaveUuids { + for enclaveUuid := range successfulDisconnectReverseProxyFromNetworkEnclaveUuids { networkInfo, found := matchingNetworkInfo[enclaveUuid] if !found { return nil, nil, stacktrace.NewError("Would have attempted to destroy enclave '%v' that didn't match the filters", enclaveUuid) } networksToDestroy[enclaveUuid] = networkInfo.dockerNetwork.GetId() } + successfulNetworkRemovalEnclaveUuids, erroredNetworkRemovalEnclaveUuids, err := destroyEnclaveNetworks(ctx, backend.dockerManager, networksToDestroy) if err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred destroying the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go new file mode 100644 index 0000000000..e3261de577 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go @@ -0,0 +1,106 @@ +package reverse_proxy_functions + +import ( + "context" + "net" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_operation_parallelizer" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" +) + +const ( + emptyAliasForLogsCollector = "" +) + +var ( + autoAssignIpAddressToLogsCollector net.IP = nil +) + +func ConnectReverseProxyToNetwork(ctx context.Context, dockerManager *docker_manager.DockerManager, networkId string) error { + _, maybeReverseProxyContainerId, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + logrus.Warnf("Attempted to connect reverse proxy to a network but no reverse proxy container was found.") + return nil + } + + if maybeReverseProxyContainerId == "" { + return nil + } + + if err = dockerManager.ConnectContainerToNetwork(ctx, networkId, maybeReverseProxyContainerId, autoAssignIpAddressToLogsCollector, emptyAliasForLogsCollector); err != nil { + return stacktrace.Propagate(err, "An error occurred while connecting container '%v' to the enclave network '%v'", maybeReverseProxyContainerId, networkId) + } + + return nil +} + +func DisconnectReverseProxyFromNetwork(ctx context.Context, dockerManager *docker_manager.DockerManager, networkId string) error { + _, maybeReverseProxyContainerId, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + logrus.Warnf("Attempted to disconnect reverse proxy from a network but no reverse proxy container was found.") + return nil + } + + if maybeReverseProxyContainerId == "" { + return nil + } + + if err = dockerManager.DisconnectContainerFromNetwork(ctx, maybeReverseProxyContainerId, networkId); err != nil { + return stacktrace.Propagate(err, "An error occurred while disconnecting container '%v' from the enclave network '%v'", maybeReverseProxyContainerId, networkId) + } + + return nil +} + +func DisconnectReverseProxyFromEnclaveNetworks( + ctx context.Context, + dockerManager *docker_manager.DockerManager, + enclaveNetworkIds map[enclave.EnclaveUUID]string, +) ( + map[enclave.EnclaveUUID]bool, + map[enclave.EnclaveUUID]error, + error, +) { + networkIdsToRemove := map[string]bool{} + enclaveUuidsForNetworkIds := map[string]enclave.EnclaveUUID{} + for enclaveUuid, networkId := range enclaveNetworkIds { + networkIdsToRemove[networkId] = true + enclaveUuidsForNetworkIds[networkId] = enclaveUuid + } + + var disconnectNetworkOperation docker_operation_parallelizer.DockerOperation = func(ctx context.Context, dockerManager *docker_manager.DockerManager, dockerObjectId string) error { + if err := DisconnectReverseProxyFromNetwork(ctx, dockerManager, dockerObjectId); err != nil { + return stacktrace.Propagate(err, "An error occurred disconnecting the reverse proxy from the enclave network with ID '%v'", dockerObjectId) + } + return nil + } + + successfulNetworkIds, erroredNetworkIds := docker_operation_parallelizer.RunDockerOperationInParallel( + ctx, + networkIdsToRemove, + dockerManager, + disconnectNetworkOperation, + ) + + successfulEnclaveUuids := map[enclave.EnclaveUUID]bool{} + for networkId := range successfulNetworkIds { + enclaveUuid, found := enclaveUuidsForNetworkIds[networkId] + if !found { + return nil, nil, stacktrace.NewError("The reverse proxy was successfully disconnected from the Docker network '%v', but wasn't requested to be disconnected", networkId) + } + successfulEnclaveUuids[enclaveUuid] = true + } + + erroredEnclaveUuids := map[enclave.EnclaveUUID]error{} + for networkId, networkRemovalErr := range erroredNetworkIds { + enclaveUuid, found := enclaveUuidsForNetworkIds[networkId] + if !found { + return nil, nil, stacktrace.NewError("Docker network '%v' had the following error during disconnect, but wasn't requested to be disconnected:\n%v", networkId, networkRemovalErr) + } + erroredEnclaveUuids[enclaveUuid] = networkRemovalErr + } + return successfulEnclaveUuids, erroredEnclaveUuids, nil +} diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go index 21f839f42f..ee58651bfd 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go @@ -18,6 +18,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" @@ -459,6 +460,23 @@ func (backend *KubernetesKurtosisBackend) DestroyLogsCollectorForEnclave(ctx con return stacktrace.NewError("Destroy the logs collector for enclave isn't yet implemented on Kubernetes") } +func (backend *KubernetesKurtosisBackend) GetReverseProxy( + ctx context.Context, +) (*reverse_proxy.ReverseProxy, error) { + // TODO IMPLEMENT + return nil, stacktrace.NewError("Getting the reverse proxy isn't yet implemented on Kubernetes") +} + +func (backend *KubernetesKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + // TODO IMPLEMENT + return nil, stacktrace.NewError("Creating the reverse proxy isn't yet implemented on Kubernetes") +} + +func (backend *KubernetesKurtosisBackend) DestroyReverseProxy(ctx context.Context) error { + // TODO IMPLEMENT + return stacktrace.NewError("Destroying the reverse proxy isn't yet implemented on Kubernetes") +} + // ==================================================================================================== // // Private Helper Functions diff --git a/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go index a41bb4b2c6..b55546a999 100644 --- a/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go @@ -14,6 +14,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/stacktrace" ) @@ -438,6 +439,18 @@ func (backend *MetricsReportingKurtosisBackend) DestroyLogsCollectorForEnclave(c return nil } +func (backend *MetricsReportingKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + return backend.underlying.CreateReverseProxy(ctx) +} + +func (backend *MetricsReportingKurtosisBackend) GetReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + return backend.underlying.GetReverseProxy(ctx) +} + +func (backend *MetricsReportingKurtosisBackend) DestroyReverseProxy(ctx context.Context) error { + return backend.underlying.DestroyReverseProxy(ctx) +} + func (backend *MetricsReportingKurtosisBackend) GetAvailableCPUAndMemory(ctx context.Context) (compute_resources.MemoryInMegaBytes, compute_resources.CpuMilliCores, bool, error) { availableMemory, availableCpu, isResourceInformationComplete, err := backend.underlying.GetAvailableCPUAndMemory(ctx) if err != nil { diff --git a/container-engine-lib/lib/backend_interface/kurtosis_backend.go b/container-engine-lib/lib/backend_interface/kurtosis_backend.go index ab6fc98e47..a0a1be1889 100644 --- a/container-engine-lib/lib/backend_interface/kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/kurtosis_backend.go @@ -13,6 +13,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" ) @@ -337,6 +338,13 @@ type KurtosisBackend interface { // Destroy the logs collector for enclave with UUID DestroyLogsCollectorForEnclave(ctx context.Context, enclaveUuid enclave.EnclaveUUID) error + CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) + + // Returns nil if logs aggregator was not found + GetReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) + + DestroyReverseProxy(ctx context.Context) error + // GetAvailableCPUAndMemory - gets available memory in megabytes and cpu in millicores, the boolean indicates whether the information is complete GetAvailableCPUAndMemory(ctx context.Context) (compute_resources.MemoryInMegaBytes, compute_resources.CpuMilliCores, bool, error) } diff --git a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go index 29fd97ae02..bd0bdf1f05 100644 --- a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.26.1. DO NOT EDIT. package backend_interface @@ -24,6 +24,8 @@ import ( mock "github.com/stretchr/testify/mock" + reverse_proxy "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" + service "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" time "time" @@ -373,6 +375,60 @@ func (_c *MockKurtosisBackend_CreateLogsCollectorForEnclave_Call) RunAndReturn(r return _c } +// CreateReverseProxy provides a mock function with given fields: ctx +func (_m *MockKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + ret := _m.Called(ctx) + + var r0 *reverse_proxy.ReverseProxy + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*reverse_proxy.ReverseProxy, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *reverse_proxy.ReverseProxy); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*reverse_proxy.ReverseProxy) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockKurtosisBackend_CreateReverseProxy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateReverseProxy' +type MockKurtosisBackend_CreateReverseProxy_Call struct { + *mock.Call +} + +// CreateReverseProxy is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockKurtosisBackend_Expecter) CreateReverseProxy(ctx interface{}) *MockKurtosisBackend_CreateReverseProxy_Call { + return &MockKurtosisBackend_CreateReverseProxy_Call{Call: _e.mock.On("CreateReverseProxy", ctx)} +} + +func (_c *MockKurtosisBackend_CreateReverseProxy_Call) Run(run func(ctx context.Context)) *MockKurtosisBackend_CreateReverseProxy_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockKurtosisBackend_CreateReverseProxy_Call) Return(_a0 *reverse_proxy.ReverseProxy, _a1 error) *MockKurtosisBackend_CreateReverseProxy_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockKurtosisBackend_CreateReverseProxy_Call) RunAndReturn(run func(context.Context) (*reverse_proxy.ReverseProxy, error)) *MockKurtosisBackend_CreateReverseProxy_Call { + _c.Call.Return(run) + return _c +} + // DestroyAPIContainers provides a mock function with given fields: ctx, filters func (_m *MockKurtosisBackend) DestroyAPIContainers(ctx context.Context, filters *api_container.APIContainerFilters) (map[enclave.EnclaveUUID]bool, map[enclave.EnclaveUUID]error, error) { ret := _m.Called(ctx, filters) @@ -650,6 +706,48 @@ func (_c *MockKurtosisBackend_DestroyLogsCollectorForEnclave_Call) RunAndReturn( return _c } +// DestroyReverseProxy provides a mock function with given fields: ctx +func (_m *MockKurtosisBackend) DestroyReverseProxy(ctx context.Context) error { + ret := _m.Called(ctx) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context) error); ok { + r0 = rf(ctx) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockKurtosisBackend_DestroyReverseProxy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DestroyReverseProxy' +type MockKurtosisBackend_DestroyReverseProxy_Call struct { + *mock.Call +} + +// DestroyReverseProxy is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockKurtosisBackend_Expecter) DestroyReverseProxy(ctx interface{}) *MockKurtosisBackend_DestroyReverseProxy_Call { + return &MockKurtosisBackend_DestroyReverseProxy_Call{Call: _e.mock.On("DestroyReverseProxy", ctx)} +} + +func (_c *MockKurtosisBackend_DestroyReverseProxy_Call) Run(run func(ctx context.Context)) *MockKurtosisBackend_DestroyReverseProxy_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockKurtosisBackend_DestroyReverseProxy_Call) Return(_a0 error) *MockKurtosisBackend_DestroyReverseProxy_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockKurtosisBackend_DestroyReverseProxy_Call) RunAndReturn(run func(context.Context) error) *MockKurtosisBackend_DestroyReverseProxy_Call { + _c.Call.Return(run) + return _c +} + // DestroyUserServices provides a mock function with given fields: ctx, enclaveUuid, filters func (_m *MockKurtosisBackend) DestroyUserServices(ctx context.Context, enclaveUuid enclave.EnclaveUUID, filters *service.ServiceFilters) (map[service.ServiceUUID]bool, map[service.ServiceUUID]error, error) { ret := _m.Called(ctx, enclaveUuid, filters) @@ -1246,6 +1344,60 @@ func (_c *MockKurtosisBackend_GetLogsCollectorForEnclave_Call) RunAndReturn(run return _c } +// GetReverseProxy provides a mock function with given fields: ctx +func (_m *MockKurtosisBackend) GetReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + ret := _m.Called(ctx) + + var r0 *reverse_proxy.ReverseProxy + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*reverse_proxy.ReverseProxy, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *reverse_proxy.ReverseProxy); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*reverse_proxy.ReverseProxy) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockKurtosisBackend_GetReverseProxy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetReverseProxy' +type MockKurtosisBackend_GetReverseProxy_Call struct { + *mock.Call +} + +// GetReverseProxy is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockKurtosisBackend_Expecter) GetReverseProxy(ctx interface{}) *MockKurtosisBackend_GetReverseProxy_Call { + return &MockKurtosisBackend_GetReverseProxy_Call{Call: _e.mock.On("GetReverseProxy", ctx)} +} + +func (_c *MockKurtosisBackend_GetReverseProxy_Call) Run(run func(ctx context.Context)) *MockKurtosisBackend_GetReverseProxy_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockKurtosisBackend_GetReverseProxy_Call) Return(_a0 *reverse_proxy.ReverseProxy, _a1 error) *MockKurtosisBackend_GetReverseProxy_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockKurtosisBackend_GetReverseProxy_Call) RunAndReturn(run func(context.Context) (*reverse_proxy.ReverseProxy, error)) *MockKurtosisBackend_GetReverseProxy_Call { + _c.Call.Return(run) + return _c +} + // GetShellOnUserService provides a mock function with given fields: ctx, enclaveUuid, serviceUuid func (_m *MockKurtosisBackend) GetShellOnUserService(ctx context.Context, enclaveUuid enclave.EnclaveUUID, serviceUuid service.ServiceUUID) error { ret := _m.Called(ctx, enclaveUuid, serviceUuid) From 94d6d9bef7759adc579c92f9994a36905e155b74 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Sun, 3 Dec 2023 12:52:40 -0800 Subject: [PATCH 05/35] Add the reverse proxy enclave network ip address to the list of already taken ips so there is no conflict with the APIC or user service. --- .../backend_creator/backend_creator.go | 11 +++ ...urtosis_backend_api_container_functions.go | 15 +++- ...cker_kurtosis_backend_enclave_functions.go | 78 ++++++++++++++++--- .../network_reverse_proxy.go | 58 +------------- .../reverse_proxy_functions/shared_helpers.go | 21 ++++- .../docker/docker_manager/docker_manager.go | 33 +++++++- .../objects/reverse_proxy/reverse_proxy.go | 20 +++-- 7 files changed, 162 insertions(+), 74 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go index 7695f72548..519c46a1a2 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go @@ -6,6 +6,7 @@ import ( "github.com/docker/docker/client" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts" @@ -238,11 +239,21 @@ func getDockerKurtosisBackend( return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector object for enclave '%v'; This is a bug in Kurtosis", enclaveUuid) } + reverseProxy, err := reverse_proxy_functions.GetReverseProxy(ctx, dockerManager) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while getting the reverse proxy, This is a bug in Kurtosis") + } + reverseProxyEnclaveNetworkIpAddress, found := reverseProxy.GetEnclaveNetworksIpAddress()[network.GetId()] + if !found { + return nil, stacktrace.NewError("An error occured while getting the reverse proxy enclave network IP address for enclave '%v', This is a bug in Kurtosis", enclaveUuid) + } + alreadyTakenIps := map[string]bool{ networkIp.String(): true, network.GetGatewayIp(): true, apiContainerIp.String(): true, logsCollectorObj.GetEnclaveNetworkIpAddress().String(): true, + reverseProxyEnclaveNetworkIpAddress.String(): true, } freeIpAddrProvider, err := free_ip_addr_tracker.GetOrCreateNewFreeIpAddrTracker( diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go index b79a98bca7..31ab5ae664 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go @@ -2,10 +2,11 @@ package docker_kurtosis_backend import ( "context" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" "net" "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" + "github.com/docker/go-connections/nat" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" @@ -73,7 +74,16 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( enclaveLogsCollector, err := backend.GetLogsCollectorForEnclave(ctx, enclaveUuid) if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector for enclave '%v; This is a bug in Kurtosis'", enclaveUuid) + return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector for enclave '%v'; This is a bug in Kurtosis", enclaveUuid) + } + + reverseProxy, err := backend.GetReverseProxy(ctx) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while getting the reverse proxy, This is a bug in Kurtosis") + } + reverseProxyEnclaveNetworkIpAddress, found := reverseProxy.GetEnclaveNetworksIpAddress()[enclaveNetwork.GetId()] + if !found { + return nil, stacktrace.NewError("An error occured while getting the reverse proxy enclave network IP address for enclave '%v', This is a bug in Kurtosis", enclaveUuid) } networkCidr := enclaveNetwork.GetIpAndMask() @@ -81,6 +91,7 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( networkCidr.IP.String(): true, enclaveNetwork.GetGatewayIp(): true, enclaveLogsCollector.GetEnclaveNetworkIpAddress().String(): true, + reverseProxyEnclaveNetworkIpAddress.String(): true, } ipAddr, err := network_helpers.GetFreeIpAddrFromSubnet(alreadyTakenIps, networkCidr) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go index f676005123..72902d9b4f 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go @@ -8,7 +8,6 @@ import ( "github.com/docker/docker/api/types/volume" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" @@ -21,9 +20,9 @@ import ( ) const ( - shouldFetchStoppedContainersWhenGettingEnclaveStatus = true - - shouldFetchStoppedContainersWhenDumpingEnclave = true + shouldFetchStoppedContainersWhenGettingEnclaveStatus = true + shouldFetchStoppedContainersWhenDumpingEnclave = true + shouldFetchStoppedContainersWhenDisconnectingFromEnclaveNetworks = false defaultHttpLogsCollectorPortNum = uint16(9712) defaultTcpLogsCollectorPortNum = uint16(9713) @@ -118,7 +117,7 @@ func (backend *DockerKurtosisBackend) CreateEnclave(ctx context.Context, enclave } } }() - logrus.Debugf("Docker network '%v' created successfully with ID '%v'", enclaveUuid, networkId) + logrus.Debugf("Docker network for enclave '%v' created successfully with ID '%v'", enclaveUuid, networkId) enclaveDataVolumeNameStr := enclaveDataVolumeAttrs.GetName().GetString() enclaveDataVolumeLabelStrs := map[string]string{} @@ -400,17 +399,17 @@ func (backend *DockerKurtosisBackend) DestroyEnclaves( networksToDisconnect[enclaveUuid] = networkInfo.dockerNetwork.GetId() } - successfulDisconnectReverseProxyFromNetworkEnclaveUuids, erroredDisconnectReverseProxyFromNetworkEnclaveUuids, err := reverse_proxy_functions.DisconnectReverseProxyFromEnclaveNetworks(ctx, backend.dockerManager, networksToDisconnect) + successfulDisconnectContainersFromNetworkEnclaveUuids, erroredDisconnectContainersFromNetworkEnclaveUuids, err := backend.disconnectContainersFromEnclaveNetworks(ctx, backend.dockerManager, networksToDisconnect) if err != nil { - return nil, nil, stacktrace.Propagate(err, "An error occurred disconnecting the reverse proxy from the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids) + return nil, nil, stacktrace.Propagate(err, "An error occurred disconnecting the containers from the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids) } - for enclaveUuid, networkDisconnectErr := range erroredDisconnectReverseProxyFromNetworkEnclaveUuids { + for enclaveUuid, networkDisconnectErr := range erroredDisconnectContainersFromNetworkEnclaveUuids { erroredEnclaveUuids[enclaveUuid] = networkDisconnectErr } // Remove the networks networksToDestroy := map[enclave.EnclaveUUID]string{} - for enclaveUuid := range successfulDisconnectReverseProxyFromNetworkEnclaveUuids { + for enclaveUuid := range successfulDisconnectContainersFromNetworkEnclaveUuids { networkInfo, found := matchingNetworkInfo[enclaveUuid] if !found { return nil, nil, stacktrace.NewError("Would have attempted to destroy enclave '%v' that didn't match the filters", enclaveUuid) @@ -556,6 +555,67 @@ func (backend *DockerKurtosisBackend) getAllEnclaveContainers( return containers, nil } +func (backend *DockerKurtosisBackend) disconnectContainersFromEnclaveNetworks( + ctx context.Context, + dockerManager *docker_manager.DockerManager, + enclaveNetworkIds map[enclave.EnclaveUUID]string, +) ( + map[enclave.EnclaveUUID]bool, + map[enclave.EnclaveUUID]error, + error, +) { + networkIdsToRemove := map[string]bool{} + enclaveUuidsForNetworkIds := map[string]enclave.EnclaveUUID{} + for enclaveUuid, networkId := range enclaveNetworkIds { + networkIdsToRemove[networkId] = true + enclaveUuidsForNetworkIds[networkId] = enclaveUuid + } + + var disconnectNetworkOperation docker_operation_parallelizer.DockerOperation = func(ctx context.Context, dockerManager *docker_manager.DockerManager, dockerObjectId string) error { + // Get containers connected to this network id (dockerObjectId here) + containers, err := backend.dockerManager.GetContainersByNetworkId(ctx, dockerObjectId, shouldFetchStoppedContainersWhenDisconnectingFromEnclaveNetworks) + if err != nil { + return stacktrace.Propagate( + err, + "An error occurred getting the containers with enclave network '%v'", + dockerObjectId, + ) + } + for _, container := range containers { + if err = dockerManager.DisconnectContainerFromNetwork(ctx, container.GetId(), dockerObjectId); err != nil { + return stacktrace.Propagate(err, "An error occurred while disconnecting container '%v' from the enclave network '%v'", container.GetId(), dockerObjectId) + } + } + return nil + } + + successfulNetworkIds, erroredNetworkIds := docker_operation_parallelizer.RunDockerOperationInParallel( + ctx, + networkIdsToRemove, + dockerManager, + disconnectNetworkOperation, + ) + + successfulEnclaveUuids := map[enclave.EnclaveUUID]bool{} + for networkId := range successfulNetworkIds { + enclaveUuid, found := enclaveUuidsForNetworkIds[networkId] + if !found { + return nil, nil, stacktrace.NewError("The containers were successfully disconnected from the enclave network '%v', but wasn't requested to be disconnected", networkId) + } + successfulEnclaveUuids[enclaveUuid] = true + } + + erroredEnclaveUuids := map[enclave.EnclaveUUID]error{} + for networkId, networkRemovalErr := range erroredNetworkIds { + enclaveUuid, found := enclaveUuidsForNetworkIds[networkId] + if !found { + return nil, nil, stacktrace.NewError("Docker network '%v' had the following error during disconnect, but wasn't requested to be disconnected:\n%v", networkId, networkRemovalErr) + } + erroredEnclaveUuids[enclaveUuid] = networkRemovalErr + } + return successfulEnclaveUuids, erroredEnclaveUuids, nil +} + func getAllEnclaveVolumes( ctx context.Context, dockerManager *docker_manager.DockerManager, diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go index e3261de577..1a74fcbebc 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go @@ -5,18 +5,16 @@ import ( "net" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_operation_parallelizer" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" ) const ( - emptyAliasForLogsCollector = "" + emptyAliasForReverseProxy = "" ) var ( - autoAssignIpAddressToLogsCollector net.IP = nil + autoAssignIpAddressToReverseProxy net.IP = nil ) func ConnectReverseProxyToNetwork(ctx context.Context, dockerManager *docker_manager.DockerManager, networkId string) error { @@ -30,7 +28,7 @@ func ConnectReverseProxyToNetwork(ctx context.Context, dockerManager *docker_man return nil } - if err = dockerManager.ConnectContainerToNetwork(ctx, networkId, maybeReverseProxyContainerId, autoAssignIpAddressToLogsCollector, emptyAliasForLogsCollector); err != nil { + if err = dockerManager.ConnectContainerToNetwork(ctx, networkId, maybeReverseProxyContainerId, autoAssignIpAddressToReverseProxy, emptyAliasForReverseProxy); err != nil { return stacktrace.Propagate(err, "An error occurred while connecting container '%v' to the enclave network '%v'", maybeReverseProxyContainerId, networkId) } @@ -54,53 +52,3 @@ func DisconnectReverseProxyFromNetwork(ctx context.Context, dockerManager *docke return nil } - -func DisconnectReverseProxyFromEnclaveNetworks( - ctx context.Context, - dockerManager *docker_manager.DockerManager, - enclaveNetworkIds map[enclave.EnclaveUUID]string, -) ( - map[enclave.EnclaveUUID]bool, - map[enclave.EnclaveUUID]error, - error, -) { - networkIdsToRemove := map[string]bool{} - enclaveUuidsForNetworkIds := map[string]enclave.EnclaveUUID{} - for enclaveUuid, networkId := range enclaveNetworkIds { - networkIdsToRemove[networkId] = true - enclaveUuidsForNetworkIds[networkId] = enclaveUuid - } - - var disconnectNetworkOperation docker_operation_parallelizer.DockerOperation = func(ctx context.Context, dockerManager *docker_manager.DockerManager, dockerObjectId string) error { - if err := DisconnectReverseProxyFromNetwork(ctx, dockerManager, dockerObjectId); err != nil { - return stacktrace.Propagate(err, "An error occurred disconnecting the reverse proxy from the enclave network with ID '%v'", dockerObjectId) - } - return nil - } - - successfulNetworkIds, erroredNetworkIds := docker_operation_parallelizer.RunDockerOperationInParallel( - ctx, - networkIdsToRemove, - dockerManager, - disconnectNetworkOperation, - ) - - successfulEnclaveUuids := map[enclave.EnclaveUUID]bool{} - for networkId := range successfulNetworkIds { - enclaveUuid, found := enclaveUuidsForNetworkIds[networkId] - if !found { - return nil, nil, stacktrace.NewError("The reverse proxy was successfully disconnected from the Docker network '%v', but wasn't requested to be disconnected", networkId) - } - successfulEnclaveUuids[enclaveUuid] = true - } - - erroredEnclaveUuids := map[enclave.EnclaveUUID]error{} - for networkId, networkRemovalErr := range erroredNetworkIds { - enclaveUuid, found := enclaveUuidsForNetworkIds[networkId] - if !found { - return nil, nil, stacktrace.NewError("Docker network '%v' had the following error during disconnect, but wasn't requested to be disconnected:\n%v", networkId, networkRemovalErr) - } - erroredEnclaveUuids[enclaveUuid] = networkRemovalErr - } - return successfulEnclaveUuids, erroredEnclaveUuids, nil -} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go index 079e30e071..f0b497b252 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go @@ -12,6 +12,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" ) const ( @@ -25,7 +26,7 @@ func getReverseProxyObjectAndContainerId( ) (*reverse_proxy.ReverseProxy, string, error) { reverseProxyContainer, found, err := getReverseProxyContainer(ctx, dockerManager) if err != nil { - return nil, "", stacktrace.Propagate(err, "An error occurred getting all reverse proxy containers") + return nil, "", stacktrace.Propagate(err, "An error occurred getting the reverse proxy container") } if !found { return nil, "", nil @@ -74,6 +75,7 @@ func getReverseProxyObjectFromContainerInfo( dockerManager *docker_manager.DockerManager, ) (*reverse_proxy.ReverseProxy, error) { var privateIpAddr net.IP + var enclaveNetworksIpAddress map[string]net.IP isContainerRunning, found := consts.IsContainerRunningDeterminer[containerStatus] if !found { @@ -93,6 +95,22 @@ func getReverseProxyObjectFromContainerInfo( if privateIpAddr == nil { return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", privateIpAddrStr) } + + networksIpAddressStr, err := dockerManager.GetContainerIps(ctx, containerId) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the networks private IP address of container '%v'", containerId) + } + enclaveNetworksIpAddress = map[string]net.IP{} + for networkId, networkIpAddressStr := range networksIpAddressStr { + if networkIpAddressStr != privateIpAddrStr { + networkIpAddress := net.ParseIP(networkIpAddressStr) + if networkIpAddress == nil { + return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", networkIpAddress) + } + enclaveNetworksIpAddress[networkId] = networkIpAddress + } + } + logrus.Debugf("Enclave networks: '%v'", enclaveNetworksIpAddress) } else { reverseProxyStatus = container.ContainerStatus_Stopped } @@ -100,6 +118,7 @@ func getReverseProxyObjectFromContainerInfo( reverseProxyObj := reverse_proxy.NewReverseProxy( reverseProxyStatus, privateIpAddr, + enclaveNetworksIpAddress, defaultReverseProxyHttpPortNum, defaultReverseProxyDashboardPortNum, ) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index f550a5e2e7..8288603961 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -73,8 +73,9 @@ const ( // ------------------ Filter Search Keys ---------------------- // All these defined in https://docs.docker.com/engine/api/v1.24 - containerNameSearchFilterKey = "name" - containerLabelSearchFilterKey = "label" + containerNameSearchFilterKey = "name" + containerLabelSearchFilterKey = "label" + containerNetworkIdSearchFilterKey = "network" volumeNameSearchFilterKey = "name" volumeLabelSearchFilterKey = "label" @@ -777,6 +778,24 @@ func (manager *DockerManager) GetContainerIP(ctx context.Context, networkName st return networkInfo.IPAddress, nil } +/* +GetContainerIps +Gets the container's IPs on all networks +Returns a map of network ID : network IP address +*/ +func (manager *DockerManager) GetContainerIps(ctx context.Context, containerId string) (map[string]string, error) { + containerIps := map[string]string{} + resp, err := manager.dockerClient.ContainerInspect(ctx, containerId) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred inspecting container with ID '%v'", containerId) + } + allNetworkInfo := resp.NetworkSettings.Networks + for _, networkInfo := range allNetworkInfo { + containerIps[networkInfo.NetworkID]= networkInfo.IPAddress + } + return containerIps, nil +} + func (manager *DockerManager) AttachToContainer(ctx context.Context, containerId string) (types.HijackedResponse, error) { attachOpts := types.ContainerAttachOptions{ Stream: true, @@ -1184,6 +1203,16 @@ func (manager *DockerManager) GetContainersByLabels(ctx context.Context, labels return result, nil } +func (manager *DockerManager) GetContainersByNetworkId(ctx context.Context, networkId string, shouldShowStoppedContainers bool) ([]*docker_manager_types.Container, error) { + filterArg := filters.Arg(containerNetworkIdSearchFilterKey, networkId) + networkIdFilterList := filters.NewArgs(filterArg) + result, err := manager.getContainersByFilterArgs(ctx, networkIdFilterList, shouldShowStoppedContainers) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting containers with network id '%+v'", networkIdFilterList) + } + return result, nil +} + // [FetchImageIfMissing] uses the local [dockerImage] if it's available. // If unavailable, will attempt to fetch the latest image. // Returns error if local [dockerImage] is unavailable and pulling image fails. diff --git a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go index ae2821313c..7a6a466a3c 100644 --- a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go +++ b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go @@ -1,8 +1,9 @@ package reverse_proxy import ( - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" "net" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" ) // This component is responsible for routing http traffic to the services @@ -12,6 +13,9 @@ type ReverseProxy struct { // This will be nil if the container is not running maybePrivateIpAddr net.IP + // IP address for each enclave network ID + maybeEnclaveNetworksIpAddress map[string]net.IP + // HTTP port httpPort uint16 @@ -22,13 +26,15 @@ type ReverseProxy struct { func NewReverseProxy( status container.ContainerStatus, maybePrivateIpAddr net.IP, + maybeEnclaveNetworksIpAddress map[string]net.IP, httpPort uint16, dashboardPort uint16) *ReverseProxy { return &ReverseProxy{ - status: status, - maybePrivateIpAddr: maybePrivateIpAddr, - httpPort: httpPort, - dashboardPort: dashboardPort, + status: status, + maybePrivateIpAddr: maybePrivateIpAddr, + maybeEnclaveNetworksIpAddress: maybeEnclaveNetworksIpAddress, + httpPort: httpPort, + dashboardPort: dashboardPort, } } @@ -40,6 +46,10 @@ func (reverseProxy *ReverseProxy) GetMaybePrivateIpAddr() net.IP { return reverseProxy.maybePrivateIpAddr } +func (reverseProxy *ReverseProxy) GetEnclaveNetworksIpAddress() map[string]net.IP { + return reverseProxy.maybeEnclaveNetworksIpAddress +} + func (reverseProxy *ReverseProxy) GetHttpPort() uint16 { return reverseProxy.httpPort } From e5f396d64deb5b7b41d205e2b86306a7f3009fd0 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Sun, 3 Dec 2023 17:47:32 -0800 Subject: [PATCH 06/35] Connect reverse proxy to enclave networks at startup. --- .../engine_functions/create_engine.go | 3 +++ .../implementations/traefik/consts.go | 4 ++++ .../network_reverse_proxy.go | 20 +++++++++++++++++++ .../reverse_proxy_functions/shared_helpers.go | 5 +++-- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index 58c52ce28f..6bd6cb7a06 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -128,6 +128,9 @@ func CreateEngine( removeReverseProxyFunc() } }() + if err = reverse_proxy_functions.ConnectReverseProxyToEnclaveNetworks(ctx, dockerManager); err != nil { + return nil, stacktrace.Propagate(err, "An error occured connecting the reverse proxy to the enclave networks") + } logrus.Infof("Reverse proxy started.") enclaveManagerUIPortSpec, err := port_spec.NewPortSpec(uint16(enclaveManagerUIPort), consts.EngineTransportProtocol, consts.HttpApplicationProtocol, defaultWait) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go index 78f8571cf8..7df8087c81 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -11,7 +11,11 @@ const ( ////////////////////////--TRAEFIK CONFIGURATION SECTION--///////////////////////////// configFileTemplate = ` +accesslog: {} +log: + level: DEBUG api: + debug: true dashboard: true insecure: true disabledashboardad: true diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go index 1a74fcbebc..fb8263f0a3 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go @@ -5,6 +5,8 @@ import ( "net" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" ) @@ -52,3 +54,21 @@ func DisconnectReverseProxyFromNetwork(ctx context.Context, dockerManager *docke return nil } + +func ConnectReverseProxyToEnclaveNetworks(ctx context.Context, dockerManager *docker_manager.DockerManager) error { + kurtosisNetworkLabels := map[string]string{ + docker_label_key.AppIDDockerLabelKey.GetString(): label_value_consts.AppIDDockerLabelValue.GetString(), + } + enclaveNetworks, err := dockerManager.GetNetworksByLabels(ctx, kurtosisNetworkLabels) + if err != nil { + return stacktrace.Propagate(err, "An error occurred getting Kurtosis networks") + } + + for _, enclaveNetwork := range enclaveNetworks { + if err = ConnectReverseProxyToNetwork(ctx, dockerManager, enclaveNetwork.GetId()); err != nil { + return stacktrace.Propagate(err, "An error occurred connecting the reverse proxy to the enclave network with id '%v'", enclaveNetwork.GetId()) + } + } + + return nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go index f0b497b252..1399189714 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go @@ -2,9 +2,10 @@ package reverse_proxy_functions import ( "context" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" "net" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" @@ -95,7 +96,7 @@ func getReverseProxyObjectFromContainerInfo( if privateIpAddr == nil { return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", privateIpAddrStr) } - + networksIpAddressStr, err := dockerManager.GetContainerIps(ctx, containerId) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting the networks private IP address of container '%v'", containerId) From 005f2a933b736179b48733beeef76582524aacf5 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Mon, 4 Dec 2023 15:11:44 -0800 Subject: [PATCH 07/35] Update Traefik port numbers. --- .../docker_kurtosis_backend/reverse_proxy_functions/consts.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go index 50c3144bab..71c596a5ee 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go @@ -1,6 +1,6 @@ package reverse_proxy_functions const ( - defaultReverseProxyHttpPortNum = uint16(9720) - defaultReverseProxyDashboardPortNum = uint16(9721) + defaultReverseProxyHttpPortNum = uint16(9730) + defaultReverseProxyDashboardPortNum = uint16(9731) ) From 1b2d3738c4ab2b3939bf38eadab5ee2865afc073 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Tue, 5 Dec 2023 10:46:56 -0800 Subject: [PATCH 08/35] Update comments. --- .../docker_kurtosis_backend/docker_kurtosis_backend.go | 6 +++--- .../docker_kurtosis_backend_enclave_functions.go | 3 +-- .../implementations/traefik/consts.go | 6 ++++++ .../traefik/traefik_container_config_provider.go | 6 +++--- .../reverse_proxy_functions/network_reverse_proxy.go | 2 +- .../reverse_proxy_functions/shared_helpers.go | 2 -- .../backend_impls/docker/docker_manager/docker_manager.go | 2 +- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go index 0642430ee6..3ff2bca9c2 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go @@ -484,7 +484,7 @@ func (backend *DockerKurtosisBackend) CreateReverseProxy(ctx context.Context) (* backend.objAttrsProvider, ) if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred creating the logs aggregator using the logs aggregator container '%+v'.", reverseProxyContainer) + return nil, stacktrace.Propagate(err, "An error occurred creating the reverse proxy using the reverse proxy container '%+v'.", reverseProxyContainer) } return reverseProxy, nil } @@ -495,7 +495,7 @@ func (backend *DockerKurtosisBackend) GetReverseProxy(ctx context.Context) (*rev backend.dockerManager, ) if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred getting the logs aggregator") + return nil, stacktrace.Propagate(err, "An error occurred getting the reverse proxy") } return maybeReverseProxy, nil @@ -503,7 +503,7 @@ func (backend *DockerKurtosisBackend) GetReverseProxy(ctx context.Context) (*rev func (backend *DockerKurtosisBackend) DestroyReverseProxy(ctx context.Context) error { if err := reverse_proxy_functions.DestroyReverseProxy(ctx, backend.dockerManager); err != nil { - return stacktrace.Propagate(err, "An error occurred destroying the logs aggregator") + return stacktrace.Propagate(err, "An error occurred destroying the reverse proxy") } return nil diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go index a5f3d26a3b..333baf3ec5 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go @@ -372,7 +372,7 @@ func (backend *DockerKurtosisBackend) DestroyEnclaves( erroredEnclaveUuids[enclaveUuid] = volumeRemovalErr } - // Disconnect the reverse proxy from the networks + // Disconnect the containers from the enclave networks being removed networksToDisconnect := map[enclave.EnclaveUUID]string{} for enclaveUuid := range successfulVolumeRemovalEnclaveUuids { networkInfo, found := matchingNetworkInfo[enclaveUuid] @@ -381,7 +381,6 @@ func (backend *DockerKurtosisBackend) DestroyEnclaves( } networksToDisconnect[enclaveUuid] = networkInfo.dockerNetwork.GetId() } - successfulDisconnectContainersFromNetworkEnclaveUuids, erroredDisconnectContainersFromNetworkEnclaveUuids, err := backend.disconnectContainersFromEnclaveNetworks(ctx, backend.dockerManager, networksToDisconnect) if err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred disconnecting the containers from the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go index 7df8087c81..f2827253a0 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -17,19 +17,25 @@ log: api: debug: true dashboard: true + # api over the traefik endpoint insecure: true disabledashboardad: true entryPoints: + # http traffic web: address: ":{{ .HttpPort }}" + # API endpoint traefik: address: ":{{ .DashboardPort }}" providers: docker: endpoint: "unix:///var/run/docker.sock" + # we don't want the containers to be exposed by default. + # we are enabling Traefik at the container level instead. exposedByDefault: false + # Docker network to start Traefik in. network: "{{ .NetworkId }}" ` ////////////////////////--FINISH--TRAEFIK CONFIGURATION SECTION--///////////////////////////// diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go index 4246c22f4c..b9de2a05c9 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go @@ -36,7 +36,7 @@ func (traefik *traefikContainerConfigProvider) GetContainerArgs( ) (*docker_manager.CreateAndStartContainerArgs, error) { bindMounts := map[string]string{ - // Necessary so that the engine server can interact with the Docker engine + // Necessary so that the reverse proxy can interact with the Docker engine consts.DockerSocketFilepath: consts.DockerSocketFilepath, } @@ -70,7 +70,7 @@ func (traefik *traefikContainerConfigProvider) GetContainerArgs( return nil, stacktrace.Propagate(err, "An error occurred creating a wait with default values") } - // Publish HTTP and Dashboard ports + // Publish HTTP and Dashboard entrypoint ports privateHttpPortSpec, err := port_spec.NewPortSpec(httpPort, port_spec.TransportProtocol_TCP, consts.HttpApplicationProtocol, defaultWait) if err != nil { return nil, stacktrace.Propagate( @@ -95,7 +95,7 @@ func (traefik *traefikContainerConfigProvider) GetContainerArgs( } privateDashboardDockerPort, err := shared_helpers.TransformPortSpecToDockerPort(privateDashboardPortSpec) if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred transforming the private http port spec to a Docker port") + return nil, stacktrace.Propagate(err, "An error occurred transforming the private dashboard port spec to a Docker port") } usedPorts := map[nat.Port]docker_manager.PortPublishSpec{ privateHttpDockerPort: docker_manager.NewManualPublishingSpec(httpPort), diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go index fb8263f0a3..f638d86959 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go @@ -61,7 +61,7 @@ func ConnectReverseProxyToEnclaveNetworks(ctx context.Context, dockerManager *do } enclaveNetworks, err := dockerManager.GetNetworksByLabels(ctx, kurtosisNetworkLabels) if err != nil { - return stacktrace.Propagate(err, "An error occurred getting Kurtosis networks") + return stacktrace.Propagate(err, "An error occurred getting enclave networks") } for _, enclaveNetwork := range enclaveNetworks { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go index 1399189714..37aff5c97e 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go @@ -20,7 +20,6 @@ const ( shouldShowStoppedReverseProxyContainers = true ) -// Returns nil [ReverseProxy] object if no container is found func getReverseProxyObjectAndContainerId( ctx context.Context, dockerManager *docker_manager.DockerManager, @@ -48,7 +47,6 @@ func getReverseProxyObjectAndContainerId( return reverseProxyObject, reverseProxyContainerID, nil } -// Returns nil [Container] object and false if no reverse proxy container is found func getReverseProxyContainer(ctx context.Context, dockerManager *docker_manager.DockerManager) (*types.Container, bool, error) { reverseProxyContainerSearchLabels := map[string]string{ docker_label_key.AppIDDockerLabelKey.GetString(): label_value_consts.AppIDDockerLabelValue.GetString(), diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 8288603961..20fec381c3 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -781,7 +781,7 @@ func (manager *DockerManager) GetContainerIP(ctx context.Context, networkName st /* GetContainerIps Gets the container's IPs on all networks -Returns a map of network ID : network IP address +Returns a map of network ID to network IP address */ func (manager *DockerManager) GetContainerIps(ctx context.Context, containerId string) (map[string]string, error) { containerIps := map[string]string{} From ff2a518fce9af13eb3ed962b56fc18b5aa1a6da4 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Tue, 5 Dec 2023 15:07:47 -0800 Subject: [PATCH 09/35] Add reverse proxy circle ci test. --- .circleci/config.yml | 64 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6f1e2979e..28cbcd5f14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,14 +80,17 @@ steps_prepare_testing_k8s_k3s: &steps_prepare_testing_k8s_k3s command: "${KURTOSIS_BINPATH} gateway" background: true +run_prepare_testing_docker: &run_prepare_testing_docker + - run: | + docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.core-server-image-filename >>" + docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.engine-server-image-filename >>" + docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.file-artifacts-expander-image-filename >>" + - run: "${KURTOSIS_BINPATH} engine start --cli-log-level trace" + # Steps to prepare a job for Docker testing steps_prepare_testing_docker: &steps_prepare_testing_docker steps: - - run: | - docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.core-server-image-filename >>" - docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.engine-server-image-filename >>" - docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.file-artifacts-expander-image-filename >>" - - run: "${KURTOSIS_BINPATH} engine start --cli-log-level trace" + <<: *run_prepare_testing_docker # Run steps to dump kurtosis enclaves from docker run_dump_kurtosis_enclaves: &run_dump_kurtosis_enclaves @@ -116,6 +119,14 @@ abort_job_if_only_docs_changes: &abort_job_if_only_docs_changes circleci-agent step halt fi +abort_job_if_kubernetes_backend: &abort_job_if_kubernetes_backend + when: + condition: + and: + - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] + steps: + - run: circleci-agent step halt + ############## # CircleCI ############## @@ -216,6 +227,9 @@ parameters: kurtosis-cluster-setting-abs-filepath: type: string default: "/home/circleci/.local/share/kurtosis/cluster-setting" + reverse-proxy-entrypoint-web-port: + type: string + default: "9730" @@ -904,6 +918,46 @@ jobs: false fi + test_reverse_proxy: + executor: ubuntu_vm + parameters: + <<: *param_cli_cluster_backend + steps: + - <<: *abort_job_if_kubernetes_backend + + - checkout + + - <<: *abort_job_if_only_docs_changes + + # Set up Kurtosis + - attach_workspace: + at: "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>" + + - <<: *steps_install_go + + - run: | + echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list + sudo apt update + sudo apt install kurtosis-cli curl + # We don't send metrics to avoid polluting our logs + - run: | + echo 'export KURTOSIS_BINPATH="<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.cli-dist-home-relative-dirpath >>/<< pipeline.parameters.cli-linux-amd-64-binary-relative-filepath >>"' >> "${BASH_ENV}" + - run: "${KURTOSIS_BINPATH} analytics disable" + + - <<: *run_prepare_testing_docker + + # Start a service and send an http request to it via the reverse proxy + - run: | + ${KURTOSIS_BINPATH} enclave add --name test-enclave + ${KURTOSIS_BINPATH} service add test-enclave test1 httpd --ports http=http:80/tcp + ENCLAVE_UUID=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') + SERVICE_UUID=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') + STATUS_CODE=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${SERVICE_UUID}-${ENCLAVE_UUID}"| head -1 | awk '{print $2}') + if ! [ "${STATUS_CODE}" -eq "200" ]; then + echo 'HTTP request status code returned is ${STATUS_CODE} instead of 200' + false + fi + test_ci_for_failure: executor: ubuntu_vm parameters: From f3162d0416cef4ebee39e7a5add479935e456e64 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Tue, 5 Dec 2023 15:14:40 -0800 Subject: [PATCH 10/35] Add reverse proxy test to workflow --- .circleci/config.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 28cbcd5f14..ac09d4008e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1406,6 +1406,18 @@ workflows: - build_files_artifacts_expander <<: *filters_ignore_main + - test_reverse_proxy: + name: "Test reverse proxy against Docker" + cli-cluster-backend: "docker" + context: + - docker-user + requires: + - build_cli + - build_api_container_server + - build_engine_server + - build_files_artifacts_expander + <<: *filters_ignore_main + # -- Artifact-publishing jobs -------------------------------- - publish_kurtosis_sdk_rust: context: From 53686a414a18515a2aa99e940aae195271b280be Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Tue, 5 Dec 2023 18:06:04 -0800 Subject: [PATCH 11/35] Fix Traefik config template. --- .../implementations/traefik/consts.go | 6 ------ .../reverse_proxy_functions/shared_helpers.go | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go index f2827253a0..7df8087c81 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -17,25 +17,19 @@ log: api: debug: true dashboard: true - # api over the traefik endpoint insecure: true disabledashboardad: true entryPoints: - # http traffic web: address: ":{{ .HttpPort }}" - # API endpoint traefik: address: ":{{ .DashboardPort }}" providers: docker: endpoint: "unix:///var/run/docker.sock" - # we don't want the containers to be exposed by default. - # we are enabling Traefik at the container level instead. exposedByDefault: false - # Docker network to start Traefik in. network: "{{ .NetworkId }}" ` ////////////////////////--FINISH--TRAEFIK CONFIGURATION SECTION--///////////////////////////// diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go index 37aff5c97e..7ec08dde1a 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go @@ -104,7 +104,7 @@ func getReverseProxyObjectFromContainerInfo( if networkIpAddressStr != privateIpAddrStr { networkIpAddress := net.ParseIP(networkIpAddressStr) if networkIpAddress == nil { - return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", networkIpAddress) + return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", networkIpAddressStr) } enclaveNetworksIpAddress[networkId] = networkIpAddress } From 1114c75b73bf585f751bcea182fd5900602cce4c Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Tue, 5 Dec 2023 20:33:11 -0800 Subject: [PATCH 12/35] Debug info for the reverse proxy ci test. --- .circleci/config.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac09d4008e..60287e8047 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -950,11 +950,14 @@ jobs: - run: | ${KURTOSIS_BINPATH} enclave add --name test-enclave ${KURTOSIS_BINPATH} service add test-enclave test1 httpd --ports http=http:80/tcp - ENCLAVE_UUID=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') - SERVICE_UUID=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') - STATUS_CODE=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${SERVICE_UUID}-${ENCLAVE_UUID}"| head -1 | awk '{print $2}') - if ! [ "${STATUS_CODE}" -eq "200" ]; then - echo 'HTTP request status code returned is ${STATUS_CODE} instead of 200' + enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') + echo ${enclave_uuid} + service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') + echo ${service_uuid} + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${service_uuid}-${enclave_uuid}"| head -1 | awk '{print $2}') + echo ${status_code} + if ! [ "${status_code}" -eq "200" ]; then + echo 'HTTP request status code returned is ${status_code} instead of 200' false fi From 02a62b2d4e337ea27fb9b7e9e2e92b0c7ca5d5e2 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 07:31:03 -0800 Subject: [PATCH 13/35] Fix run prepare testing docker include. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 60287e8047..da33ab0def 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,7 +90,7 @@ run_prepare_testing_docker: &run_prepare_testing_docker # Steps to prepare a job for Docker testing steps_prepare_testing_docker: &steps_prepare_testing_docker steps: - <<: *run_prepare_testing_docker + - <<: *run_prepare_testing_docker # Run steps to dump kurtosis enclaves from docker run_dump_kurtosis_enclaves: &run_dump_kurtosis_enclaves From f95e9c31ff50d5adfdd81363a5998f89c6c93376 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 07:55:49 -0800 Subject: [PATCH 14/35] Working on fixing CI tests. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index da33ab0def..53759c7969 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,7 +85,7 @@ run_prepare_testing_docker: &run_prepare_testing_docker docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.core-server-image-filename >>" docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.engine-server-image-filename >>" docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.file-artifacts-expander-image-filename >>" - - run: "${KURTOSIS_BINPATH} engine start --cli-log-level trace" + "${KURTOSIS_BINPATH} engine start --cli-log-level trace" # Steps to prepare a job for Docker testing steps_prepare_testing_docker: &steps_prepare_testing_docker @@ -954,7 +954,7 @@ jobs: echo ${enclave_uuid} service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') echo ${service_uuid} - status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${service_uuid}-${enclave_uuid}"| head -1 | awk '{print $2}') + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${service_uuid}-${enclave_uuid}" | head -1 | awk '{print $2}') echo ${status_code} if ! [ "${status_code}" -eq "200" ]; then echo 'HTTP request status code returned is ${status_code} instead of 200' From a7b5525283ee733144cb55901b08c5e33491933b Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 08:08:34 -0800 Subject: [PATCH 15/35] Working on fixing CI tests. --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53759c7969..dfef25abcd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -950,10 +950,12 @@ jobs: - run: | ${KURTOSIS_BINPATH} enclave add --name test-enclave ${KURTOSIS_BINPATH} service add test-enclave test1 httpd --ports http=http:80/tcp + ${KURTOSIS_BINPATH} enclave inspect test-enclave enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') echo ${enclave_uuid} service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') echo ${service_uuid} + echo 'curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${service_uuid}-${enclave_uuid}"' status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${service_uuid}-${enclave_uuid}" | head -1 | awk '{print $2}') echo ${status_code} if ! [ "${status_code}" -eq "200" ]; then From 47ad28a1fc9fe0ca96a166b1358baf70af4445fa Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 09:58:32 -0800 Subject: [PATCH 16/35] Working on fixing CI tests. --- .circleci/config.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dfef25abcd..c86b6a2aa9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,11 +81,15 @@ steps_prepare_testing_k8s_k3s: &steps_prepare_testing_k8s_k3s background: true run_prepare_testing_docker: &run_prepare_testing_docker - - run: | - docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.core-server-image-filename >>" - docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.engine-server-image-filename >>" - docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.file-artifacts-expander-image-filename >>" - "${KURTOSIS_BINPATH} engine start --cli-log-level trace" + - run: + name: Load the engine, apic and file artifacts images + command: | + docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.core-server-image-filename >>" + docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.engine-server-image-filename >>" + docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.file-artifacts-expander-image-filename >>" + - run: + name: Start the engine + command: "${KURTOSIS_BINPATH} engine start --cli-log-level trace" # Steps to prepare a job for Docker testing steps_prepare_testing_docker: &steps_prepare_testing_docker From fbf87d686609db57aae21cbc3cb68b0b0b614171 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 10:48:07 -0800 Subject: [PATCH 17/35] Working on fixing CI tests. --- .circleci/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c86b6a2aa9..bec87c3306 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,14 +82,12 @@ steps_prepare_testing_k8s_k3s: &steps_prepare_testing_k8s_k3s run_prepare_testing_docker: &run_prepare_testing_docker - run: - name: Load the engine, apic and file artifacts images + name: Load the engine, apic and file artifacts images and start the engine command: | docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.core-server-image-filename >>" docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.engine-server-image-filename >>" docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.file-artifacts-expander-image-filename >>" - - run: - name: Start the engine - command: "${KURTOSIS_BINPATH} engine start --cli-log-level trace" + ${KURTOSIS_BINPATH} engine start --cli-log-level trace # Steps to prepare a job for Docker testing steps_prepare_testing_docker: &steps_prepare_testing_docker From 6e752927c2c01fbf534e48d55e69b0725ebc0d38 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 10:53:28 -0800 Subject: [PATCH 18/35] Working on fixing CI tests. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bec87c3306..3bc48ad69c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -958,7 +958,7 @@ jobs: service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') echo ${service_uuid} echo 'curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${service_uuid}-${enclave_uuid}"' - status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${service_uuid}-${enclave_uuid}" | head -1 | awk '{print $2}') + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-"${service_uuid}"-"${enclave_uuid}"" | head -1 | awk '{print $2}') echo ${status_code} if ! [ "${status_code}" -eq "200" ]; then echo 'HTTP request status code returned is ${status_code} instead of 200' From 89a598bb13e14273fcad557e264d4fb7c920f003 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 10:54:47 -0800 Subject: [PATCH 19/35] Working on fixing CI tests. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3bc48ad69c..b886e463dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -957,7 +957,7 @@ jobs: echo ${enclave_uuid} service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') echo ${service_uuid} - echo 'curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-${service_uuid}-${enclave_uuid}"' + echo 'curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-"${service_uuid}"-"${enclave_uuid}""' status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-"${service_uuid}"-"${enclave_uuid}"" | head -1 | awk '{print $2}') echo ${status_code} if ! [ "${status_code}" -eq "200" ]; then From 47252876f596c890b74bff73d2c0d0e0d93fb849 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 11:25:46 -0800 Subject: [PATCH 20/35] Working on fixing CI tests. --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b886e463dc..e2aeb099e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -957,11 +957,10 @@ jobs: echo ${enclave_uuid} service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') echo ${service_uuid} - echo 'curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-"${service_uuid}"-"${enclave_uuid}""' - status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-"${service_uuid}"-"${enclave_uuid}"" | head -1 | awk '{print $2}') + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}') echo ${status_code} if ! [ "${status_code}" -eq "200" ]; then - echo 'HTTP request status code returned is ${status_code} instead of 200' + echo 'HTTP request status code returned is '${status_code}' instead of 200' false fi From 4b9650b5e842bd42920d4621e5be63f635484e0f Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 11:48:12 -0800 Subject: [PATCH 21/35] Working on fixing CI tests. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e2aeb099e3..5429ca897b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -957,6 +957,7 @@ jobs: echo ${enclave_uuid} service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') echo ${service_uuid} + curl -v -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}') echo ${status_code} if ! [ "${status_code}" -eq "200" ]; then From e4d36d995344dc0cecbd45eed6df6164f0eeb8c6 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 13:35:57 -0800 Subject: [PATCH 22/35] Give enough time to Traefik to discover the user service - Diable the old enclave ci test for now. --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5429ca897b..f25ddc9fe7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,6 +129,9 @@ abort_job_if_kubernetes_backend: &abort_job_if_kubernetes_backend steps: - run: circleci-agent step halt +abort_job: &abort_job + run: circleci-agent step halt + ############## # CircleCI ############## @@ -655,6 +658,8 @@ jobs: test_old_enclave_continuity: executor: ubuntu_vm steps: + - <<: *abort_job + - checkout - <<: *abort_job_if_only_docs_changes @@ -957,6 +962,8 @@ jobs: echo ${enclave_uuid} service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') echo ${service_uuid} + # Give the reverse proxy enough time to discover the httpd user service + sleep 10 curl -v -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}') echo ${status_code} From 508bae7b4082164bf73139ad4fe5d7cde38eec9d Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 15:14:46 -0800 Subject: [PATCH 23/35] Remove debug msgs. --- .circleci/config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f25ddc9fe7..bc14448e6c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -957,16 +957,11 @@ jobs: - run: | ${KURTOSIS_BINPATH} enclave add --name test-enclave ${KURTOSIS_BINPATH} service add test-enclave test1 httpd --ports http=http:80/tcp - ${KURTOSIS_BINPATH} enclave inspect test-enclave enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') - echo ${enclave_uuid} service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') - echo ${service_uuid} # Give the reverse proxy enough time to discover the httpd user service sleep 10 - curl -v -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}') - echo ${status_code} if ! [ "${status_code}" -eq "200" ]; then echo 'HTTP request status code returned is '${status_code}' instead of 200' false From eaf423ebde4cb3dc16c4ec68c1b7d383ab564a2d Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 15:24:38 -0800 Subject: [PATCH 24/35] Linting. --- .../backend_creator/backend_creator.go | 9 +++++---- .../docker_kurtosis_backend/docker_kurtosis_backend.go | 2 +- .../engine_functions/stop_engines.go | 3 ++- .../reverse_proxy_functions/network_reverse_proxy.go | 2 +- .../docker/docker_manager/docker_manager.go | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go index 519c46a1a2..e664e23542 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go @@ -3,6 +3,10 @@ package backend_creator import ( "context" "fmt" + "net" + "os" + "path" + "github.com/docker/docker/client" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" @@ -19,9 +23,6 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/service_registration" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" - "net" - "os" - "path" ) const ( @@ -253,7 +254,7 @@ func getDockerKurtosisBackend( network.GetGatewayIp(): true, apiContainerIp.String(): true, logsCollectorObj.GetEnclaveNetworkIpAddress().String(): true, - reverseProxyEnclaveNetworkIpAddress.String(): true, + reverseProxyEnclaveNetworkIpAddress.String(): true, } freeIpAddrProvider, err := free_ip_addr_tracker.GetOrCreateNewFreeIpAddrTracker( diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go index 3ff2bca9c2..a040ede0b3 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go @@ -475,7 +475,7 @@ func (backend *DockerKurtosisBackend) DestroyLogsCollectorForEnclave(ctx context } func (backend *DockerKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { - reverseProxyContainer := traefik.NewTraefikReverseProxyContainer() //Declaring the implementation + reverseProxyContainer := traefik.NewTraefikReverseProxyContainer() reverseProxy, _, err := reverse_proxy_functions.CreateReverseProxy( ctx, diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go index 9ffbe75852..b6050fbc7b 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go @@ -2,6 +2,7 @@ package engine_functions import ( "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" @@ -72,7 +73,7 @@ func StopEngines( if err := logs_aggregator_functions.DestroyLogsAggregator(ctx, dockerManager); err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred removing the logging components.") } - + // Stop reverse proxy if err := reverse_proxy_functions.DestroyReverseProxy(ctx, dockerManager); err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred removing the reverse proxy.") diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go index f638d86959..3659f4af35 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go @@ -69,6 +69,6 @@ func ConnectReverseProxyToEnclaveNetworks(ctx context.Context, dockerManager *do return stacktrace.Propagate(err, "An error occurred connecting the reverse proxy to the enclave network with id '%v'", enclaveNetwork.GetId()) } } - + return nil } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 20fec381c3..cf668f2abb 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -791,7 +791,7 @@ func (manager *DockerManager) GetContainerIps(ctx context.Context, containerId s } allNetworkInfo := resp.NetworkSettings.Networks for _, networkInfo := range allNetworkInfo { - containerIps[networkInfo.NetworkID]= networkInfo.IPAddress + containerIps[networkInfo.NetworkID] = networkInfo.IPAddress } return containerIps, nil } From 50c1400e6760661fb68d0a9aee11f9f7d95e2034 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 15:29:29 -0800 Subject: [PATCH 25/35] Add reminder to re-enable the old enclave continuity ci test once this is released. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc14448e6c..867782c2b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -658,6 +658,7 @@ jobs: test_old_enclave_continuity: executor: ubuntu_vm steps: + # TODO: Re-enable once Traefik has been released. - <<: *abort_job - checkout From 70b68f3e410418073184b78eb8f57bda66c74910 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Wed, 6 Dec 2023 18:32:28 -0800 Subject: [PATCH 26/35] Clarify the reverse proxy ip addresses. --- .../objects/reverse_proxy/reverse_proxy.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go index 7a6a466a3c..6b450ce23c 100644 --- a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go +++ b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go @@ -10,10 +10,12 @@ import ( type ReverseProxy struct { status container.ContainerStatus + // IP address of the reverse proxy container in its network // This will be nil if the container is not running maybePrivateIpAddr net.IP - // IP address for each enclave network ID + // IP address of the reverse proxy container in each enclave network + // This will be nil if the container is not running maybeEnclaveNetworksIpAddress map[string]net.IP // HTTP port @@ -42,7 +44,7 @@ func (reverseProxy *ReverseProxy) GetStatus() container.ContainerStatus { return reverseProxy.status } -func (reverseProxy *ReverseProxy) GetMaybePrivateIpAddr() net.IP { +func (reverseProxy *ReverseProxy) GetPrivateIpAddr() net.IP { return reverseProxy.maybePrivateIpAddr } From ca0ad73ab727d656aa4d47bcfd400789f55b47f1 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Thu, 7 Dec 2023 13:43:02 -0800 Subject: [PATCH 27/35] Use header instead of host for Traefik routing. --- .../enclave_object_attributes_provider.go | 12 +++++++----- .../enclave_object_attributes_provider_test.go | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index d9e482bd54..732ccc71d2 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -33,6 +33,8 @@ const ( logsCollectorFragment = "kurtosis-logs-collector" // The collector is per enclave so this is a suffix logsCollectorVolumeFragment = logsCollectorFragment + "-vol" + + reverseProxyServicePortHeader = "X-Kurtosis-Service-Port" ) type DockerEnclaveObjectAttributesProvider interface { @@ -529,7 +531,7 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb } // Return Traefik labels -// Including the labels required to route traffic to the user service ports based on the Host header: +// Including the labels required to route traffic to the user service ports based on the header X-Kurtosis-Service-Port: // -- // The Traefik service name format is: -- // With the following input: @@ -539,10 +541,10 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb // HTTP Port 2 number: 81 // the following labels are returned: // "traefik.enable": "true", -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Host(`80-3771c85af16a-65d2fb6d6732`)", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Headers(`X-Kurtosis-Service-Port`, `80-3771c85af16a-65d2fb6d6732`)", // "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.service": "65d2fb6d6732-3771c85af16a-80", // "traefik.http.services.65d2fb6d6732-3771c85af16a-80.loadbalancer.server.port": "80" -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.rule": "Host(`81-3771c85af16a-65d2fb6d6732`)", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.rule": "Headers(`X-Kurtosis-Service-Port`, `81-3771c85af16a-65d2fb6d6732`)", // "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.service": "65d2fb6d6732-3771c85af16a-81", // "traefik.http.services.65d2fb6d6732-3771c85af16a-81.loadbalancer.server.port": "81" func (provider *dockerEnclaveObjectAttributesProviderImpl) getTraefikLabelsForEnclaveObject(serviceUuid string, ports map[string]*port_spec.PortSpec) (map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue, error) { @@ -558,13 +560,13 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getTraefikLabelsForEn shortServiceUuid := uuid_generator.ShortenedUUIDString(serviceUuid) servicePortStr := fmt.Sprintf("%s-%s-%d", shortEnclaveUuid, shortServiceUuid, portSpec.GetNumber()) - // Host rule + // Header X-Kurtosis-Service-Port rule ruleKeySuffix := fmt.Sprintf("http.routers.%s.rule", servicePortStr) ruleLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey(ruleKeySuffix) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting the traefik rule label key with suffix '%v'", ruleKeySuffix) } - ruleValue := fmt.Sprintf("Host(`%d-%s-%s`)", portSpec.GetNumber(), shortServiceUuid, shortEnclaveUuid) + ruleValue := fmt.Sprintf("Headers(`%s`, `%d-%s-%s`)", reverseProxyServicePortHeader, portSpec.GetNumber(), shortServiceUuid, shortEnclaveUuid) ruleLabelValue, err := docker_label_value.CreateNewDockerLabelValue(ruleValue) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating the traefik rule label value with value '%v'", ruleValue) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go index 1b91923413..5ccf18ad09 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go @@ -68,7 +68,7 @@ func TestForUserServiceContainer(t *testing.T) { case "traefik.http.routers.65d2fb6d6732-3771c85af16a-23.rule": require.Fail(t, "A traefik label for port 23 should not be present") case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.rule": - require.Equal(t, labelValue.GetString(), "Host(`45-3771c85af16a-65d2fb6d6732`)") + require.Equal(t, labelValue.GetString(), "Headers(`X-Kurtosis-Service-Port`, `45-3771c85af16a-65d2fb6d6732`)") case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.service": require.Equal(t, labelValue.GetString(), "65d2fb6d6732-3771c85af16a-45") case "traefik.http.services.65d2fb6d6732-3771c85af16a-45.loadbalancer.server.port": From 4856979d8a4e714e81b0a524f873a6dbc4bfa83d Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Thu, 7 Dec 2023 13:48:49 -0800 Subject: [PATCH 28/35] Add reverse proxy engine restart test. --- .circleci/config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 867782c2b6..9f70c50667 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -968,6 +968,19 @@ jobs: false fi + # Restart the engine and make sure Traefik restarted properly + - run: | + ${KURTOSIS_BINPATH} engine restart + enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') + service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') + # Give the reverse proxy enough time to discover the httpd user service + sleep 10 + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}') + if ! [ "${status_code}" -eq "200" ]; then + echo 'HTTP request status code returned is '${status_code}' instead of 200' + false + fi + test_ci_for_failure: executor: ubuntu_vm parameters: From c68c167f99a3cc8c67ff743fe122dc84844c817e Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Thu, 7 Dec 2023 16:13:58 -0800 Subject: [PATCH 29/35] Split the service port header into three headers: enclave uuid, service uuid and service port. --- .../enclave_object_attributes_provider.go | 31 +++++++++++-------- ...enclave_object_attributes_provider_test.go | 2 +- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index 732ccc71d2..666f923c4e 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -34,7 +34,9 @@ const ( // The collector is per enclave so this is a suffix logsCollectorVolumeFragment = logsCollectorFragment + "-vol" - reverseProxyServicePortHeader = "X-Kurtosis-Service-Port" + reverseProxyEnclaveUuidHeader = "X-Kurtosis-Enclave-UUID" + reverseProxyServiceUuidHeader = "X-Kurtosis-Service-UUID" + reverseProxyServicePortNumberHeader = "X-Kurtosis-Service-Port-Number" ) type DockerEnclaveObjectAttributesProvider interface { @@ -535,18 +537,21 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb // -- // The Traefik service name format is: -- // With the following input: -// Enclave short UUID: 65d2fb6d6732 -// Service short UUID: 3771c85af16a -// HTTP Port 1 number: 80 -// HTTP Port 2 number: 81 +// +// Enclave short UUID: 65d2fb6d6732 +// Service short UUID: 3771c85af16a +// HTTP Port 1 number: 80 +// HTTP Port 2 number: 81 +// // the following labels are returned: -// "traefik.enable": "true", -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Headers(`X-Kurtosis-Service-Port`, `80-3771c85af16a-65d2fb6d6732`)", -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.service": "65d2fb6d6732-3771c85af16a-80", -// "traefik.http.services.65d2fb6d6732-3771c85af16a-80.loadbalancer.server.port": "80" -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.rule": "Headers(`X-Kurtosis-Service-Port`, `81-3771c85af16a-65d2fb6d6732`)", -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.service": "65d2fb6d6732-3771c85af16a-81", -// "traefik.http.services.65d2fb6d6732-3771c85af16a-81.loadbalancer.server.port": "81" +// +// "traefik.enable": "true", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Headers(`X-Kurtosis-Enclave-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Port-Number`, `80`)", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.service": "65d2fb6d6732-3771c85af16a-80", +// "traefik.http.services.65d2fb6d6732-3771c85af16a-80.loadbalancer.server.port": "80" +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.rule": "Headers(`X-Kurtosis-Enclave-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Port-Number`, `81`)", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.service": "65d2fb6d6732-3771c85af16a-81", +// "traefik.http.services.65d2fb6d6732-3771c85af16a-81.loadbalancer.server.port": "81" func (provider *dockerEnclaveObjectAttributesProviderImpl) getTraefikLabelsForEnclaveObject(serviceUuid string, ports map[string]*port_spec.PortSpec) (map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue, error) { labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{} @@ -566,7 +571,7 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getTraefikLabelsForEn if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting the traefik rule label key with suffix '%v'", ruleKeySuffix) } - ruleValue := fmt.Sprintf("Headers(`%s`, `%d-%s-%s`)", reverseProxyServicePortHeader, portSpec.GetNumber(), shortServiceUuid, shortEnclaveUuid) + ruleValue := fmt.Sprintf("Headers(`%s`, `%s`) && Headers(`%s`, `%s`) && Headers(`%s`, `%d`)", reverseProxyEnclaveUuidHeader, shortEnclaveUuid, reverseProxyServiceUuidHeader, shortServiceUuid, reverseProxyServicePortNumberHeader, portSpec.GetNumber()) ruleLabelValue, err := docker_label_value.CreateNewDockerLabelValue(ruleValue) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating the traefik rule label value with value '%v'", ruleValue) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go index 5ccf18ad09..fcd88f5573 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go @@ -68,7 +68,7 @@ func TestForUserServiceContainer(t *testing.T) { case "traefik.http.routers.65d2fb6d6732-3771c85af16a-23.rule": require.Fail(t, "A traefik label for port 23 should not be present") case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.rule": - require.Equal(t, labelValue.GetString(), "Headers(`X-Kurtosis-Service-Port`, `45-3771c85af16a-65d2fb6d6732`)") + require.Equal(t, labelValue.GetString(), "Headers(`X-Kurtosis-Enclave-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Service-Port-Number`, `45`)") case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.service": require.Equal(t, labelValue.GetString(), "65d2fb6d6732-3771c85af16a-45") case "traefik.http.services.65d2fb6d6732-3771c85af16a-45.loadbalancer.server.port": From 045622e859659493eab49c7b5666644a4ff5c42f Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Thu, 7 Dec 2023 16:14:31 -0800 Subject: [PATCH 30/35] Sync oapi-codegen file headers --- api/golang/http_rest/api_types/api_types.gen.go | 2 +- api/golang/http_rest/core_rest_api/api_container_server.gen.go | 2 +- api/golang/http_rest/engine_rest_api/engine_server.gen.go | 2 +- api/golang/http_rest/websocket_api/websocket_server.gen.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/golang/http_rest/api_types/api_types.gen.go b/api/golang/http_rest/api_types/api_types.gen.go index 1a31729aae..1bf7dfdaf4 100644 --- a/api/golang/http_rest/api_types/api_types.gen.go +++ b/api/golang/http_rest/api_types/api_types.gen.go @@ -1,6 +1,6 @@ // Package api_types provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/deepmap/oapi-codegen version (devel) DO NOT EDIT. +// Code generated by github.com/deepmap/oapi-codegen version v1.12.4 DO NOT EDIT. package api_types import ( diff --git a/api/golang/http_rest/core_rest_api/api_container_server.gen.go b/api/golang/http_rest/core_rest_api/api_container_server.gen.go index f7dad9cb8c..dc21918440 100644 --- a/api/golang/http_rest/core_rest_api/api_container_server.gen.go +++ b/api/golang/http_rest/core_rest_api/api_container_server.gen.go @@ -1,6 +1,6 @@ // Package core_rest_api provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/deepmap/oapi-codegen version (devel) DO NOT EDIT. +// Code generated by github.com/deepmap/oapi-codegen version v1.12.4 DO NOT EDIT. package core_rest_api import ( diff --git a/api/golang/http_rest/engine_rest_api/engine_server.gen.go b/api/golang/http_rest/engine_rest_api/engine_server.gen.go index 2c11fa71d0..74209c73a8 100644 --- a/api/golang/http_rest/engine_rest_api/engine_server.gen.go +++ b/api/golang/http_rest/engine_rest_api/engine_server.gen.go @@ -1,6 +1,6 @@ // Package engine_rest_api provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/deepmap/oapi-codegen version (devel) DO NOT EDIT. +// Code generated by github.com/deepmap/oapi-codegen version v1.12.4 DO NOT EDIT. package engine_rest_api import ( diff --git a/api/golang/http_rest/websocket_api/websocket_server.gen.go b/api/golang/http_rest/websocket_api/websocket_server.gen.go index f5541f40a7..c25cbfb2b2 100644 --- a/api/golang/http_rest/websocket_api/websocket_server.gen.go +++ b/api/golang/http_rest/websocket_api/websocket_server.gen.go @@ -1,6 +1,6 @@ // Package websocket_api provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/deepmap/oapi-codegen version (devel) DO NOT EDIT. +// Code generated by github.com/deepmap/oapi-codegen version v1.12.4 DO NOT EDIT. package websocket_api import ( From df280b173584dd64257f0ac89b5b85550197e14e Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Thu, 7 Dec 2023 16:19:26 -0800 Subject: [PATCH 31/35] Update CI tests. --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea7bd25415..815ab1cfcd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -962,20 +962,20 @@ jobs: service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') # Give the reverse proxy enough time to discover the httpd user service sleep 10 - status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}') + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}') if ! [ "${status_code}" -eq "200" ]; then echo 'HTTP request status code returned is '${status_code}' instead of 200' false fi - # Restart the engine and make sure Traefik restarted properly + # Restart the engine and make sure Traefik restarted and reconfigured properly - run: | ${KURTOSIS_BINPATH} engine restart enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') # Give the reverse proxy enough time to discover the httpd user service sleep 10 - status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "Host: 80-$(echo $service_uuid)-$(echo $enclave_uuid)" | head -1 | awk '{print $2}') + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}') if ! [ "${status_code}" -eq "200" ]; then echo 'HTTP request status code returned is '${status_code}' instead of 200' false From 645d187bc5e111204de63804a8053474582dec98 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Thu, 7 Dec 2023 16:27:25 -0800 Subject: [PATCH 32/35] Update headers to specify that those are short uuids. --- .circleci/config.yml | 4 ++-- .../enclave_object_attributes_provider.go | 8 ++++---- .../enclave_object_attributes_provider_test.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 815ab1cfcd..deb929ee33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -962,7 +962,7 @@ jobs: service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') # Give the reverse proxy enough time to discover the httpd user service sleep 10 - status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}') + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-Short-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-Short-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}') if ! [ "${status_code}" -eq "200" ]; then echo 'HTTP request status code returned is '${status_code}' instead of 200' false @@ -975,7 +975,7 @@ jobs: service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') # Give the reverse proxy enough time to discover the httpd user service sleep 10 - status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}') + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-Short-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-Short-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}') if ! [ "${status_code}" -eq "200" ]; then echo 'HTTP request status code returned is '${status_code}' instead of 200' false diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index 666f923c4e..c4ba964959 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -34,8 +34,8 @@ const ( // The collector is per enclave so this is a suffix logsCollectorVolumeFragment = logsCollectorFragment + "-vol" - reverseProxyEnclaveUuidHeader = "X-Kurtosis-Enclave-UUID" - reverseProxyServiceUuidHeader = "X-Kurtosis-Service-UUID" + reverseProxyEnclaveUuidHeader = "X-Kurtosis-Enclave-Short-UUID" + reverseProxyServiceUuidHeader = "X-Kurtosis-Service-Short-UUID" reverseProxyServicePortNumberHeader = "X-Kurtosis-Service-Port-Number" ) @@ -546,10 +546,10 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb // the following labels are returned: // // "traefik.enable": "true", -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Headers(`X-Kurtosis-Enclave-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Port-Number`, `80`)", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Headers(`X-Kurtosis-Enclave-Short-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-Short-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Port-Number`, `80`)", // "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.service": "65d2fb6d6732-3771c85af16a-80", // "traefik.http.services.65d2fb6d6732-3771c85af16a-80.loadbalancer.server.port": "80" -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.rule": "Headers(`X-Kurtosis-Enclave-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Port-Number`, `81`)", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.rule": "Headers(`X-Kurtosis-Enclave-Short-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-Short-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Port-Number`, `81`)", // "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.service": "65d2fb6d6732-3771c85af16a-81", // "traefik.http.services.65d2fb6d6732-3771c85af16a-81.loadbalancer.server.port": "81" func (provider *dockerEnclaveObjectAttributesProviderImpl) getTraefikLabelsForEnclaveObject(serviceUuid string, ports map[string]*port_spec.PortSpec) (map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue, error) { diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go index fcd88f5573..64d3cf4678 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go @@ -68,7 +68,7 @@ func TestForUserServiceContainer(t *testing.T) { case "traefik.http.routers.65d2fb6d6732-3771c85af16a-23.rule": require.Fail(t, "A traefik label for port 23 should not be present") case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.rule": - require.Equal(t, labelValue.GetString(), "Headers(`X-Kurtosis-Enclave-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Service-Port-Number`, `45`)") + require.Equal(t, labelValue.GetString(), "Headers(`X-Kurtosis-Enclave-Short-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-Short-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Service-Port-Number`, `45`)") case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.service": require.Equal(t, labelValue.GetString(), "65d2fb6d6732-3771c85af16a-45") case "traefik.http.services.65d2fb6d6732-3771c85af16a-45.loadbalancer.server.port": From 9b28968bdc41709c7afff2bbb7ef5d3092de8023 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Thu, 7 Dec 2023 16:29:13 -0800 Subject: [PATCH 33/35] Update headers to specify that those are short uuids. --- .../enclave_object_attributes_provider.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index c4ba964959..a36dc7cd7a 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -34,8 +34,8 @@ const ( // The collector is per enclave so this is a suffix logsCollectorVolumeFragment = logsCollectorFragment + "-vol" - reverseProxyEnclaveUuidHeader = "X-Kurtosis-Enclave-Short-UUID" - reverseProxyServiceUuidHeader = "X-Kurtosis-Service-Short-UUID" + reverseProxyEnclaveShortUuidHeader = "X-Kurtosis-Enclave-Short-UUID" + reverseProxyServiceShortUuidHeader = "X-Kurtosis-Service-Short-UUID" reverseProxyServicePortNumberHeader = "X-Kurtosis-Service-Port-Number" ) @@ -571,7 +571,7 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getTraefikLabelsForEn if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting the traefik rule label key with suffix '%v'", ruleKeySuffix) } - ruleValue := fmt.Sprintf("Headers(`%s`, `%s`) && Headers(`%s`, `%s`) && Headers(`%s`, `%d`)", reverseProxyEnclaveUuidHeader, shortEnclaveUuid, reverseProxyServiceUuidHeader, shortServiceUuid, reverseProxyServicePortNumberHeader, portSpec.GetNumber()) + ruleValue := fmt.Sprintf("Headers(`%s`, `%s`) && Headers(`%s`, `%s`) && Headers(`%s`, `%d`)", reverseProxyEnclaveShortUuidHeader, shortEnclaveUuid, reverseProxyServiceShortUuidHeader, shortServiceUuid, reverseProxyServicePortNumberHeader, portSpec.GetNumber()) ruleLabelValue, err := docker_label_value.CreateNewDockerLabelValue(ruleValue) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating the traefik rule label value with value '%v'", ruleValue) From 122df2f2a6cd93816ef89ac36017e6fb7d685e37 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Fri, 8 Dec 2023 12:52:29 -0800 Subject: [PATCH 34/35] Address Tedi's comments. --- ...er_kurtosis_backend_api_container_functions.go | 3 +++ .../docker_kurtosis_backend_enclave_functions.go | 15 ++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go index 31ab5ae664..0693f4ed40 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go @@ -78,6 +78,9 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( } reverseProxy, err := backend.GetReverseProxy(ctx) + if reverseProxy == nil { + return nil, stacktrace.Propagate(err, "The reverse proxy is not running, This is a bug in Kurtosis") + } if err != nil { return nil, stacktrace.Propagate(err, "An error occurred while getting the reverse proxy, This is a bug in Kurtosis") } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go index 333baf3ec5..388497f8d2 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go @@ -372,26 +372,26 @@ func (backend *DockerKurtosisBackend) DestroyEnclaves( erroredEnclaveUuids[enclaveUuid] = volumeRemovalErr } - // Disconnect the containers from the enclave networks being removed + // Disconnect the external containers from the enclave networks being removed networksToDisconnect := map[enclave.EnclaveUUID]string{} for enclaveUuid := range successfulVolumeRemovalEnclaveUuids { networkInfo, found := matchingNetworkInfo[enclaveUuid] if !found { - return nil, nil, stacktrace.NewError("Would have attempted to disconnect enclave network '%v' that didn't match the filters", enclaveUuid) + return nil, nil, stacktrace.NewError("Attempt was made to disconnect enclave that did not match filters. This is likely a bug in Kurtosis.", enclaveUuid) } networksToDisconnect[enclaveUuid] = networkInfo.dockerNetwork.GetId() } - successfulDisconnectContainersFromNetworkEnclaveUuids, erroredDisconnectContainersFromNetworkEnclaveUuids, err := backend.disconnectContainersFromEnclaveNetworks(ctx, backend.dockerManager, networksToDisconnect) + successfulDisconnectExternalContainersFromNetworkEnclaveUuids, erroredDisconnectExternalContainersFromNetworkEnclaveUuids, err := backend.disconnectExternalContainersFromEnclaveNetworks(ctx, backend.dockerManager, networksToDisconnect) if err != nil { - return nil, nil, stacktrace.Propagate(err, "An error occurred disconnecting the containers from the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids) + return nil, nil, stacktrace.Propagate(err, "An error occurred disconnecting the external containers from the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids) } - for enclaveUuid, networkDisconnectErr := range erroredDisconnectContainersFromNetworkEnclaveUuids { + for enclaveUuid, networkDisconnectErr := range erroredDisconnectExternalContainersFromNetworkEnclaveUuids { erroredEnclaveUuids[enclaveUuid] = networkDisconnectErr } // Remove the networks networksToDestroy := map[enclave.EnclaveUUID]string{} - for enclaveUuid := range successfulDisconnectContainersFromNetworkEnclaveUuids { + for enclaveUuid := range successfulDisconnectExternalContainersFromNetworkEnclaveUuids { networkInfo, found := matchingNetworkInfo[enclaveUuid] if !found { return nil, nil, stacktrace.NewError("Would have attempted to destroy enclave '%v' that didn't match the filters", enclaveUuid) @@ -537,7 +537,8 @@ func (backend *DockerKurtosisBackend) getAllEnclaveContainers( return containers, nil } -func (backend *DockerKurtosisBackend) disconnectContainersFromEnclaveNetworks( +// Disconnect containers not in the enclave from the enclave networks +func (backend *DockerKurtosisBackend) disconnectExternalContainersFromEnclaveNetworks( ctx context.Context, dockerManager *docker_manager.DockerManager, enclaveNetworkIds map[enclave.EnclaveUUID]string, From eebbc5135f57549a313601d15f8133150bbf7733 Mon Sep 17 00:00:00 2001 From: Laurent Luce Date: Fri, 8 Dec 2023 12:56:15 -0800 Subject: [PATCH 35/35] Linting. --- .../docker_kurtosis_backend_enclave_functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go index 388497f8d2..ae8e0b2a40 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go @@ -377,7 +377,7 @@ func (backend *DockerKurtosisBackend) DestroyEnclaves( for enclaveUuid := range successfulVolumeRemovalEnclaveUuids { networkInfo, found := matchingNetworkInfo[enclaveUuid] if !found { - return nil, nil, stacktrace.NewError("Attempt was made to disconnect enclave that did not match filters. This is likely a bug in Kurtosis.", enclaveUuid) + return nil, nil, stacktrace.NewError("Attempt was made to disconnect enclave '%v' that did not match filters. This is likely a bug in Kurtosis.", enclaveUuid) } networksToDisconnect[enclaveUuid] = networkInfo.dockerNetwork.GetId() }