diff --git a/.circleci/config.yml b/.circleci/config.yml index 42d910c48e..d53f12c984 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,14 +80,19 @@ steps_prepare_testing_k8s_k3s: &steps_prepare_testing_k8s_k3s command: "${KURTOSIS_BINPATH} gateway" background: true -# Steps to prepare a job for Docker testing -steps_prepare_testing_docker: &steps_prepare_testing_docker - steps: - - run: | +run_prepare_testing_docker: &run_prepare_testing_docker + - run: + name: Load the engine, apic and file artifacts images and start the engine + command: | docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.core-server-image-filename >>" docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.engine-server-image-filename >>" docker load -i "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.file-artifacts-expander-image-filename >>" - - run: "${KURTOSIS_BINPATH} engine start --cli-log-level trace" + ${KURTOSIS_BINPATH} engine start --cli-log-level trace + +# Steps to prepare a job for Docker testing +steps_prepare_testing_docker: &steps_prepare_testing_docker + steps: + - <<: *run_prepare_testing_docker # Run steps to dump kurtosis enclaves from docker run_dump_kurtosis_enclaves: &run_dump_kurtosis_enclaves @@ -116,6 +121,17 @@ abort_job_if_only_docs_changes: &abort_job_if_only_docs_changes circleci-agent step halt fi +abort_job_if_kubernetes_backend: &abort_job_if_kubernetes_backend + when: + condition: + and: + - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ] + steps: + - run: circleci-agent step halt + +abort_job: &abort_job + run: circleci-agent step halt + ############## # CircleCI ############## @@ -216,6 +232,9 @@ parameters: kurtosis-cluster-setting-abs-filepath: type: string default: "/home/circleci/.local/share/kurtosis/cluster-setting" + reverse-proxy-entrypoint-web-port: + type: string + default: "9730" @@ -639,6 +658,9 @@ jobs: test_old_enclave_continuity: executor: ubuntu_vm steps: + # TODO: Re-enable once Traefik has been released. + - <<: *abort_job + - checkout - <<: *abort_job_if_only_docs_changes @@ -878,9 +900,9 @@ jobs: equal: [ "docker", << parameters.cli-cluster-backend >> ] steps: - run: - name: "Verify only the engine container and logs aggregator remains after the clean" + name: "Verify only the engine, logs aggregator and reverse proxy remain after the clean" command: | - if ! [ "$(docker container ls -a | tail -n+2 | wc -l)" -eq 2 ]; then + if ! [ "$(docker container ls -a | tail -n+2 | wc -l)" -eq 3 ]; then docker container ls -a false fi @@ -904,6 +926,61 @@ jobs: false fi + test_reverse_proxy: + executor: ubuntu_vm + parameters: + <<: *param_cli_cluster_backend + steps: + - <<: *abort_job_if_kubernetes_backend + + - checkout + + - <<: *abort_job_if_only_docs_changes + + # Set up Kurtosis + - attach_workspace: + at: "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>" + + - <<: *steps_install_go + + - run: | + echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list + sudo apt update + sudo apt install kurtosis-cli curl + # We don't send metrics to avoid polluting our logs + - run: | + echo 'export KURTOSIS_BINPATH="<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.cli-dist-home-relative-dirpath >>/<< pipeline.parameters.cli-linux-amd-64-binary-relative-filepath >>"' >> "${BASH_ENV}" + - run: "${KURTOSIS_BINPATH} analytics disable" + + - <<: *run_prepare_testing_docker + + # Start a service and send an http request to it via the reverse proxy + - run: | + ${KURTOSIS_BINPATH} enclave add --name test-enclave + ${KURTOSIS_BINPATH} service add test-enclave test1 httpd --ports http=http:80/tcp + enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') + service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') + # Give the reverse proxy enough time to discover the httpd user service + sleep 10 + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-Short-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-Short-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}') + if ! [ "${status_code}" -eq "200" ]; then + echo 'HTTP request status code returned is '${status_code}' instead of 200' + false + fi + + # Restart the engine and make sure Traefik restarted and reconfigured properly + - run: | + ${KURTOSIS_BINPATH} engine restart + enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}') + service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}') + # Give the reverse proxy enough time to discover the httpd user service + sleep 10 + status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-Short-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-Short-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}') + if ! [ "${status_code}" -eq "200" ]; then + echo 'HTTP request status code returned is '${status_code}' instead of 200' + false + fi + test_ci_for_failure: executor: ubuntu_vm parameters: @@ -1352,6 +1429,18 @@ workflows: - build_files_artifacts_expander <<: *filters_ignore_main + - test_reverse_proxy: + name: "Test reverse proxy against Docker" + cli-cluster-backend: "docker" + context: + - docker-user + requires: + - build_cli + - build_api_container_server + - build_engine_server + - build_files_artifacts_expander + <<: *filters_ignore_main + # -- Artifact-publishing jobs -------------------------------- - publish_kurtosis_sdk_rust: context: diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go index 7695f72548..e664e23542 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go @@ -3,9 +3,14 @@ package backend_creator import ( "context" "fmt" + "net" + "os" + "path" + "github.com/docker/docker/client" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts" @@ -18,9 +23,6 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/service_registration" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" - "net" - "os" - "path" ) const ( @@ -238,11 +240,21 @@ func getDockerKurtosisBackend( return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector object for enclave '%v'; This is a bug in Kurtosis", enclaveUuid) } + reverseProxy, err := reverse_proxy_functions.GetReverseProxy(ctx, dockerManager) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while getting the reverse proxy, This is a bug in Kurtosis") + } + reverseProxyEnclaveNetworkIpAddress, found := reverseProxy.GetEnclaveNetworksIpAddress()[network.GetId()] + if !found { + return nil, stacktrace.NewError("An error occured while getting the reverse proxy enclave network IP address for enclave '%v', This is a bug in Kurtosis", enclaveUuid) + } + alreadyTakenIps := map[string]bool{ networkIp.String(): true, network.GetGatewayIp(): true, apiContainerIp.String(): true, logsCollectorObj.GetEnclaveNetworkIpAddress().String(): true, + reverseProxyEnclaveNetworkIpAddress.String(): true, } freeIpAddrProvider, err := free_ip_addr_tracker.GetOrCreateNewFreeIpAddrTracker( diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go index e3aed786fa..c0c1ab1ced 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go @@ -13,6 +13,8 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions/implementations/fluentbit" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik" user_service_functions "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" @@ -28,6 +30,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/free_ip_addr_tracker" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/service_registration" @@ -472,6 +475,57 @@ func (backend *DockerKurtosisBackend) DestroyLogsCollectorForEnclave(ctx context return nil } +func (backend *DockerKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + reverseProxyContainer := traefik.NewTraefikReverseProxyContainer() + + reverseProxy, _, err := reverse_proxy_functions.CreateReverseProxy( + ctx, + reverseProxyContainer, + backend.dockerManager, + backend.objAttrsProvider, + ) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating the reverse proxy using the reverse proxy container '%+v'.", reverseProxyContainer) + } + return reverseProxy, nil +} + +func (backend *DockerKurtosisBackend) GetReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + maybeReverseProxy, err := reverse_proxy_functions.GetReverseProxy( + ctx, + backend.dockerManager, + ) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the reverse proxy") + } + + return maybeReverseProxy, nil +} + +func (backend *DockerKurtosisBackend) DestroyReverseProxy(ctx context.Context) error { + if err := reverse_proxy_functions.DestroyReverseProxy(ctx, backend.dockerManager); err != nil { + return stacktrace.Propagate(err, "An error occurred destroying the reverse proxy") + } + + return nil +} + +func (backend *DockerKurtosisBackend) ConnectReverseProxyToNetwork(ctx context.Context, networkId string) error { + if err := reverse_proxy_functions.ConnectReverseProxyToNetwork(ctx, backend.dockerManager, networkId); err != nil { + return stacktrace.Propagate(err, "An error occurred connecting the reverse proxy to the network with ID '%v'", networkId) + } + + return nil +} + +func (backend *DockerKurtosisBackend) DisconnectReverseProxyFromNetwork(ctx context.Context, networkId string) error { + if err := reverse_proxy_functions.DisconnectReverseProxyFromNetwork(ctx, backend.dockerManager, networkId); err != nil { + return stacktrace.Propagate(err, "An error occurred disconnecting the reverse proxy from the network with ID '%v'", networkId) + } + + return nil +} + func (backend *DockerKurtosisBackend) GetAvailableCPUAndMemory(ctx context.Context) (compute_resources.MemoryInMegaBytes, compute_resources.CpuMilliCores, bool, error) { availableMemory, availableCpu, err := backend.dockerManager.GetAvailableCPUAndMemory(ctx) if err != nil { diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go index b79a98bca7..0693f4ed40 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go @@ -2,10 +2,11 @@ package docker_kurtosis_backend import ( "context" - "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" "net" "time" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" + "github.com/docker/go-connections/nat" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" @@ -73,7 +74,19 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( enclaveLogsCollector, err := backend.GetLogsCollectorForEnclave(ctx, enclaveUuid) if err != nil { - return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector for enclave '%v; This is a bug in Kurtosis'", enclaveUuid) + return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector for enclave '%v'; This is a bug in Kurtosis", enclaveUuid) + } + + reverseProxy, err := backend.GetReverseProxy(ctx) + if reverseProxy == nil { + return nil, stacktrace.Propagate(err, "The reverse proxy is not running, This is a bug in Kurtosis") + } + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while getting the reverse proxy, This is a bug in Kurtosis") + } + reverseProxyEnclaveNetworkIpAddress, found := reverseProxy.GetEnclaveNetworksIpAddress()[enclaveNetwork.GetId()] + if !found { + return nil, stacktrace.NewError("An error occured while getting the reverse proxy enclave network IP address for enclave '%v', This is a bug in Kurtosis", enclaveUuid) } networkCidr := enclaveNetwork.GetIpAndMask() @@ -81,6 +94,7 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer( networkCidr.IP.String(): true, enclaveNetwork.GetGatewayIp(): true, enclaveLogsCollector.GetEnclaveNetworkIpAddress().String(): true, + reverseProxyEnclaveNetworkIpAddress.String(): true, } ipAddr, err := network_helpers.GetFreeIpAddrFromSubnet(alreadyTakenIps, networkCidr) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go index e6fecd4d3b..ae8e0b2a40 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go @@ -3,6 +3,9 @@ package docker_kurtosis_backend import ( "context" "encoding/json" + "strings" + "time" + "github.com/docker/docker/api/types/volume" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" @@ -14,14 +17,12 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" - "strings" - "time" ) const ( - shouldFetchStoppedContainersWhenGettingEnclaveStatus = true - - shouldFetchStoppedContainersWhenDumpingEnclave = true + shouldFetchStoppedContainersWhenGettingEnclaveStatus = true + shouldFetchStoppedContainersWhenDumpingEnclave = true + shouldFetchStoppedContainersWhenDisconnectingFromEnclaveNetworks = false serializedArgs = "SERIALIZED_ARGS" ) @@ -113,7 +114,7 @@ func (backend *DockerKurtosisBackend) CreateEnclave(ctx context.Context, enclave } } }() - logrus.Debugf("Docker network '%v' created successfully with ID '%v'", enclaveUuid, networkId) + logrus.Debugf("Docker network for enclave '%v' created successfully with ID '%v'", enclaveUuid, networkId) enclaveDataVolumeNameStr := enclaveDataVolumeAttrs.GetName().GetString() enclaveDataVolumeLabelStrs := map[string]string{} @@ -146,8 +147,22 @@ func (backend *DockerKurtosisBackend) CreateEnclave(ctx context.Context, enclave // TODO: return production mode for create enclave request as well newEnclave := enclave.NewEnclave(enclaveUuid, enclaveName, enclave.EnclaveStatus_Empty, &creationTime, false) + if err := backend.ConnectReverseProxyToNetwork(ctx, networkId); err != nil { + return nil, stacktrace.Propagate(err, "An error occurred connecting the reverse proxy to the enclave network with ID '%v'", networkId) + } + shouldDisconnectReverseProxyFromNetwork := true + defer func() { + if shouldDisconnectReverseProxyFromNetwork { + err = backend.DisconnectReverseProxyFromNetwork(ctx, networkId) + if err != nil { + logrus.Errorf("Couldn't disconnect the reverse proxy from the enclave network with ID '%v'", networkId) + } + } + }() + shouldDeleteNetwork = false shouldDeleteVolume = false + shouldDisconnectReverseProxyFromNetwork = false return newEnclave, nil } @@ -357,15 +372,33 @@ func (backend *DockerKurtosisBackend) DestroyEnclaves( erroredEnclaveUuids[enclaveUuid] = volumeRemovalErr } + // Disconnect the external containers from the enclave networks being removed + networksToDisconnect := map[enclave.EnclaveUUID]string{} + for enclaveUuid := range successfulVolumeRemovalEnclaveUuids { + networkInfo, found := matchingNetworkInfo[enclaveUuid] + if !found { + return nil, nil, stacktrace.NewError("Attempt was made to disconnect enclave '%v' that did not match filters. This is likely a bug in Kurtosis.", enclaveUuid) + } + networksToDisconnect[enclaveUuid] = networkInfo.dockerNetwork.GetId() + } + successfulDisconnectExternalContainersFromNetworkEnclaveUuids, erroredDisconnectExternalContainersFromNetworkEnclaveUuids, err := backend.disconnectExternalContainersFromEnclaveNetworks(ctx, backend.dockerManager, networksToDisconnect) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred disconnecting the external containers from the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids) + } + for enclaveUuid, networkDisconnectErr := range erroredDisconnectExternalContainersFromNetworkEnclaveUuids { + erroredEnclaveUuids[enclaveUuid] = networkDisconnectErr + } + // Remove the networks networksToDestroy := map[enclave.EnclaveUUID]string{} - for enclaveUuid := range successfulVolumeRemovalEnclaveUuids { + for enclaveUuid := range successfulDisconnectExternalContainersFromNetworkEnclaveUuids { networkInfo, found := matchingNetworkInfo[enclaveUuid] if !found { return nil, nil, stacktrace.NewError("Would have attempted to destroy enclave '%v' that didn't match the filters", enclaveUuid) } networksToDestroy[enclaveUuid] = networkInfo.dockerNetwork.GetId() } + successfulNetworkRemovalEnclaveUuids, erroredNetworkRemovalEnclaveUuids, err := destroyEnclaveNetworks(ctx, backend.dockerManager, networksToDestroy) if err != nil { return nil, nil, stacktrace.Propagate(err, "An error occurred destroying the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids) @@ -504,6 +537,68 @@ func (backend *DockerKurtosisBackend) getAllEnclaveContainers( return containers, nil } +// Disconnect containers not in the enclave from the enclave networks +func (backend *DockerKurtosisBackend) disconnectExternalContainersFromEnclaveNetworks( + ctx context.Context, + dockerManager *docker_manager.DockerManager, + enclaveNetworkIds map[enclave.EnclaveUUID]string, +) ( + map[enclave.EnclaveUUID]bool, + map[enclave.EnclaveUUID]error, + error, +) { + networkIdsToRemove := map[string]bool{} + enclaveUuidsForNetworkIds := map[string]enclave.EnclaveUUID{} + for enclaveUuid, networkId := range enclaveNetworkIds { + networkIdsToRemove[networkId] = true + enclaveUuidsForNetworkIds[networkId] = enclaveUuid + } + + var disconnectNetworkOperation docker_operation_parallelizer.DockerOperation = func(ctx context.Context, dockerManager *docker_manager.DockerManager, dockerObjectId string) error { + // Get containers connected to this network id (dockerObjectId here) + containers, err := backend.dockerManager.GetContainersByNetworkId(ctx, dockerObjectId, shouldFetchStoppedContainersWhenDisconnectingFromEnclaveNetworks) + if err != nil { + return stacktrace.Propagate( + err, + "An error occurred getting the containers with enclave network '%v'", + dockerObjectId, + ) + } + for _, container := range containers { + if err = dockerManager.DisconnectContainerFromNetwork(ctx, container.GetId(), dockerObjectId); err != nil { + return stacktrace.Propagate(err, "An error occurred while disconnecting container '%v' from the enclave network '%v'", container.GetId(), dockerObjectId) + } + } + return nil + } + + successfulNetworkIds, erroredNetworkIds := docker_operation_parallelizer.RunDockerOperationInParallel( + ctx, + networkIdsToRemove, + dockerManager, + disconnectNetworkOperation, + ) + + successfulEnclaveUuids := map[enclave.EnclaveUUID]bool{} + for networkId := range successfulNetworkIds { + enclaveUuid, found := enclaveUuidsForNetworkIds[networkId] + if !found { + return nil, nil, stacktrace.NewError("The containers were successfully disconnected from the enclave network '%v', but wasn't requested to be disconnected", networkId) + } + successfulEnclaveUuids[enclaveUuid] = true + } + + erroredEnclaveUuids := map[enclave.EnclaveUUID]error{} + for networkId, networkRemovalErr := range erroredNetworkIds { + enclaveUuid, found := enclaveUuidsForNetworkIds[networkId] + if !found { + return nil, nil, stacktrace.NewError("Docker network '%v' had the following error during disconnect, but wasn't requested to be disconnected:\n%v", networkId, networkRemovalErr) + } + erroredEnclaveUuids[enclaveUuid] = networkRemovalErr + } + return successfulEnclaveUuids, erroredEnclaveUuids, nil +} + func getAllEnclaveVolumes( ctx context.Context, dockerManager *docker_manager.DockerManager, diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go index c054eac180..6a1d021e1e 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go @@ -9,6 +9,8 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" @@ -111,6 +113,27 @@ func CreateEngine( }() logrus.Infof("Centralized logs components started.") + reverseProxyContainer := traefik.NewTraefikReverseProxyContainer() + _, removeReverseProxyFunc, err := reverse_proxy_functions.CreateReverseProxy( + ctx, + reverseProxyContainer, + dockerManager, + objAttrsProvider) + if err != nil { + return nil, stacktrace.Propagate(err, + "An error occurred attempting to create reverse proxy for engine with GUID '%v' in Docker network with network id '%v'.", engineGuidStr, targetNetworkId) + } + shouldRemoveReverseProxy := true + defer func() { + if shouldRemoveReverseProxy { + removeReverseProxyFunc() + } + }() + if err = reverse_proxy_functions.ConnectReverseProxyToEnclaveNetworks(ctx, dockerManager); err != nil { + return nil, stacktrace.Propagate(err, "An error occured connecting the reverse proxy to the enclave networks") + } + logrus.Infof("Reverse proxy started.") + enclaveManagerUIPortSpec, err := port_spec.NewPortSpec(uint16(enclaveManagerUIPort), consts.EngineTransportProtocol, consts.HttpApplicationProtocol, defaultWait) if err != nil { return nil, stacktrace.Propagate( @@ -266,6 +289,7 @@ func CreateEngine( } shouldRemoveLogsAggregator = false + shouldRemoveReverseProxy = false shouldKillEngineContainer = false return result, nil } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go index 0ba2fef9b0..b6050fbc7b 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/stop_engines.go @@ -2,7 +2,9 @@ package engine_functions import ( "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_operation_parallelizer" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/engine" @@ -72,5 +74,10 @@ func StopEngines( return nil, nil, stacktrace.Propagate(err, "An error occurred removing the logging components.") } + // Stop reverse proxy + if err := reverse_proxy_functions.DestroyReverseProxy(ctx, dockerManager); err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred removing the reverse proxy.") + } + return successfulGuids, erroredGuids, nil } diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go new file mode 100644 index 0000000000..71c596a5ee --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/consts.go @@ -0,0 +1,6 @@ +package reverse_proxy_functions + +const ( + defaultReverseProxyHttpPortNum = uint16(9730) + defaultReverseProxyDashboardPortNum = uint16(9731) +) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/create_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/create_reverse_proxy.go new file mode 100644 index 0000000000..c432263b40 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/create_reverse_proxy.go @@ -0,0 +1,84 @@ +package reverse_proxy_functions + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" +) + +const ( + defaultContainerStatusForNewReverseProxyContainer = types.ContainerStatus_Running +) + +// Create reverse proxy idempotently, if existing reverse proxy is found, then it is returned +func CreateReverseProxy( + ctx context.Context, + reverseProxyContainer ReverseProxyContainer, + dockerManager *docker_manager.DockerManager, + objAttrsProvider object_attributes_provider.DockerObjectAttributesProvider, +) ( + *reverse_proxy.ReverseProxy, + func(), + error, +) { + _, found, err := getReverseProxyContainer(ctx, dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred getting reverse proxy container.") + } + if found { + logrus.Debugf("Found existing reverse proxy; cannot start a new one.") + reverseProxyObj, _, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred getting existing reverse proxy.") + } + return reverseProxyObj, nil, nil + } + + reverseProxyNetwork, err := shared_helpers.GetEngineAndLogsComponentsNetwork(ctx, dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred getting the reverse proxy network.") + } + targetNetworkId := reverseProxyNetwork.GetId() + + containerId, containerLabels, removeReverseProxyContainerFunc, err := reverseProxyContainer.CreateAndStart( + ctx, + defaultReverseProxyHttpPortNum, + defaultReverseProxyDashboardPortNum, + targetNetworkId, + objAttrsProvider, + dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate( + err, + "An error occurred creating the reverse proxy container in Docker network with ID '%v'", + targetNetworkId, + ) + } + shouldRemoveReverseProxyContainer := true + defer func() { + if shouldRemoveReverseProxyContainer { + removeReverseProxyContainerFunc() + } + }() + + reverseProxy, err := getReverseProxyObjectFromContainerInfo( + ctx, + containerId, + defaultContainerStatusForNewReverseProxyContainer, + dockerManager) + if err != nil { + return nil, nil, stacktrace.Propagate(err, "An error occurred getting reverse proxy object using container ID '%v', labels '%+v', status '%v'.", containerId, containerLabels, defaultContainerStatusForNewReverseProxyContainer) + } + + removeReverseProxyFunc := func() { + removeReverseProxyContainerFunc() + } + + shouldRemoveReverseProxyContainer = false + return reverseProxy, removeReverseProxyFunc, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/destroy_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/destroy_reverse_proxy.go new file mode 100644 index 0000000000..267605d743 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/destroy_reverse_proxy.go @@ -0,0 +1,36 @@ +package reverse_proxy_functions + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" + "time" +) + +const ( + stopReverseProxyContainerTimeout = 2 * time.Second +) + +// Destroys reverse proxy idempotently, returns nil if no reverse proxy reverse proxy container was found +func DestroyReverseProxy(ctx context.Context, dockerManager *docker_manager.DockerManager) error { + _, maybeReverseProxyContainerId, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + logrus.Warnf("Attempted to destroy reverse proxy but no reverse proxy container was found.") + return nil + } + + if maybeReverseProxyContainerId == "" { + return nil + } + + if err := dockerManager.StopContainer(ctx, maybeReverseProxyContainerId, stopReverseProxyContainerTimeout); err != nil { + return stacktrace.Propagate(err, "An error occurred stopping the reverse proxy container with ID '%v'", maybeReverseProxyContainerId) + } + + if err := dockerManager.RemoveContainer(ctx, maybeReverseProxyContainerId); err != nil { + return stacktrace.Propagate(err, "An error occurred removing the reverse proxy container with ID '%v'", maybeReverseProxyContainerId) + } + + return nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/get_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/get_reverse_proxy.go new file mode 100644 index 0000000000..a38df8fd71 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/get_reverse_proxy.go @@ -0,0 +1,21 @@ +package reverse_proxy_functions + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" + "github.com/kurtosis-tech/stacktrace" +) + +func GetReverseProxy( + ctx context.Context, + dockerManager *docker_manager.DockerManager, +) (*reverse_proxy.ReverseProxy, error) { + + maybeReverseProxyObject, _, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the reverse proxy") + } + + return maybeReverseProxyObject, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go new file mode 100644 index 0000000000..7df8087c81 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/consts.go @@ -0,0 +1,36 @@ +package traefik + +const ( + ////////////////////////--TRAEFIK CONTAINER CONFIGURATION SECTION--///////////////////////////// + containerImage = "traefik:2.10.6" + + configDirpath = "/etc/traefik/" + configFilepath = configDirpath + "traefik.yml" + binaryFilepath = "/usr/local/bin/traefik" + ////////////////////////--FINISH TRAEFIK CONTAINER CONFIGURATION SECTION--///////////////////////////// + + ////////////////////////--TRAEFIK CONFIGURATION SECTION--///////////////////////////// + configFileTemplate = ` +accesslog: {} +log: + level: DEBUG +api: + debug: true + dashboard: true + insecure: true + disabledashboardad: true + +entryPoints: + web: + address: ":{{ .HttpPort }}" + traefik: + address: ":{{ .DashboardPort }}" + +providers: + docker: + endpoint: "unix:///var/run/docker.sock" + exposedByDefault: false + network: "{{ .NetworkId }}" +` + ////////////////////////--FINISH--TRAEFIK CONFIGURATION SECTION--///////////////////////////// +) diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go new file mode 100644 index 0000000000..b9de2a05c9 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider.go @@ -0,0 +1,126 @@ +package traefik + +import ( + "fmt" + + "github.com/docker/go-connections/nat" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" + "github.com/kurtosis-tech/stacktrace" +) + +const ( + shBinaryFilepath = "/bin/sh" + printfCmdName = "printf" + mkdirCmdName = "mkdir" + shCmdFlag = "-c" +) + +type traefikContainerConfigProvider struct { + config *reverse_proxy.ReverseProxyConfig +} + +func newTraefikContainerConfigProvider(config *reverse_proxy.ReverseProxyConfig) *traefikContainerConfigProvider { + return &traefikContainerConfigProvider{config: config} +} + +func (traefik *traefikContainerConfigProvider) GetContainerArgs( + containerName string, + containerLabels map[string]string, + httpPort uint16, + dashboardPort uint16, + networkId string, +) (*docker_manager.CreateAndStartContainerArgs, error) { + + bindMounts := map[string]string{ + // Necessary so that the reverse proxy can interact with the Docker engine + consts.DockerSocketFilepath: consts.DockerSocketFilepath, + } + + traefikConfigContentStr, err := traefik.config.GetConfigFileContent(configFileTemplate) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the traefik configuration content") + } + + // Create cmd to + // 1. create config file in appropriate location in the traefik container + // 2. start traefik with the config file + overrideCmd := []string{ + shCmdFlag, + fmt.Sprintf( + "%v '%v' && %v '%v' > %v && %v", + mkdirCmdName, + configDirpath, + printfCmdName, + traefikConfigContentStr, + configFilepath, + binaryFilepath, + ), + } + + // Traefik should ALWAYS be running + // Thus, instruct docker to restart the container if it exits with non-zero status code for whatever reason + restartPolicy := docker_manager.RestartPolicy(docker_manager.RestartAlways) + + defaultWait, err := port_spec.CreateWaitWithDefaultValues() + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating a wait with default values") + } + + // Publish HTTP and Dashboard entrypoint ports + privateHttpPortSpec, err := port_spec.NewPortSpec(httpPort, port_spec.TransportProtocol_TCP, consts.HttpApplicationProtocol, defaultWait) + if err != nil { + return nil, stacktrace.Propagate( + err, + "An error occurred creating Traefik private http port spec object using number '%v' and protocol '%v'", + httpPort, + consts.EngineTransportProtocol.String(), + ) + } + privateHttpDockerPort, err := shared_helpers.TransformPortSpecToDockerPort(privateHttpPortSpec) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred transforming the private http port spec to a Docker port") + } + privateDashboardPortSpec, err := port_spec.NewPortSpec(dashboardPort, port_spec.TransportProtocol_TCP, consts.HttpApplicationProtocol, defaultWait) + if err != nil { + return nil, stacktrace.Propagate( + err, + "An error occurred creating Traefik private dashboard port spec object using number '%v' and protocol '%v'", + dashboardPort, + consts.EngineTransportProtocol.String(), + ) + } + privateDashboardDockerPort, err := shared_helpers.TransformPortSpecToDockerPort(privateDashboardPortSpec) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred transforming the private dashboard port spec to a Docker port") + } + usedPorts := map[nat.Port]docker_manager.PortPublishSpec{ + privateHttpDockerPort: docker_manager.NewManualPublishingSpec(httpPort), + privateDashboardDockerPort: docker_manager.NewManualPublishingSpec(dashboardPort), + } + + createAndStartArgs := docker_manager.NewCreateAndStartContainerArgsBuilder( + containerImage, + containerName, + networkId, + ).WithLabels( + containerLabels, + ).WithBindMounts( + bindMounts, + ).WithUsedPorts( + usedPorts, + ).WithEntrypointArgs( + []string{ + shBinaryFilepath, + }, + ).WithCmdArgs( + overrideCmd, + ).WithRestartPolicy( + restartPolicy, + ).Build() + + return createAndStartArgs, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go new file mode 100644 index 0000000000..2b0b30f25e --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_container_config_provider_factory.go @@ -0,0 +1,10 @@ +package traefik + +import ( + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" +) + +func createTraefikContainerConfigProvider(httpPort uint16, dashboardPort uint16, networkId string) *traefikContainerConfigProvider { + config := reverse_proxy.NewDefaultReverseProxyConfig(httpPort, dashboardPort, networkId) + return newTraefikContainerConfigProvider(config) +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go new file mode 100644 index 0000000000..ebe3f1ac48 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik/traefik_reverse_proxy_container.go @@ -0,0 +1,68 @@ +package traefik + +import ( + "context" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" +) + +type traefikReverseProxyContainer struct{} + +func NewTraefikReverseProxyContainer() *traefikReverseProxyContainer { + return &traefikReverseProxyContainer{} +} + +func (traefikContainer *traefikReverseProxyContainer) CreateAndStart( + ctx context.Context, + httpPort uint16, + dashboardPort uint16, + targetNetworkId string, + objAttrsProvider object_attributes_provider.DockerObjectAttributesProvider, + dockerManager *docker_manager.DockerManager, +) (string, map[string]string, func(), error) { + traefikContainerConfigProviderObj := createTraefikContainerConfigProvider(httpPort, dashboardPort, targetNetworkId) + + reverseProxyAttrs, err := objAttrsProvider.ForReverseProxy() + if err != nil { + return "", nil, nil, stacktrace.Propagate(err, "An error occurred getting the reverse proxy container attributes.") + } + containerName := reverseProxyAttrs.GetName().GetString() + containerLabelStrs := map[string]string{} + for labelKey, labelValue := range reverseProxyAttrs.GetLabels() { + containerLabelStrs[labelKey.GetString()] = labelValue.GetString() + } + + createAndStartArgs, err := traefikContainerConfigProviderObj.GetContainerArgs(containerName, containerLabelStrs, httpPort, dashboardPort, targetNetworkId) + if err != nil { + return "", nil, nil, err + } + + containerId, _, err := dockerManager.CreateAndStartContainer(ctx, createAndStartArgs) + if err != nil { + return "", nil, nil, stacktrace.Propagate(err, "An error occurred starting the reverse proxy container with these args '%+v'", createAndStartArgs) + } + removeContainerFunc := func() { + removeCtx := context.Background() + + if err := dockerManager.RemoveContainer(removeCtx, containerId); err != nil { + logrus.Errorf( + "Launching the reverse proxy server with container ID '%v' didn't complete successfully so we "+ + "tried to remove the container we started, but doing so exited with an error:\n%v", + containerId, + err) + logrus.Errorf("ACTION REQUIRED: You'll need to manually remove the reverse proxy server with Docker container ID '%v'!!!!!!", containerId) + } + } + shouldRemoveReverseProxyContainer := true + defer func() { + if shouldRemoveReverseProxyContainer { + removeContainerFunc() + } + }() + + shouldRemoveReverseProxyContainer = false + return containerId, containerLabelStrs, removeContainerFunc, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go new file mode 100644 index 0000000000..3659f4af35 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go @@ -0,0 +1,74 @@ +package reverse_proxy_functions + +import ( + "context" + "net" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" +) + +const ( + emptyAliasForReverseProxy = "" +) + +var ( + autoAssignIpAddressToReverseProxy net.IP = nil +) + +func ConnectReverseProxyToNetwork(ctx context.Context, dockerManager *docker_manager.DockerManager, networkId string) error { + _, maybeReverseProxyContainerId, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + logrus.Warnf("Attempted to connect reverse proxy to a network but no reverse proxy container was found.") + return nil + } + + if maybeReverseProxyContainerId == "" { + return nil + } + + if err = dockerManager.ConnectContainerToNetwork(ctx, networkId, maybeReverseProxyContainerId, autoAssignIpAddressToReverseProxy, emptyAliasForReverseProxy); err != nil { + return stacktrace.Propagate(err, "An error occurred while connecting container '%v' to the enclave network '%v'", maybeReverseProxyContainerId, networkId) + } + + return nil +} + +func DisconnectReverseProxyFromNetwork(ctx context.Context, dockerManager *docker_manager.DockerManager, networkId string) error { + _, maybeReverseProxyContainerId, err := getReverseProxyObjectAndContainerId(ctx, dockerManager) + if err != nil { + logrus.Warnf("Attempted to disconnect reverse proxy from a network but no reverse proxy container was found.") + return nil + } + + if maybeReverseProxyContainerId == "" { + return nil + } + + if err = dockerManager.DisconnectContainerFromNetwork(ctx, maybeReverseProxyContainerId, networkId); err != nil { + return stacktrace.Propagate(err, "An error occurred while disconnecting container '%v' from the enclave network '%v'", maybeReverseProxyContainerId, networkId) + } + + return nil +} + +func ConnectReverseProxyToEnclaveNetworks(ctx context.Context, dockerManager *docker_manager.DockerManager) error { + kurtosisNetworkLabels := map[string]string{ + docker_label_key.AppIDDockerLabelKey.GetString(): label_value_consts.AppIDDockerLabelValue.GetString(), + } + enclaveNetworks, err := dockerManager.GetNetworksByLabels(ctx, kurtosisNetworkLabels) + if err != nil { + return stacktrace.Propagate(err, "An error occurred getting enclave networks") + } + + for _, enclaveNetwork := range enclaveNetworks { + if err = ConnectReverseProxyToNetwork(ctx, dockerManager, enclaveNetwork.GetId()); err != nil { + return stacktrace.Propagate(err, "An error occurred connecting the reverse proxy to the enclave network with id '%v'", enclaveNetwork.GetId()) + } + } + + return nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/reverse_proxy_container.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/reverse_proxy_container.go new file mode 100644 index 0000000000..5de87e2534 --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/reverse_proxy_container.go @@ -0,0 +1,18 @@ +package reverse_proxy_functions + +import ( + "context" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider" +) + +type ReverseProxyContainer interface { + CreateAndStart( + ctx context.Context, + httpPort uint16, + dashboardPort uint16, + targetNetworkId string, + objAttrsProvider object_attributes_provider.DockerObjectAttributesProvider, + dockerManager *docker_manager.DockerManager, + ) (string, map[string]string, func(), error) +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go new file mode 100644 index 0000000000..7ec08dde1a --- /dev/null +++ b/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go @@ -0,0 +1,126 @@ +package reverse_proxy_functions + +import ( + "context" + "net" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" + "github.com/kurtosis-tech/stacktrace" + "github.com/sirupsen/logrus" +) + +const ( + shouldShowStoppedReverseProxyContainers = true +) + +func getReverseProxyObjectAndContainerId( + ctx context.Context, + dockerManager *docker_manager.DockerManager, +) (*reverse_proxy.ReverseProxy, string, error) { + reverseProxyContainer, found, err := getReverseProxyContainer(ctx, dockerManager) + if err != nil { + return nil, "", stacktrace.Propagate(err, "An error occurred getting the reverse proxy container") + } + if !found { + return nil, "", nil + } + + reverseProxyContainerID := reverseProxyContainer.GetId() + + reverseProxyObject, err := getReverseProxyObjectFromContainerInfo( + ctx, + reverseProxyContainerID, + reverseProxyContainer.GetStatus(), + dockerManager, + ) + if err != nil { + return nil, "", stacktrace.Propagate(err, "An error occurred getting the reverse proxy object using container ID '%v', labels '%+v' and the status '%v'", reverseProxyContainer.GetId(), reverseProxyContainer.GetLabels(), reverseProxyContainer.GetStatus()) + } + + return reverseProxyObject, reverseProxyContainerID, nil +} + +func getReverseProxyContainer(ctx context.Context, dockerManager *docker_manager.DockerManager) (*types.Container, bool, error) { + reverseProxyContainerSearchLabels := map[string]string{ + docker_label_key.AppIDDockerLabelKey.GetString(): label_value_consts.AppIDDockerLabelValue.GetString(), + docker_label_key.ContainerTypeDockerLabelKey.GetString(): label_value_consts.ReverseProxyTypeDockerLabelValue.GetString(), + } + + matchingReverseProxyContainers, err := dockerManager.GetContainersByLabels(ctx, reverseProxyContainerSearchLabels, shouldShowStoppedReverseProxyContainers) + if err != nil { + return nil, false, stacktrace.Propagate(err, "An error occurred fetching the reverse proxy container using labels: %+v", reverseProxyContainerSearchLabels) + } + + if len(matchingReverseProxyContainers) == 0 { + return nil, false, nil + } + if len(matchingReverseProxyContainers) > 1 { + return nil, false, stacktrace.NewError("Found more than one reverse proxy Docker container'; this is a bug in Kurtosis") + } + return matchingReverseProxyContainers[0], true, nil +} + +func getReverseProxyObjectFromContainerInfo( + ctx context.Context, + containerId string, + containerStatus types.ContainerStatus, + dockerManager *docker_manager.DockerManager, +) (*reverse_proxy.ReverseProxy, error) { + var privateIpAddr net.IP + var enclaveNetworksIpAddress map[string]net.IP + + isContainerRunning, found := consts.IsContainerRunningDeterminer[containerStatus] + if !found { + // This should never happen because we enforce completeness in a unit test + return nil, stacktrace.NewError("No is-running designation found for reverse proxy container status '%v'; this is a bug in Kurtosis!", containerStatus.String()) + } + + var reverseProxyStatus container.ContainerStatus + if isContainerRunning { + reverseProxyStatus = container.ContainerStatus_Running + + privateIpAddrStr, err := dockerManager.GetContainerIP(ctx, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn, containerId) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the private IP address of container '%v' in network '%v'", containerId, consts.NameOfNetworkToStartEngineAndLogServiceContainersIn) + } + privateIpAddr = net.ParseIP(privateIpAddrStr) + if privateIpAddr == nil { + return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", privateIpAddrStr) + } + + networksIpAddressStr, err := dockerManager.GetContainerIps(ctx, containerId) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting the networks private IP address of container '%v'", containerId) + } + enclaveNetworksIpAddress = map[string]net.IP{} + for networkId, networkIpAddressStr := range networksIpAddressStr { + if networkIpAddressStr != privateIpAddrStr { + networkIpAddress := net.ParseIP(networkIpAddressStr) + if networkIpAddress == nil { + return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", networkIpAddressStr) + } + enclaveNetworksIpAddress[networkId] = networkIpAddress + } + } + logrus.Debugf("Enclave networks: '%v'", enclaveNetworksIpAddress) + } else { + reverseProxyStatus = container.ContainerStatus_Stopped + } + + reverseProxyObj := reverse_proxy.NewReverseProxy( + reverseProxyStatus, + privateIpAddr, + enclaveNetworksIpAddress, + defaultReverseProxyHttpPortNum, + defaultReverseProxyDashboardPortNum, + ) + + return reverseProxyObj, nil +} diff --git a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go index 874272bb99..e69b7e908f 100644 --- a/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go +++ b/container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go @@ -82,8 +82,9 @@ const ( // ------------------ Filter Search Keys ---------------------- // All these defined in https://docs.docker.com/engine/api/v1.24 - containerNameSearchFilterKey = "name" - containerLabelSearchFilterKey = "label" + containerNameSearchFilterKey = "name" + containerLabelSearchFilterKey = "label" + containerNetworkIdSearchFilterKey = "network" volumeNameSearchFilterKey = "name" volumeLabelSearchFilterKey = "label" @@ -792,6 +793,24 @@ func (manager *DockerManager) GetContainerIP(ctx context.Context, networkName st return networkInfo.IPAddress, nil } +/* +GetContainerIps +Gets the container's IPs on all networks +Returns a map of network ID to network IP address +*/ +func (manager *DockerManager) GetContainerIps(ctx context.Context, containerId string) (map[string]string, error) { + containerIps := map[string]string{} + resp, err := manager.dockerClient.ContainerInspect(ctx, containerId) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred inspecting container with ID '%v'", containerId) + } + allNetworkInfo := resp.NetworkSettings.Networks + for _, networkInfo := range allNetworkInfo { + containerIps[networkInfo.NetworkID] = networkInfo.IPAddress + } + return containerIps, nil +} + func (manager *DockerManager) AttachToContainer(ctx context.Context, containerId string) (types.HijackedResponse, error) { attachOpts := types.ContainerAttachOptions{ Stream: true, @@ -1199,6 +1218,16 @@ func (manager *DockerManager) GetContainersByLabels(ctx context.Context, labels return result, nil } +func (manager *DockerManager) GetContainersByNetworkId(ctx context.Context, networkId string, shouldShowStoppedContainers bool) ([]*docker_manager_types.Container, error) { + filterArg := filters.Arg(containerNetworkIdSearchFilterKey, networkId) + networkIdFilterList := filters.NewArgs(filterArg) + result, err := manager.getContainersByFilterArgs(ctx, networkIdFilterList, shouldShowStoppedContainers) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred getting containers with network id '%+v'", networkIdFilterList) + } + return result, nil +} + // [FetchImageIfMissing] uses the local [dockerImage] if it's available. // If unavailable, will attempt to fetch the latest image. // Returns error if local [dockerImage] is unavailable and pulling image fails. diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go index d9e482bd54..a36dc7cd7a 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider.go @@ -33,6 +33,10 @@ const ( logsCollectorFragment = "kurtosis-logs-collector" // The collector is per enclave so this is a suffix logsCollectorVolumeFragment = logsCollectorFragment + "-vol" + + reverseProxyEnclaveShortUuidHeader = "X-Kurtosis-Enclave-Short-UUID" + reverseProxyServiceShortUuidHeader = "X-Kurtosis-Service-Short-UUID" + reverseProxyServicePortNumberHeader = "X-Kurtosis-Service-Port-Number" ) type DockerEnclaveObjectAttributesProvider interface { @@ -529,22 +533,25 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getLabelsForEnclaveOb } // Return Traefik labels -// Including the labels required to route traffic to the user service ports based on the Host header: +// Including the labels required to route traffic to the user service ports based on the header X-Kurtosis-Service-Port: // -- // The Traefik service name format is: -- // With the following input: -// Enclave short UUID: 65d2fb6d6732 -// Service short UUID: 3771c85af16a -// HTTP Port 1 number: 80 -// HTTP Port 2 number: 81 +// +// Enclave short UUID: 65d2fb6d6732 +// Service short UUID: 3771c85af16a +// HTTP Port 1 number: 80 +// HTTP Port 2 number: 81 +// // the following labels are returned: -// "traefik.enable": "true", -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Host(`80-3771c85af16a-65d2fb6d6732`)", -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.service": "65d2fb6d6732-3771c85af16a-80", -// "traefik.http.services.65d2fb6d6732-3771c85af16a-80.loadbalancer.server.port": "80" -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.rule": "Host(`81-3771c85af16a-65d2fb6d6732`)", -// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.service": "65d2fb6d6732-3771c85af16a-81", -// "traefik.http.services.65d2fb6d6732-3771c85af16a-81.loadbalancer.server.port": "81" +// +// "traefik.enable": "true", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.rule": "Headers(`X-Kurtosis-Enclave-Short-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-Short-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Port-Number`, `80`)", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-80.service": "65d2fb6d6732-3771c85af16a-80", +// "traefik.http.services.65d2fb6d6732-3771c85af16a-80.loadbalancer.server.port": "80" +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.rule": "Headers(`X-Kurtosis-Enclave-Short-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-Short-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Port-Number`, `81`)", +// "traefik.http.routers.65d2fb6d6732-3771c85af16a-81.service": "65d2fb6d6732-3771c85af16a-81", +// "traefik.http.services.65d2fb6d6732-3771c85af16a-81.loadbalancer.server.port": "81" func (provider *dockerEnclaveObjectAttributesProviderImpl) getTraefikLabelsForEnclaveObject(serviceUuid string, ports map[string]*port_spec.PortSpec) (map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue, error) { labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{} @@ -558,13 +565,13 @@ func (provider *dockerEnclaveObjectAttributesProviderImpl) getTraefikLabelsForEn shortServiceUuid := uuid_generator.ShortenedUUIDString(serviceUuid) servicePortStr := fmt.Sprintf("%s-%s-%d", shortEnclaveUuid, shortServiceUuid, portSpec.GetNumber()) - // Host rule + // Header X-Kurtosis-Service-Port rule ruleKeySuffix := fmt.Sprintf("http.routers.%s.rule", servicePortStr) ruleLabelKey, err := docker_label_key.CreateNewDockerTraefikLabelKey(ruleKeySuffix) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred getting the traefik rule label key with suffix '%v'", ruleKeySuffix) } - ruleValue := fmt.Sprintf("Host(`%d-%s-%s`)", portSpec.GetNumber(), shortServiceUuid, shortEnclaveUuid) + ruleValue := fmt.Sprintf("Headers(`%s`, `%s`) && Headers(`%s`, `%s`) && Headers(`%s`, `%d`)", reverseProxyEnclaveShortUuidHeader, shortEnclaveUuid, reverseProxyServiceShortUuidHeader, shortServiceUuid, reverseProxyServicePortNumberHeader, portSpec.GetNumber()) ruleLabelValue, err := docker_label_value.CreateNewDockerLabelValue(ruleValue) if err != nil { return nil, stacktrace.Propagate(err, "An error occurred creating the traefik rule label value with value '%v'", ruleValue) diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go index 1b91923413..64d3cf4678 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/enclave_object_attributes_provider_test.go @@ -68,7 +68,7 @@ func TestForUserServiceContainer(t *testing.T) { case "traefik.http.routers.65d2fb6d6732-3771c85af16a-23.rule": require.Fail(t, "A traefik label for port 23 should not be present") case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.rule": - require.Equal(t, labelValue.GetString(), "Host(`45-3771c85af16a-65d2fb6d6732`)") + require.Equal(t, labelValue.GetString(), "Headers(`X-Kurtosis-Enclave-Short-UUID`, `65d2fb6d6732`) && Headers(`X-Kurtosis-Service-Short-UUID`, `3771c85af16a`) && Headers(`X-Kurtosis-Service-Port-Number`, `45`)") case "traefik.http.routers.65d2fb6d6732-3771c85af16a-45.service": require.Equal(t, labelValue.GetString(), "65d2fb6d6732-3771c85af16a-45") case "traefik.http.services.65d2fb6d6732-3771c85af16a-45.loadbalancer.server.port": diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go index f44294d616..9b830cf813 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts.go @@ -15,6 +15,7 @@ const ( engineContainerTypeLabelValueStr = "kurtosis-engine" logsAggregatorContainerTypeLabelValueStr = "kurtosis-logs-aggregator" logsCollectorContainerTypeLabelValueStr = "kurtosis-logs-collector" + reverseProxyContainerTypeLabelValueStr = "kurtosis-reverse-proxy" // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!! apiContainerContainerTypeLabelValueStr = "api-container" @@ -38,6 +39,7 @@ var AppIDDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(app var EngineContainerTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(engineContainerTypeLabelValueStr) var LogsAggregatorTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsAggregatorContainerTypeLabelValueStr) var LogsCollectorTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(logsCollectorContainerTypeLabelValueStr) +var ReverseProxyTypeDockerLabelValue = docker_label_value.MustCreateNewDockerLabelValue(reverseProxyContainerTypeLabelValueStr) // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! DO NOT CHANGE THESE VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts_test.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts_test.go index b4d1df6886..79cddc9d5f 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts_test.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts/label_value_consts_test.go @@ -11,6 +11,7 @@ var labelValueStrsToEnsure = map[string]string{ engineContainerTypeLabelValueStr: "kurtosis-engine", logsCollectorContainerTypeLabelValueStr: "kurtosis-logs-collector", logsAggregatorContainerTypeLabelValueStr: "kurtosis-logs-aggregator", + reverseProxyContainerTypeLabelValueStr: "kurtosis-reverse-proxy", } var labelValuesToEnsure = map[*docker_label_value.DockerLabelValue]string{ @@ -18,6 +19,7 @@ var labelValuesToEnsure = map[*docker_label_value.DockerLabelValue]string{ EngineContainerTypeDockerLabelValue: "kurtosis-engine", LogsAggregatorTypeDockerLabelValue: "kurtosis-logs-aggregator", LogsCollectorTypeDockerLabelValue: "kurtosis-logs-collector", + ReverseProxyTypeDockerLabelValue: "kurtosis-reverse-proxy", } // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! IMPORTANT !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go index a68ee396f2..52fcbc54d6 100644 --- a/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go +++ b/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/object_attributes_provider.go @@ -17,6 +17,7 @@ const ( engineServerNamePrefix = "kurtosis-engine" logsAggregatorName = "kurtosis-logs-aggregator" logsStorageVolumeName = "kurtosis-logs-storage" + reverseProxyName = "kurtosis-reverse-proxy" ) type DockerObjectAttributesProvider interface { @@ -28,6 +29,7 @@ type DockerObjectAttributesProvider interface { ForEnclave(enclaveUuid enclave.EnclaveUUID) (DockerEnclaveObjectAttributesProvider, error) ForLogsAggregator() (DockerObjectAttributes, error) ForLogsStorageVolume() (DockerObjectAttributes, error) + ForReverseProxy() (DockerObjectAttributes, error) } func GetDockerObjectAttributesProvider() DockerObjectAttributesProvider { @@ -134,3 +136,20 @@ func (provider *dockerObjectAttributesProviderImpl) ForLogsStorageVolume() (Dock } return objectAttributes, nil } + +func (provider *dockerObjectAttributesProviderImpl) ForReverseProxy() (DockerObjectAttributes, error) { + name, err := docker_object_name.CreateNewDockerObjectName(reverseProxyName) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred creating a Docker object name object from string '%v'", reverseProxyName) + } + + labels := map[*docker_label_key.DockerLabelKey]*docker_label_value.DockerLabelValue{ + docker_label_key.ContainerTypeDockerLabelKey: label_value_consts.ReverseProxyTypeDockerLabelValue, + } + + objectAttributes, err := newDockerObjectAttributesImpl(name, labels) + if err != nil { + return nil, stacktrace.Propagate(err, "An error occurred while creating the ObjectAttributesImpl with the name '%s' and labels '%+v'", name, labels) + } + return objectAttributes, nil +} diff --git a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go index e9d77f605d..4cbfaefce9 100644 --- a/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend.go @@ -19,6 +19,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" @@ -460,6 +461,23 @@ func (backend *KubernetesKurtosisBackend) DestroyLogsCollectorForEnclave(ctx con return stacktrace.NewError("Destroy the logs collector for enclave isn't yet implemented on Kubernetes") } +func (backend *KubernetesKurtosisBackend) GetReverseProxy( + ctx context.Context, +) (*reverse_proxy.ReverseProxy, error) { + // TODO IMPLEMENT + return nil, stacktrace.NewError("Getting the reverse proxy isn't yet implemented on Kubernetes") +} + +func (backend *KubernetesKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + // TODO IMPLEMENT + return nil, stacktrace.NewError("Creating the reverse proxy isn't yet implemented on Kubernetes") +} + +func (backend *KubernetesKurtosisBackend) DestroyReverseProxy(ctx context.Context) error { + // TODO IMPLEMENT + return stacktrace.NewError("Destroying the reverse proxy isn't yet implemented on Kubernetes") +} + func (backend *KubernetesKurtosisBackend) BuildImage(ctx context.Context, imageName string, imageBuildSpec *image_build_spec.ImageBuildSpec) error { // TODO IMPLEMENT return stacktrace.NewError("Building images isn't yet implemented in Kubernetes.") diff --git a/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go b/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go index 550a99c585..5d1baa95d8 100644 --- a/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_impls/metrics_reporting/metrics_reporting_kurtosis_backend.go @@ -15,6 +15,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/stacktrace" ) @@ -439,6 +440,18 @@ func (backend *MetricsReportingKurtosisBackend) DestroyLogsCollectorForEnclave(c return nil } +func (backend *MetricsReportingKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + return backend.underlying.CreateReverseProxy(ctx) +} + +func (backend *MetricsReportingKurtosisBackend) GetReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + return backend.underlying.GetReverseProxy(ctx) +} + +func (backend *MetricsReportingKurtosisBackend) DestroyReverseProxy(ctx context.Context) error { + return backend.underlying.DestroyReverseProxy(ctx) +} + func (backend *MetricsReportingKurtosisBackend) GetAvailableCPUAndMemory(ctx context.Context) (compute_resources.MemoryInMegaBytes, compute_resources.CpuMilliCores, bool, error) { availableMemory, availableCpu, isResourceInformationComplete, err := backend.underlying.GetAvailableCPUAndMemory(ctx) if err != nil { diff --git a/container-engine-lib/lib/backend_interface/kurtosis_backend.go b/container-engine-lib/lib/backend_interface/kurtosis_backend.go index b012ed21c9..9114dfd2de 100644 --- a/container-engine-lib/lib/backend_interface/kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/kurtosis_backend.go @@ -14,6 +14,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector" + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" ) @@ -338,6 +339,13 @@ type KurtosisBackend interface { // Destroy the logs collector for enclave with UUID DestroyLogsCollectorForEnclave(ctx context.Context, enclaveUuid enclave.EnclaveUUID) error + CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) + + // Returns nil if logs aggregator was not found + GetReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) + + DestroyReverseProxy(ctx context.Context) error + // GetAvailableCPUAndMemory - gets available memory in megabytes and cpu in millicores, the boolean indicates whether the information is complete GetAvailableCPUAndMemory(ctx context.Context) (compute_resources.MemoryInMegaBytes, compute_resources.CpuMilliCores, bool, error) diff --git a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go index 3d51d33571..9090d97802 100644 --- a/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go +++ b/container-engine-lib/lib/backend_interface/mock_kurtosis_backend.go @@ -26,6 +26,8 @@ import ( mock "github.com/stretchr/testify/mock" + reverse_proxy "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy" + service "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" time "time" @@ -419,6 +421,60 @@ func (_c *MockKurtosisBackend_CreateLogsCollectorForEnclave_Call) RunAndReturn(r return _c } +// CreateReverseProxy provides a mock function with given fields: ctx +func (_m *MockKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + ret := _m.Called(ctx) + + var r0 *reverse_proxy.ReverseProxy + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*reverse_proxy.ReverseProxy, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *reverse_proxy.ReverseProxy); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*reverse_proxy.ReverseProxy) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockKurtosisBackend_CreateReverseProxy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateReverseProxy' +type MockKurtosisBackend_CreateReverseProxy_Call struct { + *mock.Call +} + +// CreateReverseProxy is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockKurtosisBackend_Expecter) CreateReverseProxy(ctx interface{}) *MockKurtosisBackend_CreateReverseProxy_Call { + return &MockKurtosisBackend_CreateReverseProxy_Call{Call: _e.mock.On("CreateReverseProxy", ctx)} +} + +func (_c *MockKurtosisBackend_CreateReverseProxy_Call) Run(run func(ctx context.Context)) *MockKurtosisBackend_CreateReverseProxy_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockKurtosisBackend_CreateReverseProxy_Call) Return(_a0 *reverse_proxy.ReverseProxy, _a1 error) *MockKurtosisBackend_CreateReverseProxy_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockKurtosisBackend_CreateReverseProxy_Call) RunAndReturn(run func(context.Context) (*reverse_proxy.ReverseProxy, error)) *MockKurtosisBackend_CreateReverseProxy_Call { + _c.Call.Return(run) + return _c +} + // DestroyAPIContainers provides a mock function with given fields: ctx, filters func (_m *MockKurtosisBackend) DestroyAPIContainers(ctx context.Context, filters *api_container.APIContainerFilters) (map[enclave.EnclaveUUID]bool, map[enclave.EnclaveUUID]error, error) { ret := _m.Called(ctx, filters) @@ -696,6 +752,48 @@ func (_c *MockKurtosisBackend_DestroyLogsCollectorForEnclave_Call) RunAndReturn( return _c } +// DestroyReverseProxy provides a mock function with given fields: ctx +func (_m *MockKurtosisBackend) DestroyReverseProxy(ctx context.Context) error { + ret := _m.Called(ctx) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context) error); ok { + r0 = rf(ctx) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockKurtosisBackend_DestroyReverseProxy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DestroyReverseProxy' +type MockKurtosisBackend_DestroyReverseProxy_Call struct { + *mock.Call +} + +// DestroyReverseProxy is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockKurtosisBackend_Expecter) DestroyReverseProxy(ctx interface{}) *MockKurtosisBackend_DestroyReverseProxy_Call { + return &MockKurtosisBackend_DestroyReverseProxy_Call{Call: _e.mock.On("DestroyReverseProxy", ctx)} +} + +func (_c *MockKurtosisBackend_DestroyReverseProxy_Call) Run(run func(ctx context.Context)) *MockKurtosisBackend_DestroyReverseProxy_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockKurtosisBackend_DestroyReverseProxy_Call) Return(_a0 error) *MockKurtosisBackend_DestroyReverseProxy_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockKurtosisBackend_DestroyReverseProxy_Call) RunAndReturn(run func(context.Context) error) *MockKurtosisBackend_DestroyReverseProxy_Call { + _c.Call.Return(run) + return _c +} + // DestroyUserServices provides a mock function with given fields: ctx, enclaveUuid, filters func (_m *MockKurtosisBackend) DestroyUserServices(ctx context.Context, enclaveUuid enclave.EnclaveUUID, filters *service.ServiceFilters) (map[service.ServiceUUID]bool, map[service.ServiceUUID]error, error) { ret := _m.Called(ctx, enclaveUuid, filters) @@ -1292,6 +1390,60 @@ func (_c *MockKurtosisBackend_GetLogsCollectorForEnclave_Call) RunAndReturn(run return _c } +// GetReverseProxy provides a mock function with given fields: ctx +func (_m *MockKurtosisBackend) GetReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) { + ret := _m.Called(ctx) + + var r0 *reverse_proxy.ReverseProxy + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*reverse_proxy.ReverseProxy, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *reverse_proxy.ReverseProxy); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*reverse_proxy.ReverseProxy) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockKurtosisBackend_GetReverseProxy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetReverseProxy' +type MockKurtosisBackend_GetReverseProxy_Call struct { + *mock.Call +} + +// GetReverseProxy is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockKurtosisBackend_Expecter) GetReverseProxy(ctx interface{}) *MockKurtosisBackend_GetReverseProxy_Call { + return &MockKurtosisBackend_GetReverseProxy_Call{Call: _e.mock.On("GetReverseProxy", ctx)} +} + +func (_c *MockKurtosisBackend_GetReverseProxy_Call) Run(run func(ctx context.Context)) *MockKurtosisBackend_GetReverseProxy_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockKurtosisBackend_GetReverseProxy_Call) Return(_a0 *reverse_proxy.ReverseProxy, _a1 error) *MockKurtosisBackend_GetReverseProxy_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockKurtosisBackend_GetReverseProxy_Call) RunAndReturn(run func(context.Context) (*reverse_proxy.ReverseProxy, error)) *MockKurtosisBackend_GetReverseProxy_Call { + _c.Call.Return(run) + return _c +} + // GetShellOnUserService provides a mock function with given fields: ctx, enclaveUuid, serviceUuid func (_m *MockKurtosisBackend) GetShellOnUserService(ctx context.Context, enclaveUuid enclave.EnclaveUUID, serviceUuid service.ServiceUUID) error { ret := _m.Called(ctx, enclaveUuid, serviceUuid) diff --git a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/consts.go b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/consts.go new file mode 100644 index 0000000000..0044efe43e --- /dev/null +++ b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/consts.go @@ -0,0 +1,5 @@ +package reverse_proxy + +const ( + configFileTemplateName = "configFileTemplate" +) diff --git a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go new file mode 100644 index 0000000000..6b450ce23c --- /dev/null +++ b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy.go @@ -0,0 +1,61 @@ +package reverse_proxy + +import ( + "net" + + "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container" +) + +// This component is responsible for routing http traffic to the services +type ReverseProxy struct { + status container.ContainerStatus + + // IP address of the reverse proxy container in its network + // This will be nil if the container is not running + maybePrivateIpAddr net.IP + + // IP address of the reverse proxy container in each enclave network + // This will be nil if the container is not running + maybeEnclaveNetworksIpAddress map[string]net.IP + + // HTTP port + httpPort uint16 + + // Dashboard port + dashboardPort uint16 +} + +func NewReverseProxy( + status container.ContainerStatus, + maybePrivateIpAddr net.IP, + maybeEnclaveNetworksIpAddress map[string]net.IP, + httpPort uint16, + dashboardPort uint16) *ReverseProxy { + return &ReverseProxy{ + status: status, + maybePrivateIpAddr: maybePrivateIpAddr, + maybeEnclaveNetworksIpAddress: maybeEnclaveNetworksIpAddress, + httpPort: httpPort, + dashboardPort: dashboardPort, + } +} + +func (reverseProxy *ReverseProxy) GetStatus() container.ContainerStatus { + return reverseProxy.status +} + +func (reverseProxy *ReverseProxy) GetPrivateIpAddr() net.IP { + return reverseProxy.maybePrivateIpAddr +} + +func (reverseProxy *ReverseProxy) GetEnclaveNetworksIpAddress() map[string]net.IP { + return reverseProxy.maybeEnclaveNetworksIpAddress +} + +func (reverseProxy *ReverseProxy) GetHttpPort() uint16 { + return reverseProxy.httpPort +} + +func (reverseProxy *ReverseProxy) GetDashboardPort() uint16 { + return reverseProxy.dashboardPort +} diff --git a/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy_config.go b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy_config.go new file mode 100644 index 0000000000..7035ba9013 --- /dev/null +++ b/container-engine-lib/lib/backend_interface/objects/reverse_proxy/reverse_proxy_config.go @@ -0,0 +1,37 @@ +package reverse_proxy + +import ( + "bytes" + "text/template" + + "github.com/kurtosis-tech/stacktrace" +) + +type ReverseProxyConfig struct { + HttpPort uint16 + DashboardPort uint16 + NetworkId string +} + +func NewDefaultReverseProxyConfig(httpPort uint16, dashboardPort uint16, networkId string) *ReverseProxyConfig { + return &ReverseProxyConfig{ + HttpPort: httpPort, + DashboardPort: dashboardPort, + NetworkId: networkId, + } +} + +func (cfg *ReverseProxyConfig) GetConfigFileContent(configFileTemplate string) (string, error) { + cfgFileTemplate, err := template.New(configFileTemplateName).Parse(configFileTemplate) + if err != nil { + return "", stacktrace.Propagate(err, "An error occurred parsing the reverse proxy's config template.") + } + + templateStrBuffer := &bytes.Buffer{} + if err := cfgFileTemplate.Execute(templateStrBuffer, cfg); err != nil { + return "", stacktrace.Propagate(err, "An error occurred executing the reverse proxy's config file template.") + } + templateStr := templateStrBuffer.String() + + return templateStr, nil +}