feat: Engine K8S ingress for REST API reverse proxy routing

container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/engine_functions/create_engine.go

@@ -27,7 +27,6 @@ const (
 	//TODO: pass this parameter
 	enclaveManagerUIPort                        = 9711
 	enclaveManagerAPIPort                       = 8081
-	restAPIPort                                 = 9779 //TODO: pass this parameter
 	maxWaitForEngineAvailabilityRetries         = 10
 	timeBetweenWaitForEngineAvailabilityRetries = 1 * time.Second
 	logsStorageDirpath                          = "/var/log/kurtosis/"
@@ -159,12 +158,12 @@ func CreateEngine(
 		)
 	}
 
-	restAPIPortSpec, err := port_spec.NewPortSpec(uint16(restAPIPort), consts.EngineTransportProtocol, consts.HttpApplicationProtocol, defaultWait)
+	restAPIPortSpec, err := port_spec.NewPortSpec(engine.RESTAPIPortAddr, consts.EngineTransportProtocol, consts.HttpApplicationProtocol, defaultWait)
 	if err != nil {
 		return nil, stacktrace.Propagate(
 			err,
 			"An error occurred creating the REST API server's http port spec object using number '%v' and protocol '%v'",
-			restAPIPort,
+			engine.RESTAPIPortAddr,
 			consts.EngineTransportProtocol.String(),
 		)
 	}
@@ -207,7 +206,7 @@ func CreateEngine(
 		privateGrpcDockerPort:       docker_manager.NewManualPublishingSpec(grpcPortNum),
 		enclaveManagerUIDockerPort:  docker_manager.NewManualPublishingSpec(uint16(enclaveManagerUIPort)),
 		enclaveManagerAPIDockerPort: docker_manager.NewManualPublishingSpec(uint16(enclaveManagerAPIPort)),
-		restAPIDockerPort:           docker_manager.NewManualPublishingSpec(uint16(restAPIPort)),
+		restAPIDockerPort:           docker_manager.NewManualPublishingSpec(engine.RESTAPIPortAddr),
 	}
 
 	bindMounts := map[string]string{

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/consts/consts.go

@@ -2,7 +2,8 @@ package consts
 
 import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1"
 )
 
 const (
@@ -14,11 +15,16 @@ const (
 	//  be stored in the port spec label
 	KurtosisInternalContainerGrpcPortSpecId = "grpc"
 
-	// The ID of the GRPC proxy port for Kurtosis-internal containers. This is necessary because
-	// Typescript's grpc-web cannot communicate directly with GRPC ports, so Kurtosis-internal containers
-	// need a proxy  that will translate grpc-web requests before they hit the main GRPC server
-	KurtosisInternalContainerGrpcProxyPortSpecId = "grpc-proxy"
-	HttpApplicationProtocol                      = "http"
+	// The ID of the REST API port
+	KurtosisInternalContainerRESTAPIPortSpecId = "rest-api"
+
+	HttpApplicationProtocol = "http"
+
+	IngressRulePathAllPaths = "/"
+)
+
+var (
+	IngressRulePathTypePrefix = netv1.PathTypePrefix
 )
 
 // This maps a Kubernetes pod's phase to a binary "is the pod considered running?" determiner

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/create_engine.go

@@ -3,6 +3,8 @@ package engine_functions
 import (
 	"context"
 	"fmt"
+	"time"
+
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/consts"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager"
@@ -16,8 +18,8 @@ import (
 	"github.com/kurtosis-tech/stacktrace"
 	"github.com/sirupsen/logrus"
 	apiv1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	"time"
 )
 
 const (
@@ -26,6 +28,8 @@ const (
 	maxWaitForEngineContainerAvailabilityRetries         = 30
 	timeBetweenWaitForEngineContainerAvailabilityRetries = 1 * time.Second
 	httpApplicationProtocol                              = "http"
+
+	restAPIPortHost = "engine"
 )
 
 var noWait *port_spec.Wait = nil
@@ -65,6 +69,15 @@ func CreateEngine(
 			consts.KurtosisServersTransportProtocol.String(),
 		)
 	}
+	privateRESTAPIPortSpec, err := port_spec.NewPortSpec(engine.RESTAPIPortAddr, consts.KurtosisServersTransportProtocol, httpApplicationProtocol, noWait)
+	if err != nil {
+		return nil, stacktrace.Propagate(
+			err,
+			"An error occurred creating the engine's private rest api port spec object using number '%v' and protocol '%v'",
+			engine.RESTAPIPortAddr,
+			consts.KurtosisServersTransportProtocol.String(),
+		)
+	}
 	privatePortSpecs := map[string]*port_spec.PortSpec{
 		consts.KurtosisInternalContainerGrpcPortSpecId: privateGrpcPortSpec,
 	}
@@ -149,6 +162,7 @@ func CreateEngine(
 		namespaceName,
 		engineAttributesProvider,
 		privateGrpcPortSpec,
+		privateRESTAPIPortSpec,
 		enginePodLabels,
 		kubernetesManager,
 	)
@@ -165,13 +179,34 @@ func CreateEngine(
 		}
 	}()
 
+	engineIngress, err := createEngineIngress(
+		ctx,
+		namespaceName,
+		engineAttributesProvider,
+		privateRESTAPIPortSpec,
+		kubernetesManager,
+	)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred creating the engine ingress")
+	}
+	var shouldRemoveIngress = true
+	defer func() {
+		if shouldRemoveIngress {
+			if err := kubernetesManager.RemoveIngress(ctx, engineIngress); err != nil {
+				logrus.Errorf("Creating the engine didn't complete successfully, so we tried to delete Kubernetes ingress '%v' that we created but an error was thrown:\n%v", engineIngress.Name, err)
+				logrus.Errorf("ACTION REQUIRED: You'll need to manually remove Kubernetes ingress with name '%v'!!!!!!!", engineIngress.Name)
+			}
+		}
+	}()
+
 	engineResources := &engineKubernetesResources{
 		clusterRole:        clusterRole,
 		clusterRoleBinding: clusterRoleBindings,
 		namespace:          namespace,
 		serviceAccount:     serviceAccount,
 		service:            engineService,
 		pod:                enginePod,
+		ingress:            engineIngress,
 	}
 	engineObjsById, err := getEngineObjectsFromKubernetesResources(map[engine.EngineGUID]*engineKubernetesResources{
 		engineGuid: engineResources,
@@ -216,6 +251,7 @@ func CreateEngine(
 	shouldRemoveClusterRoleBinding = false
 	shouldRemovePod = false
 	shouldRemoveService = false
+	shouldRemoveIngress = false
 	return resultEngine, nil
 }
 
@@ -448,18 +484,21 @@ func createEngineService(
 	namespace string,
 	engineAttributesProvider object_attributes_provider.KubernetesEngineObjectAttributesProvider,
 	privateGrpcPortSpec *port_spec.PortSpec,
+	privateRESTAPIPortSpec *port_spec.PortSpec,
 	podMatchLabels map[*kubernetes_label_key.KubernetesLabelKey]*kubernetes_label_value.KubernetesLabelValue,
 	kubernetesManager *kubernetes_manager.KubernetesManager,
 ) (*apiv1.Service, error) {
 	engineServiceAttributes, err := engineAttributesProvider.ForEngineService(
 		consts.KurtosisInternalContainerGrpcPortSpecId,
 		privateGrpcPortSpec,
-		consts.KurtosisInternalContainerGrpcProxyPortSpecId, nil)
+		consts.KurtosisInternalContainerRESTAPIPortSpecId,
+		privateRESTAPIPortSpec)
 	if err != nil {
 		return nil, stacktrace.Propagate(
 			err,
-			"An error occurred getting the engine service attributes using private grpc port spec '%+v'",
+			"An error occurred getting the engine service attributes using private grpc port spec '%+v' and private REST API port spec '%+v'",
 			privateGrpcPortSpec,
+			privateRESTAPIPortSpec,
 		)
 	}
 	engineServiceName := engineServiceAttributes.GetName().GetString()
@@ -468,7 +507,8 @@ func createEngineService(
 
 	// Define service ports. These hook up to ports on the containers running in the engine pod
 	servicePorts, err := shared_helpers.GetKubernetesServicePortsFromPrivatePortSpecs(map[string]*port_spec.PortSpec{
-		consts.KurtosisInternalContainerGrpcPortSpecId: privateGrpcPortSpec,
+		consts.KurtosisInternalContainerGrpcPortSpecId:    privateGrpcPortSpec,
+		consts.KurtosisInternalContainerRESTAPIPortSpecId: privateRESTAPIPortSpec,
 	})
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "An error occurred getting the engine service's ports using the engine private port specs")
@@ -490,11 +530,83 @@ func createEngineService(
 	if err != nil {
 		return nil, stacktrace.Propagate(
 			err,
-			"An error occurred while creating the service with name '%s' in namespace '%s' with ports '%v'",
+			"An error occurred while creating the service with name '%s' in namespace '%s' with ports '%v' and '%v'",
 			engineServiceName,
 			namespace,
 			privateGrpcPortSpec.GetNumber(),
+			privateRESTAPIPortSpec.GetNumber(),
 		)
 	}
 	return service, nil
 }
+
+func createEngineIngress(
+	ctx context.Context,
+	namespace string,
+	engineAttributesProvider object_attributes_provider.KubernetesEngineObjectAttributesProvider,
+	privateRESTAPIPortSpec *port_spec.PortSpec,
+	kubernetesManager *kubernetes_manager.KubernetesManager,
+) (*netv1.Ingress, error) {
+	engineIngressAttributes, err := engineAttributesProvider.ForEngineIngress()
+	if err != nil {
+		return nil, stacktrace.Propagate(
+			err,
+			"An error occurred getting the engine ingress attributes",
+		)
+	}
+	engineIngressName := engineIngressAttributes.GetName().GetString()
+	engineIngressLabels := shared_helpers.GetStringMapFromLabelMap(engineIngressAttributes.GetLabels())
+	engineIngressAnnotations := shared_helpers.GetStringMapFromAnnotationMap(engineIngressAttributes.GetAnnotations())
+
+	engineIngressRules, err := getEngineIngressRules(engineIngressName, privateRESTAPIPortSpec)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred creating the user service ingress rules for ingress service with name '%v'", engineIngressName)
+	}
+
+	createdIngress, err := kubernetesManager.CreateIngress(
+		ctx,
+		namespace,
+		engineIngressName,
+		engineIngressLabels,
+		engineIngressAnnotations,
+		engineIngressRules,
+	)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred while creating the ingress with name '%s' in namespace '%s'", engineIngressName, namespace)
+	}
+
+	return createdIngress, nil
+}
+
+func getEngineIngressRules(
+	engineIngressName string,
+	privateRESTAPIPortSpec *port_spec.PortSpec,
+) ([]netv1.IngressRule, error) {
+	var ingressRules []netv1.IngressRule
+	ingressRule := netv1.IngressRule{
+		Host: restAPIPortHost,
+		IngressRuleValue: netv1.IngressRuleValue{
+			HTTP: &netv1.HTTPIngressRuleValue{
+				Paths: []netv1.HTTPIngressPath{
+					{
+						Path:     consts.IngressRulePathAllPaths,
+						PathType: &consts.IngressRulePathTypePrefix,
+						Backend: netv1.IngressBackend{
+							Service: &netv1.IngressServiceBackend{
+								Name: engineIngressName,
+								Port: netv1.ServiceBackendPort{
+									Name:   "",
+									Number: int32(privateRESTAPIPortSpec.GetNumber()),
+								},
+							},
+							Resource: nil,
+						},
+					},
+				},
+			},
+		},
+	}
+	ingressRules = append(ingressRules, ingressRule)
+
+	return ingressRules, nil
+}

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/engine_kubernetes_resources.go

@@ -2,6 +2,7 @@ package engine_functions
 
 import (
 	apiv1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 )
 
@@ -21,4 +22,7 @@ type engineKubernetesResources struct {
 
 	// Should always be nil if namespace is nil
 	pod *apiv1.Pod
+
+	// Should always be nil if namespace is nil
+	ingress *netv1.Ingress
 }

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/engine_functions/shared_helpers.go

@@ -2,6 +2,8 @@ package engine_functions
 
 import (
 	"context"
+	"net"
+
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/shared_helpers"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_manager"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/kubernetes/kubernetes_resource_collectors"
@@ -11,7 +13,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec"
 	"github.com/kurtosis-tech/stacktrace"
 	apiv1 "k8s.io/api/core/v1"
-	"net"
+	netv1 "k8s.io/api/networking/v1"
 )
 
 func getEngineObjectsFromKubernetesResources(allResources map[engine.EngineGUID]*engineKubernetesResources) (map[engine.EngineGUID]*engine.Engine, error) {
@@ -128,6 +130,7 @@ func getMatchingEngineKubernetesResources(
 				serviceAccount:     nil,
 				service:            nil,
 				pod:                nil,
+				ingress:            nil,
 			}
 		}
 		engineResources.namespace = namespacesForId[0]
@@ -163,6 +166,7 @@ func getMatchingEngineKubernetesResources(
 				serviceAccount:     nil,
 				service:            nil,
 				pod:                nil,
+				ingress:            nil,
 			}
 		}
 		engineResources.clusterRole = clusterRolesForId[0]
@@ -198,6 +202,7 @@ func getMatchingEngineKubernetesResources(
 				serviceAccount:     nil,
 				service:            nil,
 				pod:                nil,
+				ingress:            nil,
 			}
 		}
 		engineResources.clusterRoleBinding = clusterRoleBindingsForId[0]
@@ -297,9 +302,37 @@ func getMatchingEngineKubernetesResources(
 			pod = podsForId[0]
 		}
 
+		// Ingress
+		ingresses, err := kubernetes_resource_collectors.CollectMatchingIngresses(
+			ctx,
+			kubernetesManager,
+			namespaceName,
+			engineMatchLabels,
+			kubernetes_label_key.IDKubernetesLabelKey.GetString(),
+			map[string]bool{
+				engineGuidStr: true,
+			},
+		)
+		if err != nil {
+			return nil, stacktrace.Propagate(err, "An error occurred getting ingresses matching engine GUID '%v' in namespace '%v'", engineGuid, namespaceName)
+		}
+		var ingress *netv1.Ingress
+		if ingressesForId, found := ingresses[engineGuidStr]; found {
+			if len(ingressesForId) > 1 {
+				return nil, stacktrace.NewError(
+					"Expected at most one engine ingress in namespace '%v' for engine with GUID '%v' but found '%v'",
+					namespaceName,
+					engineGuid,
+					len(ingresses),
+				)
+			}
+			ingress = ingressesForId[0]
+		}
+
 		engineResources.service = service
 		engineResources.pod = pod
 		engineResources.serviceAccount = serviceAccount
+		engineResources.ingress = ingress
 	}
 
 	return result, nil

container-engine-lib/lib/backend_impls/kubernetes/kubernetes_kurtosis_backend/kubernetes_kurtosis_backend_api_container_functions.go

@@ -130,9 +130,7 @@ func (backend *KubernetesKurtosisBackend) CreateAPIContainer(
 	// Get Service Attributes
 	apiContainerServiceAttributes, err := apiContainerAttributesProvider.ForApiContainerService(
 		consts.KurtosisInternalContainerGrpcPortSpecId,
-		privateGrpcPortSpec,
-		consts.KurtosisInternalContainerGrpcProxyPortSpecId,
-		nil)
+		privateGrpcPortSpec)
 	if err != nil {
 		return nil, stacktrace.Propagate(
 			err,