Skip to content

Commit

Permalink
Add the reverse proxy enclave network ip address to the list of alrea…
Browse files Browse the repository at this point in the history 
…dy taken ips so there is no conflict with the APIC or user service.
  • Loading branch information
@laurentluce
laurentluce committed Dec 3, 2023
1 parent e9982ed commit 94d6d9b
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 74 deletions.
11 changes: 11 additions & 0 deletions ...e-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"github.com/docker/docker/client"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts"
Expand Down Expand Up @@ -238,11 +239,21 @@ func getDockerKurtosisBackend(
return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector object for enclave '%v'; This is a bug in Kurtosis", enclaveUuid)
}

reverseProxy, err := reverse_proxy_functions.GetReverseProxy(ctx, dockerManager)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred while getting the reverse proxy, This is a bug in Kurtosis")
}
reverseProxyEnclaveNetworkIpAddress, found := reverseProxy.GetEnclaveNetworksIpAddress()[network.GetId()]
if !found {
return nil, stacktrace.NewError("An error occured while getting the reverse proxy enclave network IP address for enclave '%v', This is a bug in Kurtosis", enclaveUuid)
}

alreadyTakenIps := map[string]bool{
networkIp.String(): true,
network.GetGatewayIp(): true,
apiContainerIp.String(): true,
logsCollectorObj.GetEnclaveNetworkIpAddress().String(): true,
reverseProxyEnclaveNetworkIpAddress.String(): true,
}

freeIpAddrProvider, err := free_ip_addr_tracker.GetOrCreateNewFreeIpAddrTracker(
Expand Down
15 changes: 13 additions & 2 deletions ...d_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@ package docker_kurtosis_backend

import (
"context"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"
"net"
"time"

"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"

"github.com/docker/go-connections/nat"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers"
Expand Down Expand Up @@ -73,14 +74,24 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer(

enclaveLogsCollector, err := backend.GetLogsCollectorForEnclave(ctx, enclaveUuid)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector for enclave '%v; This is a bug in Kurtosis'", enclaveUuid)
return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector for enclave '%v'; This is a bug in Kurtosis", enclaveUuid)
}

reverseProxy, err := backend.GetReverseProxy(ctx)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred while getting the reverse proxy, This is a bug in Kurtosis")
}
reverseProxyEnclaveNetworkIpAddress, found := reverseProxy.GetEnclaveNetworksIpAddress()[enclaveNetwork.GetId()]
if !found {
return nil, stacktrace.NewError("An error occured while getting the reverse proxy enclave network IP address for enclave '%v', This is a bug in Kurtosis", enclaveUuid)
}

networkCidr := enclaveNetwork.GetIpAndMask()
alreadyTakenIps := map[string]bool{
networkCidr.IP.String(): true,
enclaveNetwork.GetGatewayIp(): true,
enclaveLogsCollector.GetEnclaveNetworkIpAddress().String(): true,
reverseProxyEnclaveNetworkIpAddress.String(): true,
}

ipAddr, err := network_helpers.GetFreeIpAddrFromSubnet(alreadyTakenIps, networkCidr)
Expand Down
78 changes: 69 additions & 9 deletions ...backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_enclave_functions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

"github.com/docker/docker/api/types/volume"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types"
Expand All @@ -21,9 +20,9 @@ import (
)

const (
shouldFetchStoppedContainersWhenGettingEnclaveStatus = true

shouldFetchStoppedContainersWhenDumpingEnclave = true
shouldFetchStoppedContainersWhenGettingEnclaveStatus = true
shouldFetchStoppedContainersWhenDumpingEnclave = true
shouldFetchStoppedContainersWhenDisconnectingFromEnclaveNetworks = false

defaultHttpLogsCollectorPortNum = uint16(9712)
defaultTcpLogsCollectorPortNum = uint16(9713)
Expand Down Expand Up @@ -118,7 +117,7 @@ func (backend *DockerKurtosisBackend) CreateEnclave(ctx context.Context, enclave
}
}
}()
logrus.Debugf("Docker network '%v' created successfully with ID '%v'", enclaveUuid, networkId)
logrus.Debugf("Docker network for enclave '%v' created successfully with ID '%v'", enclaveUuid, networkId)

enclaveDataVolumeNameStr := enclaveDataVolumeAttrs.GetName().GetString()
enclaveDataVolumeLabelStrs := map[string]string{}
Expand Down Expand Up @@ -400,17 +399,17 @@ func (backend *DockerKurtosisBackend) DestroyEnclaves(
networksToDisconnect[enclaveUuid] = networkInfo.dockerNetwork.GetId()
}

successfulDisconnectReverseProxyFromNetworkEnclaveUuids, erroredDisconnectReverseProxyFromNetworkEnclaveUuids, err := reverse_proxy_functions.DisconnectReverseProxyFromEnclaveNetworks(ctx, backend.dockerManager, networksToDisconnect)
successfulDisconnectContainersFromNetworkEnclaveUuids, erroredDisconnectContainersFromNetworkEnclaveUuids, err := backend.disconnectContainersFromEnclaveNetworks(ctx, backend.dockerManager, networksToDisconnect)
if err != nil {
return nil, nil, stacktrace.Propagate(err, "An error occurred disconnecting the reverse proxy from the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids)
return nil, nil, stacktrace.Propagate(err, "An error occurred disconnecting the containers from the networks for enclaves whose volumes were successfully destroyed: %+v", successfulVolumeRemovalEnclaveUuids)
}
for enclaveUuid, networkDisconnectErr := range erroredDisconnectReverseProxyFromNetworkEnclaveUuids {
for enclaveUuid, networkDisconnectErr := range erroredDisconnectContainersFromNetworkEnclaveUuids {
erroredEnclaveUuids[enclaveUuid] = networkDisconnectErr
}

// Remove the networks
networksToDestroy := map[enclave.EnclaveUUID]string{}
for enclaveUuid := range successfulDisconnectReverseProxyFromNetworkEnclaveUuids {
for enclaveUuid := range successfulDisconnectContainersFromNetworkEnclaveUuids {
networkInfo, found := matchingNetworkInfo[enclaveUuid]
if !found {
return nil, nil, stacktrace.NewError("Would have attempted to destroy enclave '%v' that didn't match the filters", enclaveUuid)
Expand Down Expand Up @@ -556,6 +555,67 @@ func (backend *DockerKurtosisBackend) getAllEnclaveContainers(
return containers, nil
}

func (backend *DockerKurtosisBackend) disconnectContainersFromEnclaveNetworks(
ctx context.Context,
dockerManager *docker_manager.DockerManager,
enclaveNetworkIds map[enclave.EnclaveUUID]string,
) (
map[enclave.EnclaveUUID]bool,
map[enclave.EnclaveUUID]error,
error,
) {
networkIdsToRemove := map[string]bool{}
enclaveUuidsForNetworkIds := map[string]enclave.EnclaveUUID{}
for enclaveUuid, networkId := range enclaveNetworkIds {
networkIdsToRemove[networkId] = true
enclaveUuidsForNetworkIds[networkId] = enclaveUuid
}

var disconnectNetworkOperation docker_operation_parallelizer.DockerOperation = func(ctx context.Context, dockerManager *docker_manager.DockerManager, dockerObjectId string) error {
// Get containers connected to this network id (dockerObjectId here)
containers, err := backend.dockerManager.GetContainersByNetworkId(ctx, dockerObjectId, shouldFetchStoppedContainersWhenDisconnectingFromEnclaveNetworks)
if err != nil {
return stacktrace.Propagate(
err,
"An error occurred getting the containers with enclave network '%v'",
dockerObjectId,
)
}
for _, container := range containers {
if err = dockerManager.DisconnectContainerFromNetwork(ctx, container.GetId(), dockerObjectId); err != nil {
return stacktrace.Propagate(err, "An error occurred while disconnecting container '%v' from the enclave network '%v'", container.GetId(), dockerObjectId)
}
}
return nil
}

successfulNetworkIds, erroredNetworkIds := docker_operation_parallelizer.RunDockerOperationInParallel(
ctx,
networkIdsToRemove,
dockerManager,
disconnectNetworkOperation,
)

successfulEnclaveUuids := map[enclave.EnclaveUUID]bool{}
for networkId := range successfulNetworkIds {
enclaveUuid, found := enclaveUuidsForNetworkIds[networkId]
if !found {
return nil, nil, stacktrace.NewError("The containers were successfully disconnected from the enclave network '%v', but wasn't requested to be disconnected", networkId)
}
successfulEnclaveUuids[enclaveUuid] = true
}

erroredEnclaveUuids := map[enclave.EnclaveUUID]error{}
for networkId, networkRemovalErr := range erroredNetworkIds {
enclaveUuid, found := enclaveUuidsForNetworkIds[networkId]
if !found {
return nil, nil, stacktrace.NewError("Docker network '%v' had the following error during disconnect, but wasn't requested to be disconnected:\n%v", networkId, networkRemovalErr)
}
erroredEnclaveUuids[enclaveUuid] = networkRemovalErr
}
return successfulEnclaveUuids, erroredEnclaveUuids, nil
}

func getAllEnclaveVolumes(
ctx context.Context,
dockerManager *docker_manager.DockerManager,
Expand Down
58 changes: 3 additions & 55 deletions ...end_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/network_reverse_proxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,16 @@ import (
"net"

"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_operation_parallelizer"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave"
"github.com/kurtosis-tech/stacktrace"
"github.com/sirupsen/logrus"
)

const (
emptyAliasForLogsCollector = ""
emptyAliasForReverseProxy = ""
)

var (
autoAssignIpAddressToLogsCollector net.IP = nil
autoAssignIpAddressToReverseProxy net.IP = nil
)

func ConnectReverseProxyToNetwork(ctx context.Context, dockerManager *docker_manager.DockerManager, networkId string) error {
Expand All @@ -30,7 +28,7 @@ func ConnectReverseProxyToNetwork(ctx context.Context, dockerManager *docker_man
return nil
}

if err = dockerManager.ConnectContainerToNetwork(ctx, networkId, maybeReverseProxyContainerId, autoAssignIpAddressToLogsCollector, emptyAliasForLogsCollector); err != nil {
if err = dockerManager.ConnectContainerToNetwork(ctx, networkId, maybeReverseProxyContainerId, autoAssignIpAddressToReverseProxy, emptyAliasForReverseProxy); err != nil {
return stacktrace.Propagate(err, "An error occurred while connecting container '%v' to the enclave network '%v'", maybeReverseProxyContainerId, networkId)
}

Expand All @@ -54,53 +52,3 @@ func DisconnectReverseProxyFromNetwork(ctx context.Context, dockerManager *docke

return nil
}

func DisconnectReverseProxyFromEnclaveNetworks(
ctx context.Context,
dockerManager *docker_manager.DockerManager,
enclaveNetworkIds map[enclave.EnclaveUUID]string,
) (
map[enclave.EnclaveUUID]bool,
map[enclave.EnclaveUUID]error,
error,
) {
networkIdsToRemove := map[string]bool{}
enclaveUuidsForNetworkIds := map[string]enclave.EnclaveUUID{}
for enclaveUuid, networkId := range enclaveNetworkIds {
networkIdsToRemove[networkId] = true
enclaveUuidsForNetworkIds[networkId] = enclaveUuid
}

var disconnectNetworkOperation docker_operation_parallelizer.DockerOperation = func(ctx context.Context, dockerManager *docker_manager.DockerManager, dockerObjectId string) error {
if err := DisconnectReverseProxyFromNetwork(ctx, dockerManager, dockerObjectId); err != nil {
return stacktrace.Propagate(err, "An error occurred disconnecting the reverse proxy from the enclave network with ID '%v'", dockerObjectId)
}
return nil
}

successfulNetworkIds, erroredNetworkIds := docker_operation_parallelizer.RunDockerOperationInParallel(
ctx,
networkIdsToRemove,
dockerManager,
disconnectNetworkOperation,
)

successfulEnclaveUuids := map[enclave.EnclaveUUID]bool{}
for networkId := range successfulNetworkIds {
enclaveUuid, found := enclaveUuidsForNetworkIds[networkId]
if !found {
return nil, nil, stacktrace.NewError("The reverse proxy was successfully disconnected from the Docker network '%v', but wasn't requested to be disconnected", networkId)
}
successfulEnclaveUuids[enclaveUuid] = true
}

erroredEnclaveUuids := map[enclave.EnclaveUUID]error{}
for networkId, networkRemovalErr := range erroredNetworkIds {
enclaveUuid, found := enclaveUuidsForNetworkIds[networkId]
if !found {
return nil, nil, stacktrace.NewError("Docker network '%v' had the following error during disconnect, but wasn't requested to be disconnected:\n%v", networkId, networkRemovalErr)
}
erroredEnclaveUuids[enclaveUuid] = networkRemovalErr
}
return successfulEnclaveUuids, erroredEnclaveUuids, nil
}
21 changes: 20 additions & 1 deletion ...ib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/shared_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/container"
"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy"
"github.com/kurtosis-tech/stacktrace"
"github.com/sirupsen/logrus"
)

const (
Expand All @@ -25,7 +26,7 @@ func getReverseProxyObjectAndContainerId(
) (*reverse_proxy.ReverseProxy, string, error) {
reverseProxyContainer, found, err := getReverseProxyContainer(ctx, dockerManager)
if err != nil {
return nil, "", stacktrace.Propagate(err, "An error occurred getting all reverse proxy containers")
return nil, "", stacktrace.Propagate(err, "An error occurred getting the reverse proxy container")
}
if !found {
return nil, "", nil
Expand Down Expand Up @@ -74,6 +75,7 @@ func getReverseProxyObjectFromContainerInfo(
dockerManager *docker_manager.DockerManager,
) (*reverse_proxy.ReverseProxy, error) {
var privateIpAddr net.IP
var enclaveNetworksIpAddress map[string]net.IP

isContainerRunning, found := consts.IsContainerRunningDeterminer[containerStatus]
if !found {
Expand All @@ -93,13 +95,30 @@ func getReverseProxyObjectFromContainerInfo(
if privateIpAddr == nil {
return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", privateIpAddrStr)
}

networksIpAddressStr, err := dockerManager.GetContainerIps(ctx, containerId)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred getting the networks private IP address of container '%v'", containerId)
}
enclaveNetworksIpAddress = map[string]net.IP{}
for networkId, networkIpAddressStr := range networksIpAddressStr {
if networkIpAddressStr != privateIpAddrStr {
networkIpAddress := net.ParseIP(networkIpAddressStr)
if networkIpAddress == nil {
return nil, stacktrace.NewError("Couldn't parse private IP address string '%v' to an IP", networkIpAddress)
}
enclaveNetworksIpAddress[networkId] = networkIpAddress
}
}
logrus.Debugf("Enclave networks: '%v'", enclaveNetworksIpAddress)
} else {
reverseProxyStatus = container.ContainerStatus_Stopped
}

reverseProxyObj := reverse_proxy.NewReverseProxy(
reverseProxyStatus,
privateIpAddr,
enclaveNetworksIpAddress,
defaultReverseProxyHttpPortNum,
defaultReverseProxyDashboardPortNum,
)
Expand Down
33 changes: 31 additions & 2 deletions container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,9 @@ const (
// ------------------ Filter Search Keys ----------------------
// All these defined in https://docs.docker.com/engine/api/v1.24

containerNameSearchFilterKey = "name"
containerLabelSearchFilterKey = "label"
containerNameSearchFilterKey = "name"
containerLabelSearchFilterKey = "label"
containerNetworkIdSearchFilterKey = "network"

volumeNameSearchFilterKey = "name"
volumeLabelSearchFilterKey = "label"
Expand Down Expand Up @@ -777,6 +778,24 @@ func (manager *DockerManager) GetContainerIP(ctx context.Context, networkName st
return networkInfo.IPAddress, nil
}

/*
GetContainerIps
Gets the container's IPs on all networks
Returns a map of network ID : network IP address
*/
func (manager *DockerManager) GetContainerIps(ctx context.Context, containerId string) (map[string]string, error) {
containerIps := map[string]string{}
resp, err := manager.dockerClient.ContainerInspect(ctx, containerId)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred inspecting container with ID '%v'", containerId)
}
allNetworkInfo := resp.NetworkSettings.Networks
for _, networkInfo := range allNetworkInfo {
containerIps[networkInfo.NetworkID]= networkInfo.IPAddress
}
return containerIps, nil
}

func (manager *DockerManager) AttachToContainer(ctx context.Context, containerId string) (types.HijackedResponse, error) {
attachOpts := types.ContainerAttachOptions{
Stream: true,
Expand Down Expand Up @@ -1184,6 +1203,16 @@ func (manager *DockerManager) GetContainersByLabels(ctx context.Context, labels
return result, nil
}

func (manager *DockerManager) GetContainersByNetworkId(ctx context.Context, networkId string, shouldShowStoppedContainers bool) ([]*docker_manager_types.Container, error) {
filterArg := filters.Arg(containerNetworkIdSearchFilterKey, networkId)
networkIdFilterList := filters.NewArgs(filterArg)
result, err := manager.getContainersByFilterArgs(ctx, networkIdFilterList, shouldShowStoppedContainers)
if err != nil {
return nil, stacktrace.Propagate(err, "An error occurred getting containers with network id '%+v'", networkIdFilterList)
}
return result, nil
}

// [FetchImageIfMissing] uses the local [dockerImage] if it's available.
// If unavailable, will attempt to fetch the latest image.
// Returns error if local [dockerImage] is unavailable and pulling image fails.
Expand Down
Loading

0 comments on commit 94d6d9b

Please sign in to comment.