From 27200e3884d55ffcee122e09a26592de863ece37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Pa=C3=9F?= <22845248+mpass99@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:15:47 +0100 Subject: [PATCH] Fix Runner DNS resolution by adding public nameservers to the CNI secure bridge configuration. --- configuration.example.yaml | 8 +++++++ internal/config/config.go | 6 ++++++ internal/environment/nomad_environment.go | 21 ++++++++----------- .../environment/nomad_environment_test.go | 9 ++++---- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/configuration.example.yaml b/configuration.example.yaml index 36144fc9..43e65b5c 100644 --- a/configuration.example.yaml +++ b/configuration.example.yaml @@ -65,6 +65,14 @@ nomad: # namespace: poseidon # Prefer local Docker images over pulling them from a registry. Images with the `latest` tag will always be force pulled by Nomad regardless of this configuration. disableforcepull: true + # Network configuration for network-enabled runners. See https://developer.hashicorp.com/nomad/docs/job-specification/network. + network: + # Available Modes: "none", "bridge", "host", "cni/*". + # "none": Even the network-enabled runners will be isolated. + # "bridge": Isolated network namespace with bridged interface. Linux-only. + # "host": Using the host network namespace. Less-secure. + # "cni/*": Configure an isolated network namespace using CNI. For example, this can be a more secured bridge network. + mode: "cni/secure-bridge" aws: # Specifies whether AWS should be used as executor. diff --git a/internal/config/config.go b/internal/config/config.go index f0202d6c..caf4892a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,6 +8,7 @@ import ( "flag" "fmt" "github.com/getsentry/sentry-go" + nomadApi "github.com/hashicorp/nomad/api" "github.com/openHPI/poseidon/pkg/dto" "github.com/openHPI/poseidon/pkg/logging" "github.com/sirupsen/logrus" @@ -54,6 +55,10 @@ var ( }, Namespace: "default", DisableForcePull: false, + Network: nomadApi.NetworkResource{ + Mode: "bridge", + DNS: nil, + }, }, AWS: AWS{ Enabled: false, @@ -120,6 +125,7 @@ type Nomad struct { TLS TLS Namespace string DisableForcePull bool + Network nomadApi.NetworkResource } // URL returns the URL for the configured Nomad cluster. diff --git a/internal/environment/nomad_environment.go b/internal/environment/nomad_environment.go index bcc6da33..8fc20562 100644 --- a/internal/environment/nomad_environment.go +++ b/internal/environment/nomad_environment.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" nomadApi "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/jobspec2" + "github.com/openHPI/poseidon/internal/config" "github.com/openHPI/poseidon/internal/nomad" "github.com/openHPI/poseidon/internal/runner" "github.com/openHPI/poseidon/pkg/dto" @@ -170,16 +171,7 @@ func (n *NomadEnvironment) SetNetworkAccess(allow bool, exposedPorts []uint16) { } if allow { - var networkResource *nomadApi.NetworkResource - if len(defaultTaskGroup.Networks) == 0 { - networkResource = &nomadApi.NetworkResource{} - defaultTaskGroup.Networks = []*nomadApi.NetworkResource{networkResource} - } else { - networkResource = defaultTaskGroup.Networks[0] - } - // Prefer "bridge" network over "host" to have an isolated network namespace with bridged interface - // instead of joining the host network namespace. - networkResource.Mode = "cni/secure-bridge" + networkResource := config.Config.Nomad.Network for _, portNumber := range exposedPorts { port := nomadApi.Port{ Label: strconv.FormatUint(uint64(portNumber), portNumberBase), @@ -187,6 +179,11 @@ func (n *NomadEnvironment) SetNetworkAccess(allow bool, exposedPorts []uint16) { } networkResource.DynamicPorts = append(networkResource.DynamicPorts, port) } + if len(defaultTaskGroup.Networks) == 0 { + defaultTaskGroup.Networks = []*nomadApi.NetworkResource{&networkResource} + } else { + defaultTaskGroup.Networks[0] = &networkResource + } // Explicitly set mode to override existing settings when updating job from without to with network. // Don't use bridge as it collides with the bridge mode above. This results in Docker using 'bridge' @@ -332,12 +329,12 @@ func (n *NomadEnvironment) SetConfigFrom(environment runner.ExecutionEnvironment } func parseJob(jobHCL string) (*nomadApi.Job, error) { - config := jobspec2.ParseConfig{ + jobConfig := jobspec2.ParseConfig{ Body: []byte(jobHCL), AllowFS: false, Strict: true, } - job, err := jobspec2.ParseWithConfig(&config) + job, err := jobspec2.ParseWithConfig(&jobConfig) if err != nil { return job, fmt.Errorf("couldn't parse job HCL: %w", err) } diff --git a/internal/environment/nomad_environment_test.go b/internal/environment/nomad_environment_test.go index ae43d080..dd713197 100644 --- a/internal/environment/nomad_environment_test.go +++ b/internal/environment/nomad_environment_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" nomadApi "github.com/hashicorp/nomad/api" + "github.com/openHPI/poseidon/internal/config" "github.com/openHPI/poseidon/internal/nomad" "github.com/openHPI/poseidon/internal/runner" "github.com/openHPI/poseidon/pkg/storage" @@ -32,14 +33,14 @@ func (s *MainTestSuite) TestConfigureNetworkDoesNotCreateNewNetworkWhenNetworkEx defaultTaskGroup := nomad.FindAndValidateDefaultTaskGroup(job) environment := &NomadEnvironment{nil, "", job, nil, context.Background(), nil} - networkResource := &nomadApi.NetworkResource{Mode: "cni/secure-bridge"} - defaultTaskGroup.Networks = []*nomadApi.NetworkResource{networkResource} + networkResource := config.Config.Nomad.Network + defaultTaskGroup.Networks = []*nomadApi.NetworkResource{&networkResource} if s.Equal(1, len(defaultTaskGroup.Networks)) { environment.SetNetworkAccess(true, []uint16{}) s.Equal(1, len(defaultTaskGroup.Networks)) - s.Equal(networkResource, defaultTaskGroup.Networks[0]) + s.Equal(&networkResource, defaultTaskGroup.Networks[0]) } } @@ -80,7 +81,7 @@ func (s *MainTestSuite) TestConfigureNetworkSetsCorrectValues() { s.Require().Equal(1, len(testTaskGroup.Networks)) networkResource := testTaskGroup.Networks[0] - s.Equal("cni/secure-bridge", networkResource.Mode) + s.Equal(config.Config.Nomad.Network.Mode, networkResource.Mode) s.Require().Equal(len(ports), len(networkResource.DynamicPorts)) assertExpectedPorts(s.T(), ports, networkResource)