From 36596b85512a6d31386f312d1bda833c8350867d Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Thu, 21 Nov 2024 10:09:23 +0100 Subject: [PATCH] caa: upgrade to v0.10.0 Changes to the network configuration of Azure pod VMs require a NAT gateway in the subnet. Having that is desirable anyway, because default outbound access is deprecated and scheduled for removal in 2025. --- infra/azure-peerpods/main.tf | 26 ++++++++++++++- ...eplace-routes-instead-of-adding-them.patch | 30 +++++++++++++++++ .../by-name/cloud-api-adaptor/package.nix | 32 ++++++++++++++----- 3 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 packages/by-name/cloud-api-adaptor/0001-netops-replace-routes-instead-of-adding-them.patch diff --git a/infra/azure-peerpods/main.tf b/infra/azure-peerpods/main.tf index 19d31e390..83e980d39 100644 --- a/infra/azure-peerpods/main.tf +++ b/infra/azure-peerpods/main.tf @@ -42,6 +42,30 @@ resource "azurerm_virtual_network" "main" { } } +resource "azurerm_public_ip" "nat_ip" { + name = "${local.name}_nat_ip" + location = data.azurerm_resource_group.rg.location + resource_group_name = data.azurerm_resource_group.rg.name + allocation_method = "Static" + sku = "Standard" +} + +resource "azurerm_nat_gateway" "nat" { + name = "${local.name}_nat" + location = data.azurerm_resource_group.rg.location + resource_group_name = data.azurerm_resource_group.rg.name +} + +resource "azurerm_nat_gateway_public_ip_association" "ip_to_nat" { + nat_gateway_id = azurerm_nat_gateway.nat.id + public_ip_address_id = azurerm_public_ip.nat_ip.id +} + +resource "azurerm_subnet_nat_gateway_association" "subnet_to_nat" { + subnet_id = one(azurerm_virtual_network.main.subnet.*.id) + nat_gateway_id = azurerm_nat_gateway.nat.id +} + resource "azurerm_kubernetes_cluster" "cluster" { name = "${local.name}_aks" resource_group_name = data.azurerm_resource_group.rg.name @@ -88,7 +112,7 @@ bases: images: - name: cloud-api-adaptor newName: quay.io/confidential-containers/cloud-api-adaptor - newTag: v0.9.0-amd64 + newTag: v0.10.0-amd64 generatorOptions: disableNameSuffixHash: true configMapGenerator: diff --git a/packages/by-name/cloud-api-adaptor/0001-netops-replace-routes-instead-of-adding-them.patch b/packages/by-name/cloud-api-adaptor/0001-netops-replace-routes-instead-of-adding-them.patch new file mode 100644 index 000000000..159fc5694 --- /dev/null +++ b/packages/by-name/cloud-api-adaptor/0001-netops-replace-routes-instead-of-adding-them.patch @@ -0,0 +1,30 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Markus Rudy +Date: Tue, 19 Nov 2024 18:00:47 +0100 +Subject: [PATCH] netops: replace routes instead of adding them + +Some systems create a route if an IP address with prefix mask is added +to an interface. If this route is then also copied from the worker node, +a conflict may occur. + +A simple fix is to replace the route instead of adding it. The behaviour +is the same when the route does not exist. When it exists, we are either +setting the same route again, or overriding a route that's not desirable +in the first place. +--- + src/cloud-api-adaptor/pkg/util/netops/netops.go | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/cloud-api-adaptor/pkg/util/netops/netops.go b/src/cloud-api-adaptor/pkg/util/netops/netops.go +index 6a761dfef10fcf127ab914ebf0b759e76add3435..f0eaea5713c4a92479eed6a4ca46ff7b4abfe3c5 100644 +--- a/src/cloud-api-adaptor/pkg/util/netops/netops.go ++++ b/src/cloud-api-adaptor/pkg/util/netops/netops.go +@@ -623,7 +623,7 @@ func (ns *namespace) RouteAdd(route *Route) error { + if !route.Gateway.IsValid() { + nlRoute.Scope = netlink.SCOPE_LINK + } +- if err := ns.handle.RouteAdd(nlRoute); err != nil { ++ if err := ns.handle.RouteReplace(nlRoute); err != nil { + return fmt.Errorf("failed to create a route (table: %d, dest: %s, gw: %s) with flags %d: %w", nlRoute.Table, nlRoute.Dst.String(), nlRoute.Gw.String(), nlRoute.Flags, err) + } + return nil diff --git a/packages/by-name/cloud-api-adaptor/package.nix b/packages/by-name/cloud-api-adaptor/package.nix index ab008d7ba..1e1e91520 100644 --- a/packages/by-name/cloud-api-adaptor/package.nix +++ b/packages/by-name/cloud-api-adaptor/package.nix @@ -9,7 +9,10 @@ libvirt, writeShellApplication, gnugrep, + iptables, runCommand, + applyPatches, + makeWrapper, # List of supported cloud providers builtinCloudProviders ? [ @@ -31,21 +34,30 @@ in buildGoModule rec { pname = "cloud-api-adaptor"; - version = "0.9.0"; + version = "0.10.0"; + + src = applyPatches { + src = fetchFromGitHub { + owner = "confidential-containers"; + repo = "cloud-api-adaptor"; + rev = "v${version}"; + hash = "sha256-OaSIO26nlkeI2olSx0o8xdwhLMZ8eH753pUbyHypI+E="; + }; - src = fetchFromGitHub { - owner = "confidential-containers"; - repo = "cloud-api-adaptor"; - rev = "v${version}"; - hash = "sha256-5tDG0sEiRAsb259lPui5ntR6DVHDdcXhb04UESJzHhE="; + patches = [ + # This fixes a route setting problem we see with our NixOS image that + # does not seem to occur with the upstream image. + # TODO(burgerdev): upstream + ./0001-netops-replace-routes-instead-of-adding-them.patch + ]; }; sourceRoot = "${src.name}/src/cloud-api-adaptor"; proxyVendor = true; - vendorHash = "sha256-kqzi7jRF3tQ4/yLkJXfZly4EvVKFb400/WXlN0WjYm8="; + vendorHash = "sha256-FsckYZAiBfTEp25+dDNqPpB/550NqeEsutWC34s+GmE="; - nativeBuildInputs = lib.optional withLibvirt pkg-config; + nativeBuildInputs = [ makeWrapper ] ++ lib.optional withLibvirt pkg-config; buildInputs = lib.optional withLibvirt libvirt; @@ -63,6 +75,10 @@ buildGoModule rec { "-X 'github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd.VERSION=${version}'" ]; + postInstall = '' + wrapProgram $out/bin/agent-protocol-forwarder --prefix PATH : ${lib.makeBinPath [ iptables ]} + ''; + passthru = { kata-agent-clean = writeShellApplication { name = "kata-agent-clean";