nodeinstaller: add nydus-pull container

.github/workflows/release.yml

@@ -246,6 +246,7 @@ jobs:
           serviceMeshImg=$(nix run .#containers.push-service-mesh-proxy -- "$container_registry/contrast/service-mesh-proxy")
           tardevSnapshotterImg=$(nix run .#containers.push-tardev-snapshotter -- "$container_registry/contrast/tardev-snapshotter")
           nydusSnapshotterImg=$(nix run .#containers.push-nydus-snapshotter -- "$container_registry/contrast/nydus-snapshotter")
+          nydusPullImg=$(nix run .#containers.push-nydus-pull -- "$container_registry/contrast/nydus-pull")
           cryptsetupImg=$(nix run .#containers.push-cryptsetup -- "$container_registry/contrast/cryptsetup")
           echo "coordinatorImg=$coordinatorImg" | tee -a "$GITHUB_ENV"
           echo "nodeInstallerMsftImg=$nodeInstallerMsftImg" | tee -a "$GITHUB_ENV"
@@ -254,6 +255,7 @@ jobs:
           echo "serviceMeshImg=$serviceMeshImg" | tee -a "$GITHUB_ENV"
           echo "tardevSnapshotterImg=$tardevSnapshotterImg" | tee -a "$GITHUB_ENV"
           echo "nydusSnapshotterImg=$nydusSnapshotterImg" | tee -a "$GITHUB_ENV"
+          echo "nydusPullImg=$nydusPullImg" | tee -a "$GITHUB_ENV"
           echo "cryptsetupImg=$cryptsetupImg" | tee -a "$GITHUB_ENV"
       - name: Add tag to Coordinator image
         run: |
@@ -267,6 +269,7 @@ jobs:
           echo "nodeInstallerKataImgTagged=$(tag "$nodeInstallerKataImg")" | tee -a "$GITHUB_ENV"
           echo "initializerImgTagged=$(tag "$initializerImg")" | tee -a "$GITHUB_ENV"
           echo "serviceMeshImgTagged=$(tag "$serviceMeshImg")" | tee -a "$GITHUB_ENV"
+          echo "nydusPullImgTagged=$(tag "$nydusPullImg")" | tee -a "$GITHUB_ENV"
           echo "cryptsetupImgTagged=$(tag "$cryptsetupImg")" | tee -a "$GITHUB_ENV"
 
           tardevVer=$(nix eval --impure --raw --expr "(builtins.getFlake \"git+file://$(pwd)?shallow=1\").outputs.legacyPackages.x86_64-linux.microsoft.tardev-snapshotter.version")
@@ -288,6 +291,7 @@ jobs:
             echo "ghcr.io/edgelesssys/contrast/node-installer-kata:latest=$nodeInstallerKataImgTagged"
             echo "ghcr.io/edgelesssys/contrast/tardev-snapshotter:latest=$tardevSnapshotterImgTagged"
             echo "ghcr.io/edgelesssys/contrast/nydus-snapshotter:latest=$nydusSnapshotterImgTagged"
+            echo "ghcr.io/edgelesssys/contrast/nydus-pull:latest=$nydusPullImgTagged"
             echo "ghcr.io/edgelesssys/contrast/cryptsetup:latest=$cryptsetupImgTagged"
           } > image-replacements.txt
       - name: Upload image replacements file (for main branch PR)

go.work

@@ -4,4 +4,5 @@ use (
 	.
 	./service-mesh
 	./tools/tdx-measure
+	./tools/nydus-pull
 )

internal/kuberesource/mutators.go

@@ -13,6 +13,7 @@ import (
 	applybatchv1 "k8s.io/client-go/applyconfigurations/batch/v1"
 	applycorev1 "k8s.io/client-go/applyconfigurations/core/v1"
 	applymetav1 "k8s.io/client-go/applyconfigurations/meta/v1"
+	applyrbacv1 "k8s.io/client-go/applyconfigurations/rbac/v1"
 )
 
 const (
@@ -293,6 +294,10 @@ func PatchNamespaces(resources []any, namespace string) []any {
 			r.Namespace = nsPtr
 		case *applycorev1.ServiceAccountApplyConfiguration:
 			r.Namespace = nsPtr
+		case *applyrbacv1.ClusterRoleBindingApplyConfiguration:
+			for i := range len(r.Subjects) {
+				r.Subjects[i].Namespace = nsPtr
+			}
 		}
 	}
 	return resources

internal/kuberesource/parts.go

@@ -15,6 +15,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	applyappsv1 "k8s.io/client-go/applyconfigurations/apps/v1"
 	applycorev1 "k8s.io/client-go/applyconfigurations/core/v1"
+	applyrbacv1 "k8s.io/client-go/applyconfigurations/rbac/v1"
 )
 
 // ContrastRuntimeClass creates a new RuntimeClassConfig.
@@ -39,6 +40,9 @@ func ContrastRuntimeClass(platform platforms.Platform) (*RuntimeClassConfig, err
 // NodeInstallerConfig wraps a DaemonSetApplyConfiguration for a node installer.
 type NodeInstallerConfig struct {
 	*applyappsv1.DaemonSetApplyConfiguration
+	*applycorev1.ServiceAccountApplyConfiguration
+	*applyrbacv1.ClusterRoleApplyConfiguration
+	*applyrbacv1.ClusterRoleBindingApplyConfiguration
 }
 
 // NodeInstaller constructs a node installer daemon set.
@@ -116,19 +120,46 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 				WithPath(fmt.Sprintf("/var/lib/nydus-snapshotter/%s", runtimeHandler)).
 				WithType(corev1.HostPathDirectoryOrCreate),
 			),
+		Volume().
+			WithName("containerd-socket").
+			WithHostPath(HostPathVolumeSource().
+				WithPath("/run/k3s/containerd/containerd.sock").
+				WithType(corev1.HostPathSocket),
+			),
 	}
 
+	nydusPull := Container().
+		WithName("nydus-pull").
+		WithImage("ghcr.io/edgelesssys/contrast/nydus-pull:latest").
+		WithArgs(runtimeHandler).
+		WithEnv(
+			EnvVar().
+				WithName("NODE_NAME").
+				WithValueFrom(
+					applycorev1.EnvVarSource().
+						WithFieldRef(
+							applycorev1.ObjectFieldSelector().
+								WithFieldPath("spec.nodeName"),
+						),
+				),
+		).
+		WithVolumeMounts(
+			VolumeMount().
+				WithName("containerd-socket").
+				WithMountPath("/run/containerd/containerd.sock"),
+		)
+
 	var nodeInstallerImageURL string
-	var snapshotter *applycorev1.ContainerApplyConfiguration
+	var containers []*applycorev1.ContainerApplyConfiguration
 	var snapshotterVolumes []*applycorev1.VolumeApplyConfiguration
 	switch platform {
 	case platforms.AKSCloudHypervisorSNP:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-microsoft:latest"
-		snapshotter = tardevSnapshotter
+		containers = append(containers, tardevSnapshotter)
 		snapshotterVolumes = tardevSnapshotterVolumes
 	case platforms.MetalQEMUSNP, platforms.MetalQEMUTDX:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
-		snapshotter = nydusSnapshotter
+		containers = append(containers, nydusSnapshotter, nydusPull)
 		nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume().
 			WithName("var-lib-containerd").
 			WithHostPath(HostPathVolumeSource().
@@ -138,7 +169,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 		snapshotterVolumes = nydusSnapshotterVolumes
 	case platforms.K3sQEMUTDX, platforms.K3sQEMUSNP, platforms.RKE2QEMUTDX:
 		nodeInstallerImageURL = "ghcr.io/edgelesssys/contrast/node-installer-kata:latest"
-		snapshotter = nydusSnapshotter
+		containers = append(containers, nydusSnapshotter, nydusPull)
 		nydusSnapshotterVolumes = append(nydusSnapshotterVolumes, Volume().
 			WithName("var-lib-containerd").
 			WithHostPath(HostPathVolumeSource().
@@ -163,6 +194,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 					"contrast.edgeless.systems/platform": platform.String(),
 				}).
 				WithSpec(PodSpec().
+					WithServiceAccountName("nodeinstaller-serviceaccount").
 					WithHostPID(true).
 					WithInitContainers(Container().
 						WithName("installer").
@@ -177,7 +209,7 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 						WithCommand("/bin/node-installer", platform.String()),
 					).
 					WithContainers(
-						snapshotter,
+						containers...,
 					).
 					WithVolumes(append(
 						snapshotterVolumes,
@@ -193,7 +225,36 @@ func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstalle
 			),
 		)
 
-	return &NodeInstallerConfig{d}, nil
+	serviceAccount := applycorev1.ServiceAccount("nodeinstaller-serviceaccount", "")
+
+	clusterRole := applyrbacv1.ClusterRole("nodeinstaller-clusterrole").
+		WithRules(
+			applyrbacv1.PolicyRule().
+				WithAPIGroups("").
+				WithResources("pods").
+				WithVerbs("watch"),
+		)
+
+	clusterRoleBinding := applyrbacv1.ClusterRoleBinding("nodeinstaller-clusterrole-binding").
+		WithSubjects(
+			applyrbacv1.Subject().
+				WithKind("ServiceAccount").
+				WithName("nodeinstaller-serviceaccount").
+				WithNamespace(namespace),
+		).
+		WithRoleRef(
+			applyrbacv1.RoleRef().
+				WithKind("ClusterRole").
+				WithName("nodeinstaller-clusterrole").
+				WithAPIGroup("rbac.authorization.k8s.io"),
+		)
+
+	return &NodeInstallerConfig{
+		DaemonSetApplyConfiguration:          d,
+		ServiceAccountApplyConfiguration:     serviceAccount,
+		ClusterRoleApplyConfiguration:        clusterRole,
+		ClusterRoleBindingApplyConfiguration: clusterRoleBinding,
+	}, nil
 }
 
 // PortForwarderConfig wraps a PodApplyConfiguration for a port forwarder.

internal/kuberesource/sets.go

@@ -47,6 +47,9 @@ func Runtime(platform platforms.Platform) ([]any, error) {
 	return []any{
 		runtimeClassApplyConfig,
 		nodeInstaller.DaemonSetApplyConfiguration,
+		nodeInstaller.ServiceAccountApplyConfiguration,
+		nodeInstaller.ClusterRoleApplyConfiguration,
+		nodeInstaller.ClusterRoleBindingApplyConfiguration,
 	}, nil
 }
 

justfile

@@ -33,6 +33,9 @@ tardev-snapshotter: (push "tardev-snapshotter")
 # Build the nydus-snapshotter, containerize and push it.
 nydus-snapshotter: (push "nydus-snapshotter")
 
+# Build the nydus-pull container and push it.
+nydus-pull: (push "nydus-pull")
+
 default_cli := "contrast.cli"
 default_deploy_target := "openssl"
 default_platform := "${default_platform}"
@@ -49,6 +52,7 @@ node-installer platform=default_platform:
         ;;
         "Metal-QEMU-SNP"|"Metal-QEMU-TDX"|"K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX")
             just push "nydus-snapshotter"
+            just push "nydus-pull"
             just push "node-installer-kata"
         ;;
         "AKS-PEER-SNP")

packages/by-name/contrast/package.nix

@@ -174,6 +174,7 @@ buildGoModule rec {
           fileset.unions [
             (path.append root "service-mesh")
             (path.append root "tools/tdx-measure")
+            (path.append root "tools/nydus-pull")
           ]
         ))
       ];

packages/by-name/nydus-pull/package.nix

@@ -0,0 +1,35 @@
+# Copyright 2024 Edgeless Systems GmbH
+# SPDX-License-Identifier: AGPL-3.0-only
+
+{ buildGoModule }:
+
+buildGoModule rec {
+  pname = "nydus-pull";
+  version = builtins.readFile ../../../version.txt;
+
+  src = ../../../tools/nydus-pull;
+
+  proxyVendor = true;
+  vendorHash = "sha256-UFrQPzK2/yWRI5kwGwMVfl3MGBKWAtQOCnpNOYapvQs=";
+
+  subPackages = [ "." ];
+
+  CGO_ENABLED = 0;
+  ldflags = [
+    "-s"
+    "-w"
+    "-X main.version=v${version}"
+  ];
+
+  preCheck = ''
+    export CGO_ENABLED=1
+  '';
+
+  checkPhase = ''
+    runHook preCheck
+    go test -race ./...
+    runHook postCheck
+  '';
+
+  meta.mainProgram = "nydus-pull";
+}

packages/containers.nix

@@ -177,6 +177,16 @@ let
         Cmd = [ "${lib.getExe pkgs.cloud-api-adaptor.entrypoint}" ];
       };
     };
+
+    nydus-pull = dockerTools.buildImage {
+      name = "nydus-pull";
+      tag = "v${pkgs.nydus-pull.version}";
+      copyToRoot = with dockerTools; [ caCertificates ];
+      config = {
+        Entrypoint = [ "${lib.getExe pkgs.nydus-pull}" ];
+        Env = [ "PATH=/bin" ]; # This is only here for policy generation.
+      };
+    };
   };
 in
 containers

packages/scripts.nix

@@ -122,6 +122,8 @@
       nix-update --version=skip --flake legacyPackages.x86_64-linux.service-mesh
       echo "Updating vendorHash of contrast.cli package" >&2
       nix-update --version=skip --flake legacyPackages.x86_64-linux.contrast
+      echo "Updating vendorHash of nydus-pull package" >&2
+      nix-update --version=skip --flake legacyPackages.x86_64-linux.nydus-pull
 
       echo "Updating src hash of kata.kata-kernel-uvm.configfile" >&2
       nix-update --version=skip --flake legacyPackages.x86_64-linux.kata.kata-kernel-uvm.configfile

tools/nydus-pull/go.mod

@@ -0,0 +1,95 @@
+module github.com/edgelesssys/contrast/nydus-pull
+
+go 1.23.0
+
+require (
+	github.com/containerd/containerd v1.7.24
+	github.com/edgelesssys/contrast v1.2.1
+	k8s.io/api v0.32.0
+	k8s.io/apimachinery v0.32.0
+	k8s.io/client-go v0.32.0
+)
+
+require (
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
+	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.11.7 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
+	github.com/containerd/containerd/api v1.7.19 // indirect
+	github.com/containerd/continuity v0.4.2 // indirect
+	github.com/containerd/errdefs v0.3.0 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/containerd/ttrpc v1.2.5 // indirect
+	github.com/containerd/typeurl/v2 v2.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-sev-guest v0.12.1 // indirect
+	github.com/google/go-tdx-guest v0.3.1 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/logger v1.1.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/sys/mountinfo v0.6.2 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/signal v0.7.0 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
+	github.com/opencontainers/runtime-spec v1.1.0 // indirect
+	github.com/opencontainers/selinux v1.11.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
+	go.opentelemetry.io/otel v1.31.0 // indirect
+	go.opentelemetry.io/otel/metric v1.31.0 // indirect
+	go.opentelemetry.io/otel/trace v1.31.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/net v0.32.0 // indirect
+	golang.org/x/oauth2 v0.24.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/time v0.7.0 // indirect
+	google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
+	google.golang.org/grpc v1.69.0 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
+	k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+)