Replace Kaniko with Buildkit (#503)

This commit replaces the use of Kaniko build tool with Buildkit 
which provides higher security standards 

Signed-off-by: Paolo Di Tommaso <[email protected]>
Co-authored-by: Paolo Di Tommaso <[email protected]>

configuration.md

@@ -81,11 +81,11 @@ Below are the standard format for known registries, but you can change registry
 
 - **`wave.build.timeout`**: the timeout for the build process. Its default value is `5m` (5 minutes), providing a reasonable time frame for the build operation. *Optional*.
 
-- **`wave.build.workspace`**: defines the path to the directory used by Wave to store artifacts such as Dockerfiles, Trivy cache for scan, Kaniko context, authentication configuration files, etc. For example, `/efs/wave/build`. *Mandatory*.
+- **`wave.build.workspace`**: defines the path to the directory used by Wave to store artifacts such as Dockerfiles, Trivy cache for scan, Buildkit context, authentication configuration files, etc. For example, `/efs/wave/build`. *Mandatory*.
 
 - **`wave.build.cleanup`**: determines the cleanup strategy after the build process. Options include `OnSuccess`, meaning cleanup occurs only if the build is successful. *Optional*.
 
-- **`wave.build.kaniko-image`**: specifies the [Kaniko](https://github.com/GoogleContainerTools/kaniko) Docker image used in the Wave build process. The default is `gcr.io/kaniko-project/executor:v1.19.2`. *Optional*.
+- **`wave.build.buildkit-image`**: specifies the [Buildkit](https://github.com/moby/buildkit) Docker image used in the Wave build process. The default is `moby/buildkit:v0.13.2-rootless`. *Optional*.
 
 - **`wave.build.singularity-image`**: sets the [Singularity](https://quay.io/repository/singularity/singularity?tab=tags) image used in the build process. The default is `quay.io/singularity/singularity:v3.11.4-slim`. *Optional*.
 
@@ -101,8 +101,11 @@ Below are the standard format for known registries, but you can change registry
 
 - **`wave.build.public`**: indicates whether the Docker container repository is public. If set to true, Wave freeze will prefer this public repository over `wave.build.repo`. *Optional*.
 
-- **`wave.build.compress-caching`**: determines whether to compress cache layers produced by the build process. The default is `true`, enabling compression for more efficient storage. *Optional*.
+- **`wave.build.oci-mediatypes`**: defines whether to use OCI mediatypes in exported manifests. its default value is `true`. *Optional*.
 
+- **`wave.build.compression`**: defines which type of compression will be applied to cache layers. its default value is `gzip` and other options are `uncompressed|estargz|zstd`. *Optional*.
+
+- **`wave.build.force-compression`**: determines whether to force the compression for each cache layers produced by the build process. The default is `false`, enabling compression for more efficient storage. *Optional*.
 
 ### Spack configuration for wave build process
 

src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy

@@ -36,8 +36,8 @@ import jakarta.inject.Singleton
 @Slf4j
 class BuildConfig {
 
-    @Value('${wave.build.kaniko-image}')
-    String kanikoImage
+    @Value('${wave.build.buildkit-image}')
+    String buildkitImage
 
     @Value('${wave.build.singularity-image}')
     String singularityImage
@@ -75,19 +75,26 @@ class BuildConfig {
     @Nullable
     String cleanup
 
-    @Value('${wave.build.compress-caching:true}')
-    Boolean compressCaching = true
-
     @Value('${wave.build.reserved-words:[]}')
     Set<String> reservedWords
 
     @Value('${wave.build.record.duration:5d}')
     Duration recordDuration
 
+    @Value('${wave.build.oci-mediatypes:true}')
+    Boolean ociMediatypes
+
+    //check here for other options https://github.com/moby/buildkit?tab=readme-ov-file#registry-push-image-and-cache-separately
+    @Value('${wave.build.compression:gzip}')
+    String compression
+
+    @Value('${wave.build.force-compression:false}')
+    Boolean forceCompression
+
     @PostConstruct
     private void init() {
         log.debug("Builder config: " +
-                "kaniko-image=${kanikoImage}; " +
+                "buildkit-image=${buildkitImage}; " +
                 "singularity-image=${singularityImage}; " +
                 "singularity-image-amr64=${singularityImageArm64}; " +
                 "default-build-repository=${defaultBuildRepository}; " +
@@ -97,9 +104,11 @@ class BuildConfig {
                 "build-timeout=${buildTimeout}; " +
                 "status-delay=${statusDelay}; " +
                 "status-duration=${statusDuration}; " +
-                "compress-caching=$compressCaching; " +
                 "record-duration=${recordDuration}; " +
-                "cleanup=${cleanup}; ")
+                "cleanup=${cleanup}; "+
+                "oci-mediatypes=${ociMediatypes}; " +
+                "compression=${compression}; " +
+                "force-compression=${forceCompression}; ")
     }
 
     String singularityImage(ContainerPlatform containerPlatform){
@@ -111,4 +120,5 @@ class BuildConfig {
     String getSingularityImageArm64(){
         return singularityImageArm64 ?: singularityImage + "-arm64"
     }
+
 }

src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy

@@ -37,6 +37,8 @@ abstract class BuildStrategy {
 
     abstract BuildResult build(BuildRequest req)
 
+    static final public String BUILDKIT_ENTRYPOINT = 'buildctl-daemonless.sh'
+
     void cleanup(BuildRequest req) {
         req.workDir?.deleteDir()
     }
@@ -55,35 +57,48 @@ abstract class BuildStrategy {
     protected List<String> dockerLaunchCmd(BuildRequest req) {
         final result = new ArrayList(10)
         result
-                << "--dockerfile"
-                << "$req.workDir/Containerfile".toString()
-                << "--context"
-                << "$req.workDir/context".toString()
-                << "--destination"
-                << req.targetImage
-                << "--cache=true"
-                << "--custom-platform"
-                << req.platform.toString()
+                << "build"
+                << "--frontend"
+                << "dockerfile.v0"
+                << "--local"
+                << "dockerfile=$req.workDir".toString()
+                << "--opt"
+                << "filename=Containerfile"
+                << "--local"
+                << "context=$req.workDir/context".toString()
+                << "--output"
+                << "type=image,name=$req.targetImage,push=true,oci-mediatypes=${buildConfig.ociMediatypes}".toString()
+                << "--opt"
+                << "platform=$req.platform".toString()
 
         if( req.cacheRepository ) {
-            result << "--cache-repo" << req.cacheRepository
-        }
+            result << "--export-cache"
+            def exportCache = new StringBuilder()
+            exportCache << "type=registry,"
+            exportCache << "image-manifest=true,"
+            exportCache << "ref=${req.cacheRepository}:${req.containerId},"
+            exportCache << "mode=max,"
+            exportCache << "ignore-error=true,"
+            exportCache << "oci-mediatypes=${buildConfig.ociMediatypes},"
+            exportCache << "compression=${buildConfig.compression},"
+            exportCache << "force-compression=${buildConfig.forceCompression}"
+            result << exportCache.toString()
 
-        if( !buildConfig.compressCaching ){
-            result << "--compressed-caching=false"
+            result << "--import-cache"
+            result << "type=registry,ref=$req.cacheRepository:$req.containerId".toString()
         }
 
         if(req.spackFile){
-            result << '--build-arg'
-            result << 'AWS_STS_REGIONAL_ENDPOINTS=$(AWS_STS_REGIONAL_ENDPOINTS)'
-            result << '--build-arg'
-            result << 'AWS_REGION=$(AWS_REGION)'
-            result << '--build-arg'
-            result << 'AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION)'
-            result << '--build-arg'
-            result << 'AWS_ROLE_ARN=$(AWS_ROLE_ARN)'
-            result << '--build-arg'
-            result << 'AWS_WEB_IDENTITY_TOKEN_FILE=$(AWS_WEB_IDENTITY_TOKEN_FILE)'
+            result << '--opt'
+            result << 'build-arg:AWS_STS_REGIONAL_ENDPOINTS=$(AWS_STS_REGIONAL_ENDPOINTS)'
+            result << '--opt'
+            result << 'build-arg:AWS_REGION=$(AWS_REGION)'
+            result << '--opt'
+            result << 'build-arg:AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION)'
+            result << '--opt'
+            result << 'build-arg:AWS_ROLE_ARN=$(AWS_ROLE_ARN)'
+            result << '--opt'
+            result << 'build-arg:AWS_WEB_IDENTITY_TOKEN_FILE=$(AWS_WEB_IDENTITY_TOKEN_FILE)'
         }
 
         return result
@@ -97,4 +112,5 @@ abstract class BuildStrategy {
             << "singularity build image.sif ${req.workDir}/Containerfile && singularity push image.sif ${req.targetImage}".toString()
         return result
     }
+
 }

src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy

@@ -110,21 +110,25 @@ class DockerBuildStrategy extends BuildStrategy {
         final spack = req.isSpackBuild ? spackConfig : null
 
         final dockerCmd = req.formatDocker()
-                ? cmdForKaniko( req.workDir, credsFile, spack, req.platform)
+                ? cmdForBuildkit( req.workDir, credsFile, spack, req.platform)
                 : cmdForSingularity( req.workDir, credsFile, spack, req.platform)
 
         return dockerCmd + launchCmd(req)
     }
 
-    protected List<String> cmdForKaniko(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) {
+    protected List<String> cmdForBuildkit(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) {
+        //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker
         final wrapper = ['docker',
                          'run',
                          '--rm',
-                         '-v', "$workDir:$workDir".toString()]
+                         '--privileged',
+                         '-v', "$workDir:$workDir".toString(),
+                         '--entrypoint',
+                         BUILDKIT_ENTRYPOINT]
 
         if( credsFile ) {
             wrapper.add('-v')
-            wrapper.add("$credsFile:/kaniko/.docker/config.json:ro".toString())
+            wrapper.add("$credsFile:/home/user/.docker/config.json:ro".toString())
         }
 
         if( spackConfig ) {
@@ -137,8 +141,9 @@ class DockerBuildStrategy extends BuildStrategy {
             wrapper.add('--platform')
             wrapper.add(platform.toString())
         }
-        // the container image to be used t
-        wrapper.add( buildConfig.kanikoImage )
+
+        // the container image to be used to build
+        wrapper.add( buildConfig.buildkitImage )
         // return it
         return wrapper
     }

src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy

@@ -117,7 +117,7 @@ class KubeBuildStrategy extends BuildStrategy {
 
     protected String getBuildImage(BuildRequest buildRequest){
         if( buildRequest.formatDocker() ) {
-            return buildConfig.kanikoImage
+            return buildConfig.buildkitImage
         }
 
         if( buildRequest.formatSingularity() ) {

src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy

@@ -50,6 +50,7 @@ import io.seqera.wave.core.ContainerPlatform
 import io.seqera.wave.service.scan.Trivy
 import jakarta.inject.Inject
 import jakarta.inject.Singleton
+import static io.seqera.wave.service.builder.BuildStrategy.BUILDKIT_ENTRYPOINT
 /**
  * implements the support for Kubernetes cluster
  *
@@ -104,6 +105,16 @@ class K8sServiceImpl implements K8sService {
     @Inject
     private BuildConfig buildConfig
 
+    // check this link to know more about these options https://github.com/moby/buildkit/tree/master/examples/kubernetes#kubernetes-manifests-for-buildkit
+    private final static Map<String,String> BUILDKIT_FLAGS = ['BUILDKITD_FLAGS': '--oci-worker-no-process-sandbox']
+
+    private Map<String, String> getBuildkitAnnotations(String containerName, boolean singularity) {
+        if( singularity )
+            return null
+        final key = "container.apparmor.security.beta.kubernetes.io/${containerName}".toString()
+        return Map.of(key, "unconfined")
+    }
+
     /**
      * Validate config setting
      */
@@ -304,7 +315,7 @@ class K8sServiceImpl implements K8sService {
     }
 
     /**
-     * Create a container for container image building via Kaniko
+     * Create a container for container image building via buildkit
      *
      * @param name
      *      The name of pod
@@ -341,7 +352,7 @@ class K8sServiceImpl implements K8sService {
 
         if( credsFile ){
             if( !singularity ) {
-                mounts.add(0, mountHostPath(credsFile, storageMountPath, '/kaniko/.docker/config.json'))
+                mounts.add(0, mountHostPath(credsFile, storageMountPath, '/home/user/.docker/config.json'))
             }
             else {
                 final remoteFile = credsFile.resolveSibling('singularity-remote.yaml')
@@ -361,6 +372,7 @@ class K8sServiceImpl implements K8sService {
                 .withNamespace(namespace)
                 .withName(name)
                 .addToLabels(labels)
+                .addToAnnotations(getBuildkitAnnotations(name,singularity))
                 .endMetadata()
 
         //spec section
@@ -391,10 +403,13 @@ class K8sServiceImpl implements K8sService {
             // use 'command' to override the entrypoint of the container
                     .withCommand(args)
                     .withNewSecurityContext().withPrivileged(true).endSecurityContext()
-        }
-        else {
-            // use 'arg' to avoid overriding the entrypoint of the container set by kaniko
-            container.withArgs(args)
+        } else {
+            container
+                    //required by buildkit rootless container
+                    .withEnv(toEnvList(BUILDKIT_FLAGS))
+                    // buildCommand is to set entrypoint for buildkit
+                    .withCommand(BUILDKIT_ENTRYPOINT)
+                    .withArgs(args)
         }
 
         // spec section
@@ -582,4 +597,5 @@ class K8sServiceImpl implements K8sService {
             result.add( new V1EnvVar().name(it.key).value(it.value) )
         return result
     }
+
 }

src/main/resources/application.yml

@@ -64,7 +64,7 @@ wave:
   server:
       url: "${WAVE_SERVER_URL:`http://localhost:9090`}"
   build:
-    kaniko-image: "gcr.io/kaniko-project/executor:v1.22.0"
+    buildkit-image: "moby/buildkit:v0.13.2-rootless"
     singularity-image: "quay.io/singularity/singularity:v3.11.4-slim"
     singularity-image-arm64: "quay.io/singularity/singularity:v3.11.4-slim-arm64"
     repo: "195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build/dev"