From a26f5dfe2efade08aca4b86c5f1362ca1d16970c Mon Sep 17 00:00:00 2001 From: Munish Chouhan Date: Mon, 10 Jun 2024 12:05:28 +0200 Subject: [PATCH] Replace Kaniko with Buildkit (#503) This commit replaces the use of Kaniko build tool with Buildkit which provides higher security standards Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- configuration.md | 9 +- .../wave/configuration/BuildConfig.groovy | 26 ++- .../wave/service/builder/BuildStrategy.groovy | 62 +++--- .../builder/DockerBuildStrategy.groovy | 17 +- .../service/builder/KubeBuildStrategy.groovy | 2 +- .../wave/service/k8s/K8sServiceImpl.groovy | 28 ++- src/main/resources/application.yml | 2 +- .../service/builder/BuildStrategyTest.groovy | 87 ++++---- .../builder/DockerBuildStrategyTest.groovy | 84 ++++---- .../builder/KubeBuildStrategyTest.groovy | 4 +- .../ContainerInspectServiceImplTest.groovy | 6 +- .../wave/service/k8s/K8sClientTest.groovy | 2 +- .../service/k8s/K8sServiceImplTest.groovy | 188 +++++++++--------- 13 files changed, 284 insertions(+), 233 deletions(-) diff --git a/configuration.md b/configuration.md index 7bccee1c1..359d93ae2 100644 --- a/configuration.md +++ b/configuration.md @@ -81,11 +81,11 @@ Below are the standard format for known registries, but you can change registry - **`wave.build.timeout`**: the timeout for the build process. Its default value is `5m` (5 minutes), providing a reasonable time frame for the build operation. *Optional*. -- **`wave.build.workspace`**: defines the path to the directory used by Wave to store artifacts such as Dockerfiles, Trivy cache for scan, Kaniko context, authentication configuration files, etc. For example, `/efs/wave/build`. *Mandatory*. +- **`wave.build.workspace`**: defines the path to the directory used by Wave to store artifacts such as Dockerfiles, Trivy cache for scan, Buildkit context, authentication configuration files, etc. For example, `/efs/wave/build`. *Mandatory*. - **`wave.build.cleanup`**: determines the cleanup strategy after the build process. Options include `OnSuccess`, meaning cleanup occurs only if the build is successful. *Optional*. -- **`wave.build.kaniko-image`**: specifies the [Kaniko](https://github.com/GoogleContainerTools/kaniko) Docker image used in the Wave build process. The default is `gcr.io/kaniko-project/executor:v1.19.2`. *Optional*. +- **`wave.build.buildkit-image`**: specifies the [Buildkit](https://github.com/moby/buildkit) Docker image used in the Wave build process. The default is `moby/buildkit:v0.13.2-rootless`. *Optional*. - **`wave.build.singularity-image`**: sets the [Singularity](https://quay.io/repository/singularity/singularity?tab=tags) image used in the build process. The default is `quay.io/singularity/singularity:v3.11.4-slim`. *Optional*. @@ -101,8 +101,11 @@ Below are the standard format for known registries, but you can change registry - **`wave.build.public`**: indicates whether the Docker container repository is public. If set to true, Wave freeze will prefer this public repository over `wave.build.repo`. *Optional*. -- **`wave.build.compress-caching`**: determines whether to compress cache layers produced by the build process. The default is `true`, enabling compression for more efficient storage. *Optional*. +- **`wave.build.oci-mediatypes`**: defines whether to use OCI mediatypes in exported manifests. its default value is `true`. *Optional*. +- **`wave.build.compression`**: defines which type of compression will be applied to cache layers. its default value is `gzip` and other options are `uncompressed|estargz|zstd`. *Optional*. + +- **`wave.build.force-compression`**: determines whether to force the compression for each cache layers produced by the build process. The default is `false`, enabling compression for more efficient storage. *Optional*. ### Spack configuration for wave build process diff --git a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy index 41afebd35..acc21afb8 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy @@ -36,8 +36,8 @@ import jakarta.inject.Singleton @Slf4j class BuildConfig { - @Value('${wave.build.kaniko-image}') - String kanikoImage + @Value('${wave.build.buildkit-image}') + String buildkitImage @Value('${wave.build.singularity-image}') String singularityImage @@ -75,19 +75,26 @@ class BuildConfig { @Nullable String cleanup - @Value('${wave.build.compress-caching:true}') - Boolean compressCaching = true - @Value('${wave.build.reserved-words:[]}') Set reservedWords @Value('${wave.build.record.duration:5d}') Duration recordDuration + @Value('${wave.build.oci-mediatypes:true}') + Boolean ociMediatypes + + //check here for other options https://github.com/moby/buildkit?tab=readme-ov-file#registry-push-image-and-cache-separately + @Value('${wave.build.compression:gzip}') + String compression + + @Value('${wave.build.force-compression:false}') + Boolean forceCompression + @PostConstruct private void init() { log.debug("Builder config: " + - "kaniko-image=${kanikoImage}; " + + "buildkit-image=${buildkitImage}; " + "singularity-image=${singularityImage}; " + "singularity-image-amr64=${singularityImageArm64}; " + "default-build-repository=${defaultBuildRepository}; " + @@ -97,9 +104,11 @@ class BuildConfig { "build-timeout=${buildTimeout}; " + "status-delay=${statusDelay}; " + "status-duration=${statusDuration}; " + - "compress-caching=$compressCaching; " + "record-duration=${recordDuration}; " + - "cleanup=${cleanup}; ") + "cleanup=${cleanup}; "+ + "oci-mediatypes=${ociMediatypes}; " + + "compression=${compression}; " + + "force-compression=${forceCompression}; ") } String singularityImage(ContainerPlatform containerPlatform){ @@ -111,4 +120,5 @@ class BuildConfig { String getSingularityImageArm64(){ return singularityImageArm64 ?: singularityImage + "-arm64" } + } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index 0d8c7dc5b..723d2611a 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -37,6 +37,8 @@ abstract class BuildStrategy { abstract BuildResult build(BuildRequest req) + static final public String BUILDKIT_ENTRYPOINT = 'buildctl-daemonless.sh' + void cleanup(BuildRequest req) { req.workDir?.deleteDir() } @@ -55,35 +57,48 @@ abstract class BuildStrategy { protected List dockerLaunchCmd(BuildRequest req) { final result = new ArrayList(10) result - << "--dockerfile" - << "$req.workDir/Containerfile".toString() - << "--context" - << "$req.workDir/context".toString() - << "--destination" - << req.targetImage - << "--cache=true" - << "--custom-platform" - << req.platform.toString() + << "build" + << "--frontend" + << "dockerfile.v0" + << "--local" + << "dockerfile=$req.workDir".toString() + << "--opt" + << "filename=Containerfile" + << "--local" + << "context=$req.workDir/context".toString() + << "--output" + << "type=image,name=$req.targetImage,push=true,oci-mediatypes=${buildConfig.ociMediatypes}".toString() + << "--opt" + << "platform=$req.platform".toString() if( req.cacheRepository ) { - result << "--cache-repo" << req.cacheRepository - } + result << "--export-cache" + def exportCache = new StringBuilder() + exportCache << "type=registry," + exportCache << "image-manifest=true," + exportCache << "ref=${req.cacheRepository}:${req.containerId}," + exportCache << "mode=max," + exportCache << "ignore-error=true," + exportCache << "oci-mediatypes=${buildConfig.ociMediatypes}," + exportCache << "compression=${buildConfig.compression}," + exportCache << "force-compression=${buildConfig.forceCompression}" + result << exportCache.toString() - if( !buildConfig.compressCaching ){ - result << "--compressed-caching=false" + result << "--import-cache" + result << "type=registry,ref=$req.cacheRepository:$req.containerId".toString() } if(req.spackFile){ - result << '--build-arg' - result << 'AWS_STS_REGIONAL_ENDPOINTS=$(AWS_STS_REGIONAL_ENDPOINTS)' - result << '--build-arg' - result << 'AWS_REGION=$(AWS_REGION)' - result << '--build-arg' - result << 'AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION)' - result << '--build-arg' - result << 'AWS_ROLE_ARN=$(AWS_ROLE_ARN)' - result << '--build-arg' - result << 'AWS_WEB_IDENTITY_TOKEN_FILE=$(AWS_WEB_IDENTITY_TOKEN_FILE)' + result << '--opt' + result << 'build-arg:AWS_STS_REGIONAL_ENDPOINTS=$(AWS_STS_REGIONAL_ENDPOINTS)' + result << '--opt' + result << 'build-arg:AWS_REGION=$(AWS_REGION)' + result << '--opt' + result << 'build-arg:AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION)' + result << '--opt' + result << 'build-arg:AWS_ROLE_ARN=$(AWS_ROLE_ARN)' + result << '--opt' + result << 'build-arg:AWS_WEB_IDENTITY_TOKEN_FILE=$(AWS_WEB_IDENTITY_TOKEN_FILE)' } return result @@ -97,4 +112,5 @@ abstract class BuildStrategy { << "singularity build image.sif ${req.workDir}/Containerfile && singularity push image.sif ${req.targetImage}".toString() return result } + } diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index a4ed1e745..5c5f13804 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -110,21 +110,25 @@ class DockerBuildStrategy extends BuildStrategy { final spack = req.isSpackBuild ? spackConfig : null final dockerCmd = req.formatDocker() - ? cmdForKaniko( req.workDir, credsFile, spack, req.platform) + ? cmdForBuildkit( req.workDir, credsFile, spack, req.platform) : cmdForSingularity( req.workDir, credsFile, spack, req.platform) return dockerCmd + launchCmd(req) } - protected List cmdForKaniko(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { + protected List cmdForBuildkit(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { + //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker final wrapper = ['docker', 'run', '--rm', - '-v', "$workDir:$workDir".toString()] + '--privileged', + '-v', "$workDir:$workDir".toString(), + '--entrypoint', + BUILDKIT_ENTRYPOINT] if( credsFile ) { wrapper.add('-v') - wrapper.add("$credsFile:/kaniko/.docker/config.json:ro".toString()) + wrapper.add("$credsFile:/home/user/.docker/config.json:ro".toString()) } if( spackConfig ) { @@ -137,8 +141,9 @@ class DockerBuildStrategy extends BuildStrategy { wrapper.add('--platform') wrapper.add(platform.toString()) } - // the container image to be used t - wrapper.add( buildConfig.kanikoImage ) + + // the container image to be used to build + wrapper.add( buildConfig.buildkitImage ) // return it return wrapper } diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index be7fdc178..54e6431f0 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -117,7 +117,7 @@ class KubeBuildStrategy extends BuildStrategy { protected String getBuildImage(BuildRequest buildRequest){ if( buildRequest.formatDocker() ) { - return buildConfig.kanikoImage + return buildConfig.buildkitImage } if( buildRequest.formatSingularity() ) { diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index df64fc33b..01d03dd98 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -50,6 +50,7 @@ import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton +import static io.seqera.wave.service.builder.BuildStrategy.BUILDKIT_ENTRYPOINT /** * implements the support for Kubernetes cluster * @@ -104,6 +105,16 @@ class K8sServiceImpl implements K8sService { @Inject private BuildConfig buildConfig + // check this link to know more about these options https://github.com/moby/buildkit/tree/master/examples/kubernetes#kubernetes-manifests-for-buildkit + private final static Map BUILDKIT_FLAGS = ['BUILDKITD_FLAGS': '--oci-worker-no-process-sandbox'] + + private Map getBuildkitAnnotations(String containerName, boolean singularity) { + if( singularity ) + return null + final key = "container.apparmor.security.beta.kubernetes.io/${containerName}".toString() + return Map.of(key, "unconfined") + } + /** * Validate config setting */ @@ -304,7 +315,7 @@ class K8sServiceImpl implements K8sService { } /** - * Create a container for container image building via Kaniko + * Create a container for container image building via buildkit * * @param name * The name of pod @@ -341,7 +352,7 @@ class K8sServiceImpl implements K8sService { if( credsFile ){ if( !singularity ) { - mounts.add(0, mountHostPath(credsFile, storageMountPath, '/kaniko/.docker/config.json')) + mounts.add(0, mountHostPath(credsFile, storageMountPath, '/home/user/.docker/config.json')) } else { final remoteFile = credsFile.resolveSibling('singularity-remote.yaml') @@ -361,6 +372,7 @@ class K8sServiceImpl implements K8sService { .withNamespace(namespace) .withName(name) .addToLabels(labels) + .addToAnnotations(getBuildkitAnnotations(name,singularity)) .endMetadata() //spec section @@ -391,10 +403,13 @@ class K8sServiceImpl implements K8sService { // use 'command' to override the entrypoint of the container .withCommand(args) .withNewSecurityContext().withPrivileged(true).endSecurityContext() - } - else { - // use 'arg' to avoid overriding the entrypoint of the container set by kaniko - container.withArgs(args) + } else { + container + //required by buildkit rootless container + .withEnv(toEnvList(BUILDKIT_FLAGS)) + // buildCommand is to set entrypoint for buildkit + .withCommand(BUILDKIT_ENTRYPOINT) + .withArgs(args) } // spec section @@ -582,4 +597,5 @@ class K8sServiceImpl implements K8sService { result.add( new V1EnvVar().name(it.key).value(it.value) ) return result } + } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 5acc31b06..14ae48515 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -64,7 +64,7 @@ wave: server: url: "${WAVE_SERVER_URL:`http://localhost:9090`}" build: - kaniko-image: "gcr.io/kaniko-project/executor:v1.22.0" + buildkit-image: "moby/buildkit:v0.13.2-rootless" singularity-image: "quay.io/singularity/singularity:v3.11.4-slim" singularity-image-arm64: "quay.io/singularity/singularity:v3.11.4-slim-arm64" repo: "195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build/dev" diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy index 5eda1fe44..db621a613 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildStrategyTest.groovy @@ -22,81 +22,89 @@ import spock.lang.Specification import java.nio.file.Path -import io.seqera.wave.configuration.BuildConfig +import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.ContainerHelper +import jakarta.inject.Inject /** * * @author Paolo Di Tommaso */ +@MicronautTest class BuildStrategyTest extends Specification { - def 'should get kaniko command' () { + @Inject + BuildStrategy strategy + + def 'should get buildkit command' () { given: - def cache = 'reg.io/wave/build/cache' - def service = Spy(BuildStrategy) - service.@buildConfig = new BuildConfig() - and: def req = new BuildRequest( + id: 'c168dba125e28777', workDir: Path.of('/work/foo/c168dba125e28777'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'quay.io/wave:c168dba125e28777', cacheRepository: 'reg.io/wave/build/cache' ) when: - def cmd = service.launchCmd(req) + def cmd = strategy.launchCmd(req) then: cmd == [ - '--dockerfile', - '/work/foo/c168dba125e28777/Containerfile', - '--context', - '/work/foo/c168dba125e28777/context', - '--destination', - 'quay.io/wave:c168dba125e28777', - '--cache=true', - '--custom-platform', - 'linux/amd64', - '--cache-repo', - 'reg.io/wave/build/cache', + 'build', + '--frontend', + 'dockerfile.v0', + '--local', + 'dockerfile=/work/foo/c168dba125e28777', + '--opt', + 'filename=Containerfile', + '--local', + 'context=/work/foo/c168dba125e28777/context', + '--output', + 'type=image,name=quay.io/wave:c168dba125e28777,push=true,oci-mediatypes=true', + '--opt', + 'platform=linux/amd64', + '--export-cache', + 'type=registry,image-manifest=true,ref=reg.io/wave/build/cache:c168dba125e28777,mode=max,ignore-error=true,oci-mediatypes=true,compression=gzip,force-compression=false', + '--import-cache', + 'type=registry,ref=reg.io/wave/build/cache:c168dba125e28777' ] } - def 'should get kaniko command with build context' () { + def 'should get buildkit command with build context' () { given: - def cache = 'reg.io/wave/build/cache' - def service = Spy(BuildStrategy) - service.@buildConfig = new BuildConfig() - and: def req = new BuildRequest( + id: 'c168dba125e28777', workDir: Path.of('/work/foo/3980470531b4a52a'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'quay.io/wave:3980470531b4a52a', cacheRepository: 'reg.io/wave/build/cache' ) when: - def cmd = service.launchCmd(req) + def cmd = strategy.launchCmd(req) then: cmd == [ - '--dockerfile', - '/work/foo/3980470531b4a52a/Containerfile', - '--context', - '/work/foo/3980470531b4a52a/context', - '--destination', - 'quay.io/wave:3980470531b4a52a', - '--cache=true', - '--custom-platform', - 'linux/amd64', - '--cache-repo', - 'reg.io/wave/build/cache', + 'build', + '--frontend', + 'dockerfile.v0', + '--local', + 'dockerfile=/work/foo/3980470531b4a52a', + '--opt', + 'filename=Containerfile', + '--local', + 'context=/work/foo/3980470531b4a52a/context', + '--output', + 'type=image,name=quay.io/wave:3980470531b4a52a,push=true,oci-mediatypes=true', + '--opt', + 'platform=linux/amd64', + '--export-cache', + 'type=registry,image-manifest=true,ref=reg.io/wave/build/cache:c168dba125e28777,mode=max,ignore-error=true,oci-mediatypes=true,compression=gzip,force-compression=false', + '--import-cache', + 'type=registry,ref=reg.io/wave/build/cache:c168dba125e28777' ] } def 'should get singularity command' () { given: - def cache = 'reg.io/wave/build/cache' - def service = Spy(BuildStrategy) - and: def req = new BuildRequest( workDir: Path.of('/work/foo/c168dba125e28777'), platform: ContainerPlatform.of('linux/amd64'), @@ -104,7 +112,7 @@ class BuildStrategyTest extends Specification { format: BuildFormat.SINGULARITY, cacheRepository: 'reg.io/wave/build/cache' ) when: - def cmd = service.launchCmd(req) + def cmd = strategy.launchCmd(req) then: cmd == [ "sh", @@ -155,5 +163,4 @@ class BuildStrategyTest extends Specification { build.buildId == 'af15cb0a413a2d48_100' build.workDir == Path.of('.').toRealPath().resolve('some/path/af15cb0a413a2d48_100') } - } diff --git a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy index 658066ae1..a6a26a693 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/DockerBuildStrategyTest.groovy @@ -45,41 +45,50 @@ class DockerBuildStrategyTest extends Specification { and: def work = Path.of('/work/foo') when: - def cmd = service.cmdForKaniko(work, null, null, null) + def cmd = service.cmdForBuildkit(work, null, null, null) then: cmd == ['docker', 'run', '--rm', + '--privileged', '-v', '/work/foo:/work/foo', - 'gcr.io/kaniko-project/executor:v1.22.0'] + '--entrypoint', + 'buildctl-daemonless.sh', + 'moby/buildkit:v0.13.2-rootless'] when: - cmd = service.cmdForKaniko(work, Path.of('/foo/creds.json'), null, ContainerPlatform.of('arm64')) + cmd = service.cmdForBuildkit(work, Path.of('/foo/creds.json'), null, ContainerPlatform.of('arm64')) then: cmd == ['docker', 'run', '--rm', + '--privileged', '-v', '/work/foo:/work/foo', - '-v', '/foo/creds.json:/kaniko/.docker/config.json:ro', + '--entrypoint', + 'buildctl-daemonless.sh', + '-v', '/foo/creds.json:/home/user/.docker/config.json:ro', '--platform', 'linux/arm64', - 'gcr.io/kaniko-project/executor:v1.22.0'] + 'moby/buildkit:v0.13.2-rootless'] when: - cmd = service.cmdForKaniko(work, Path.of('/foo/creds.json'), spackConfig, null) + cmd = service.cmdForBuildkit(work, Path.of('/foo/creds.json'), spackConfig, null) then: cmd == ['docker', 'run', '--rm', + '--privileged', '-v', '/work/foo:/work/foo', - '-v', '/foo/creds.json:/kaniko/.docker/config.json:ro', + '--entrypoint', + 'buildctl-daemonless.sh', + '-v', '/foo/creds.json:/home/user/.docker/config.json:ro', '-v', '/host/spack/key:/opt/spack/key:ro', - 'gcr.io/kaniko-project/executor:v1.22.0'] + 'moby/buildkit:v0.13.2-rootless'] cleanup: ctx.close() } - def 'should get kaniko build command' () { + def 'should get buildkit build command' () { given: def ctx = ApplicationContext.run() def service = ctx.getBean(DockerBuildStrategy) @@ -87,6 +96,7 @@ class DockerBuildStrategyTest extends Specification { def creds = Path.of('/work/creds.json') and: def req = new BuildRequest( + id: '89fb83ce6ec8627b', workDir: Path.of('/work/foo/89fb83ce6ec8627b'), platform: ContainerPlatform.of('linux/amd64'), targetImage: 'repo:89fb83ce6ec8627b', @@ -97,42 +107,30 @@ class DockerBuildStrategyTest extends Specification { cmd == ['docker', 'run', '--rm', + '--privileged', '-v', '/work/foo/89fb83ce6ec8627b:/work/foo/89fb83ce6ec8627b', - '-v', '/work/creds.json:/kaniko/.docker/config.json:ro', + '--entrypoint', + 'buildctl-daemonless.sh', + '-v', '/work/creds.json:/home/user/.docker/config.json:ro', '--platform', 'linux/amd64', - 'gcr.io/kaniko-project/executor:v1.22.0', - '--dockerfile', '/work/foo/89fb83ce6ec8627b/Containerfile', - '--context', '/work/foo/89fb83ce6ec8627b/context', - '--destination', 'repo:89fb83ce6ec8627b', - '--cache=true', - '--custom-platform', 'linux/amd64', - '--cache-repo', 'reg.io/wave/build/cache' ] - - cleanup: - ctx.close() - } - - def 'should disable compress-caching' () { - given: - def ctx = ApplicationContext.run(['wave.build.compress-caching': false]) - def service = ctx.getBean(DockerBuildStrategy) - and: - def req = new BuildRequest( - workDir: Path.of('/work/foo/89fb83ce6ec8627b'), - platform: ContainerPlatform.of('linux/amd64'), - targetImage: 'repo:89fb83ce6ec8627b', - cacheRepository: 'reg.io/wave/build/cache' ) - when: - def cmd = service.launchCmd(req) - then: - cmd == [ - '--dockerfile', '/work/foo/89fb83ce6ec8627b/Containerfile', - '--context', '/work/foo/89fb83ce6ec8627b/context', - '--destination', 'repo:89fb83ce6ec8627b', - '--cache=true', - '--custom-platform', 'linux/amd64', - '--cache-repo', 'reg.io/wave/build/cache', - '--compressed-caching=false' ] + 'moby/buildkit:v0.13.2-rootless', + 'build', + '--frontend', + 'dockerfile.v0', + '--local', + 'dockerfile=/work/foo/89fb83ce6ec8627b', + '--opt', + 'filename=Containerfile', + '--local', + 'context=/work/foo/89fb83ce6ec8627b/context', + '--output', + 'type=image,name=repo:89fb83ce6ec8627b,push=true,oci-mediatypes=true', + '--opt', + 'platform=linux/amd64', + '--export-cache', + 'type=registry,image-manifest=true,ref=reg.io/wave/build/cache:89fb83ce6ec8627b,mode=max,ignore-error=true,oci-mediatypes=true,compression=gzip,force-compression=false', + '--import-cache', + 'type=registry,ref=reg.io/wave/build/cache:89fb83ce6ec8627b' ] cleanup: ctx.close() diff --git a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy index 6f4af98e8..ca5e79227 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy @@ -103,8 +103,8 @@ class KubeBuildStrategyTest extends Specification { def targetImage = ContainerHelper.makeTargetImage(BuildFormat.DOCKER, repo, containerId, null, null, null) def req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{"config":"json"}', null,null , null, null, BuildFormat.DOCKER).withBuildId('1') - then: 'should return kaniko image' - strategy.getBuildImage(req) == 'gcr.io/kaniko-project/executor:v1.22.0' + then: 'should return buildkit image' + strategy.getBuildImage(req) == 'moby/buildkit:v0.13.2-rootless' when:'getting singularity with amd64 arch in build request' req = new BuildRequest(containerId, dockerfile, null, null, PATH, targetImage, USER, ContainerPlatform.of('amd64'), cache, "10.20.30.40", '{}', null,null , null, null, BuildFormat.SINGULARITY).withBuildId('1') diff --git a/src/test/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImplTest.groovy index fbe4dab1c..09c2a4397 100644 --- a/src/test/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/inspect/ContainerInspectServiceImplTest.groovy @@ -90,11 +90,11 @@ class ContainerInspectServiceImplTest extends Specification { and: ContainerInspectServiceImpl.findRepositories(''' - FROM gcr.io/kaniko-project/executor:latest AS knk + FROM moby/buildkit:v0.13.2-rootless AS bkt RUN this and that FROM amazoncorretto:17.0.4 - COPY --from=knk /kaniko/executor /kaniko/executor - ''') == ['gcr.io/kaniko-project/executor:latest', 'amazoncorretto:17.0.4'] + COPY --from=bkt /usr/bin/buildctl /usr/bin/buildctl + ''') == ['moby/buildkit:v0.13.2-rootless', 'amazoncorretto:17.0.4'] } diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy index bf6e8ac56..27ef124ac 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sClientTest.groovy @@ -56,7 +56,7 @@ class K8sClientTest extends Specification { def pod = k8sService.buildContainer( 'my-pod', 'busybox', - ['cat','/kaniko/.docker/config.json'], + ['cat','/home/user/.docker/config.json'], Path.of('/work/dir'), Path.of('/creds'), Path.of('/spack/dir'), diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 16184693c..39e2f585c 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -155,10 +155,10 @@ class K8sServiceImplTest extends Specification { def k8sService = ctx.getBean(K8sServiceImpl) when: - def mount = k8sService.mountHostPath(Path.of('/foo/work/x1/config.json'), '/foo','/kaniko/.docker/config.json') + def mount = k8sService.mountHostPath(Path.of('/foo/work/x1/config.json'), '/foo','/home/user/.docker/config.json') then: mount.name == 'build-data' - mount.mountPath == '/kaniko/.docker/config.json' + mount.mountPath == '/home/user/.docker/config.json' mount.readOnly mount.subPath == 'work/x1/config.json' @@ -190,42 +190,42 @@ class K8sServiceImplTest extends Specification { ctx.close() } - def 'should create build pod for kaniko' () { + def 'should create build pod for buildkit' () { given: def PROPS = [ - 'wave.build.workspace': '/build/work', - 'wave.build.timeout': '10s', - 'wave.build.k8s.namespace': 'my-ns', - 'wave.build.k8s.configPath': '/home/kube.config', + 'wave.build.workspace' : '/build/work', + 'wave.build.timeout' : '10s', + 'wave.build.k8s.namespace' : 'my-ns', + 'wave.build.k8s.configPath' : '/home/kube.config', 'wave.build.k8s.storage.claimName': 'build-claim', - 'wave.build.k8s.storage.mountPath': '/build' ] + 'wave.build.k8s.storage.mountPath': '/build'] and: def ctx = ApplicationContext.run(PROPS) def k8sService = ctx.getBean(K8sServiceImpl) when: - def result = k8sService.buildSpec('foo', 'my-image:latest', ['this','that'], Path.of('/build/work/xyz'), Path.of('/build/work/xyz/config.json'), null, [:]) + def result = k8sService.buildSpec('foo', 'my-image:latest', ['this', 'that'], Path.of('/build/work/xyz'), Path.of('/build/work/xyz/config.json'), null, [:]) then: result.metadata.name == 'foo' result.metadata.namespace == 'my-ns' and: result.spec.activeDeadlineSeconds == 10 and: - result.spec.containers.get(0).name == 'foo' - result.spec.containers.get(0).image == 'my-image:latest' - result.spec.containers.get(0).args == ['this','that'] - result.spec.containers.get(0).command == null - and: - result.spec.containers.get(0).volumeMounts.size() == 2 - and: - result.spec.containers.get(0).volumeMounts.get(0).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(0).mountPath == '/kaniko/.docker/config.json' - result.spec.containers.get(0).volumeMounts.get(0).subPath == 'work/xyz/config.json' - and: - result.spec.containers.get(0).volumeMounts.get(1).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(1).mountPath == '/build/work/xyz' - result.spec.containers.get(0).volumeMounts.get(1).subPath == 'work/xyz' - + verifyAll(result.spec.containers.get(0)) { + name == 'foo' + image == 'my-image:latest' + args == ['this', 'that'] + env.name == ['BUILDKITD_FLAGS'] + env.value == ['--oci-worker-no-process-sandbox'] + command == ['buildctl-daemonless.sh'] + volumeMounts.size() == 2 + volumeMounts.get(0).name == 'build-data' + volumeMounts.get(0).mountPath == '/home/user/.docker/config.json' + volumeMounts.get(0).subPath == 'work/xyz/config.json' + volumeMounts.get(1).name == 'build-data' + volumeMounts.get(1).mountPath == '/build/work/xyz' + volumeMounts.get(1).subPath == 'work/xyz' + } and: result.spec.volumes.get(0).name == 'build-data' result.spec.volumes.get(0).persistentVolumeClaim.claimName == 'build-claim' @@ -255,28 +255,25 @@ class K8sServiceImplTest extends Specification { and: result.spec.activeDeadlineSeconds == 10 and: - result.spec.containers.get(0).name == 'foo' - result.spec.containers.get(0).image == 'singularity:latest' - result.spec.containers.get(0).command == ['this','that'] - result.spec.containers.get(0).args == null - and: - result.spec.containers.get(0).volumeMounts.size() == 3 - and: - result.spec.containers.get(0).volumeMounts.get(0).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(0).mountPath == '/root/.singularity/docker-config.json' - result.spec.containers.get(0).volumeMounts.get(0).subPath == 'work/xyz/config.json' - and: - result.spec.containers.get(0).volumeMounts.get(1).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(1).mountPath == '/root/.singularity/remote.yaml' - result.spec.containers.get(0).volumeMounts.get(1).subPath == 'work/xyz/singularity-remote.yaml' - and: - result.spec.containers.get(0).volumeMounts.get(2).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(2).mountPath == '/build/work/xyz' - result.spec.containers.get(0).volumeMounts.get(2).subPath == 'work/xyz' - and: - result.spec.containers.get(0).getWorkingDir() == null - result.spec.containers.get(0).getSecurityContext().privileged + verifyAll(result.spec.containers.get(0)) { + name == 'foo' + image == 'singularity:latest' + command == ['this', 'that'] + args == null + volumeMounts.size() == 3 + volumeMounts.get(0).name == 'build-data' + volumeMounts.get(0).mountPath == '/root/.singularity/docker-config.json' + volumeMounts.get(0).subPath == 'work/xyz/config.json' + volumeMounts.get(1).name == 'build-data' + volumeMounts.get(1).mountPath == '/root/.singularity/remote.yaml' + volumeMounts.get(1).subPath == 'work/xyz/singularity-remote.yaml' + volumeMounts.get(2).name == 'build-data' + volumeMounts.get(2).mountPath == '/build/work/xyz' + volumeMounts.get(2).subPath == 'work/xyz' + getWorkingDir() == null + getSecurityContext().privileged + } and: result.spec.volumes.get(0).name == 'build-data' result.spec.volumes.get(0).persistentVolumeClaim.claimName == 'build-claim' @@ -309,21 +306,21 @@ class K8sServiceImplTest extends Specification { and: result.spec.activeDeadlineSeconds == 10 and: - result.spec.containers.get(0).name == 'foo' - result.spec.containers.get(0).image == 'my-image:latest' - result.spec.containers.get(0).args == ['this','that'] - and: - result.spec.containers.get(0).volumeMounts.size() == 2 - and: - result.spec.containers.get(0).volumeMounts.get(0).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(0).mountPath == '/build/work/xyz' - result.spec.containers.get(0).volumeMounts.get(0).subPath == 'work/xyz' - and: - result.spec.containers.get(0).volumeMounts.get(1).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(1).mountPath == '/opt/container/spack/key' - result.spec.containers.get(0).volumeMounts.get(1).subPath == 'host/spack/key' - result.spec.containers.get(0).volumeMounts.get(1).readOnly - + verifyAll(result.spec.containers.get(0)) { + name == 'foo' + image == 'my-image:latest' + args == ['this', 'that'] + env.name == ['BUILDKITD_FLAGS'] + env.value == ['--oci-worker-no-process-sandbox'] + volumeMounts.size() == 2 + volumeMounts.get(0).name == 'build-data' + volumeMounts.get(0).mountPath == '/build/work/xyz' + volumeMounts.get(0).subPath == 'work/xyz' + volumeMounts.get(1).name == 'build-data' + volumeMounts.get(1).mountPath == '/opt/container/spack/key' + volumeMounts.get(1).subPath == 'host/spack/key' + volumeMounts.get(1).readOnly + } and: result.spec.volumes.get(0).name == 'build-data' result.spec.volumes.get(0).persistentVolumeClaim.claimName == 'build-claim' @@ -355,16 +352,17 @@ class K8sServiceImplTest extends Specification { and: !result.spec.initContainers and: - result.spec.containers.get(0).name == 'foo' - result.spec.containers.get(0).image == 'my-image:latest' - result.spec.containers.get(0).args == ['this','that'] - and: - result.spec.containers.get(0).volumeMounts.size() == 1 - and: - result.spec.containers.get(0).volumeMounts.get(0).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(0).mountPath == '/build/work/xyz' - result.spec.containers.get(0).volumeMounts.get(0).subPath == 'work/xyz' - + verifyAll(result.spec.containers.get(0)) { + name == 'foo' + image == 'my-image:latest' + args == ['this', 'that'] + env.name == ['BUILDKITD_FLAGS'] + env.value == ['--oci-worker-no-process-sandbox'] + volumeMounts.size() == 1 + volumeMounts.get(0).name == 'build-data' + volumeMounts.get(0).mountPath == '/build/work/xyz' + volumeMounts.get(0).subPath == 'work/xyz' + } and: result.spec.volumes.get(0).name == 'build-data' result.spec.volumes.get(0).persistentVolumeClaim.claimName == 'build-claim' @@ -473,24 +471,21 @@ class K8sServiceImplTest extends Specification { and: result.spec.activeDeadlineSeconds == 10 and: - result.spec.containers.get(0).name == 'foo' - result.spec.containers.get(0).image == 'my-image:latest' - result.spec.containers.get(0).args == ['this','that'] - and: - result.spec.containers.get(0).volumeMounts.size() == 3 - and: - result.spec.containers.get(0).volumeMounts.get(0).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(0).mountPath == '/root/.docker/config.json' - result.spec.containers.get(0).volumeMounts.get(0).subPath == 'work/xyz/config.json' - and: - result.spec.containers.get(0).volumeMounts.get(1).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(1).mountPath == '/build/work/xyz' - result.spec.containers.get(0).volumeMounts.get(1).subPath == 'work/xyz' - and: - result.spec.containers.get(0).volumeMounts.get(2).name == 'build-data' - result.spec.containers.get(0).volumeMounts.get(2).mountPath == '/root/.cache/' - result.spec.containers.get(0).volumeMounts.get(2).subPath == 'work/.trivy' - + verifyAll(result.spec.containers.get(0)) { + name == 'foo' + image == 'my-image:latest' + args == ['this', 'that'] + volumeMounts.size() == 3 + volumeMounts.get(0).name == 'build-data' + volumeMounts.get(0).mountPath == '/root/.docker/config.json' + volumeMounts.get(0).subPath == 'work/xyz/config.json' + volumeMounts.get(1).name == 'build-data' + volumeMounts.get(1).mountPath == '/build/work/xyz' + volumeMounts.get(1).subPath == 'work/xyz' + volumeMounts.get(2).name == 'build-data' + volumeMounts.get(2).mountPath == '/root/.cache/' + volumeMounts.get(2).subPath == 'work/.trivy' + } and: result.spec.volumes.get(0).name == 'build-data' result.spec.volumes.get(0).persistentVolumeClaim.claimName == 'build-claim' @@ -560,14 +555,15 @@ class K8sServiceImplTest extends Specification { result.spec.activeDeadlineSeconds == 20 result.spec.serviceAccount == 'foo-sa' and: - result.spec.containers.get(0).name == 'foo' - result.spec.containers.get(0).image == 'my-image:latest' - result.spec.containers.get(0).args == ['this','that'] - result.spec.containers.get(0).getEnv().get(0) == new V1EnvVar().name('FOO').value('one') - result.spec.containers.get(0).getEnv().get(1) == new V1EnvVar().name('BAR').value('two') - and: - result.spec.containers.get(0).getResources().requests.get('cpu') == new Quantity('2') - result.spec.containers.get(0).getResources().requests.get('memory') == new Quantity('8Gi') + verifyAll(result.spec.containers.get(0)) { + name == 'foo' + image == 'my-image:latest' + args == ['this', 'that'] + getEnv().get(0) == new V1EnvVar().name('FOO').value('one') + getEnv().get(1) == new V1EnvVar().name('BAR').value('two') + getResources().requests.get('cpu') == new Quantity('2') + getResources().requests.get('memory') == new Quantity('8Gi') + } and: !result.spec.containers.get(0).getResources().limits