From 690f09b43d6c89d9c2a0a58368b95671ffc4a8dc Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Thu, 5 Oct 2023 18:18:45 +0200 Subject: [PATCH 01/21] define tekton tasks in Go Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/task_defs.go | 597 ++++++++++++++++++++++++++++++ 1 file changed, 597 insertions(+) create mode 100644 pkg/pipelines/tekton/task_defs.go diff --git a/pkg/pipelines/tekton/task_defs.go b/pkg/pipelines/tekton/task_defs.go new file mode 100644 index 0000000000..1b6f901cfa --- /dev/null +++ b/pkg/pipelines/tekton/task_defs.go @@ -0,0 +1,597 @@ +package tekton + +import ( + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + v1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + BashImage = "docker.io/library/bash:5.1.4@sha256:b208215a4655538be652b2769d82e576bc4d0a2bb132144c060efc5be8c3f5d6" + S2IImage = "quay.io/boson/s2i:latest" + FuncImage = "ghcr.io/knative/func/func:latest" + BuildahImage = "quay.io/buildah/stable:v1.31.0" +) + +var BuildpackTask = v1beta1.Task{ + TypeMeta: metaV1.TypeMeta{ + Kind: "Task", + APIVersion: "tekton.dev/v1beta1", + }, + ObjectMeta: metaV1.ObjectMeta{ + Name: "func-buildpacks", + Labels: map[string]string{ + "app.kubernetes.io/version": "0.1", + }, + Annotations: map[string]string{ + "tekton.dev/displayName": "Knative Functions Buildpacks", + "tekton.dev/pipelines.minVersion": "0.17.0", + "tekton.dev/platforms": "linux/amd64", + "tekton.dev/tags": "image-build", + "tekton.dev/categories": "Image Build", + }, + }, + Spec: v1beta1.TaskSpec{ + Params: []v1beta1.ParamSpec{ + v1beta1.ParamSpec{ + Name: "APP_IMAGE", + Description: "The name of where to store the app image.", + }, + v1beta1.ParamSpec{ + Name: "REGISTRY", + Description: "The registry associated with the function image.", + }, + v1beta1.ParamSpec{ + Name: "BUILDER_IMAGE", + Description: "The image on which builds will run (must include lifecycle and compatible buildpacks).", + }, + v1beta1.ParamSpec{ + Name: "SOURCE_SUBPATH", + Description: "A subpath within the `source` input where the source to build is located.", + Default: &v1beta1.ParamValue{ + Type: "string", + }, + }, + v1beta1.ParamSpec{ + Name: "ENV_VARS", + Type: "array", + Description: "Environment variables to set during _build-time_.", + Default: &v1beta1.ParamValue{ + Type: "array", + ArrayVal: []string{}, + }, + }, + v1beta1.ParamSpec{ + Name: "RUN_IMAGE", + Description: "Reference to a run image to use.", + Default: &v1beta1.ParamValue{ + Type: "string", + }, + }, + v1beta1.ParamSpec{ + Name: "CACHE_IMAGE", + Description: "The name of the persistent app cache image (if no cache workspace is provided).", + Default: &v1beta1.ParamValue{ + Type: "string", + }, + }, + v1beta1.ParamSpec{ + Name: "SKIP_RESTORE", + Description: "Do not write layer metadata or restore cached layers.", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: "false", + }, + }, + v1beta1.ParamSpec{ + Name: "USER_ID", + Description: "The user ID of the builder image user.", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: "1001", + }, + }, + v1beta1.ParamSpec{ + Name: "GROUP_ID", + Description: "The group ID of the builder image user.", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: "0", + }, + }, + v1beta1.ParamSpec{ + Name: "PLATFORM_DIR", + Description: "The name of the platform directory.", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: "empty-dir", + }, + }, + }, + Description: "The Knative Functions Buildpacks task builds source into a container image and pushes it to a registry, using Cloud Native Buildpacks. This task is based on the Buildpacks Tekton task v 0.4.", + Steps: []v1beta1.Step{ + v1beta1.Step{ + Name: "prepare", + Image: BashImage, + Args: []string{ + "--env-vars", + "$(params.ENV_VARS[*])", + }, + VolumeMounts: []v1.VolumeMount{ + v1.VolumeMount{ + Name: "layers-dir", + MountPath: "/layers", + }, + v1.VolumeMount{ + Name: "$(params.PLATFORM_DIR)", + MountPath: "/platform", + }, + v1.VolumeMount{ + Name: "empty-dir", + MountPath: "/emptyDir", + }, + }, + Script: `#!/usr/bin/env bash +set -e + +if [[ "$(workspaces.cache.bound)" == "true" ]]; then + echo "> Setting permissions on '$(workspaces.cache.path)'..." + chown -R "$(params.USER_ID):$(params.GROUP_ID)" "$(workspaces.cache.path)" +fi + +####################################################### +##### "/emptyDir" has been added for Knative Functions +for path in "/tekton/home" "/layers" "/emptyDir" "$(workspaces.source.path)"; do + echo "> Setting permissions on '$path'..." + chown -R "$(params.USER_ID):$(params.GROUP_ID)" "$path" + + if [[ "$path" == "$(workspaces.source.path)" ]]; then + chmod 775 "$(workspaces.source.path)" + fi +done + +echo "> Parsing additional configuration..." +parsing_flag="" +envs=() +for arg in "$@"; do + if [[ "$arg" == "--env-vars" ]]; then + echo "-> Parsing env variables..." + parsing_flag="env-vars" + elif [[ "$parsing_flag" == "env-vars" ]]; then + envs+=("$arg") + fi +done + +echo "> Processing any environment variables..." +ENV_DIR="/platform/env" + +echo "--> Creating 'env' directory: $ENV_DIR" +mkdir -p "$ENV_DIR" + +for env in "${envs[@]}"; do + IFS='=' read -r key value <<< "$env" + if [[ "$key" != "" && "$value" != "" ]]; then + path="${ENV_DIR}/${key}" + echo "--> Writing ${path}..." + echo -n "$value" > "$path" + fi +done + +############################################ +##### Added part for Knative Functions ##### +############################################ + +func_file="$(workspaces.source.path)/func.yaml" +if [ "$(params.SOURCE_SUBPATH)" != "" ]; then + func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml" +fi +echo "--> Saving 'func.yaml'" +cp $func_file /emptyDir/func.yaml + +############################################ +`, + }, + v1beta1.Step{ + Name: "create", + Image: "$(params.BUILDER_IMAGE)", + Command: []string{ + "/cnb/lifecycle/creator", + }, + Args: []string{ + "-app=$(workspaces.source.path)/$(params.SOURCE_SUBPATH)", + "-cache-dir=$(workspaces.cache.path)", + "-cache-image=$(params.CACHE_IMAGE)", + "-uid=$(params.USER_ID)", + "-gid=$(params.GROUP_ID)", + "-layers=/layers", + "-platform=/platform", + "-report=/layers/report.toml", + "-skip-restore=$(params.SKIP_RESTORE)", + "-previous-image=$(params.APP_IMAGE)", + "-run-image=$(params.RUN_IMAGE)", + "$(params.APP_IMAGE)", + }, + Env: []v1.EnvVar{ + v1.EnvVar{ + Name: "DOCKER_CONFIG", + Value: "$(workspaces.dockerconfig.path)", + }, + }, + VolumeMounts: []v1.VolumeMount{ + v1.VolumeMount{ + Name: "layers-dir", + MountPath: "/layers", + }, + v1.VolumeMount{ + Name: "$(params.PLATFORM_DIR)", + MountPath: "/platform", + }, + }, + ImagePullPolicy: "Always", + SecurityContext: &v1.SecurityContext{ + RunAsUser: ptr(int64(1001)), + RunAsGroup: ptr(int64(0)), + }, + }, + v1beta1.Step{ + Name: "results", + Image: BashImage, + VolumeMounts: []v1.VolumeMount{ + v1.VolumeMount{ + Name: "layers-dir", + MountPath: "/layers", + }, + v1.VolumeMount{ + Name: "empty-dir", + MountPath: "/emptyDir", + }, + }, + Script: `#!/usr/bin/env bash +set -e +cat /layers/report.toml | grep "digest" | cut -d'"' -f2 | cut -d'"' -f2 | tr -d '\n' | tee $(results.IMAGE_DIGEST.path) + +############################################ +##### Added part for Knative Functions ##### +############################################ + +digest=$(cat $(results.IMAGE_DIGEST.path)) + +func_file="$(workspaces.source.path)/func.yaml" +if [ "$(params.SOURCE_SUBPATH)" != "" ]; then + func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml" +fi + +if [[ ! -f "$func_file" ]]; then + echo "--> Restoring 'func.yaml'" + mkdir -p "$(workspaces.source.path)/$(params.SOURCE_SUBPATH)" + cp /emptyDir/func.yaml $func_file +fi + +echo "" +sed -i "s|^image:.*$|image: $(params.APP_IMAGE)|" "$func_file" +echo "Function image name: $(params.APP_IMAGE)" + +sed -i "s/^imageDigest:.*$/imageDigest: $digest/" "$func_file" +echo "Function image digest: $digest" + +sed -i "s|^registry:.*$|registry: $(params.REGISTRY)|" "$func_file" +echo "Function image registry: $(params.REGISTRY)" + +############################################ +`, + }, + }, + Volumes: []v1.Volume{ + v1.Volume{ + Name: "empty-dir", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + v1.Volume{ + Name: "layers-dir", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + StepTemplate: &v1beta1.StepTemplate{ + Env: []v1.EnvVar{ + v1.EnvVar{ + Name: "CNB_PLATFORM_API", + Value: "0.10", + }, + }, + }, + Workspaces: []v1beta1.WorkspaceDeclaration{ + v1beta1.WorkspaceDeclaration{ + Name: "source", + Description: "Directory where application source is located.", + }, + v1beta1.WorkspaceDeclaration{ + Name: "cache", + Description: "Directory where cache is stored (when no cache image is provided).", + Optional: true, + }, + v1beta1.WorkspaceDeclaration{ + Name: "dockerconfig", + Description: "An optional workspace that allows providing a .docker/config.json file for Buildpacks lifecycle binary to access the container registry. The file should be placed at the root of the Workspace with name config.json.", + Optional: true, + }, + }, + Results: []v1beta1.TaskResult{ + v1beta1.TaskResult{ + Name: "IMAGE_DIGEST", + Description: "The digest of the built `APP_IMAGE`.", + }, + }, + }, +} + +var S2ITask = v1beta1.Task{ + TypeMeta: metaV1.TypeMeta{ + Kind: "Task", + APIVersion: "tekton.dev/v1beta1", + }, + ObjectMeta: metaV1.ObjectMeta{ + Name: "func-s2i", + Labels: map[string]string{ + "app.kubernetes.io/version": "0.1", + }, + Annotations: map[string]string{ + "tekton.dev/pipelines.minVersion": "0.17.0", + "tekton.dev/platforms": "linux/amd64", + "tekton.dev/tags": "image-build", + "tekton.dev/categories": "Image Build", + }, + }, + Spec: v1beta1.TaskSpec{ + Params: []v1beta1.ParamSpec{ + v1beta1.ParamSpec{ + Name: "BUILDER_IMAGE", + Description: "The location of the s2i builder image.", + }, + v1beta1.ParamSpec{ + Name: "IMAGE", + Description: "Reference of the image S2I will produce.", + }, + v1beta1.ParamSpec{ + Name: "REGISTRY", + Description: "The registry associated with the function image.", + }, + v1beta1.ParamSpec{ + Name: "PATH_CONTEXT", + Description: "The location of the path to run s2i from.", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: ".", + }, + }, + v1beta1.ParamSpec{ + Name: "TLSVERIFY", + Description: "Verify the TLS on the registry endpoint (for push/pull to a non-TLS registry)", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: "true", + }, + }, + v1beta1.ParamSpec{ + Name: "LOGLEVEL", + Description: "Log level when running the S2I binary", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: "0", + }, + }, + v1beta1.ParamSpec{ + Name: "ENV_VARS", + Type: "array", + Description: "Environment variables to set during _build-time_.", + Default: &v1beta1.ParamValue{ + Type: "array", + ArrayVal: []string{}, + }, + }, + v1beta1.ParamSpec{ + Name: "S2I_IMAGE_SCRIPTS_URL", + Description: "The URL containing the default assemble and run scripts for the builder image.", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: "image:///usr/libexec/s2i", + }, + }, + }, + Description: `Knative Functions Source-to-Image (S2I) is a toolkit and workflow for building reproducible container images from source code +S2I produces images by injecting source code into a base S2I container image and letting the container prepare that source code for execution. The base S2I container images contains the language runtime and build tools needed for building and running the source code.`, + Steps: []v1beta1.Step{ + v1beta1.Step{ + Name: "generate", + Image: S2IImage, + Args: []string{ + "$(params.ENV_VARS[*])", + }, + WorkingDir: "$(workspaces.source.path)", + VolumeMounts: []v1.VolumeMount{ + v1.VolumeMount{ + Name: "gen-source", + MountPath: "/gen-source", + }, + v1.VolumeMount{ + Name: "env-vars", + MountPath: "/env-vars", + }, + }, + Script: `echo "Processing Build Environment Variables" +echo "" > /env-vars/env-file +for var in "$@" +do + if [[ "$var" != "=" ]]; then + echo "$var" >> /env-vars/env-file + fi +done + +echo "Generated Build Env Var file" +echo "------------------------------" +cat /env-vars/env-file +echo "------------------------------" + +/usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.PATH_CONTEXT) $(params.BUILDER_IMAGE) \ +--image-scripts-url $(params.S2I_IMAGE_SCRIPTS_URL) \ +--as-dockerfile /gen-source/Dockerfile.gen --environment-file /env-vars/env-file + +echo "Preparing func.yaml for later deployment" +func_file="$(workspaces.source.path)/func.yaml" +if [ "$(params.PATH_CONTEXT)" != "" ]; then + func_file="$(workspaces.source.path)/$(params.PATH_CONTEXT)/func.yaml" +fi +sed -i "s|^registry:.*$|registry: $(params.REGISTRY)|" "$func_file" +echo "Function image registry: $(params.REGISTRY)" + +s2iignore_file="$(dirname "$func_file")/.s2iignore" +[ -f "$s2iignore_file" ] || echo "node_modules" >> "$s2iignore_file" +`, + }, + v1beta1.Step{ + Name: "build", + Image: BuildahImage, + WorkingDir: "/gen-source", + VolumeMounts: []v1.VolumeMount{ + v1.VolumeMount{ + Name: "varlibcontainers", + MountPath: "/var/lib/containers", + }, + v1.VolumeMount{ + Name: "gen-source", + MountPath: "/gen-source", + }, + }, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{ + "SETFCAP", + }, + }, + }, + Script: `TLS_VERIFY_FLAG="" +if [ "$(params.TLSVERIFY)" = "false" ] || [ "$(params.TLSVERIFY)" = "0" ]; then + TLS_VERIFY_FLAG="--tls-verify=false" +fi + +[[ "$(workspaces.sslcertdir.bound)" == "true" ]] && CERT_DIR_FLAG="--cert-dir $(workspaces.sslcertdir.path)" +ARTIFACTS_CACHE_PATH="$(workspaces.cache.path)/mvn-artifacts" +[ -d "${ARTIFACTS_CACHE_PATH}" ] || mkdir "${ARTIFACTS_CACHE_PATH}" +buildah ${CERT_DIR_FLAG} bud --storage-driver=vfs ${TLS_VERIFY_FLAG} --layers \ + -v "${ARTIFACTS_CACHE_PATH}:/tmp/artifacts/:rw,z,U" \ + -f /gen-source/Dockerfile.gen -t $(params.IMAGE) . + +[[ "$(workspaces.dockerconfig.bound)" == "true" ]] && export DOCKER_CONFIG="$(workspaces.dockerconfig.path)" +buildah ${CERT_DIR_FLAG} push --storage-driver=vfs ${TLS_VERIFY_FLAG} --digestfile $(workspaces.source.path)/image-digest \ + $(params.IMAGE) docker://$(params.IMAGE) + +cat $(workspaces.source.path)/image-digest | tee /tekton/results/IMAGE_DIGEST +`, + }, + }, + Volumes: []v1.Volume{ + v1.Volume{ + Name: "varlibcontainers", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + v1.Volume{ + Name: "gen-source", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + v1.Volume{ + Name: "env-vars", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + Workspaces: []v1beta1.WorkspaceDeclaration{ + v1beta1.WorkspaceDeclaration{ + Name: "source", + }, + v1beta1.WorkspaceDeclaration{ + Name: "cache", + Description: "Directory where cache is stored (e.g. local mvn repo).", + Optional: true, + }, + v1beta1.WorkspaceDeclaration{ + Name: "sslcertdir", + Optional: true, + }, + v1beta1.WorkspaceDeclaration{ + Name: "dockerconfig", + Description: "An optional workspace that allows providing a .docker/config.json file for Buildah to access the container registry. The file should be placed at the root of the Workspace with name config.json.", + Optional: true, + }, + }, + Results: []v1beta1.TaskResult{ + v1beta1.TaskResult{ + Name: "IMAGE_DIGEST", + Description: "Digest of the image just built.", + }, + }, + }, +} + +var DeployTask = v1beta1.Task{ + TypeMeta: metaV1.TypeMeta{ + Kind: "Task", + APIVersion: "tekton.dev/v1beta1", + }, + ObjectMeta: metaV1.ObjectMeta{ + Name: "func-deploy", + Labels: map[string]string{ + "app.kubernetes.io/version": "0.1", + }, + Annotations: map[string]string{ + "tekton.dev/pipelines.minVersion": "0.12.1", + "tekton.dev/platforms": "linux/amd64", + "tekton.dev/tags": "cli", + "tekton.dev/categories": "CLI", + }, + }, + Spec: v1beta1.TaskSpec{ + Params: []v1beta1.ParamSpec{ + v1beta1.ParamSpec{ + Name: "path", + Description: "Path to the function project", + Default: &v1beta1.ParamValue{ + Type: "string", + }, + }, + v1beta1.ParamSpec{ + Name: "image", + Description: "Container image to be deployed", + Default: &v1beta1.ParamValue{ + Type: "string", + }, + }, + }, + Description: "This Task performs a deploy operation using the Knative `func` CLI", + Steps: []v1beta1.Step{ + v1beta1.Step{ + Name: "func-deploy", + Image: FuncImage, + Script: `func deploy --verbose --build=false --push=false --path=$(params.path) --remote=false --image="$(params.image)" +`, + }, + }, + Workspaces: []v1beta1.WorkspaceDeclaration{ + v1beta1.WorkspaceDeclaration{ + Name: "source", + Description: "The workspace containing the function project", + }, + }, + }, +} + +func ptr[T any](val T) *T { + return &val +} From 2afb4fb456df6502a50277e09605bcdfd1d4cba6 Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Fri, 6 Oct 2023 13:37:23 +0200 Subject: [PATCH 02/21] create s2i pipeline more programatically Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates.go | 21 ++- pkg/pipelines/tekton/templates_s2i.go | 252 ++++++++++++++++++++++++++ 2 files changed, 268 insertions(+), 5 deletions(-) diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index b4f6f22eb6..5d2cdff1ee 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -2,6 +2,7 @@ package tekton import ( "bytes" + "context" "fmt" "net/http" "os" @@ -12,6 +13,7 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/manifestival/manifestival" "gopkg.in/yaml.v3" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/func/pkg/builders" fn "knative.dev/func/pkg/functions" @@ -354,16 +356,25 @@ func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[ *val.field = ts } - var template string if f.Build.Builder == builders.Pack { - template = packPipelineTemplate + return createAndApplyResource(f.Root, pipelineFileName, packPipelineTemplate, "pipeline", getPipelineName(f), namespace, data) } else if f.Build.Builder == builders.S2I { - template = s2iPipelineTemplate + cli, err := NewTektonClients() + if err != nil { + return fmt.Errorf("cannot create tekton client: %w", err) + } + pipeline, err := GetS2IPipeline(f) + if err != nil { + return fmt.Errorf("cannot generate pipeline: %w", err) + } + _, err = cli.Tekton.TektonV1beta1().Pipelines(namespace).Create(context.TODO(), pipeline, v1.CreateOptions{}) + if err != nil { + err = fmt.Errorf("cannot create pipeline in cluster: %w", err) + } + return err } else { return builders.ErrBuilderNotSupported{Builder: f.Build.Builder} } - - return createAndApplyResource(f.Root, pipelineFileName, template, "pipeline", getPipelineName(f), namespace, data) } // createAndApplyPipelineRunTemplate creates and applies PipelineRun template for a standard on-cluster build diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index b78fc47a90..522e0ea62f 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -1,5 +1,257 @@ package tekton +import ( + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fn "knative.dev/func/pkg/functions" +) + +func GetS2IPipeline(f fn.Function) (*v1beta1.Pipeline, error) { + + labels, err := f.LabelsMap() + if err != nil { + return nil, fmt.Errorf("cannot generate labels: %w", err) + } + + tasks := []v1beta1.PipelineTask{ + v1beta1.PipelineTask{ + Name: "fetch-sources", + TaskRef: &v1beta1.TaskRef{ + ResolverRef: v1beta1.ResolverRef{ + Resolver: "hub", + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "kind", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "task", + }, + }, + v1beta1.Param{ + Name: "name", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "git-clone", + }, + }, + v1beta1.Param{ + Name: "version", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "0.4", + }, + }, + }, + }, + }, + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "url", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.gitRepository)", + }, + }, + v1beta1.Param{ + Name: "revision", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.gitRevision)", + }, + }, + }, + Workspaces: []v1beta1.WorkspacePipelineTaskBinding{ + v1beta1.WorkspacePipelineTaskBinding{ + Name: "output", + Workspace: "source-workspace", + }, + }, + }, + v1beta1.PipelineTask{ + Name: "build", + TaskSpec: &v1beta1.EmbeddedTask{ + TaskSpec: *S2ITask.Spec.DeepCopy(), + }, + RunAfter: []string{"fetch-sources"}, + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "IMAGE", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.imageName)", + }, + }, + v1beta1.Param{ + Name: "REGISTRY", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.registry)", + }, + }, + v1beta1.Param{ + Name: "PATH_CONTEXT", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.contextDir)", + }, + }, + v1beta1.Param{ + Name: "BUILDER_IMAGE", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.builderImage)", + }, + }, + v1beta1.Param{ + Name: "ENV_VARS", + Value: v1beta1.ParamValue{ + Type: "array", + ArrayVal: []string{ + "$(params.buildEnvs[*])", + }, + }, + }, + v1beta1.Param{ + Name: "S2I_IMAGE_SCRIPTS_URL", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.s2iImageScriptsUrl)", + }, + }, + }, + Workspaces: []v1beta1.WorkspacePipelineTaskBinding{ + v1beta1.WorkspacePipelineTaskBinding{ + Name: "source", + Workspace: "source-workspace", + }, + v1beta1.WorkspacePipelineTaskBinding{ + Name: "cache", + Workspace: "cache-workspace", + }, + v1beta1.WorkspacePipelineTaskBinding{ + Name: "dockerconfig", + Workspace: "dockerconfig-workspace", + }, + }, + }, + v1beta1.PipelineTask{ + Name: "deploy", + TaskSpec: &v1beta1.EmbeddedTask{ + TaskSpec: *DeployTask.Spec.DeepCopy(), + }, + RunAfter: []string{"build"}, + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "path", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(workspaces.source.path)/$(params.contextDir)", + }, + }, + v1beta1.Param{ + Name: "image", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.imageName)@$(tasks.build.results.IMAGE_DIGEST)", + }, + }, + }, + Workspaces: []v1beta1.WorkspacePipelineTaskBinding{ + v1beta1.WorkspacePipelineTaskBinding{ + Name: "source", + Workspace: "source-workspace", + }, + }, + }, + } + + if f.Build.Git.URL == "" { + tasks = tasks[1:] + tasks[0].RunAfter = nil + } + + result := v1beta1.Pipeline{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: getPipelineName(f), + Labels: labels, + Annotations: f.Deploy.Annotations, + }, + Spec: v1beta1.PipelineSpec{ + Tasks: tasks, + Params: []v1beta1.ParamSpec{ + v1beta1.ParamSpec{ + Name: "gitRepository", + Type: "string", + Description: "Git repository that hosts the function project", + Default: &v1beta1.ParamValue{ + Type: "string", + }, + }, + v1beta1.ParamSpec{ + Name: "gitRevision", + Type: "string", + Description: "Git revision to build", + }, + v1beta1.ParamSpec{ + Name: "contextDir", + Type: "string", + Description: "Path where the function project is", + Default: &v1beta1.ParamValue{ + Type: "string", + }, + }, + v1beta1.ParamSpec{ + Name: "imageName", + Type: "string", + Description: "Function image name", + }, + v1beta1.ParamSpec{ + Name: "registry", + Type: "string", + Description: "The registry associated with the function image", + }, + v1beta1.ParamSpec{ + Name: "builderImage", + Type: "string", + Description: "Builder image to be used", + }, + v1beta1.ParamSpec{ + Name: "buildEnvs", + Type: "array", + Description: "Environment variables to set during build time", + }, + v1beta1.ParamSpec{ + Name: "s2iImageScriptsUrl", + Type: "string", + Description: "URL containing the default assemble and run scripts for the builder image", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: "image:///usr/libexec/s2i", + }, + }, + }, + Workspaces: []v1beta1.PipelineWorkspaceDeclaration{ + v1beta1.PipelineWorkspaceDeclaration{ + Name: "source-workspace", + Description: "Directory where function source is located.", + }, + v1beta1.PipelineWorkspaceDeclaration{ + Name: "cache-workspace", + Description: "Directory where build cache is stored.", + }, + v1beta1.PipelineWorkspaceDeclaration{ + Name: "dockerconfig-workspace", + Description: "Directory containing image registry credentials stored in config.json file.", + Optional: true, + }, + }, + }, + } + + return &result, nil +} + const ( // s2iPipelineTemplate contains the S2I template used for both Tekton standard and PAC Pipeline s2iPipelineTemplate = ` From b325c457fb71026b8187009982bb4c388a69b323 Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Fri, 6 Oct 2023 14:57:55 +0200 Subject: [PATCH 03/21] create s2i pipeline run more programatically Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates.go | 19 +++- pkg/pipelines/tekton/templates_s2i.go | 136 ++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 5 deletions(-) diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index 5d2cdff1ee..133179a68e 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -427,16 +427,25 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m Revision: pipelinesTargetBranch, } - var template string if f.Build.Builder == builders.Pack { - template = packRunTemplate + return createAndApplyResource(f.Root, pipelineFileName, packRunTemplate, "pipelinerun", getPipelineRunGenerateName(f), namespace, data) } else if f.Build.Builder == builders.S2I { - template = s2iRunTemplate + cli, err := NewTektonClients() + if err != nil { + return fmt.Errorf("cannot create tekton client: %w", err) + } + piplineRun, err := GetS2IPipelineRun(f) + if err != nil { + return fmt.Errorf("cannot generate pipeline run: %w", err) + } + _, err = cli.Tekton.TektonV1beta1().PipelineRuns(namespace).Create(context.Background(), piplineRun, v1.CreateOptions{}) + if err != nil { + err = fmt.Errorf("cannot create pipeline run in cluster: %w", err) + } + return err } else { return builders.ErrBuilderNotSupported{Builder: f.Build.Builder} } - - return createAndApplyResource(f.Root, pipelineFileName, template, "pipelinerun", getPipelineRunGenerateName(f), namespace, data) } // allows simple mocking in unit tests diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index 522e0ea62f..e003aee0f6 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -1,8 +1,12 @@ package tekton import ( + "fmt" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + coreV1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/func/pkg/builders" fn "knative.dev/func/pkg/functions" ) @@ -252,6 +256,138 @@ func GetS2IPipeline(f fn.Function) (*v1beta1.Pipeline, error) { return &result, nil } +func GetS2IPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { + labels, err := f.LabelsMap() + if err != nil { + return nil, fmt.Errorf("cannot generate labels: %w", err) + } + labels["tekton.dev/pipeline"] = getPipelineName(f) + + pipelinesTargetBranch := f.Build.Git.Revision + if pipelinesTargetBranch == "" { + pipelinesTargetBranch = defaultPipelinesTargetBranch + } + + contextDir := f.Build.Git.ContextDir + if contextDir == "" && f.Build.Builder == builders.S2I { + // TODO(lkingland): could instead update S2I to interpret empty string + // as cwd, such that builder-specific code can be kept out of here. + contextDir = "." + } + + buildEnvs := []string{} + if len(f.Build.BuildEnvs) == 0 { + buildEnvs = []string{"="} + } else { + for i := range f.Build.BuildEnvs { + buildEnvs = append(buildEnvs, f.Build.BuildEnvs[i].KeyValuePair()) + } + } + + s2iImageScriptsUrl := defaultS2iImageScriptsUrl + if f.Runtime == "quarkus" { + s2iImageScriptsUrl = quarkusS2iImageScriptsUrl + } + + result := v1beta1.PipelineRun{ + TypeMeta: metav1.TypeMeta{ + Kind: "PipelineRun", + APIVersion: "tekton.dev/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: getPipelineRunGenerateName(f), + Labels: labels, + Annotations: f.Deploy.Annotations, + }, + Spec: v1beta1.PipelineRunSpec{ + PipelineRef: &v1beta1.PipelineRef{ + Name: getPipelineName(f), + }, + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "gitRepository", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: f.Build.Git.URL, + }, + }, + v1beta1.Param{ + Name: "gitRevision", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: pipelinesTargetBranch, + }, + }, + v1beta1.Param{ + Name: "contextDir", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: contextDir, + }, + }, + v1beta1.Param{ + Name: "imageName", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: f.Image, + }, + }, + v1beta1.Param{ + Name: "registry", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: f.Registry, + }, + }, + v1beta1.Param{ + Name: "builderImage", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: getBuilderImage(f), + }, + }, + v1beta1.Param{ + Name: "buildEnvs", + Value: v1beta1.ParamValue{ + Type: "array", + ArrayVal: buildEnvs, + }, + }, + v1beta1.Param{ + Name: "s2iImageScriptsUrl", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: s2iImageScriptsUrl, + }, + }, + }, + Workspaces: []v1beta1.WorkspaceBinding{ + v1beta1.WorkspaceBinding{ + Name: "source-workspace", + SubPath: "source", + PersistentVolumeClaim: &coreV1.PersistentVolumeClaimVolumeSource{ + ClaimName: getPipelinePvcName(f), + }, + }, + v1beta1.WorkspaceBinding{ + Name: "cache-workspace", + SubPath: "cache", + PersistentVolumeClaim: &coreV1.PersistentVolumeClaimVolumeSource{ + ClaimName: getPipelinePvcName(f), + }, + }, + v1beta1.WorkspaceBinding{ + Name: "dockerconfig-workspace", + Secret: &coreV1.SecretVolumeSource{ + SecretName: getPipelineSecretName(f), + }, + }, + }, + }, + } + return &result, nil +} + const ( // s2iPipelineTemplate contains the S2I template used for both Tekton standard and PAC Pipeline s2iPipelineTemplate = ` From 6b9b92dcb74248e38241783971777671866a5019 Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Fri, 6 Oct 2023 15:09:23 +0200 Subject: [PATCH 04/21] disable createAndApplyPipelineRunTemplate for s2i Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates_test.go | 112 ++++++++++++------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/pkg/pipelines/tekton/templates_test.go b/pkg/pipelines/tekton/templates_test.go index 4fbc5e2863..0b524433f0 100644 --- a/pkg/pipelines/tekton/templates_test.go +++ b/pkg/pipelines/tekton/templates_test.go @@ -209,62 +209,62 @@ var testData = []struct { namespace: "test-ns", wantErr: false, }, - { - name: "correct - s2i & node", - root: "testdata/testCreatePipelineS2INode", - runtime: "node", - builder: builders.S2I, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - s2i & quarkus", - root: "testdata/testCreatePipelineS2IQuarkus", - runtime: "quarkus", - builder: builders.S2I, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - s2i & go", - root: "testdata/testCreatePipelineS2IGo", - runtime: "go", - builder: builders.S2I, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - s2i & python", - root: "testdata/testCreatePipelineS2IPython", - runtime: "python", - builder: builders.S2I, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - s2i & typescript", - root: "testdata/testCreatePipelineS2ITypescript", - runtime: "typescript", - builder: builders.S2I, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - s2i & springboot", - root: "testdata/testCreatePipelineS2ISpringboot", - runtime: "springboot", - builder: builders.S2I, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - s2i & rust", - root: "testdata/testCreatePipelineS2IRust", - runtime: "rust", - builder: builders.S2I, - namespace: "test-ns", - wantErr: false, - }, + //{ + // name: "correct - s2i & node", + // root: "testdata/testCreatePipelineS2INode", + // runtime: "node", + // builder: builders.S2I, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - s2i & quarkus", + // root: "testdata/testCreatePipelineS2IQuarkus", + // runtime: "quarkus", + // builder: builders.S2I, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - s2i & go", + // root: "testdata/testCreatePipelineS2IGo", + // runtime: "go", + // builder: builders.S2I, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - s2i & python", + // root: "testdata/testCreatePipelineS2IPython", + // runtime: "python", + // builder: builders.S2I, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - s2i & typescript", + // root: "testdata/testCreatePipelineS2ITypescript", + // runtime: "typescript", + // builder: builders.S2I, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - s2i & springboot", + // root: "testdata/testCreatePipelineS2ISpringboot", + // runtime: "springboot", + // builder: builders.S2I, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - s2i & rust", + // root: "testdata/testCreatePipelineS2IRust", + // runtime: "rust", + // builder: builders.S2I, + // namespace: "test-ns", + // wantErr: false, + //}, } func Test_createAndApplyPipelineRunTemplate(t *testing.T) { From ee3ed102561d25e92f8f491fad8cbca57ab02996 Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Fri, 6 Oct 2023 16:32:49 +0200 Subject: [PATCH 05/21] unify app-image param name between s2i and pack Signed-off-by: Matej Vasek --- .../resources/tekton/task/func-s2i/0.1/func-s2i.yaml | 6 +++--- pkg/pipelines/tekton/task_defs.go | 6 +++--- pkg/pipelines/tekton/templates_s2i.go | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml b/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml index 5c30b7566c..cbf92216e6 100644 --- a/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml +++ b/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml @@ -22,7 +22,7 @@ spec: params: - name: BUILDER_IMAGE description: The location of the s2i builder image. - - name: IMAGE + - name: APP_IMAGE description: Reference of the image S2I will produce. - name: REGISTRY description: The registry associated with the function image. @@ -112,11 +112,11 @@ spec: [ -d "${ARTIFACTS_CACHE_PATH}" ] || mkdir "${ARTIFACTS_CACHE_PATH}" buildah ${CERT_DIR_FLAG} bud --storage-driver=vfs ${TLS_VERIFY_FLAG} --layers \ -v "${ARTIFACTS_CACHE_PATH}:/tmp/artifacts/:rw,z,U" \ - -f /gen-source/Dockerfile.gen -t $(params.IMAGE) . + -f /gen-source/Dockerfile.gen -t $(params.APP_IMAGE) . [[ "$(workspaces.dockerconfig.bound)" == "true" ]] && export DOCKER_CONFIG="$(workspaces.dockerconfig.path)" buildah ${CERT_DIR_FLAG} push --storage-driver=vfs ${TLS_VERIFY_FLAG} --digestfile $(workspaces.source.path)/image-digest \ - $(params.IMAGE) docker://$(params.IMAGE) + $(params.APP_IMAGE) docker://$(params.APP_IMAGE) cat $(workspaces.source.path)/image-digest | tee /tekton/results/IMAGE_DIGEST volumeMounts: diff --git a/pkg/pipelines/tekton/task_defs.go b/pkg/pipelines/tekton/task_defs.go index 1b6f901cfa..98a680dd57 100644 --- a/pkg/pipelines/tekton/task_defs.go +++ b/pkg/pipelines/tekton/task_defs.go @@ -352,7 +352,7 @@ var S2ITask = v1beta1.Task{ Description: "The location of the s2i builder image.", }, v1beta1.ParamSpec{ - Name: "IMAGE", + Name: "APP_IMAGE", Description: "Reference of the image S2I will produce.", }, v1beta1.ParamSpec{ @@ -482,11 +482,11 @@ ARTIFACTS_CACHE_PATH="$(workspaces.cache.path)/mvn-artifacts" [ -d "${ARTIFACTS_CACHE_PATH}" ] || mkdir "${ARTIFACTS_CACHE_PATH}" buildah ${CERT_DIR_FLAG} bud --storage-driver=vfs ${TLS_VERIFY_FLAG} --layers \ -v "${ARTIFACTS_CACHE_PATH}:/tmp/artifacts/:rw,z,U" \ - -f /gen-source/Dockerfile.gen -t $(params.IMAGE) . + -f /gen-source/Dockerfile.gen -t $(params.APP_IMAGE) . [[ "$(workspaces.dockerconfig.bound)" == "true" ]] && export DOCKER_CONFIG="$(workspaces.dockerconfig.path)" buildah ${CERT_DIR_FLAG} push --storage-driver=vfs ${TLS_VERIFY_FLAG} --digestfile $(workspaces.source.path)/image-digest \ - $(params.IMAGE) docker://$(params.IMAGE) + $(params.APP_IMAGE) docker://$(params.APP_IMAGE) cat $(workspaces.source.path)/image-digest | tee /tekton/results/IMAGE_DIGEST `, diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index e003aee0f6..08329ea2dc 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -79,7 +79,7 @@ func GetS2IPipeline(f fn.Function) (*v1beta1.Pipeline, error) { RunAfter: []string{"fetch-sources"}, Params: []v1beta1.Param{ v1beta1.Param{ - Name: "IMAGE", + Name: "APP_IMAGE", Value: v1beta1.ParamValue{ Type: "string", StringVal: "$(params.imageName)", @@ -436,7 +436,7 @@ spec: {{.GitCloneTaskRef}} - name: build params: - - name: IMAGE + - name: APP_IMAGE value: $(params.imageName) - name: REGISTRY value: $(params.registry) From 86337f22a05a0eb7b806f1a6c6eef00a37ed0eda Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Fri, 6 Oct 2023 16:36:53 +0200 Subject: [PATCH 06/21] unify sub-path param name between s2i and pack Signed-off-by: Matej Vasek --- .../resources/tekton/task/func-s2i/0.1/func-s2i.yaml | 8 ++++---- pkg/pipelines/tekton/task_defs.go | 8 ++++---- pkg/pipelines/tekton/templates_s2i.go | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml b/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml index cbf92216e6..9a2f7a3b92 100644 --- a/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml +++ b/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml @@ -26,7 +26,7 @@ spec: description: Reference of the image S2I will produce. - name: REGISTRY description: The registry associated with the function image. - - name: PATH_CONTEXT + - name: SOURCE_SUBPATH description: The location of the path to run s2i from. default: . - name: TLSVERIFY @@ -78,14 +78,14 @@ spec: cat /env-vars/env-file echo "------------------------------" - /usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.PATH_CONTEXT) $(params.BUILDER_IMAGE) \ + /usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.SOURCE_SUBPATH) $(params.BUILDER_IMAGE) \ --image-scripts-url $(params.S2I_IMAGE_SCRIPTS_URL) \ --as-dockerfile /gen-source/Dockerfile.gen --environment-file /env-vars/env-file echo "Preparing func.yaml for later deployment" func_file="$(workspaces.source.path)/func.yaml" - if [ "$(params.PATH_CONTEXT)" != "" ]; then - func_file="$(workspaces.source.path)/$(params.PATH_CONTEXT)/func.yaml" + if [ "$(params.SOURCE_SUBPATH)" != "" ]; then + func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml" fi sed -i "s|^registry:.*$|registry: $(params.REGISTRY)|" "$func_file" echo "Function image registry: $(params.REGISTRY)" diff --git a/pkg/pipelines/tekton/task_defs.go b/pkg/pipelines/tekton/task_defs.go index 98a680dd57..3aae85e9ef 100644 --- a/pkg/pipelines/tekton/task_defs.go +++ b/pkg/pipelines/tekton/task_defs.go @@ -360,7 +360,7 @@ var S2ITask = v1beta1.Task{ Description: "The registry associated with the function image.", }, v1beta1.ParamSpec{ - Name: "PATH_CONTEXT", + Name: "SOURCE_SUBPATH", Description: "The location of the path to run s2i from.", Default: &v1beta1.ParamValue{ Type: "string", @@ -435,14 +435,14 @@ echo "------------------------------" cat /env-vars/env-file echo "------------------------------" -/usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.PATH_CONTEXT) $(params.BUILDER_IMAGE) \ +/usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.SOURCE_SUBPATH) $(params.BUILDER_IMAGE) \ --image-scripts-url $(params.S2I_IMAGE_SCRIPTS_URL) \ --as-dockerfile /gen-source/Dockerfile.gen --environment-file /env-vars/env-file echo "Preparing func.yaml for later deployment" func_file="$(workspaces.source.path)/func.yaml" -if [ "$(params.PATH_CONTEXT)" != "" ]; then - func_file="$(workspaces.source.path)/$(params.PATH_CONTEXT)/func.yaml" +if [ "$(params.SOURCE_SUBPATH)" != "" ]; then + func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml" fi sed -i "s|^registry:.*$|registry: $(params.REGISTRY)|" "$func_file" echo "Function image registry: $(params.REGISTRY)" diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index 08329ea2dc..94baa1db8c 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -93,7 +93,7 @@ func GetS2IPipeline(f fn.Function) (*v1beta1.Pipeline, error) { }, }, v1beta1.Param{ - Name: "PATH_CONTEXT", + Name: "SOURCE_SUBPATH", Value: v1beta1.ParamValue{ Type: "string", StringVal: "$(params.contextDir)", @@ -440,7 +440,7 @@ spec: value: $(params.imageName) - name: REGISTRY value: $(params.registry) - - name: PATH_CONTEXT + - name: SOURCE_SUBPATH value: $(params.contextDir) - name: BUILDER_IMAGE value: $(params.builderImage) From 97950f73a50e5194bc46d013c07fd714af7a6e18 Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 13:20:27 +0200 Subject: [PATCH 07/21] test: use basic Go program (non-Function) in test Signed-off-by: Matej Vasek --- .../tekton/pipelines_integration_test.go | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/pkg/pipelines/tekton/pipelines_integration_test.go b/pkg/pipelines/tekton/pipelines_integration_test.go index b10f66de53..215d112c73 100644 --- a/pkg/pipelines/tekton/pipelines_integration_test.go +++ b/pkg/pipelines/tekton/pipelines_integration_test.go @@ -20,7 +20,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/func/pkg/k8s" - "knative.dev/func/pkg/builders/buildpacks" "knative.dev/func/pkg/docker" fn "knative.dev/func/pkg/functions" "knative.dev/func/pkg/pipelines/tekton" @@ -174,7 +173,7 @@ func createSimpleGoProject(t *testing.T, ns string) fn.Function { Invoke: "none", Build: fn.BuildSpec{ BuilderImages: map[string]string{ - "pack": buildpacks.DefaultTinyBuilder, + "pack": "index.docker.io/paketobuildpacks/builder-jammy-tiny", "s2i": "registry.access.redhat.com/ubi8/go-toolset", }, }, @@ -190,16 +189,40 @@ func createSimpleGoProject(t *testing.T, ns string) fn.Function { return f } -const simpleGOSvc = `package function +const simpleGOSvc = `package main import ( "context" + "net" "net/http" + "os" + "os/signal" + "syscall" ) -func Handle(ctx context.Context, resp http.ResponseWriter, req *http.Request) { - resp.Header().Add("Content-Type", "text/plain") - resp.WriteHeader(200) - _, _ = resp.Write([]byte("Hello World!\n")) +func main() { + sigs := make(chan os.Signal, 5) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + s := http.Server{ + Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + resp.Header().Add("Content-Type", "text/plain") + resp.WriteHeader(200) + _, _ = resp.Write([]byte("OK")) + }), + } + go func() { + <-sigs + _ = s.Shutdown(context.Background()) + }() + port := "8080" + if p, ok := os.LookupEnv("PORT"); ok { + port = p + } + l, err := net.Listen("tcp4", ":"+port) + if err != nil { + panic(err) + } + _ = s.Serve(l) } ` From fc8f9651d9afd14de425458bb2f6f2ae74718b18 Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 14:05:09 +0200 Subject: [PATCH 08/21] feat: create pack pipeline and pipelinerun more programatically Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates.go | 88 +-------------------------- pkg/pipelines/tekton/templates_s2i.go | 12 +++- 2 files changed, 13 insertions(+), 87 deletions(-) diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index 133179a68e..3dd449c0e1 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -323,42 +323,7 @@ func getTaskSpec(taskUrlTemplate string) (string, error) { // createAndApplyPipelineTemplate creates and applies Pipeline template for a standard on-cluster build // all resources are created on the fly, if there's a Pipeline defined in the project directory, it is used instead func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[string]string) error { - // If Git is set up create fetch task and reference it from build task, - // otherwise sources have been already uploaded to workspace PVC. - gitCloneTaskRef := "" - runAfterFetchSources := "" - if f.Build.Git.URL != "" { - runAfterFetchSources = runAfterFetchSourcesRef - gitCloneTaskRef = taskGitCloneTaskRef - } - - data := templateData{ - FunctionName: f.Name, - Annotations: f.Deploy.Annotations, - Labels: labels, - PipelineName: getPipelineName(f), - RunAfterFetchSources: runAfterFetchSources, - GitCloneTaskRef: gitCloneTaskRef, - } - - for _, val := range []struct { - ref string - field *string - }{ - {BuildpackTaskURL, &data.FuncBuildpacksTaskRef}, - {S2ITaskURL, &data.FuncS2iTaskRef}, - {DeployTaskURL, &data.FuncDeployTaskRef}, - } { - ts, err := getTaskSpec(val.ref) - if err != nil { - return err - } - *val.field = ts - } - - if f.Build.Builder == builders.Pack { - return createAndApplyResource(f.Root, pipelineFileName, packPipelineTemplate, "pipeline", getPipelineName(f), namespace, data) - } else if f.Build.Builder == builders.S2I { + if f.Build.Builder == builders.Pack || f.Build.Builder == builders.S2I { cli, err := NewTektonClients() if err != nil { return fmt.Errorf("cannot create tekton client: %w", err) @@ -380,56 +345,7 @@ func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[ // createAndApplyPipelineRunTemplate creates and applies PipelineRun template for a standard on-cluster build // all resources are created on the fly, if there's a PipelineRun defined in the project directory, it is used instead func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels map[string]string) error { - contextDir := f.Build.Git.ContextDir - if contextDir == "" && f.Build.Builder == builders.S2I { - // TODO(lkingland): could instead update S2I to interpret empty string - // as cwd, such that builder-specific code can be kept out of here. - contextDir = "." - } - - pipelinesTargetBranch := f.Build.Git.Revision - if pipelinesTargetBranch == "" { - pipelinesTargetBranch = defaultPipelinesTargetBranch - } - - buildEnvs := []string{} - if len(f.Build.BuildEnvs) == 0 { - buildEnvs = []string{"="} - } else { - for i := range f.Build.BuildEnvs { - buildEnvs = append(buildEnvs, f.Build.BuildEnvs[i].KeyValuePair()) - } - } - - s2iImageScriptsUrl := defaultS2iImageScriptsUrl - if f.Runtime == "quarkus" { - s2iImageScriptsUrl = quarkusS2iImageScriptsUrl - } - - data := templateData{ - FunctionName: f.Name, - Annotations: f.Deploy.Annotations, - Labels: labels, - ContextDir: contextDir, - FunctionImage: f.Image, - Registry: f.Registry, - BuilderImage: getBuilderImage(f), - BuildEnvs: buildEnvs, - - PipelineName: getPipelineName(f), - PipelineRunName: getPipelineRunGenerateName(f), - PvcName: getPipelinePvcName(f), - SecretName: getPipelineSecretName(f), - - S2iImageScriptsUrl: s2iImageScriptsUrl, - - RepoUrl: f.Build.Git.URL, - Revision: pipelinesTargetBranch, - } - - if f.Build.Builder == builders.Pack { - return createAndApplyResource(f.Root, pipelineFileName, packRunTemplate, "pipelinerun", getPipelineRunGenerateName(f), namespace, data) - } else if f.Build.Builder == builders.S2I { + if f.Build.Builder == builders.Pack || f.Build.Builder == builders.S2I { cli, err := NewTektonClients() if err != nil { return fmt.Errorf("cannot create tekton client: %w", err) diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index 94baa1db8c..900d89f529 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -17,6 +17,16 @@ func GetS2IPipeline(f fn.Function) (*v1beta1.Pipeline, error) { return nil, fmt.Errorf("cannot generate labels: %w", err) } + var buildTaskSpec v1beta1.TaskSpec + switch f.Build.Builder { + case builders.S2I: + buildTaskSpec = *S2ITask.Spec.DeepCopy() + case builders.Pack: + buildTaskSpec = *BuildpackTask.Spec.DeepCopy() + default: + return nil, fmt.Errorf("unsupported builder: %q", f.Build.BuilderImages) + } + tasks := []v1beta1.PipelineTask{ v1beta1.PipelineTask{ Name: "fetch-sources", @@ -74,7 +84,7 @@ func GetS2IPipeline(f fn.Function) (*v1beta1.Pipeline, error) { v1beta1.PipelineTask{ Name: "build", TaskSpec: &v1beta1.EmbeddedTask{ - TaskSpec: *S2ITask.Spec.DeepCopy(), + TaskSpec: buildTaskSpec, }, RunAfter: []string{"fetch-sources"}, Params: []v1beta1.Param{ From 68ab6eebee6fdf0e648a2d2abc47c898073724fb Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 14:30:55 +0200 Subject: [PATCH 09/21] chore: remove unused code Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates.go | 34 ------------------------------- 1 file changed, 34 deletions(-) diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index 3dd449c0e1..483a4ec9d1 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "net/http" "os" "path" "strings" @@ -12,7 +11,6 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/manifestival/manifestival" - "gopkg.in/yaml.v3" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/func/pkg/builders" @@ -288,38 +286,6 @@ func deleteAllPipelineTemplates(f fn.Function) string { return "" } -func getTaskSpec(taskUrlTemplate string) (string, error) { - resp, err := http.Get(taskUrlTemplate) - if err != nil { - return "", err - } - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("cannot get task: %q bad http code: %d", taskUrlTemplate, resp.StatusCode) - } - defer resp.Body.Close() - var data map[string]any - dec := yaml.NewDecoder(resp.Body) - err = dec.Decode(&data) - if err != nil { - return "", err - } - data = map[string]any{ - "taskSpec": data["spec"], - } - var buff bytes.Buffer - enc := yaml.NewEncoder(&buff) - enc.SetIndent(2) - err = enc.Encode(data) - if err != nil { - return "", err - } - err = enc.Close() - if err != nil { - return "", err - } - return strings.ReplaceAll(buff.String(), "\n", "\n "), nil -} - // createAndApplyPipelineTemplate creates and applies Pipeline template for a standard on-cluster build // all resources are created on the fly, if there's a Pipeline defined in the project directory, it is used instead func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[string]string) error { From b075874d7c768e597e0a58ece5ff567bd387ce3b Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 14:45:25 +0200 Subject: [PATCH 10/21] disable createAndApplyPipelineRunTemplate for pack Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates_test.go | 112 ++++++++++++------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/pkg/pipelines/tekton/templates_test.go b/pkg/pipelines/tekton/templates_test.go index 0b524433f0..cbfa08dcd9 100644 --- a/pkg/pipelines/tekton/templates_test.go +++ b/pkg/pipelines/tekton/templates_test.go @@ -153,62 +153,62 @@ var testData = []struct { labels map[string]string wantErr bool }{ - { - name: "correct - pack & node", - root: "testdata/testCreatePipelinePackNode", - runtime: "node", - builder: builders.Pack, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - pack & quarkus", - root: "testdata/testCreatePipelinePackQuarkus", - runtime: "quarkus", - builder: builders.Pack, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - pack & go", - root: "testdata/testCreatePipelinePackGo", - runtime: "go", - builder: builders.Pack, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - pack & python", - root: "testdata/testCreatePipelinePackPython", - runtime: "python", - builder: builders.Pack, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - pack & typescript", - root: "testdata/testCreatePipelinePackTypescript", - runtime: "typescript", - builder: builders.Pack, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - pack & springboot", - root: "testdata/testCreatePipelinePackSpringboot", - runtime: "springboot", - builder: builders.Pack, - namespace: "test-ns", - wantErr: false, - }, - { - name: "correct - pack & rust", - root: "testdata/testCreatePipelinePackRust", - runtime: "rust", - builder: builders.Pack, - namespace: "test-ns", - wantErr: false, - }, + //{ + // name: "correct - pack & node", + // root: "testdata/testCreatePipelinePackNode", + // runtime: "node", + // builder: builders.Pack, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - pack & quarkus", + // root: "testdata/testCreatePipelinePackQuarkus", + // runtime: "quarkus", + // builder: builders.Pack, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - pack & go", + // root: "testdata/testCreatePipelinePackGo", + // runtime: "go", + // builder: builders.Pack, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - pack & python", + // root: "testdata/testCreatePipelinePackPython", + // runtime: "python", + // builder: builders.Pack, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - pack & typescript", + // root: "testdata/testCreatePipelinePackTypescript", + // runtime: "typescript", + // builder: builders.Pack, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - pack & springboot", + // root: "testdata/testCreatePipelinePackSpringboot", + // runtime: "springboot", + // builder: builders.Pack, + // namespace: "test-ns", + // wantErr: false, + //}, + //{ + // name: "correct - pack & rust", + // root: "testdata/testCreatePipelinePackRust", + // runtime: "rust", + // builder: builders.Pack, + // namespace: "test-ns", + // wantErr: false, + //}, //{ // name: "correct - s2i & node", // root: "testdata/testCreatePipelineS2INode", From 3b9e6234899eab9b9af2096be0fb967923963f7d Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 14:55:35 +0200 Subject: [PATCH 11/21] chore: rename function approprietly Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates.go | 2 +- pkg/pipelines/tekton/templates_s2i.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index 483a4ec9d1..53b583fa34 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -294,7 +294,7 @@ func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[ if err != nil { return fmt.Errorf("cannot create tekton client: %w", err) } - pipeline, err := GetS2IPipeline(f) + pipeline, err := GetPipeline(f) if err != nil { return fmt.Errorf("cannot generate pipeline: %w", err) } diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index 900d89f529..aec3601f93 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -10,7 +10,7 @@ import ( fn "knative.dev/func/pkg/functions" ) -func GetS2IPipeline(f fn.Function) (*v1beta1.Pipeline, error) { +func GetPipeline(f fn.Function) (*v1beta1.Pipeline, error) { labels, err := f.LabelsMap() if err != nil { From c20839e6279a179fb96a7b03e8b37c83224f1d5e Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 15:22:59 +0200 Subject: [PATCH 12/21] chore: rename function approprietly Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates.go | 2 +- pkg/pipelines/tekton/templates_s2i.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index 53b583fa34..7e94497ab2 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -316,7 +316,7 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m if err != nil { return fmt.Errorf("cannot create tekton client: %w", err) } - piplineRun, err := GetS2IPipelineRun(f) + piplineRun, err := GetPipelineRun(f) if err != nil { return fmt.Errorf("cannot generate pipeline run: %w", err) } diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index aec3601f93..972cdb64f5 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -266,7 +266,7 @@ func GetPipeline(f fn.Function) (*v1beta1.Pipeline, error) { return &result, nil } -func GetS2IPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { +func GetPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { labels, err := f.LabelsMap() if err != nil { return nil, fmt.Errorf("cannot generate labels: %w", err) From 765e3f068314365a322424b26993939a913a1f7b Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 15:45:23 +0200 Subject: [PATCH 13/21] fixup: re-enable possibility to load resources from FS Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates_s2i.go | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index 972cdb64f5..d6f5f74cec 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -2,15 +2,31 @@ package tekton import ( "fmt" + "os" + "path" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" coreV1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + k8sYaml "k8s.io/apimachinery/pkg/util/yaml" + "knative.dev/func/pkg/builders" fn "knative.dev/func/pkg/functions" ) func GetPipeline(f fn.Function) (*v1beta1.Pipeline, error) { + pipelineFromFile, err := LoadResource[*v1beta1.Pipeline](path.Join(f.Root, resourcesDirectory, pipelineFileName)) + if err != nil { + return nil, fmt.Errorf("cannot load resource from file: %v", err) + } + if pipelineFromFile != nil { + name := getPipelineName(f) + if pipelineFromFile.Name != name { + return nil, fmt.Errorf("resource name missmatch: %q != %q", pipelineFromFile.Name, name) + } + return pipelineFromFile, nil + } labels, err := f.LabelsMap() if err != nil { @@ -267,6 +283,18 @@ func GetPipeline(f fn.Function) (*v1beta1.Pipeline, error) { } func GetPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { + pipelineRunFromFile, err := LoadResource[*v1beta1.PipelineRun](path.Join(f.Root, resourcesDirectory, pipelineRunFilenane)) + if err != nil { + return nil, fmt.Errorf("cannot load resource from file: %v", err) + } + if pipelineRunFromFile != nil { + generateName := getPipelineRunGenerateName(f) + if pipelineRunFromFile.GetGenerateName() != generateName { + return nil, fmt.Errorf("resource name missmatch: %q != %q", pipelineRunFromFile.GetGenerateName(), generateName) + } + return pipelineRunFromFile, nil + } + labels, err := f.LabelsMap() if err != nil { return nil, fmt.Errorf("cannot generate labels: %w", err) @@ -398,6 +426,35 @@ func GetPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { return &result, nil } +type res interface { + GetGroupVersionKind() schema.GroupVersionKind + GetObjectKind() schema.ObjectKind +} + +func LoadResource[T res](fileName string) (T, error) { + var result T + filePath := fileName + if _, err := os.Stat(filePath); !os.IsNotExist(err) { + var file *os.File + file, err = os.Open(filePath) + if err != nil { + return result, fmt.Errorf("cannot opern resource file: %w", err) + } + defer file.Close() + dec := k8sYaml.NewYAMLToJSONDecoder(file) + err = dec.Decode(&result) + if err != nil { + return result, fmt.Errorf("cannot deserialize resource: %w", err) + } + gvk := result.GetGroupVersionKind() + if gvk != result.GetObjectKind().GroupVersionKind() { + return result, fmt.Errorf("unexpected resource type: %q", result.GetObjectKind().GroupVersionKind()) + } + return result, nil + } + return result, nil +} + const ( // s2iPipelineTemplate contains the S2I template used for both Tekton standard and PAC Pipeline s2iPipelineTemplate = ` From 5241faf86c051dda5609ec308b21bfde6ae6a7ba Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 16:00:53 +0200 Subject: [PATCH 14/21] chore: remove unused symbols Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates.go | 65 -------------------------- pkg/pipelines/tekton/templates_pack.go | 50 -------------------- pkg/pipelines/tekton/templates_s2i.go | 53 +-------------------- 3 files changed, 1 insertion(+), 167 deletions(-) diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index 7e94497ab2..f418bbcac9 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -1,16 +1,13 @@ package tekton import ( - "bytes" "context" "fmt" "os" "path" - "strings" "text/template" "github.com/AlecAivazis/survey/v2" - "github.com/manifestival/manifestival" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/func/pkg/builders" @@ -332,65 +329,3 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m // allows simple mocking in unit tests var manifestivalClient = k8s.GetManifestivalClient - -// createAndApplyResource tries to create and apply a resource to the k8s cluster from the input template and data, -// if there's the same resource already created in the project directory, it is used instead -func createAndApplyResource(projectRoot, fileName, fileTemplate, kind, resourceName, namespace string, data interface{}) error { - var source manifestival.Source - - filePath := path.Join(projectRoot, resourcesDirectory, fileName) - if _, err := os.Stat(filePath); !os.IsNotExist(err) { - source = manifestival.Path(filePath) - } else { - tmpl, err := template.New("template").Parse(fileTemplate) - if err != nil { - return fmt.Errorf("error parsing template: %v", err) - } - - var buf bytes.Buffer - err = tmpl.Execute(&buf, data) - if err != nil { - return fmt.Errorf("error executing template: %v", err) - } - source = manifestival.Reader(&buf) - } - - client, err := manifestivalClient() - if err != nil { - return fmt.Errorf("error generating template: %v", err) - } - - m, err := manifestival.ManifestFrom(source, manifestival.UseClient(client)) - if err != nil { - return fmt.Errorf("error generating template: %v", err) - } - - resources := m.Resources() - if len(resources) != 1 { - return fmt.Errorf("error creating pipeline resources: there could be only a single resource in the template file %q", filePath) - } - - if strings.ToLower(resources[0].GetKind()) != kind { - return fmt.Errorf("error creating pipeline resources: expected resource kind in file %q is %q, but got %q", filePath, kind, resources[0].GetKind()) - } - - existingResourceName := resources[0].GetName() - if kind == "pipelinerun" { - existingResourceName = resources[0].GetGenerateName() - } - if existingResourceName != resourceName { - return fmt.Errorf("error creating pipeline resources: expected resource name in file %q is %q, but got %q", filePath, resourceName, existingResourceName) - } - - if resources[0].GetNamespace() != "" && resources[0].GetNamespace() != namespace { - return fmt.Errorf("error creating pipeline resources: expected resource namespace in file %q is %q, but got %q", filePath, namespace, resources[0].GetNamespace()) - } - - m, err = m.Transform(manifestival.InjectNamespace(namespace)) - if err != nil { - fmt.Printf("error procesing template: %v", err) - return err - } - - return m.Apply() -} diff --git a/pkg/pipelines/tekton/templates_pack.go b/pkg/pipelines/tekton/templates_pack.go index 8dde6ded51..16c80e87f9 100644 --- a/pkg/pipelines/tekton/templates_pack.go +++ b/pkg/pipelines/tekton/templates_pack.go @@ -86,56 +86,6 @@ spec: optional: true ` - // packRunTemplate contains the Buildpacks template used for Tekton standard PipelineRun - packRunTemplate = ` -apiVersion: tekton.dev/v1beta1 -kind: PipelineRun -metadata: - labels: - {{range $key, $value := .Labels -}} - "{{$key}}": "{{$value}}" - {{end}} - tekton.dev/pipeline: {{.PipelineName}} - annotations: - # User defined Annotations - {{range $key, $value := .Annotations -}} - "{{$key}}": "{{$value}}" - {{end}} - generateName: {{.PipelineRunName}} -spec: - params: - - name: gitRepository - value: {{.RepoUrl}} - - name: gitRevision - value: {{.Revision}} - - name: contextDir - value: {{.ContextDir}} - - name: imageName - value: {{.FunctionImage}} - - name: registry - value: {{.Registry}} - - name: builderImage - value: {{.BuilderImage}} - - name: buildEnvs - value: - {{range .BuildEnvs -}} - - {{.}} - {{end}} - pipelineRef: - name: {{.PipelineName}} - workspaces: - - name: source-workspace - persistentVolumeClaim: - claimName: {{.PvcName}} - subPath: source - - name: cache-workspace - persistentVolumeClaim: - claimName: {{.PvcName}} - subPath: cache - - name: dockerconfig-workspace - secret: - secretName: {{.SecretName}} -` // packRunTemplatePAC contains the Buildpacks template used for the Tekton PAC PipelineRun packRunTemplatePAC = ` apiVersion: tekton.dev/v1beta1 diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index d6f5f74cec..5dd7b01d40 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -546,58 +546,7 @@ spec: name: dockerconfig-workspace optional: true ` - // s2iRunTemplate contains the S2I template used for Tekton standard PipelineRun - s2iRunTemplate = ` -apiVersion: tekton.dev/v1beta1 -kind: PipelineRun -metadata: - labels: - {{range $key, $value := .Labels -}} - "{{$key}}": "{{$value}}" - {{end}} - tekton.dev/pipeline: {{.PipelineName}} - annotations: - # User defined Annotations - {{range $key, $value := .Annotations -}} - "{{$key}}": "{{$value}}" - {{end}} - generateName: {{.PipelineRunName}} -spec: - params: - - name: gitRepository - value: {{.RepoUrl}} - - name: gitRevision - value: {{.Revision}} - - name: contextDir - value: {{.ContextDir}} - - name: imageName - value: {{.FunctionImage}} - - name: registry - value: {{.Registry}} - - name: builderImage - value: {{.BuilderImage}} - - name: buildEnvs - value: - {{range .BuildEnvs -}} - - {{.}} - {{end}} - - name: s2iImageScriptsUrl - value: {{.S2iImageScriptsUrl}} - pipelineRef: - name: {{.PipelineName}} - workspaces: - - name: source-workspace - persistentVolumeClaim: - claimName: {{.PvcName}} - subPath: source - - name: cache-workspace - persistentVolumeClaim: - claimName: {{.PvcName}} - subPath: cache - - name: dockerconfig-workspace - secret: - secretName: {{.SecretName}} -` + // s2iRunTemplatePAC contains the S2I template used for Tekton PAC PipelineRun s2iRunTemplatePAC = ` apiVersion: tekton.dev/v1beta1 From e1e88445f3ba7c2fdaf0c9d57c8d1197edb4a47d Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 16:08:05 +0200 Subject: [PATCH 15/21] fixup: update codegen Signed-off-by: Matej Vasek --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index eca9126551..4422720bdf 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,6 @@ require ( golang.org/x/sync v0.4.0 golang.org/x/term v0.13.0 gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.4.0 k8s.io/api v0.27.6 k8s.io/apimachinery v0.27.6 @@ -242,6 +241,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.6 // indirect k8s.io/cli-runtime v0.25.9 // indirect k8s.io/klog/v2 v2.100.1 // indirect From 5970490d40eaa140504e81a6949b94a3bb956487 Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 16:27:08 +0200 Subject: [PATCH 16/21] fixup: re-enable tests Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/templates.go | 20 +- .../tekton/templates_integration_test.go | 12 +- pkg/pipelines/tekton/templates_test.go | 236 +++++++++--------- 3 files changed, 137 insertions(+), 131 deletions(-) diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index f418bbcac9..d7311f8052 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -8,11 +8,11 @@ import ( "text/template" "github.com/AlecAivazis/survey/v2" + "github.com/tektoncd/pipeline/pkg/client/clientset/versioned" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/func/pkg/builders" fn "knative.dev/func/pkg/functions" - "knative.dev/func/pkg/k8s" ) const ( @@ -287,7 +287,7 @@ func deleteAllPipelineTemplates(f fn.Function) string { // all resources are created on the fly, if there's a Pipeline defined in the project directory, it is used instead func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[string]string) error { if f.Build.Builder == builders.Pack || f.Build.Builder == builders.S2I { - cli, err := NewTektonClients() + iface, err := newTektonClient() if err != nil { return fmt.Errorf("cannot create tekton client: %w", err) } @@ -295,7 +295,7 @@ func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[ if err != nil { return fmt.Errorf("cannot generate pipeline: %w", err) } - _, err = cli.Tekton.TektonV1beta1().Pipelines(namespace).Create(context.TODO(), pipeline, v1.CreateOptions{}) + _, err = iface.TektonV1beta1().Pipelines(namespace).Create(context.TODO(), pipeline, v1.CreateOptions{}) if err != nil { err = fmt.Errorf("cannot create pipeline in cluster: %w", err) } @@ -309,15 +309,15 @@ func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[ // all resources are created on the fly, if there's a PipelineRun defined in the project directory, it is used instead func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels map[string]string) error { if f.Build.Builder == builders.Pack || f.Build.Builder == builders.S2I { - cli, err := NewTektonClients() + iface, err := newTektonClient() if err != nil { - return fmt.Errorf("cannot create tekton client: %w", err) + return err } piplineRun, err := GetPipelineRun(f) if err != nil { return fmt.Errorf("cannot generate pipeline run: %w", err) } - _, err = cli.Tekton.TektonV1beta1().PipelineRuns(namespace).Create(context.Background(), piplineRun, v1.CreateOptions{}) + _, err = iface.TektonV1beta1().PipelineRuns(namespace).Create(context.Background(), piplineRun, v1.CreateOptions{}) if err != nil { err = fmt.Errorf("cannot create pipeline run in cluster: %w", err) } @@ -328,4 +328,10 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m } // allows simple mocking in unit tests -var manifestivalClient = k8s.GetManifestivalClient +var newTektonClient func() (versioned.Interface, error) = func() (versioned.Interface, error) { + cli, err := NewTektonClients() + if err != nil { + return nil, err + } + return cli.Tekton, nil +} diff --git a/pkg/pipelines/tekton/templates_integration_test.go b/pkg/pipelines/tekton/templates_integration_test.go index ea6129adf8..49452a0a49 100644 --- a/pkg/pipelines/tekton/templates_integration_test.go +++ b/pkg/pipelines/tekton/templates_integration_test.go @@ -5,8 +5,8 @@ package tekton import ( "testing" - "github.com/manifestival/manifestival" - "github.com/manifestival/manifestival/fake" + "github.com/tektoncd/pipeline/pkg/client/clientset/versioned" + fakepipelineclientset "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/fake" fn "knative.dev/func/pkg/functions" . "knative.dev/func/pkg/testing" @@ -16,11 +16,11 @@ func Test_createAndApplyPipelineTemplate(t *testing.T) { for _, tt := range testData { t.Run(tt.name, func(t *testing.T) { // save current function and restore it at the end - old := manifestivalClient - defer func() { manifestivalClient = old }() + old := newTektonClient + defer func() { newTektonClient = old }() - manifestivalClient = func() (manifestival.Client, error) { - return fake.New(), nil + newTektonClient = func() (versioned.Interface, error) { + return fakepipelineclientset.NewSimpleClientset(), nil } root := tt.root diff --git a/pkg/pipelines/tekton/templates_test.go b/pkg/pipelines/tekton/templates_test.go index cbfa08dcd9..2c25714813 100644 --- a/pkg/pipelines/tekton/templates_test.go +++ b/pkg/pipelines/tekton/templates_test.go @@ -4,8 +4,8 @@ import ( "path/filepath" "testing" - "github.com/manifestival/manifestival" - "github.com/manifestival/manifestival/fake" + "github.com/tektoncd/pipeline/pkg/client/clientset/versioned" + fakepipelineclientset "github.com/tektoncd/pipeline/pkg/client/clientset/versioned/fake" "knative.dev/func/pkg/builders" fn "knative.dev/func/pkg/functions" @@ -153,129 +153,129 @@ var testData = []struct { labels map[string]string wantErr bool }{ - //{ - // name: "correct - pack & node", - // root: "testdata/testCreatePipelinePackNode", - // runtime: "node", - // builder: builders.Pack, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - pack & quarkus", - // root: "testdata/testCreatePipelinePackQuarkus", - // runtime: "quarkus", - // builder: builders.Pack, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - pack & go", - // root: "testdata/testCreatePipelinePackGo", - // runtime: "go", - // builder: builders.Pack, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - pack & python", - // root: "testdata/testCreatePipelinePackPython", - // runtime: "python", - // builder: builders.Pack, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - pack & typescript", - // root: "testdata/testCreatePipelinePackTypescript", - // runtime: "typescript", - // builder: builders.Pack, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - pack & springboot", - // root: "testdata/testCreatePipelinePackSpringboot", - // runtime: "springboot", - // builder: builders.Pack, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - pack & rust", - // root: "testdata/testCreatePipelinePackRust", - // runtime: "rust", - // builder: builders.Pack, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - s2i & node", - // root: "testdata/testCreatePipelineS2INode", - // runtime: "node", - // builder: builders.S2I, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - s2i & quarkus", - // root: "testdata/testCreatePipelineS2IQuarkus", - // runtime: "quarkus", - // builder: builders.S2I, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - s2i & go", - // root: "testdata/testCreatePipelineS2IGo", - // runtime: "go", - // builder: builders.S2I, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - s2i & python", - // root: "testdata/testCreatePipelineS2IPython", - // runtime: "python", - // builder: builders.S2I, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - s2i & typescript", - // root: "testdata/testCreatePipelineS2ITypescript", - // runtime: "typescript", - // builder: builders.S2I, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - s2i & springboot", - // root: "testdata/testCreatePipelineS2ISpringboot", - // runtime: "springboot", - // builder: builders.S2I, - // namespace: "test-ns", - // wantErr: false, - //}, - //{ - // name: "correct - s2i & rust", - // root: "testdata/testCreatePipelineS2IRust", - // runtime: "rust", - // builder: builders.S2I, - // namespace: "test-ns", - // wantErr: false, - //}, + { + name: "correct - pack & node", + root: "testdata/testCreatePipelinePackNode", + runtime: "node", + builder: builders.Pack, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - pack & quarkus", + root: "testdata/testCreatePipelinePackQuarkus", + runtime: "quarkus", + builder: builders.Pack, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - pack & go", + root: "testdata/testCreatePipelinePackGo", + runtime: "go", + builder: builders.Pack, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - pack & python", + root: "testdata/testCreatePipelinePackPython", + runtime: "python", + builder: builders.Pack, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - pack & typescript", + root: "testdata/testCreatePipelinePackTypescript", + runtime: "typescript", + builder: builders.Pack, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - pack & springboot", + root: "testdata/testCreatePipelinePackSpringboot", + runtime: "springboot", + builder: builders.Pack, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - pack & rust", + root: "testdata/testCreatePipelinePackRust", + runtime: "rust", + builder: builders.Pack, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - s2i & node", + root: "testdata/testCreatePipelineS2INode", + runtime: "node", + builder: builders.S2I, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - s2i & quarkus", + root: "testdata/testCreatePipelineS2IQuarkus", + runtime: "quarkus", + builder: builders.S2I, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - s2i & go", + root: "testdata/testCreatePipelineS2IGo", + runtime: "go", + builder: builders.S2I, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - s2i & python", + root: "testdata/testCreatePipelineS2IPython", + runtime: "python", + builder: builders.S2I, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - s2i & typescript", + root: "testdata/testCreatePipelineS2ITypescript", + runtime: "typescript", + builder: builders.S2I, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - s2i & springboot", + root: "testdata/testCreatePipelineS2ISpringboot", + runtime: "springboot", + builder: builders.S2I, + namespace: "test-ns", + wantErr: false, + }, + { + name: "correct - s2i & rust", + root: "testdata/testCreatePipelineS2IRust", + runtime: "rust", + builder: builders.S2I, + namespace: "test-ns", + wantErr: false, + }, } func Test_createAndApplyPipelineRunTemplate(t *testing.T) { for _, tt := range testData { t.Run(tt.name, func(t *testing.T) { // save current function and restore it at the end - old := manifestivalClient - defer func() { manifestivalClient = old }() + old := newTektonClient + defer func() { newTektonClient = old }() - manifestivalClient = func() (manifestival.Client, error) { - return fake.New(), nil + newTektonClient = func() (versioned.Interface, error) { + return fakepipelineclientset.NewSimpleClientset(), nil } root := tt.root + "Run" From 5b1d2ee58145bebd335a495b75721346c7dc5bfc Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 17:49:02 +0200 Subject: [PATCH 17/21] refactor: move functions Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/resources.go | 446 +++++++++++++++++++++++++ pkg/pipelines/tekton/templates_s2i.go | 455 -------------------------- 2 files changed, 446 insertions(+), 455 deletions(-) diff --git a/pkg/pipelines/tekton/resources.go b/pkg/pipelines/tekton/resources.go index 583962c73a..206f102a4a 100644 --- a/pkg/pipelines/tekton/resources.go +++ b/pkg/pipelines/tekton/resources.go @@ -3,8 +3,14 @@ package tekton import ( "context" "fmt" + "os" + "path" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + coreV1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + k8sYaml "k8s.io/apimachinery/pkg/util/yaml" "knative.dev/func/pkg/builders" "knative.dev/func/pkg/builders/buildpacks" @@ -12,6 +18,446 @@ import ( fn "knative.dev/func/pkg/functions" ) +func GetPipeline(f fn.Function) (*v1beta1.Pipeline, error) { + pipelineFromFile, err := LoadResource[*v1beta1.Pipeline](path.Join(f.Root, resourcesDirectory, pipelineFileName)) + if err != nil { + return nil, fmt.Errorf("cannot load resource from file: %v", err) + } + if pipelineFromFile != nil { + name := getPipelineName(f) + if pipelineFromFile.Name != name { + return nil, fmt.Errorf("resource name missmatch: %q != %q", pipelineFromFile.Name, name) + } + return pipelineFromFile, nil + } + + labels, err := f.LabelsMap() + if err != nil { + return nil, fmt.Errorf("cannot generate labels: %w", err) + } + + var buildTaskSpec v1beta1.TaskSpec + switch f.Build.Builder { + case builders.S2I: + buildTaskSpec = *S2ITask.Spec.DeepCopy() + case builders.Pack: + buildTaskSpec = *BuildpackTask.Spec.DeepCopy() + default: + return nil, fmt.Errorf("unsupported builder: %q", f.Build.BuilderImages) + } + + tasks := []v1beta1.PipelineTask{ + v1beta1.PipelineTask{ + Name: "fetch-sources", + TaskRef: &v1beta1.TaskRef{ + ResolverRef: v1beta1.ResolverRef{ + Resolver: "hub", + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "kind", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "task", + }, + }, + v1beta1.Param{ + Name: "name", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "git-clone", + }, + }, + v1beta1.Param{ + Name: "version", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "0.4", + }, + }, + }, + }, + }, + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "url", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.gitRepository)", + }, + }, + v1beta1.Param{ + Name: "revision", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.gitRevision)", + }, + }, + }, + Workspaces: []v1beta1.WorkspacePipelineTaskBinding{ + v1beta1.WorkspacePipelineTaskBinding{ + Name: "output", + Workspace: "source-workspace", + }, + }, + }, + v1beta1.PipelineTask{ + Name: "build", + TaskSpec: &v1beta1.EmbeddedTask{ + TaskSpec: buildTaskSpec, + }, + RunAfter: []string{"fetch-sources"}, + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "APP_IMAGE", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.imageName)", + }, + }, + v1beta1.Param{ + Name: "REGISTRY", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.registry)", + }, + }, + v1beta1.Param{ + Name: "SOURCE_SUBPATH", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.contextDir)", + }, + }, + v1beta1.Param{ + Name: "BUILDER_IMAGE", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.builderImage)", + }, + }, + v1beta1.Param{ + Name: "ENV_VARS", + Value: v1beta1.ParamValue{ + Type: "array", + ArrayVal: []string{ + "$(params.buildEnvs[*])", + }, + }, + }, + v1beta1.Param{ + Name: "S2I_IMAGE_SCRIPTS_URL", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.s2iImageScriptsUrl)", + }, + }, + }, + Workspaces: []v1beta1.WorkspacePipelineTaskBinding{ + v1beta1.WorkspacePipelineTaskBinding{ + Name: "source", + Workspace: "source-workspace", + }, + v1beta1.WorkspacePipelineTaskBinding{ + Name: "cache", + Workspace: "cache-workspace", + }, + v1beta1.WorkspacePipelineTaskBinding{ + Name: "dockerconfig", + Workspace: "dockerconfig-workspace", + }, + }, + }, + v1beta1.PipelineTask{ + Name: "deploy", + TaskSpec: &v1beta1.EmbeddedTask{ + TaskSpec: *DeployTask.Spec.DeepCopy(), + }, + RunAfter: []string{"build"}, + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "path", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(workspaces.source.path)/$(params.contextDir)", + }, + }, + v1beta1.Param{ + Name: "image", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: "$(params.imageName)@$(tasks.build.results.IMAGE_DIGEST)", + }, + }, + }, + Workspaces: []v1beta1.WorkspacePipelineTaskBinding{ + v1beta1.WorkspacePipelineTaskBinding{ + Name: "source", + Workspace: "source-workspace", + }, + }, + }, + } + + if f.Build.Git.URL == "" { + tasks = tasks[1:] + tasks[0].RunAfter = nil + } + + result := v1beta1.Pipeline{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: getPipelineName(f), + Labels: labels, + Annotations: f.Deploy.Annotations, + }, + Spec: v1beta1.PipelineSpec{ + Tasks: tasks, + Params: []v1beta1.ParamSpec{ + v1beta1.ParamSpec{ + Name: "gitRepository", + Type: "string", + Description: "Git repository that hosts the function project", + Default: &v1beta1.ParamValue{ + Type: "string", + }, + }, + v1beta1.ParamSpec{ + Name: "gitRevision", + Type: "string", + Description: "Git revision to build", + }, + v1beta1.ParamSpec{ + Name: "contextDir", + Type: "string", + Description: "Path where the function project is", + Default: &v1beta1.ParamValue{ + Type: "string", + }, + }, + v1beta1.ParamSpec{ + Name: "imageName", + Type: "string", + Description: "Function image name", + }, + v1beta1.ParamSpec{ + Name: "registry", + Type: "string", + Description: "The registry associated with the function image", + }, + v1beta1.ParamSpec{ + Name: "builderImage", + Type: "string", + Description: "Builder image to be used", + }, + v1beta1.ParamSpec{ + Name: "buildEnvs", + Type: "array", + Description: "Environment variables to set during build time", + }, + v1beta1.ParamSpec{ + Name: "s2iImageScriptsUrl", + Type: "string", + Description: "URL containing the default assemble and run scripts for the builder image", + Default: &v1beta1.ParamValue{ + Type: "string", + StringVal: "image:///usr/libexec/s2i", + }, + }, + }, + Workspaces: []v1beta1.PipelineWorkspaceDeclaration{ + v1beta1.PipelineWorkspaceDeclaration{ + Name: "source-workspace", + Description: "Directory where function source is located.", + }, + v1beta1.PipelineWorkspaceDeclaration{ + Name: "cache-workspace", + Description: "Directory where build cache is stored.", + }, + v1beta1.PipelineWorkspaceDeclaration{ + Name: "dockerconfig-workspace", + Description: "Directory containing image registry credentials stored in config.json file.", + Optional: true, + }, + }, + }, + } + + return &result, nil +} + +func GetPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { + pipelineRunFromFile, err := LoadResource[*v1beta1.PipelineRun](path.Join(f.Root, resourcesDirectory, pipelineRunFilenane)) + if err != nil { + return nil, fmt.Errorf("cannot load resource from file: %v", err) + } + if pipelineRunFromFile != nil { + generateName := getPipelineRunGenerateName(f) + if pipelineRunFromFile.GetGenerateName() != generateName { + return nil, fmt.Errorf("resource name missmatch: %q != %q", pipelineRunFromFile.GetGenerateName(), generateName) + } + return pipelineRunFromFile, nil + } + + labels, err := f.LabelsMap() + if err != nil { + return nil, fmt.Errorf("cannot generate labels: %w", err) + } + labels["tekton.dev/pipeline"] = getPipelineName(f) + + pipelinesTargetBranch := f.Build.Git.Revision + if pipelinesTargetBranch == "" { + pipelinesTargetBranch = defaultPipelinesTargetBranch + } + + contextDir := f.Build.Git.ContextDir + if contextDir == "" && f.Build.Builder == builders.S2I { + // TODO(lkingland): could instead update S2I to interpret empty string + // as cwd, such that builder-specific code can be kept out of here. + contextDir = "." + } + + buildEnvs := []string{} + if len(f.Build.BuildEnvs) == 0 { + buildEnvs = []string{"="} + } else { + for i := range f.Build.BuildEnvs { + buildEnvs = append(buildEnvs, f.Build.BuildEnvs[i].KeyValuePair()) + } + } + + s2iImageScriptsUrl := defaultS2iImageScriptsUrl + if f.Runtime == "quarkus" { + s2iImageScriptsUrl = quarkusS2iImageScriptsUrl + } + + result := v1beta1.PipelineRun{ + TypeMeta: metav1.TypeMeta{ + Kind: "PipelineRun", + APIVersion: "tekton.dev/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: getPipelineRunGenerateName(f), + Labels: labels, + Annotations: f.Deploy.Annotations, + }, + Spec: v1beta1.PipelineRunSpec{ + PipelineRef: &v1beta1.PipelineRef{ + Name: getPipelineName(f), + }, + Params: []v1beta1.Param{ + v1beta1.Param{ + Name: "gitRepository", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: f.Build.Git.URL, + }, + }, + v1beta1.Param{ + Name: "gitRevision", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: pipelinesTargetBranch, + }, + }, + v1beta1.Param{ + Name: "contextDir", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: contextDir, + }, + }, + v1beta1.Param{ + Name: "imageName", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: f.Image, + }, + }, + v1beta1.Param{ + Name: "registry", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: f.Registry, + }, + }, + v1beta1.Param{ + Name: "builderImage", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: getBuilderImage(f), + }, + }, + v1beta1.Param{ + Name: "buildEnvs", + Value: v1beta1.ParamValue{ + Type: "array", + ArrayVal: buildEnvs, + }, + }, + v1beta1.Param{ + Name: "s2iImageScriptsUrl", + Value: v1beta1.ParamValue{ + Type: "string", + StringVal: s2iImageScriptsUrl, + }, + }, + }, + Workspaces: []v1beta1.WorkspaceBinding{ + v1beta1.WorkspaceBinding{ + Name: "source-workspace", + SubPath: "source", + PersistentVolumeClaim: &coreV1.PersistentVolumeClaimVolumeSource{ + ClaimName: getPipelinePvcName(f), + }, + }, + v1beta1.WorkspaceBinding{ + Name: "cache-workspace", + SubPath: "cache", + PersistentVolumeClaim: &coreV1.PersistentVolumeClaimVolumeSource{ + ClaimName: getPipelinePvcName(f), + }, + }, + v1beta1.WorkspaceBinding{ + Name: "dockerconfig-workspace", + Secret: &coreV1.SecretVolumeSource{ + SecretName: getPipelineSecretName(f), + }, + }, + }, + }, + } + return &result, nil +} + +type res interface { + GetGroupVersionKind() schema.GroupVersionKind + GetObjectKind() schema.ObjectKind +} + +func LoadResource[T res](fileName string) (T, error) { + var result T + filePath := fileName + if _, err := os.Stat(filePath); !os.IsNotExist(err) { + var file *os.File + file, err = os.Open(filePath) + if err != nil { + return result, fmt.Errorf("cannot opern resource file: %w", err) + } + defer file.Close() + dec := k8sYaml.NewYAMLToJSONDecoder(file) + err = dec.Decode(&result) + if err != nil { + return result, fmt.Errorf("cannot deserialize resource: %w", err) + } + gvk := result.GetGroupVersionKind() + if gvk != result.GetObjectKind().GroupVersionKind() { + return result, fmt.Errorf("unexpected resource type: %q", result.GetObjectKind().GroupVersionKind()) + } + return result, nil + } + return result, nil +} + func deletePipelines(ctx context.Context, namespaceOverride string, listOptions metav1.ListOptions) (err error) { client, namespace, err := NewTektonClientAndResolvedNamespace(namespaceOverride) if err != nil { diff --git a/pkg/pipelines/tekton/templates_s2i.go b/pkg/pipelines/tekton/templates_s2i.go index 5dd7b01d40..bc2e12e012 100644 --- a/pkg/pipelines/tekton/templates_s2i.go +++ b/pkg/pipelines/tekton/templates_s2i.go @@ -1,460 +1,5 @@ package tekton -import ( - "fmt" - "os" - "path" - - "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" - coreV1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - k8sYaml "k8s.io/apimachinery/pkg/util/yaml" - - "knative.dev/func/pkg/builders" - fn "knative.dev/func/pkg/functions" -) - -func GetPipeline(f fn.Function) (*v1beta1.Pipeline, error) { - pipelineFromFile, err := LoadResource[*v1beta1.Pipeline](path.Join(f.Root, resourcesDirectory, pipelineFileName)) - if err != nil { - return nil, fmt.Errorf("cannot load resource from file: %v", err) - } - if pipelineFromFile != nil { - name := getPipelineName(f) - if pipelineFromFile.Name != name { - return nil, fmt.Errorf("resource name missmatch: %q != %q", pipelineFromFile.Name, name) - } - return pipelineFromFile, nil - } - - labels, err := f.LabelsMap() - if err != nil { - return nil, fmt.Errorf("cannot generate labels: %w", err) - } - - var buildTaskSpec v1beta1.TaskSpec - switch f.Build.Builder { - case builders.S2I: - buildTaskSpec = *S2ITask.Spec.DeepCopy() - case builders.Pack: - buildTaskSpec = *BuildpackTask.Spec.DeepCopy() - default: - return nil, fmt.Errorf("unsupported builder: %q", f.Build.BuilderImages) - } - - tasks := []v1beta1.PipelineTask{ - v1beta1.PipelineTask{ - Name: "fetch-sources", - TaskRef: &v1beta1.TaskRef{ - ResolverRef: v1beta1.ResolverRef{ - Resolver: "hub", - Params: []v1beta1.Param{ - v1beta1.Param{ - Name: "kind", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "task", - }, - }, - v1beta1.Param{ - Name: "name", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "git-clone", - }, - }, - v1beta1.Param{ - Name: "version", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "0.4", - }, - }, - }, - }, - }, - Params: []v1beta1.Param{ - v1beta1.Param{ - Name: "url", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "$(params.gitRepository)", - }, - }, - v1beta1.Param{ - Name: "revision", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "$(params.gitRevision)", - }, - }, - }, - Workspaces: []v1beta1.WorkspacePipelineTaskBinding{ - v1beta1.WorkspacePipelineTaskBinding{ - Name: "output", - Workspace: "source-workspace", - }, - }, - }, - v1beta1.PipelineTask{ - Name: "build", - TaskSpec: &v1beta1.EmbeddedTask{ - TaskSpec: buildTaskSpec, - }, - RunAfter: []string{"fetch-sources"}, - Params: []v1beta1.Param{ - v1beta1.Param{ - Name: "APP_IMAGE", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "$(params.imageName)", - }, - }, - v1beta1.Param{ - Name: "REGISTRY", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "$(params.registry)", - }, - }, - v1beta1.Param{ - Name: "SOURCE_SUBPATH", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "$(params.contextDir)", - }, - }, - v1beta1.Param{ - Name: "BUILDER_IMAGE", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "$(params.builderImage)", - }, - }, - v1beta1.Param{ - Name: "ENV_VARS", - Value: v1beta1.ParamValue{ - Type: "array", - ArrayVal: []string{ - "$(params.buildEnvs[*])", - }, - }, - }, - v1beta1.Param{ - Name: "S2I_IMAGE_SCRIPTS_URL", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "$(params.s2iImageScriptsUrl)", - }, - }, - }, - Workspaces: []v1beta1.WorkspacePipelineTaskBinding{ - v1beta1.WorkspacePipelineTaskBinding{ - Name: "source", - Workspace: "source-workspace", - }, - v1beta1.WorkspacePipelineTaskBinding{ - Name: "cache", - Workspace: "cache-workspace", - }, - v1beta1.WorkspacePipelineTaskBinding{ - Name: "dockerconfig", - Workspace: "dockerconfig-workspace", - }, - }, - }, - v1beta1.PipelineTask{ - Name: "deploy", - TaskSpec: &v1beta1.EmbeddedTask{ - TaskSpec: *DeployTask.Spec.DeepCopy(), - }, - RunAfter: []string{"build"}, - Params: []v1beta1.Param{ - v1beta1.Param{ - Name: "path", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "$(workspaces.source.path)/$(params.contextDir)", - }, - }, - v1beta1.Param{ - Name: "image", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: "$(params.imageName)@$(tasks.build.results.IMAGE_DIGEST)", - }, - }, - }, - Workspaces: []v1beta1.WorkspacePipelineTaskBinding{ - v1beta1.WorkspacePipelineTaskBinding{ - Name: "source", - Workspace: "source-workspace", - }, - }, - }, - } - - if f.Build.Git.URL == "" { - tasks = tasks[1:] - tasks[0].RunAfter = nil - } - - result := v1beta1.Pipeline{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: getPipelineName(f), - Labels: labels, - Annotations: f.Deploy.Annotations, - }, - Spec: v1beta1.PipelineSpec{ - Tasks: tasks, - Params: []v1beta1.ParamSpec{ - v1beta1.ParamSpec{ - Name: "gitRepository", - Type: "string", - Description: "Git repository that hosts the function project", - Default: &v1beta1.ParamValue{ - Type: "string", - }, - }, - v1beta1.ParamSpec{ - Name: "gitRevision", - Type: "string", - Description: "Git revision to build", - }, - v1beta1.ParamSpec{ - Name: "contextDir", - Type: "string", - Description: "Path where the function project is", - Default: &v1beta1.ParamValue{ - Type: "string", - }, - }, - v1beta1.ParamSpec{ - Name: "imageName", - Type: "string", - Description: "Function image name", - }, - v1beta1.ParamSpec{ - Name: "registry", - Type: "string", - Description: "The registry associated with the function image", - }, - v1beta1.ParamSpec{ - Name: "builderImage", - Type: "string", - Description: "Builder image to be used", - }, - v1beta1.ParamSpec{ - Name: "buildEnvs", - Type: "array", - Description: "Environment variables to set during build time", - }, - v1beta1.ParamSpec{ - Name: "s2iImageScriptsUrl", - Type: "string", - Description: "URL containing the default assemble and run scripts for the builder image", - Default: &v1beta1.ParamValue{ - Type: "string", - StringVal: "image:///usr/libexec/s2i", - }, - }, - }, - Workspaces: []v1beta1.PipelineWorkspaceDeclaration{ - v1beta1.PipelineWorkspaceDeclaration{ - Name: "source-workspace", - Description: "Directory where function source is located.", - }, - v1beta1.PipelineWorkspaceDeclaration{ - Name: "cache-workspace", - Description: "Directory where build cache is stored.", - }, - v1beta1.PipelineWorkspaceDeclaration{ - Name: "dockerconfig-workspace", - Description: "Directory containing image registry credentials stored in config.json file.", - Optional: true, - }, - }, - }, - } - - return &result, nil -} - -func GetPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { - pipelineRunFromFile, err := LoadResource[*v1beta1.PipelineRun](path.Join(f.Root, resourcesDirectory, pipelineRunFilenane)) - if err != nil { - return nil, fmt.Errorf("cannot load resource from file: %v", err) - } - if pipelineRunFromFile != nil { - generateName := getPipelineRunGenerateName(f) - if pipelineRunFromFile.GetGenerateName() != generateName { - return nil, fmt.Errorf("resource name missmatch: %q != %q", pipelineRunFromFile.GetGenerateName(), generateName) - } - return pipelineRunFromFile, nil - } - - labels, err := f.LabelsMap() - if err != nil { - return nil, fmt.Errorf("cannot generate labels: %w", err) - } - labels["tekton.dev/pipeline"] = getPipelineName(f) - - pipelinesTargetBranch := f.Build.Git.Revision - if pipelinesTargetBranch == "" { - pipelinesTargetBranch = defaultPipelinesTargetBranch - } - - contextDir := f.Build.Git.ContextDir - if contextDir == "" && f.Build.Builder == builders.S2I { - // TODO(lkingland): could instead update S2I to interpret empty string - // as cwd, such that builder-specific code can be kept out of here. - contextDir = "." - } - - buildEnvs := []string{} - if len(f.Build.BuildEnvs) == 0 { - buildEnvs = []string{"="} - } else { - for i := range f.Build.BuildEnvs { - buildEnvs = append(buildEnvs, f.Build.BuildEnvs[i].KeyValuePair()) - } - } - - s2iImageScriptsUrl := defaultS2iImageScriptsUrl - if f.Runtime == "quarkus" { - s2iImageScriptsUrl = quarkusS2iImageScriptsUrl - } - - result := v1beta1.PipelineRun{ - TypeMeta: metav1.TypeMeta{ - Kind: "PipelineRun", - APIVersion: "tekton.dev/v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: getPipelineRunGenerateName(f), - Labels: labels, - Annotations: f.Deploy.Annotations, - }, - Spec: v1beta1.PipelineRunSpec{ - PipelineRef: &v1beta1.PipelineRef{ - Name: getPipelineName(f), - }, - Params: []v1beta1.Param{ - v1beta1.Param{ - Name: "gitRepository", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: f.Build.Git.URL, - }, - }, - v1beta1.Param{ - Name: "gitRevision", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: pipelinesTargetBranch, - }, - }, - v1beta1.Param{ - Name: "contextDir", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: contextDir, - }, - }, - v1beta1.Param{ - Name: "imageName", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: f.Image, - }, - }, - v1beta1.Param{ - Name: "registry", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: f.Registry, - }, - }, - v1beta1.Param{ - Name: "builderImage", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: getBuilderImage(f), - }, - }, - v1beta1.Param{ - Name: "buildEnvs", - Value: v1beta1.ParamValue{ - Type: "array", - ArrayVal: buildEnvs, - }, - }, - v1beta1.Param{ - Name: "s2iImageScriptsUrl", - Value: v1beta1.ParamValue{ - Type: "string", - StringVal: s2iImageScriptsUrl, - }, - }, - }, - Workspaces: []v1beta1.WorkspaceBinding{ - v1beta1.WorkspaceBinding{ - Name: "source-workspace", - SubPath: "source", - PersistentVolumeClaim: &coreV1.PersistentVolumeClaimVolumeSource{ - ClaimName: getPipelinePvcName(f), - }, - }, - v1beta1.WorkspaceBinding{ - Name: "cache-workspace", - SubPath: "cache", - PersistentVolumeClaim: &coreV1.PersistentVolumeClaimVolumeSource{ - ClaimName: getPipelinePvcName(f), - }, - }, - v1beta1.WorkspaceBinding{ - Name: "dockerconfig-workspace", - Secret: &coreV1.SecretVolumeSource{ - SecretName: getPipelineSecretName(f), - }, - }, - }, - }, - } - return &result, nil -} - -type res interface { - GetGroupVersionKind() schema.GroupVersionKind - GetObjectKind() schema.ObjectKind -} - -func LoadResource[T res](fileName string) (T, error) { - var result T - filePath := fileName - if _, err := os.Stat(filePath); !os.IsNotExist(err) { - var file *os.File - file, err = os.Open(filePath) - if err != nil { - return result, fmt.Errorf("cannot opern resource file: %w", err) - } - defer file.Close() - dec := k8sYaml.NewYAMLToJSONDecoder(file) - err = dec.Decode(&result) - if err != nil { - return result, fmt.Errorf("cannot deserialize resource: %w", err) - } - gvk := result.GetGroupVersionKind() - if gvk != result.GetObjectKind().GroupVersionKind() { - return result, fmt.Errorf("unexpected resource type: %q", result.GetObjectKind().GroupVersionKind()) - } - return result, nil - } - return result, nil -} - const ( // s2iPipelineTemplate contains the S2I template used for both Tekton standard and PAC Pipeline s2iPipelineTemplate = ` From 01c5c3e3affe888ffd335aaa0966f93cb2e6313d Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 25 Oct 2023 17:50:08 +0200 Subject: [PATCH 18/21] refactor: rename -- make functions non-exported Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/resources.go | 10 +++++----- pkg/pipelines/tekton/templates.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/pipelines/tekton/resources.go b/pkg/pipelines/tekton/resources.go index 206f102a4a..adf1822807 100644 --- a/pkg/pipelines/tekton/resources.go +++ b/pkg/pipelines/tekton/resources.go @@ -18,8 +18,8 @@ import ( fn "knative.dev/func/pkg/functions" ) -func GetPipeline(f fn.Function) (*v1beta1.Pipeline, error) { - pipelineFromFile, err := LoadResource[*v1beta1.Pipeline](path.Join(f.Root, resourcesDirectory, pipelineFileName)) +func getPipeline(f fn.Function) (*v1beta1.Pipeline, error) { + pipelineFromFile, err := loadResource[*v1beta1.Pipeline](path.Join(f.Root, resourcesDirectory, pipelineFileName)) if err != nil { return nil, fmt.Errorf("cannot load resource from file: %v", err) } @@ -285,8 +285,8 @@ func GetPipeline(f fn.Function) (*v1beta1.Pipeline, error) { return &result, nil } -func GetPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { - pipelineRunFromFile, err := LoadResource[*v1beta1.PipelineRun](path.Join(f.Root, resourcesDirectory, pipelineRunFilenane)) +func getPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { + pipelineRunFromFile, err := loadResource[*v1beta1.PipelineRun](path.Join(f.Root, resourcesDirectory, pipelineRunFilenane)) if err != nil { return nil, fmt.Errorf("cannot load resource from file: %v", err) } @@ -434,7 +434,7 @@ type res interface { GetObjectKind() schema.ObjectKind } -func LoadResource[T res](fileName string) (T, error) { +func loadResource[T res](fileName string) (T, error) { var result T filePath := fileName if _, err := os.Stat(filePath); !os.IsNotExist(err) { diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index d7311f8052..5224c32ea2 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -291,7 +291,7 @@ func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[ if err != nil { return fmt.Errorf("cannot create tekton client: %w", err) } - pipeline, err := GetPipeline(f) + pipeline, err := getPipeline(f) if err != nil { return fmt.Errorf("cannot generate pipeline: %w", err) } @@ -313,7 +313,7 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m if err != nil { return err } - piplineRun, err := GetPipelineRun(f) + piplineRun, err := getPipelineRun(f) if err != nil { return fmt.Errorf("cannot generate pipeline run: %w", err) } From 9040279b346fbf5b6ad8dd932b9074593d8fe543 Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Tue, 31 Oct 2023 20:22:44 +0100 Subject: [PATCH 19/21] fix: apply labels from PipelineDecorator Signed-off-by: Matej Vasek --- .../tekton/pipelines_integration_test.go | 37 +++++++++++++++++++ pkg/pipelines/tekton/resources.go | 14 ++----- pkg/pipelines/tekton/templates.go | 4 +- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/pkg/pipelines/tekton/pipelines_integration_test.go b/pkg/pipelines/tekton/pipelines_integration_test.go index 215d112c73..d24c8847af 100644 --- a/pkg/pipelines/tekton/pipelines_integration_test.go +++ b/pkg/pipelines/tekton/pipelines_integration_test.go @@ -57,6 +57,7 @@ func TestOnClusterBuild(t *testing.T) { ns := setupNS(t) pp := tekton.NewPipelinesProvider( + tekton.WithPipelineDecorator(testDecorator{}), tekton.WithCredentialsProvider(credentialsProvider), tekton.WithNamespace(ns)) @@ -84,10 +85,46 @@ func TestOnClusterBuild(t *testing.T) { return } t.Log("call to knative service successful") + + // Check if labels are correct. + cli, err := tekton.NewTektonClients() + if err != nil { + t.Fatal(err) + } + pl, err := cli.Tekton.TektonV1beta1().Pipelines(ns).List(ctx, metav1.ListOptions{}) + if len(pl.Items) == 1 { + if val, ok := pl.Items[0].Labels["test-label-key"]; !ok || val != "test-label-value" { + t.Error("test label has not been set for pipeline") + } + } else { + t.Errorf("unexpected pipeline count: %d", len(pl.Items)) + } + prl, err := cli.Tekton.TektonV1beta1().PipelineRuns(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(prl.Items) == 1 { + if val, ok := prl.Items[0].Labels["test-label-key"]; !ok || val != "test-label-value" { + t.Error("test label has not been set for pipeline run") + } + } else { + t.Errorf("unexpected pipeline run count: %d", len(prl.Items)) + } }) } } +type testDecorator struct{} + +func (t testDecorator) UpdateLabels(function fn.Function, m map[string]string) map[string]string { + result := make(map[string]string, len(m)+1) + for k, v := range m { + result[k] = v + } + result["test-label-key"] = "test-label-value" + return result +} + func setupNS(t *testing.T) string { name := "pipeline-integration-test-" + strings.ToLower(random.AlphaString(5)) cliSet, err := k8s.NewKubernetesClientset() diff --git a/pkg/pipelines/tekton/resources.go b/pkg/pipelines/tekton/resources.go index adf1822807..2839b74be1 100644 --- a/pkg/pipelines/tekton/resources.go +++ b/pkg/pipelines/tekton/resources.go @@ -18,7 +18,7 @@ import ( fn "knative.dev/func/pkg/functions" ) -func getPipeline(f fn.Function) (*v1beta1.Pipeline, error) { +func getPipeline(f fn.Function, labels map[string]string) (*v1beta1.Pipeline, error) { pipelineFromFile, err := loadResource[*v1beta1.Pipeline](path.Join(f.Root, resourcesDirectory, pipelineFileName)) if err != nil { return nil, fmt.Errorf("cannot load resource from file: %v", err) @@ -31,11 +31,6 @@ func getPipeline(f fn.Function) (*v1beta1.Pipeline, error) { return pipelineFromFile, nil } - labels, err := f.LabelsMap() - if err != nil { - return nil, fmt.Errorf("cannot generate labels: %w", err) - } - var buildTaskSpec v1beta1.TaskSpec switch f.Build.Builder { case builders.S2I: @@ -285,7 +280,7 @@ func getPipeline(f fn.Function) (*v1beta1.Pipeline, error) { return &result, nil } -func getPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { +func getPipelineRun(f fn.Function, labels map[string]string) (*v1beta1.PipelineRun, error) { pipelineRunFromFile, err := loadResource[*v1beta1.PipelineRun](path.Join(f.Root, resourcesDirectory, pipelineRunFilenane)) if err != nil { return nil, fmt.Errorf("cannot load resource from file: %v", err) @@ -298,9 +293,8 @@ func getPipelineRun(f fn.Function) (*v1beta1.PipelineRun, error) { return pipelineRunFromFile, nil } - labels, err := f.LabelsMap() - if err != nil { - return nil, fmt.Errorf("cannot generate labels: %w", err) + if labels == nil { + labels = make(map[string]string, 1) } labels["tekton.dev/pipeline"] = getPipelineName(f) diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index 5224c32ea2..f465c57ba8 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -291,7 +291,7 @@ func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[ if err != nil { return fmt.Errorf("cannot create tekton client: %w", err) } - pipeline, err := getPipeline(f) + pipeline, err := getPipeline(f, labels) if err != nil { return fmt.Errorf("cannot generate pipeline: %w", err) } @@ -313,7 +313,7 @@ func createAndApplyPipelineRunTemplate(f fn.Function, namespace string, labels m if err != nil { return err } - piplineRun, err := getPipelineRun(f) + piplineRun, err := getPipelineRun(f, labels) if err != nil { return fmt.Errorf("cannot generate pipeline run: %w", err) } From 2ba9004b876e2c95b9335d0cf8a9ac7b872cafde Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 1 Nov 2023 14:09:34 +0100 Subject: [PATCH 20/21] fix: keep older version of task Signed-off-by: Matej Vasek --- .../tekton/task/func-s2i/0.1/func-s2i.yaml | 14 +- .../tekton/task/func-s2i/0.2/func-s2i.yaml | 136 ++++++++++++++++++ pkg/pipelines/tekton/templates.go | 2 +- 3 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 pkg/pipelines/resources/tekton/task/func-s2i/0.2/func-s2i.yaml diff --git a/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml b/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml index 9a2f7a3b92..5c30b7566c 100644 --- a/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml +++ b/pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml @@ -22,11 +22,11 @@ spec: params: - name: BUILDER_IMAGE description: The location of the s2i builder image. - - name: APP_IMAGE + - name: IMAGE description: Reference of the image S2I will produce. - name: REGISTRY description: The registry associated with the function image. - - name: SOURCE_SUBPATH + - name: PATH_CONTEXT description: The location of the path to run s2i from. default: . - name: TLSVERIFY @@ -78,14 +78,14 @@ spec: cat /env-vars/env-file echo "------------------------------" - /usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.SOURCE_SUBPATH) $(params.BUILDER_IMAGE) \ + /usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.PATH_CONTEXT) $(params.BUILDER_IMAGE) \ --image-scripts-url $(params.S2I_IMAGE_SCRIPTS_URL) \ --as-dockerfile /gen-source/Dockerfile.gen --environment-file /env-vars/env-file echo "Preparing func.yaml for later deployment" func_file="$(workspaces.source.path)/func.yaml" - if [ "$(params.SOURCE_SUBPATH)" != "" ]; then - func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml" + if [ "$(params.PATH_CONTEXT)" != "" ]; then + func_file="$(workspaces.source.path)/$(params.PATH_CONTEXT)/func.yaml" fi sed -i "s|^registry:.*$|registry: $(params.REGISTRY)|" "$func_file" echo "Function image registry: $(params.REGISTRY)" @@ -112,11 +112,11 @@ spec: [ -d "${ARTIFACTS_CACHE_PATH}" ] || mkdir "${ARTIFACTS_CACHE_PATH}" buildah ${CERT_DIR_FLAG} bud --storage-driver=vfs ${TLS_VERIFY_FLAG} --layers \ -v "${ARTIFACTS_CACHE_PATH}:/tmp/artifacts/:rw,z,U" \ - -f /gen-source/Dockerfile.gen -t $(params.APP_IMAGE) . + -f /gen-source/Dockerfile.gen -t $(params.IMAGE) . [[ "$(workspaces.dockerconfig.bound)" == "true" ]] && export DOCKER_CONFIG="$(workspaces.dockerconfig.path)" buildah ${CERT_DIR_FLAG} push --storage-driver=vfs ${TLS_VERIFY_FLAG} --digestfile $(workspaces.source.path)/image-digest \ - $(params.APP_IMAGE) docker://$(params.APP_IMAGE) + $(params.IMAGE) docker://$(params.IMAGE) cat $(workspaces.source.path)/image-digest | tee /tekton/results/IMAGE_DIGEST volumeMounts: diff --git a/pkg/pipelines/resources/tekton/task/func-s2i/0.2/func-s2i.yaml b/pkg/pipelines/resources/tekton/task/func-s2i/0.2/func-s2i.yaml new file mode 100644 index 0000000000..9a2f7a3b92 --- /dev/null +++ b/pkg/pipelines/resources/tekton/task/func-s2i/0.2/func-s2i.yaml @@ -0,0 +1,136 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: func-s2i + labels: + app.kubernetes.io/version: "0.1" + annotations: + tekton.dev/pipelines.minVersion: "0.17.0" + tekton.dev/categories: Image Build + tekton.dev/tags: image-build + tekton.dev/platforms: "linux/amd64" +spec: + description: >- + Knative Functions Source-to-Image (S2I) is a toolkit and workflow for building reproducible + container images from source code + + S2I produces images by injecting source code into a base S2I container image + and letting the container prepare that source code for execution. The base + S2I container images contains the language runtime and build tools needed for + building and running the source code. + + params: + - name: BUILDER_IMAGE + description: The location of the s2i builder image. + - name: APP_IMAGE + description: Reference of the image S2I will produce. + - name: REGISTRY + description: The registry associated with the function image. + - name: SOURCE_SUBPATH + description: The location of the path to run s2i from. + default: . + - name: TLSVERIFY + description: Verify the TLS on the registry endpoint (for push/pull to a non-TLS registry) + default: "true" + - name: LOGLEVEL + description: Log level when running the S2I binary + default: "0" + - name: ENV_VARS + type: array + description: Environment variables to set during _build-time_. + default: [] + - name: S2I_IMAGE_SCRIPTS_URL + description: The URL containing the default assemble and run scripts for the builder image. + default: "image:///usr/libexec/s2i" + workspaces: + - name: source + - name: cache + description: Directory where cache is stored (e.g. local mvn repo). + optional: true + - name: sslcertdir + optional: true + - name: dockerconfig + description: >- + An optional workspace that allows providing a .docker/config.json file + for Buildah to access the container registry. + The file should be placed at the root of the Workspace with name config.json. + optional: true + results: + - name: IMAGE_DIGEST + description: Digest of the image just built. + steps: + - name: generate + image: quay.io/boson/s2i:latest + workingDir: $(workspaces.source.path) + args: ["$(params.ENV_VARS[*])"] + script: | + echo "Processing Build Environment Variables" + echo "" > /env-vars/env-file + for var in "$@" + do + if [[ "$var" != "=" ]]; then + echo "$var" >> /env-vars/env-file + fi + done + + echo "Generated Build Env Var file" + echo "------------------------------" + cat /env-vars/env-file + echo "------------------------------" + + /usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.SOURCE_SUBPATH) $(params.BUILDER_IMAGE) \ + --image-scripts-url $(params.S2I_IMAGE_SCRIPTS_URL) \ + --as-dockerfile /gen-source/Dockerfile.gen --environment-file /env-vars/env-file + + echo "Preparing func.yaml for later deployment" + func_file="$(workspaces.source.path)/func.yaml" + if [ "$(params.SOURCE_SUBPATH)" != "" ]; then + func_file="$(workspaces.source.path)/$(params.SOURCE_SUBPATH)/func.yaml" + fi + sed -i "s|^registry:.*$|registry: $(params.REGISTRY)|" "$func_file" + echo "Function image registry: $(params.REGISTRY)" + + s2iignore_file="$(dirname "$func_file")/.s2iignore" + [ -f "$s2iignore_file" ] || echo "node_modules" >> "$s2iignore_file" + + volumeMounts: + - mountPath: /gen-source + name: gen-source + - mountPath: /env-vars + name: env-vars + - name: build + image: quay.io/buildah/stable:v1.31.0 + workingDir: /gen-source + script: | + TLS_VERIFY_FLAG="" + if [ "$(params.TLSVERIFY)" = "false" ] || [ "$(params.TLSVERIFY)" = "0" ]; then + TLS_VERIFY_FLAG="--tls-verify=false" + fi + + [[ "$(workspaces.sslcertdir.bound)" == "true" ]] && CERT_DIR_FLAG="--cert-dir $(workspaces.sslcertdir.path)" + ARTIFACTS_CACHE_PATH="$(workspaces.cache.path)/mvn-artifacts" + [ -d "${ARTIFACTS_CACHE_PATH}" ] || mkdir "${ARTIFACTS_CACHE_PATH}" + buildah ${CERT_DIR_FLAG} bud --storage-driver=vfs ${TLS_VERIFY_FLAG} --layers \ + -v "${ARTIFACTS_CACHE_PATH}:/tmp/artifacts/:rw,z,U" \ + -f /gen-source/Dockerfile.gen -t $(params.APP_IMAGE) . + + [[ "$(workspaces.dockerconfig.bound)" == "true" ]] && export DOCKER_CONFIG="$(workspaces.dockerconfig.path)" + buildah ${CERT_DIR_FLAG} push --storage-driver=vfs ${TLS_VERIFY_FLAG} --digestfile $(workspaces.source.path)/image-digest \ + $(params.APP_IMAGE) docker://$(params.APP_IMAGE) + + cat $(workspaces.source.path)/image-digest | tee /tekton/results/IMAGE_DIGEST + volumeMounts: + - name: varlibcontainers + mountPath: /var/lib/containers + - mountPath: /gen-source + name: gen-source + securityContext: + capabilities: + add: ["SETFCAP"] + volumes: + - emptyDir: {} + name: varlibcontainers + - emptyDir: {} + name: gen-source + - emptyDir: {} + name: env-vars diff --git a/pkg/pipelines/tekton/templates.go b/pkg/pipelines/tekton/templates.go index f465c57ba8..6272f69bf7 100644 --- a/pkg/pipelines/tekton/templates.go +++ b/pkg/pipelines/tekton/templates.go @@ -91,7 +91,7 @@ var ( taskBasePath = "https://raw.githubusercontent.com/" + FuncRepoRef + "/" + FuncRepoBranchRef + "/pkg/pipelines/resources/tekton/task/" BuildpackTaskURL = taskBasePath + "func-buildpacks/0.2/func-buildpacks.yaml" - S2ITaskURL = taskBasePath + "func-s2i/0.1/func-s2i.yaml" + S2ITaskURL = taskBasePath + "func-s2i/0.2/func-s2i.yaml" DeployTaskURL = taskBasePath + "func-deploy/0.1/func-deploy.yaml" ) From 3119564a02603296fc104b14f4c28f3fff4e7884 Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Wed, 1 Nov 2023 14:37:31 +0100 Subject: [PATCH 21/21] test: tkn tasks in yaml and Go code as the same Signed-off-by: Matej Vasek --- pkg/pipelines/tekton/task_defs_test.go | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 pkg/pipelines/tekton/task_defs_test.go diff --git a/pkg/pipelines/tekton/task_defs_test.go b/pkg/pipelines/tekton/task_defs_test.go new file mode 100644 index 0000000000..4f567eea32 --- /dev/null +++ b/pkg/pipelines/tekton/task_defs_test.go @@ -0,0 +1,51 @@ +package tekton_test + +import ( + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" + k8sYaml "k8s.io/apimachinery/pkg/util/yaml" + + "knative.dev/func/pkg/pipelines/tekton" +) + +func TestTaskMatch(t *testing.T) { + for _, tt := range []struct { + path string + task v1beta1.Task + }{ + { + path: "../resources/tekton/task/func-buildpacks/0.2/func-buildpacks.yaml", + task: tekton.BuildpackTask, + }, + { + path: "../resources/tekton/task/func-s2i/0.2/func-s2i.yaml", + task: tekton.S2ITask, + }, + { + path: "../resources/tekton/task/func-deploy/0.1/func-deploy.yaml", + task: tekton.DeployTask, + }, + } { + t.Run(tt.task.Name, func(t *testing.T) { + + f, err := os.Open(tt.path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + dec := k8sYaml.NewYAMLToJSONDecoder(f) + var taskFromYaml v1beta1.Task + err = dec.Decode(&taskFromYaml) + if err != nil { + t.Fatal(err) + } + if d := cmp.Diff(tt.task, taskFromYaml); d != "" { + t.Error("output missmatch (-want, +got):", d) + } + }) + } +}