Merge branch 'master' into feature/expose-k8s-prom-metrics

Signed-off-by: Daniel Rammer <[email protected]>

.github/workflows/single-binary.yml

@@ -160,7 +160,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
       - uses: unionai/[email protected]
       - name: Setup sandbox
         run: |
@@ -178,7 +178,7 @@ jobs:
       - name: Install Python dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flytekit
+          pip install flytekit flytekitplugins-deck-standard
           pip freeze
       - name: Checkout flytesnacks
         uses: actions/checkout@v3
@@ -189,7 +189,9 @@ jobs:
           # ref: ${{ env.FLYTESNACKS_VERSION }}
       - name: Register specific tests
         run: |
-          for f in basics/basics/basic_workflow.py \
+          for f in \
+                  basics/basics/basic_workflow.py \
+                  basics/basics/deck.py \
                   basics/basics/hello_world.py \
                   basics/basics/lp.py \
                   basics/basics/named_outputs.py \

.github/workflows/tests.yml

@@ -16,6 +16,8 @@ jobs:
           go-version: "1.19"
       - name: Compile
         run: make compile
+      - name: Run tests
+        run: make test_unit
 
   docs:
     runs-on: ubuntu-latest

Makefile

@@ -1,5 +1,6 @@
 export REPOSITORY=flyte
 include boilerplate/flyte/end2end/Makefile
+include boilerplate/flyte/golang_test_targets/Makefile
 
 define PIP_COMPILE
 pip-compile $(1) --upgrade --verbose --resolver=backtracking

boilerplate/flyte/end2end/functional-test-config.yaml

@@ -1,18 +1,5 @@
 admin:
+  # For GRPC endpoints you might want to use dns:///flyte.myexample.com
   endpoint: dns:///localhost:30080
-  insecure: true
   authType: Pkce
-logger:
-  show-source: true
-  level: 0
-storage:
-  connection:
-    access-key: minio
-    auth-type: accesskey
-    disable-ssl: true
-    endpoint: http://localhost:30002
-    region: us-east-1
-    secret-key: miniostorage
-  type: minio
-  container: "my-s3-bucket"
-  enable-multicontainer: true
+  insecure: true

boilerplate/flyte/end2end/run-tests.py

@@ -21,7 +21,13 @@
 # inputs. This is so we can progressively cover all priorities in the original flytesnacks manifest,
 # starting with "core".
 FLYTESNACKS_WORKFLOW_GROUPS: Mapping[str, List[Tuple[str, dict]]] = {
+    "lite": [
+        ("basics.hello_world.my_wf", {}),
+        ("basics.lp.go_greet", {"day_of_week": "5", "number": 3, "am": True}),
+    ],
     "core": [
+        ("basics.deck.wf", {}),
+        # The chain_workflows example in flytesnacks expects to be running in a sandbox.
         ("control_flow.chain_entities.chain_workflows_wf", {}),
         ("control_flow.dynamics.wf", {"s1": "Pear", "s2": "Earth"}),
         ("control_flow.map_task.my_map_workflow", {"a": [1, 2, 3, 4, 5]}),
@@ -43,10 +49,7 @@
         # ("basics.folders.download_and_rotate", {}),
         ("basics.hello_world.my_wf", {}),
         ("basics.lp.my_wf", {"val": 4}),
-        (
-            "basics.lp.go_greet",
-            {"day_of_week": "5", "number": 3, "am": True},
-        ),
+        ("basics.lp.go_greet", {"day_of_week": "5", "number": 3, "am": True}),
         ("basics.named_outputs.my_wf", {}),
         # # Getting a 403 for the wikipedia image
         # # ("basics.reference_task.wf", {}),
@@ -55,36 +58,36 @@
         # ("type_system.enums.enum_wf", {"c": "red"}),
         ("type_system.schema.df_wf", {"a": 42}),
         ("type_system.typed_schema.wf", {}),
-        # ("my.imperative.workflow.example", {"in1": "hello", "in2": "foo"}),
+        #("my.imperative.workflow.example", {"in1": "hello", "in2": "foo"}),
     ],
     "integrations-k8s-spark": [
-        ("k8s_spark.pyspark_pi.my_spark", {"triggered_date": datetime.datetime.now()}),
+        ("k8s_spark_plugin.pyspark_pi.my_spark", {"triggered_date": datetime.datetime.now()}),
     ],
     "integrations-kfpytorch": [
-        ("kfpytorch.pytorch_mnist.pytorch_training_wf", {}),
+        ("kfpytorch_plugin.pytorch_mnist.pytorch_training_wf", {}),
     ],
     "integrations-kftensorflow": [
-        ("kftensorflow.tf_mnist.mnist_tensorflow_workflow", {}),
+        ("kftensorflow_plugin.tf_mnist.mnist_tensorflow_workflow", {}),
     ],
     # "integrations-pod": [
     #     ("pod.pod.pod_workflow", {}),
     # ],
     "integrations-pandera_examples": [
-        ("pandera_examples.basic_schema_example.process_data", {}),
+        ("pandera_plugin.basic_schema_example.process_data", {}),
         # TODO: investigate type mismatch float -> numpy.float64
-        # ("pandera_examples.validating_and_testing_ml_pipelines.pipeline", {"data_random_state": 42, "model_random_state": 99}),
+        # ("pandera_plugin.validating_and_testing_ml_pipelines.pipeline", {"data_random_state": 42, "model_random_state": 99}),
     ],
     "integrations-modin_examples": [
-        ("modin_examples.knn_classifier.pipeline", {}),
+        ("modin_plugin.knn_classifier.pipeline", {}),
     ],
     "integrations-papermilltasks": [
-        ("papermilltasks.simple.nb_to_python_wf", {"f": 3.1415926535}),
+        ("papermill_plugin.simple.nb_to_python_wf", {"f": 3.1415926535}),
     ],
     "integrations-greatexpectations": [
-        ("greatexpectations.task_example.simple_wf", {}),
-        ("greatexpectations.task_example.file_wf", {}),
-        ("greatexpectations.task_example.schema_wf", {}),
-        ("greatexpectations.task_example.runtime_wf", {}),
+        ("greatexpectations_plugin.task_example.simple_wf", {}),
+        ("greatexpectations_plugin.task_example.file_wf", {}),
+        ("greatexpectations_plugin.task_example.schema_wf", {}),
+        ("greatexpectations_plugin.task_example.runtime_wf", {}),
     ],
 }
 
@@ -190,16 +193,15 @@ def run(
 
     # For a given release tag and priority, this function filters the workflow groups from the flytesnacks
     # manifest file. For example, for the release tag "v0.2.224" and the priority "P0" it returns [ "core" ].
-    manifest_url = (
-        "https://raw.githubusercontent.com/flyteorg/flytesnacks/"
-        f"{flytesnacks_release_tag}/flyte_tests_manifest.json"
-    )
+    manifest_url = "https://raw.githubusercontent.com/flyteorg/flytesnacks/" \
+                   f"{flytesnacks_release_tag}/flyte_tests_manifest.json"
     r = requests.get(manifest_url)
     parsed_manifest = r.json()
+    workflow_groups = []
+    workflow_groups = ["lite"] if "lite" in priorities else [
+            group["name"] for group in parsed_manifest if group["priority"] in priorities
+        ]
 
-    workflow_groups = [
-        group["name"] for group in parsed_manifest if group["priority"] in priorities
-    ]
     results = []
     valid_workgroups = []
     for workflow_group in workflow_groups:

boilerplate/flyte/golang_test_targets/Makefile

@@ -0,0 +1,57 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+
+.PHONY: download_tooling
+download_tooling: #download dependencies (including test deps) for the package
+	@boilerplate/flyte/golang_test_targets/download_tooling.sh
+
+.PHONY: generate
+generate: download_tooling #generate go code
+	@boilerplate/flyte/golang_test_targets/go-gen.sh
+
+.PHONY: lint
+lint: download_tooling #lints the package for common code smells
+	GL_DEBUG=linters_output,env golangci-lint run --deadline=5m --exclude deprecated -v
+
+# If code is failing goimports linter, this will fix.
+# skips 'vendor'
+.PHONY: goimports
+goimports:
+	@boilerplate/flyte/golang_test_targets/goimports
+
+.PHONY: mod_download
+mod_download: #download dependencies (including test deps) for the package
+	go mod download
+
+.PHONY: install
+install: download_tooling mod_download
+
+.PHONY: show
+show:
+	go list -m all
+
+.PHONY: test_unit
+test_unit:
+	go test -cover ./... -race
+
+.PHONY: test_benchmark
+test_benchmark:
+	go test -bench . ./...
+
+.PHONY: test_unit_cover
+test_unit_cover:
+	go test ./... -coverprofile /tmp/cover.out -covermode=count
+	go tool cover -func /tmp/cover.out
+
+.PHONY: test_unit_visual
+test_unit_visual:
+	go test ./... -coverprofile /tmp/cover.out -covermode=count
+	go tool cover -html=/tmp/cover.out
+
+.PHONY: test_unit_codecov
+test_unit_codecov:
+	go test ./... -race -coverprofile=coverage.txt -covermode=atomic
+	curl -s https://codecov.io/bash > codecov_bash.sh && bash codecov_bash.sh

boilerplate/flyte/golang_test_targets/Readme.rst

@@ -0,0 +1,31 @@
+Golang Test Targets
+~~~~~~~~~~~~~~~~~~~
+
+Provides an ``install`` make target that uses ``go mod`` to install golang dependencies.
+
+Provides a ``lint`` make target that uses golangci to lint your code.
+
+Provides a ``test_unit`` target for unit tests.
+
+Provides a ``test_unit_cover`` target for analysing coverage of unit tests, which will output the coverage of each function and total statement coverage.
+
+Provides a ``test_unit_visual`` target for visualizing coverage of unit tests through an interactive html code heat map.
+
+Provides a ``test_benchmark`` target for benchmark tests.
+
+**To Enable:**
+
+Add ``flyteorg/golang_test_targets`` to your ``boilerplate/update.cfg`` file.
+
+Make sure you're using ``go mod`` for dependency management.
+
+Provide a ``.golangci`` configuration (the lint target requires it).
+
+Add ``include boilerplate/flyte/golang_test_targets/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable
+
+::
+
+    REPOSITORY=<myreponame>
+    include boilerplate/flyte/golang_test_targets/Makefile
+
+(this ensures the extra make targets get included in your main Makefile)

boilerplate/flyte/golang_test_targets/download_tooling.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Everything in this file needs to be installed outside of current module
+# The reason we cannot turn off module entirely and install is that we need the replace statement in go.mod
+# because we are installing a mockery fork. Turning it off would result installing the original not the fork.
+# We also want to version all the other tools.  We also want to be able to run go mod tidy without removing the version
+# pins.  To facilitate this, we're maintaining two sets of go.mod/sum files - the second one only for tooling.  This is
+# the same approach that go 1.14 will take as well.
+# See:
+#   https://github.com/flyteorg/flyte/issues/129
+#   https://github.com/golang/go/issues/30515 for some background context
+#   https://github.com/go-modules-by-example/index/blob/5ec250b4b78114a55001bd7c9cb88f6e07270ea5/010_tools/README.md
+
+set -e
+
+# List of tools to go get
+# In the format of "<cli>:<package>" or ":<package>" if no cli
+tools=(
+  "github.com/EngHabu/mockery/cmd/mockery"
+  "github.com/flyteorg/flytestdlib/cli/pflags@latest"
+  "github.com/golangci/golangci-lint/cmd/golangci-lint"
+  "github.com/alvaroloes/enumer"
+  "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
+)
+
+tmp_dir=$(mktemp -d -t gotooling-XXX)
+echo "Using temp directory ${tmp_dir}"
+cp -R boilerplate/flyte/golang_support_tools/* $tmp_dir
+pushd "$tmp_dir"
+
+for tool in "${tools[@]}"
+do
+    echo "Installing ${tool}"
+    GO111MODULE=on go install $tool
+done
+
+popd

boilerplate/flyte/golang_test_targets/go-gen.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -ex
+
+echo "Running go generate"
+go generate ./...
+
+# This section is used by GitHub workflow to ensure that the generation step was run
+if [ -n "$DELTA_CHECK" ]; then
+  DIRTY=$(git status --porcelain)
+  if [ -n "$DIRTY" ]; then
+    echo "FAILED: Go code updated without commiting generated code."
+    echo "Ensure make generate has run and all changes are committed."
+    DIFF=$(git diff)
+    echo "diff detected: $DIFF"
+    DIFF=$(git diff --name-only)
+    echo "files different: $DIFF"
+    exit 1
+  else
+    echo "SUCCESS: Generated code is up to date."
+  fi
+fi

boilerplate/flyte/golang_test_targets/goimports

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+goimports -w $(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/client/*" -not -path "./boilerplate/*")

boilerplate/update.cfg

@@ -1,2 +1,3 @@
 flyte/end2end
 flyte/code_of_conduct
+flyte/golang_test_targets

charts/flyte-core/README.md

@@ -160,9 +160,13 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flyteadmin deployment |
 | flyteadmin.secrets | object | `{}` |  |
 | flyteadmin.service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"}` | Service settings for Flyteadmin |
-| flyteadmin.serviceAccount | object | `{"alwaysCreate":false,"annotations":{},"create":true,"createClusterRole":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlyteAdmin |
+| flyteadmin.serviceAccount | object | `{"alwaysCreate":false,"annotations":{},"clusterRole":{"apiGroups":["","flyte.lyft.com","rbac.authorization.k8s.io"],"resources":["configmaps","flyteworkflows","namespaces","pods","resourcequotas","roles","rolebindings","secrets","services","serviceaccounts","spark-role","limitranges"],"verbs":["*"]},"create":true,"createClusterRole":true,"imagePullSecrets":[]}` | Configuration for service accounts for FlyteAdmin |
 | flyteadmin.serviceAccount.alwaysCreate | bool | `false` | Should a service account always be created for flyteadmin even without an actual flyteadmin deployment running (e.g. for multi-cluster setups) |
 | flyteadmin.serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to Flyteadmin pods |
+| flyteadmin.serviceAccount.clusterRole | object | `{"apiGroups":["","flyte.lyft.com","rbac.authorization.k8s.io"],"resources":["configmaps","flyteworkflows","namespaces","pods","resourcequotas","roles","rolebindings","secrets","services","serviceaccounts","spark-role","limitranges"],"verbs":["*"]}` | Configuration for ClusterRole created for Flyteadmin |
+| flyteadmin.serviceAccount.clusterRole.apiGroups | list | `["","flyte.lyft.com","rbac.authorization.k8s.io"]` | Specifies the API groups that this ClusterRole can access |
+| flyteadmin.serviceAccount.clusterRole.resources | list | `["configmaps","flyteworkflows","namespaces","pods","resourcequotas","roles","rolebindings","secrets","services","serviceaccounts","spark-role","limitranges"]` | Specifies the resources that this ClusterRole can access |
+| flyteadmin.serviceAccount.clusterRole.verbs | list | `["*"]` | Specifies the verbs (actions) that this ClusterRole can perform on the specified resources |
 | flyteadmin.serviceAccount.create | bool | `true` | Should a service account be created for flyteadmin |
 | flyteadmin.serviceAccount.createClusterRole | bool | `true` | Should a ClusterRole be created for Flyteadmin |
 | flyteadmin.serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account |

charts/flyte-core/templates/admin/rbac.yaml

@@ -23,25 +23,9 @@ metadata:
   name: {{ template "flyte.namespace" . -}}-{{- template "flyteadmin.name" . }}
   labels: {{ include "flyteadmin.labels" . | nindent 4 }}
 rules:
-- apiGroups:
-  - ""
-  - flyte.lyft.com
-  - rbac.authorization.k8s.io
-  resources:
-  - configmaps
-  - flyteworkflows
-  - namespaces
-  - pods
-  - resourcequotas
-  - roles
-  - rolebindings
-  - secrets
-  - services
-  - serviceaccounts
-  - spark-role
-  verbs:
-  - '*'
-
+- apiGroups: {{ toYaml .Values.flyteadmin.serviceAccount.clusterRole.apiGroups | nindent 2 }}
+  resources: {{ toYaml .Values.flyteadmin.serviceAccount.clusterRole.resources | nindent 2 }}
+  verbs: {{ toYaml .Values.flyteadmin.serviceAccount.clusterRole.verbs | nindent 2 }}
 ---
 {{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }}
 apiVersion: rbac.authorization.k8s.io/v1