Skeleton testing framework (kubeflow#18)

* First stab at adding tests to this repo

* Add prow_config.yaml & remove test-infra dir

* Add .gitignore

* Add components.workflows.prow to params.libsonnet

Change ksonnet app name

* Add package names & EXTRA_REPOS, remove steps

* Put steps back

* Remove build step

* Remove cluster setup & teardown

prow_config.yaml

@@ -0,0 +1,7 @@
+# This file configures the workflows to trigger in our Prow jobs.
+# see kubeflow/testing/py/run_e2e_workflow.py
+workflows:
+  - app_dir: kubeflow/examples/test/workflows
+    component: workflows
+    name: examples-e2e
+

test/workflows/.ksonnet/registries/incubator/422d521c05aa905df949868143b26445f5e4eda5.yaml

@@ -0,0 +1,39 @@
+apiVersion: "0.1"
+gitVersion:
+  commitSha: 422d521c05aa905df949868143b26445f5e4eda5
+  refSpec: master
+kind: ksonnet.io/registry
+libraries:
+  apache:
+    path: apache
+    version: master
+  efk:
+    path: efk
+    version: master
+  mariadb:
+    path: mariadb
+    version: master
+  memcached:
+    path: memcached
+    version: master
+  mongodb:
+    path: mongodb
+    version: master
+  mysql:
+    path: mysql
+    version: master
+  nginx:
+    path: nginx
+    version: master
+  node:
+    path: node
+    version: master
+  postgres:
+    path: postgres
+    version: master
+  redis:
+    path: redis
+    version: master
+  tomcat:
+    path: tomcat
+    version: master

test/workflows/app.yaml

@@ -0,0 +1,11 @@
+apiVersion: 0.0.1
+kind: ksonnet.io/app
+name: test-infra
+registries:
+  incubator:
+    gitVersion:
+      commitSha: 422d521c05aa905df949868143b26445f5e4eda5
+      refSpec: master
+    protocol: github
+    uri: github.com/ksonnet/parts/tree/master/incubator
+version: 0.0.1

test/workflows/components/params.libsonnet

@@ -0,0 +1,17 @@
+{
+  global: {
+    // User-defined global parameters; accessible to all component and environments, Ex:
+    // replicas: 4,
+  },
+  components: {
+    // Component-level parameters, defined initially from 'ks prototype use ...'
+    // Each object below should correspond to a component in the components/ directory
+    workflows: {
+      bucket: "mlkube-testing_temp",
+      name: "kubeflow-examples-presubmit-test-374-6e32",
+      namespace: "kubeflow-test-infra",
+      prow: "JOB_NAME=kubeflow-examples-presubmit-test,JOB_TYPE=presubmit,PULL_NUMBER=209,REPO_NAME=examples,REPO_OWNER=kubeflow,BUILD_NUMBER=997a",
+      prow_env: "JOB_NAME=kubeflow-examples-presubmit-test,JOB_TYPE=presubmit,PULL_NUMBER=374,REPO_NAME=examples,REPO_OWNER=kubeflow,BUILD_NUMBER=6e32",
+    },
+  },
+}

test/workflows/components/workflows.jsonnet

@@ -0,0 +1,14 @@
+local params = std.extVar("__ksonnet/params").components.workflows;
+
+local k = import 'k.libsonnet';
+local workflows = import 'workflows.libsonnet';
+local namespace = params.namespace;
+
+// TODO(jlewi): Can we make name default so some random unique value?
+// I didn't see any routines in the standard library for datetime or random.
+local name = params.name;
+
+local prowEnv = workflows.parseEnv(params.prow_env);
+local bucket = params.bucket;
+std.prune(k.core.v1.list.new([workflows.parts(namespace, name).e2e(prowEnv, bucket)]))
+

test/workflows/components/workflows.libsonnet

@@ -0,0 +1,244 @@
+{
+  // TODO(https://github.com/ksonnet/ksonnet/issues/222): Taking namespace as an argument is a work around for the fact that ksonnet
+  // doesn't support automatically piping in the namespace from the environment to prototypes.
+
+  // convert a list of two items into a map representing an environment variable
+  // TODO(jlewi): Should we move this into kubeflow/core/util.libsonnet
+  listToMap:: function(v)
+    {
+      name: v[0],
+      value: v[1],
+    },
+
+  // Function to turn comma separated list of prow environment variables into a dictionary.
+  parseEnv:: function(v)
+    local pieces = std.split(v, ",");
+    if v != "" && std.length(pieces) > 0 then
+      std.map(
+        function(i) $.listToMap(std.split(i, "=")),
+        std.split(v, ",")
+      )
+    else [],
+
+  parts(namespace, name):: {
+    // Workflow to run the e2e test.
+    e2e(prow_env, bucket):
+      // mountPath is the directory where the volume to store the test data
+      // should be mounted.
+      local mountPath = "/mnt/" + "test-data-volume";
+      // testDir is the root directory for all data for a particular test run.
+      local testDir = mountPath + "/" + name;
+      // outputDir is the directory to sync to GCS to contain the output for this job.
+      local outputDir = testDir + "/output";
+      local artifactsDir = outputDir + "/artifacts";
+      local goDir = testDir + "/go";
+      // Source directory where all repos should be checked out
+      local srcRootDir = testDir + "/src";
+      // The directory containing the kubeflow/examples repo
+      local srcDir = srcRootDir + "/kubeflow/examples";
+      local image = "gcr.io/mlkube-testing/test-worker";
+      // The name of the NFS volume claim to use for test files.
+      // local nfsVolumeClaim = "kubeflow-testing";
+      local nfsVolumeClaim = "nfs-external";
+      // The name to use for the volume to use to contain test data.
+      local dataVolume = "kubeflow-test-volume";
+      local versionTag = name;
+      // The directory within the kubeflow_testing submodule containing
+      // py scripts to use.
+      local kubeflowExamplesPy = srcDir;
+      local kubeflowTestingPy = srcRootDir + "/kubeflow/testing/py";
+
+      local project = "mlkube-testing";
+      // GKE cluster to use
+      // We need to truncate the cluster to no more than 40 characters because
+      // cluster names can be a max of 40 characters.
+      // We expect the suffix of the cluster name to be unique salt.
+      // We prepend a z because cluster name must start with an alphanumeric character
+      // and if we cut the prefix we might end up starting with "-" or other invalid
+      // character for first character.
+      local cluster = 
+        if std.length(name) > 40 then
+          "z" + std.substr(name, std.length(name) - 39, 39)
+        else 
+        name;
+      local zone = "us-east1-d";
+      local chart = srcDir + "/bin/examples-chart-0.2.1-" + versionTag + ".tgz";
+      {
+        // Build an Argo template to execute a particular command.
+        // step_name: Name for the template
+        // command: List to pass as the container command.
+        buildTemplate(step_name, command):: {
+          name: step_name,
+          container: {
+            command: command,
+            image: image,
+            workingDir: srcDir,
+            env: [
+              {
+                // Add the source directories to the python path.
+                name: "PYTHONPATH",
+                value: kubeflowExamplesPy + ":" + kubeflowTestingPy,
+              },
+              {
+                // Set the GOPATH
+                name: "GOPATH",
+                value: goDir,
+              },
+              {
+                name: "GOOGLE_APPLICATION_CREDENTIALS",
+                value: "/secret/gcp-credentials/key.json",
+              },
+              {
+                name: "GIT_TOKEN",
+                valueFrom: {
+                  secretKeyRef: {
+                    name: "github-token",
+                    key: "github_token",
+                  },
+                },
+              },
+              {
+                name: "EXTRA_REPOS",
+                value: "kubeflow/testing@HEAD",
+              },
+            ] + prow_env,
+            volumeMounts: [
+              {
+                name: dataVolume,
+                mountPath: mountPath,
+              },
+              {
+                name: "github-token",
+                mountPath: "/secret/github-token",
+              },
+              {
+                name: "gcp-credentials",
+                mountPath: "/secret/gcp-credentials",
+              },
+            ],
+          },
+        },  // buildTemplate
+
+        apiVersion: "argoproj.io/v1alpha1",
+        kind: "Workflow",
+        metadata: {
+          name: name,
+          namespace: namespace,
+        },
+        // TODO(jlewi): Use OnExit to run cleanup steps.
+        spec: {
+          entrypoint: "e2e",
+          volumes: [
+            {
+              name: "github-token",
+              secret: {
+                secretName: "github-token",
+              },
+            },
+            {
+              name: "gcp-credentials",
+              secret: {
+                secretName: "kubeflow-testing-credentials",
+              },
+            },
+            {
+              name: dataVolume,
+              persistentVolumeClaim: {
+                claimName: nfsVolumeClaim,
+              },
+            },
+          ],  // volumes
+          // onExit specifies the template that should always run when the workflow completes.
+          onExit: "exit-handler",
+          templates: [
+            {
+              name: "e2e",
+              steps: [
+                [{
+                  name: "checkout",
+                  template: "checkout",
+                }],
+                [
+                  {
+                    name: "create-pr-symlink",
+                    template: "create-pr-symlink",
+                  },
+                  {
+                    name: "py-test",
+                    template: "py-test",
+                  },
+                  {
+                    name: "py-lint",
+                    template: "py-lint",
+                  },
+                ],
+              ],
+            },
+            {
+              name: "exit-handler",
+              steps: [
+                [{
+                  name: "copy-artifacts",
+                  template: "copy-artifacts",
+                }],
+              ],
+            },
+            {
+              name: "checkout",
+              container: {
+                command: [
+                  "/usr/local/bin/checkout.sh",
+                  srcRootDir,
+                ],
+                env: prow_env + [{
+                  name: "EXTRA_REPOS",
+                  value: "kubeflow/testing@HEAD",
+                }],
+                image: image,
+                volumeMounts: [
+                  {
+                    name: dataVolume,
+                    mountPath: mountPath,
+                  },
+                ],
+              },
+            },  // checkout
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("py-test", [
+              "python",
+              "-m",
+              "kubeflow.testing.py_checks",
+              "test",
+              "--src_dir=" + srcDir,
+              "--project=mlkube-testing",
+              "--junit_path=" + artifactsDir + "/junit_pycheckstest.xml",
+            ]),  // py test
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("py-lint", [
+              "python",
+              "-m",
+              "kubeflow.testing.py_checks",
+              "lint",
+              "--src_dir=" + srcDir,
+              "--project=mlkube-testing",
+              "--junit_path=" + artifactsDir + "/junit_pycheckslint.xml",
+            ]),  // py lint
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("create-pr-symlink", [
+              "python",
+              "-m",
+              "kubeflow.testing.prow_artifacts",
+              "--artifacts_dir=" + outputDir,
+              "create_pr_symlink",
+              "--bucket=" + bucket,
+            ]),  // create-pr-symlink
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("copy-artifacts", [
+              "python",
+              "-m",
+              "kubeflow.testing.prow_artifacts",
+              "--artifacts_dir=" + outputDir,
+              "copy_artifacts",
+              "--bucket=" + bucket,
+            ]),  // copy-artifacts
+          ],  // templates
+        },
+      },  // e2e
+  },  // parts
+}

test/workflows/environments/base.libsonnet

@@ -0,0 +1,4 @@
+local components = std.extVar("__ksonnet/components");
+components + {
+  // Insert user-specified overrides here.
+}