Configure Gubernator using a static, external file

As other organizations begin to use Gubernator, the need to configure links, paths and defaults becomes greater. This patch introduces a static configuration file where this configuration can live. Mutliple mappings of organization to GCS bucket and Prow instance URL should be supported to allow for one Gubernator instance to serve valid data for multiple organizations' jobs. Signed-off-by: Steve Kuznetsov <[email protected]>
gajanankulkarni-18 · Sep 1, 2017 · eb64cab · eb64cab
1 parent cbf22fb
commit eb64cab
Show file tree

Hide file tree

Showing 13 changed files with 251 additions and 103 deletions.
diff --git a/gubernator/config.yaml b/gubernator/config.yaml
@@ -0,0 +1,35 @@
+---
+default_org: kubernetes
+default_repo: kubernetes
+external_services:
+  istio:
+    gcs_pull_prefix: istio-prow/pull
+    prow_url: prow.istio.io
+  kubernetes:
+    gcs_pull_prefix: kubernetes-jenkins/pr-logs/pull
+    prow_url: prow.k8s.io
+jobs:
+  kubernetes-jenkins/logs/:
+  - ci-kubernetes-e2e-gce-etcd3
+  - ci-kubernetes-e2e-gci-gce
+  - ci-kubernetes-e2e-gci-gce-slow
+  - ci-kubernetes-e2e-gci-gke
+  - ci-kubernetes-e2e-gci-gke-slow
+  - ci-kubernetes-kubemark-500-gce
+  - ci-kubernetes-node-kubelet
+  - ci-kubernetes-test-go
+  - ci-kubernetes-verify-master
+  - kubernetes-build
+  - kubernetes-e2e-kops-aws
+  kubernetes-jenkins/pr-logs/directory/:
+  - pull-kubernetes-bazel-build
+  - pull-kubernetes-bazel-test
+  - pull-kubernetes-e2e-gce-bazel
+  - pull-kubernetes-e2e-gce-etcd3
+  - pull-kubernetes-e2e-gce-gpu
+  - pull-kubernetes-e2e-kops-aws
+  - pull-kubernetes-federation-e2e-gce
+  - pull-kubernetes-kubemark-e2e-gce
+  - pull-kubernetes-node-e2e
+  - pull-kubernetes-unit
+  - pull-kubernetes-verify
diff --git a/gubernator/main.py b/gubernator/main.py
@@ -18,6 +18,7 @@
 import logging
 import os
 
+import yaml
 import webapp2
 from webapp2_extras import security
 
@@ -64,6 +65,10 @@ def get_github_client():
         return None
 
 
+def get_app_config():
+    with open('config.yaml') as config_file:
+        return yaml.load(config_file)
+
 config = {
     'webapp2_extras.sessions': {
         'secret_key': get_session_secret(),
@@ -76,6 +81,8 @@ def get_github_client():
     'github_client': get_github_client(),
 }
 
+config.update(get_app_config())
+
 class Warmup(webapp2.RequestHandler):
     """Warms up gubernator."""
     def get(self):

diff --git a/gubernator/main_test.py b/gubernator/main_test.py
@@ -35,7 +35,6 @@
 
 app = webtest.TestApp(main.app)
 
-
 JUNIT_SUITE = '''<testsuite tests="8" failures="0" time="1000.24">
     <testcase name="First" classname="Example e2e suite" time="0">
         <skipped/>

diff --git a/gubernator/prow_jobs.yaml b/gubernator/prow_jobs.yaml
diff --git a/gubernator/test-gubernator.sh b/gubernator/test-gubernator.sh
@@ -23,3 +23,4 @@ pip install -r test_requirements.txt
 ./test.sh --nologcapture
 ./lint.sh
 mocha static/build_test.js
+./verify_config.sh
diff --git a/gubernator/update_config.py b/gubernator/update_config.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Updates the Gubernator configuration from the Prow configuration."""
+
+import argparse
+import yaml
+
+def main(prow_config, gubernator_config):
+    with open(prow_config) as prow_file:
+        prow_data = yaml.load(prow_file)
+
+    default_presubmits = []
+    for job in prow_data['presubmits']['kubernetes/kubernetes']:
+        if job.get('always_run'):
+            default_presubmits.append(job['name'])
+
+    with open(gubernator_config) as gubernator_file:
+        gubernator_data = yaml.load(gubernator_file)
+
+    gubernator_data['jobs']['kubernetes-jenkins/pr-logs/directory/'] = default_presubmits
+
+    with open(gubernator_config, 'w+') as gubernator_file:
+        yaml.dump(gubernator_data, gubernator_file, default_flow_style=False, explicit_start=True)
+
+if __name__ == '__main__':
+    PARSER = argparse.ArgumentParser()
+    PARSER.add_argument('prow_config', help="Path to Prow configuration YAML.")
+    PARSER.add_argument('gubernator_config', help="Path to Gubernator configuration YAML.")
+    ARGS = PARSER.parse_args()
+    main(ARGS.prow_config, ARGS.gubernator_config)
diff --git a/gubernator/update_config.sh b/gubernator/update_config.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script updates the Gubernator configuration
+# file to keep it in sync with Prow.
+
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+./update_config.py ./../prow/config.yaml ./config.yaml
diff --git a/gubernator/verify_config.sh b/gubernator/verify_config.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script verifies the Gubernator configuration
+# file is in sync with Prow.
+
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+config="$( mktemp )"
+trap "rm ${config}" EXIT
+
+cp ./config.yaml "${config}"
+./update_config.py ./../prow/config.yaml "${config}"
+
+if ! output="$( diff ./config.yaml "${config}" )"; then
+    echo "Gubernator configuration file is out of sync!"
+    echo "${output}"
+    echo "Run gubernator/update-config.sh"
+    exit 1
+fi
diff --git a/gubernator/view_base.py b/gubernator/view_base.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import collections
 import functools
 import logging
 import os
@@ -21,48 +20,13 @@
 import cloudstorage as gcs
 import jinja2
 import webapp2
-import yaml
 
 from google.appengine.api import urlfetch
 from google.appengine.api import memcache
 from webapp2_extras import sessions
 
 import filters as jinja_filters
 
-
-PROW_JOBS = yaml.load(open('prow_jobs.yaml'))
-
-DEFAULT_JOBS = {
-    'kubernetes-jenkins/logs/': {
-        'ci-kubernetes-e2e-gce-etcd3',
-        'ci-kubernetes-e2e-gci-gce',
-        'ci-kubernetes-e2e-gci-gce-slow',
-        'ci-kubernetes-e2e-gci-gke',
-        'ci-kubernetes-e2e-gci-gke-slow',
-        'ci-kubernetes-kubemark-500-gce',
-        'ci-kubernetes-node-kubelet',
-        'ci-kubernetes-test-go',
-        'ci-kubernetes-verify-master',
-        'kubernetes-build',
-        'kubernetes-e2e-kops-aws',
-    },
-    'kubernetes-jenkins/pr-logs/directory/': {
-        j['name'] for j in PROW_JOBS['presubmits']['kubernetes/kubernetes'] if j.get('always_run')
-    },
-}
-
-# Maps github organizations/owners to GCS paths for presubmit results
-PR_PREFIX = collections.OrderedDict([
-    ('kubernetes', 'kubernetes-jenkins/pr-logs/pull'),
-    ('google', 'kubernetes-jenkins/pr-logs/pull'),  # for cadvisor
-    ('istio', 'istio-prow/pull'),
-])
-
-PROW_INSTANCES = {
-    'istio': 'prow.istio.io',
-    'DEFAULT': 'prow.k8s.io',
-}
-
 JINJA_ENVIRONMENT = jinja2.Environment(
     loader=jinja2.FileSystemLoader(os.path.dirname(__file__) + '/templates'),
     extensions=['jinja2.ext.autoescape', 'jinja2.ext.loopcontrols'],
@@ -108,7 +72,7 @@ def render(self, template, context):
 class IndexHandler(BaseHandler):
     """Render the index."""
     def get(self):
-        self.render("index.html", {'jobs': DEFAULT_JOBS})
+        self.render("index.html", {'jobs': self.app.config['jobs']})
 
 
 def memcache_memoize(prefix, expires=60 * 60, neg_expires=60):

diff --git a/gubernator/view_build.py b/gubernator/view_build.py
@@ -100,11 +100,9 @@ def get_build_log(build_dir):
         return log_parser.digest(build_log)
 
 
-def get_running_build_log(job, build, repo):
-    org = repo and repo.split('/')[0]
+def get_running_build_log(job, build, prow_url):
     try:
-        prow_instance = view_base.PROW_INSTANCES.get(org, view_base.PROW_INSTANCES['DEFAULT'])
-        url = "https://%s/log?job=%s&id=%s" % (prow_instance, job, build)
+        url = "https://%s/log?job=%s&id=%s" % (prow_url, job, build)
         result = urlfetch.fetch(url)
         if result.status_code == 200:
             return log_parser.digest(result.content), url
@@ -151,23 +149,37 @@ def build_details(build_dir):
     return started, finished, parser.get_results()
 
 
-def parse_pr_path(prefix):
-    for org, path in view_base.PR_PREFIX.items():
-        if not prefix.startswith(path):
-            continue
-        pr = os.path.basename(prefix)
-        repo = os.path.basename(os.path.dirname(prefix))
-        if '_' in repo and not repo.startswith(org):
-            continue
-        if org == 'kubernetes' and repo == 'pull':
-            return pr, '', 'kubernetes/kubernetes'
-        pr_path = repo.replace('_', '/')
-        if '_' not in repo:
-            repo = '%s/%s' % (org, repo)
-        else:
-            repo = pr_path
-        return pr, pr_path + '/', repo
-    return None, None, None
+def parse_pr_path(gcs_path, default_org, default_repo):
+    """
+    Parse GCS bucket directory into metadata. We
+    allow for two short-form names and one long one:
+
+     gs://<pull_prefix>/<pull_number>
+      -- this fills in the default repo and org
+
+     gs://<pull_prefix>/repo/<pull_number>
+      -- this fills in the default org
+
+     gs://<pull_prefix>/org_repo/<pull_number>
+
+    :param gcs_path: GCS bucket directory for a build
+    :return: tuple of:
+     - PR number
+     - Gubernator PR link
+     - PR repo
+    """
+    pull_number = os.path.basename(gcs_path)
+    parsed_repo = os.path.basename(os.path.dirname(gcs_path))
+    if parsed_repo == 'pull':
+        pr_path = ''
+        repo = '%s/%s' % (default_org, default_repo)
+    elif '_' not in parsed_repo:
+        pr_path = parsed_repo + '/'
+        repo = '%s/%s' % (default_org, parsed_repo)
+    else:
+        pr_path = parsed_repo.replace('_', '/') + '/'
+        repo = parsed_repo.replace('_', '/')
+    return pull_number, pr_path, repo
 
 
 class BuildHandler(view_base.BaseHandler):
@@ -194,18 +206,21 @@ def get(self, prefix, job, build):
             return
         started, finished, results = details
 
-        pr, pr_path, repo = parse_pr_path(prefix)
-        pr_digest = None
-        if pr:
-            pr_digest = models.GHIssueDigest.get(repo, pr)
-
+        want_build_log = False
         build_log = ''
         build_log_src = None
         if 'log' in self.request.params or (not finished) or \
             (finished and finished.get('result') != 'SUCCESS' and len(results['failed']) <= 1):
+            want_build_log = True
             build_log = get_build_log(build_dir)
-            if not build_log:
-                build_log, build_log_src = get_running_build_log(job, build, repo)
+
+        pr, pr_path, pr_digest, repo = None, None, None, None
+        external_config = get_pr_config(prefix, self.app.config)
+        if external_config is not None:
+            pr, pr_path, pr_digest, repo = get_pr_info(prefix, self.app.config)
+            if want_build_log and not build_log:
+                build_log, build_log_src = get_running_build_log(job, build,
+                                                                 external_config["prow_url"])
 
         # 'version' might be in either started or finished.
         # prefer finished.
@@ -236,6 +251,25 @@ def get(self, prefix, job, build):
             testgrid_query=testgrid_query))
 
 
+def get_pr_config(prefix, config):
+    for item in config["external_services"].values():
+        if prefix.startswith(item["gcs_pull_prefix"]):
+            return item
+
+def get_pr_info(prefix, config):
+    if config is not None:
+        pr, pr_path, repo = parse_pr_path(
+            gcs_path=prefix,
+            default_org=config['default_org'],
+            default_repo=config['default_repo'],
+        )
+        pr_digest = models.GHIssueDigest.get(repo, pr)
+        return pr, pr_path, pr_digest, repo
+
+def get_running_pr_log(job, build, config):
+    if config is not None:
+        return get_running_build_log(job, build, config["prow_url"])
+
 def get_build_numbers(job_dir, before, indirect):
     try:
         if '/pull/' in job_dir and not indirect: