update

.github/codeql-config-inject.yml

@@ -5,6 +5,3 @@ disable-default-queries: true
 packs:
   java:
     - zbazztian/java-queries
-
-queries:
-  - uses: zbazztian/java-queries:codeql-suites/java-security-and-quality.qls

customize/cli.py

@@ -0,0 +1,108 @@
+import argparse
+import util
+import os
+from os.path import dirname, isfile, abspath, join, basename
+import sys
+import ghlib
+
+
+def upload(args):
+  git = util.Git('.')
+  gh = ghlib.GitHub('https://api.github.com', os.environ['GITHUB_TOKEN'])
+  repo = gh.getRepo(args.repo_id)
+  repo.upload_latest(git.branch(), git.revision(git.branch()), args.dist)
+
+
+def inject(args):
+  args.script = abspath(args.script)
+  args.dist = abspath(args.dist)
+  args.output = abspath(args.output)
+  scriptdir = dirname(args.script)
+
+  if not isfile(args.script):
+    util.error('Given script "%s" does not exist!' % (args.script))
+
+  os.chdir(scriptdir)
+  util.info('Working directory is "%s"!' % (scriptdir))
+
+  git = util.Git(scriptdir)
+  inputdist = util.extract_dist(args.dist)
+  customization_hash = util.hashstr(
+    util.sha1sumd(inputdist) +
+    git.revision(git.branch())
+  )
+  if customization_hash != util.get_customization_hash(args.output):
+    util.info('Customization hashes of input and output differ. Recreating output...')
+  else:
+    util.info('Customization hashes of input and output match. Nothing to do!')
+    return
+
+  # execute the user's script
+  from importlib import import_module
+  sys.path.append(scriptdir)
+  m = import_module(basename(args.script).rsplit('.', 1)[0])
+  customize = getattr(m, 'customize')
+  customize(util.ScriptUtils(inputdist))
+
+  util.info('Writing customization hash ...')
+  util.write_str(join(inputdist, 'customization_hash'), customization_hash)
+  util.info('Creating output archive "%s"...' % (args.output))
+  util.tar_czf(inputdist, args.output)
+
+
+def main():
+  parser = argparse.ArgumentParser(prog="customize")
+  subparsers = parser.add_subparsers()
+
+  # inject
+  inject_parser = subparsers.add_parser(
+    'inject',
+    help='Inject customizations into a CodeQL distribution',
+    description='Inject customizations into a CodeQL distribution',
+  )
+  inject_parser.add_argument(
+    '--dist',
+    required=True,
+    help='A .zip, .tar.gz or directory containing a CodeQL distribution',
+  )
+  inject_parser.add_argument(
+    '--output',
+    required=True,
+    help='The file to which to output the customized distribution (a .tar.gz archive)',
+  )
+  inject_parser.add_argument(
+    '--script',
+    required=True,
+    help='A python file with the customization script. It should contain a function "customize()" which takes a "Utils" object as a single parameter',
+  )
+  inject_parser.set_defaults(func=inject)
+
+  # upload
+  upload_parser = subparsers.add_parser(
+    'upload',
+    help='Upload a customized distribution as a release, using the GitHub REST API',
+    description='Upload a customized CodeQL distribution as a release, using the GitHub REST API',
+  )
+  upload_parser.add_argument(
+    '--dist',
+    required=True,
+    help='A .tar.gz file containing a CodeQL distribution',
+  )
+  upload_parser.add_argument(
+    '--repo-id',
+    required=True,
+    help='The repository id in the format of "orgoruser/reponame"',
+  )
+  upload_parser.set_defaults(func=upload)
+
+  def print_usage(args):
+    print(parser.format_usage())
+
+  parser.set_defaults(func=print_usage)
+  args = parser.parse_args()
+
+  # run the given action
+  args.func(args)
+
+
+main()

customize/customize

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -eu
+HERE="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
+python3 "${HERE}/cli.py" "$@"
+

customize/ghlib.py

@@ -0,0 +1,183 @@
+import requests
+import logging
+import json
+import util
+from os.path import join
+from requests import HTTPError
+import mimetypes
+from fnmatch import fnmatch
+
+RESULTS_PER_PAGE = 100
+
+
+def json_accept_header():
+  return {"Accept": "application/vnd.github.v3+json"}
+
+
+def sort_by_created_at(releasesorassets):
+  return sorted(
+    releasesorassets,
+    key=lambda e: parse_date(e['created_at']),
+    reverse=True
+  )
+
+
+def make_tag(branch, revision):
+  return '%s-%s' % (branch, revision)
+
+
+class GitHub:
+  def __init__(self, url, token):
+    self.url = url
+    self.token = token
+
+  def default_headers(self):
+    auth = {"Authorization": "token " + self.token}
+    auth.update(json_accept_header())
+    return auth
+
+  def getRepo(self, repo_id):
+    return Repo(self, repo_id)
+
+
+class Repo:
+
+  def __init__(self, github, repo_id):
+    self.gh = github
+    self.repo_id = repo_id
+
+
+  def latest_asset(tagfilter, assetfilter):
+    ''' note that this will not necessarily return a file
+        from the _latest_ release. It will return a file
+        from the newest release which matches the given
+        tagfilter and assetfilter. This is important, since
+        a download would fail if someone would be in the
+        process of uploading an asset, which first requires
+        to create an empty release (race condition)'''
+    for r in sort_by_created_at(self.list_releases()):
+      if fnmatch(r['name'], tagfilter):
+        for a in sort_by_created_at(r['assets']):
+          if fnmatch(a['name'], assetfilter):
+            return a
+    return None
+
+
+  def download_asset(self, asset, directory):
+    headers=self.gh.default_headers()
+    headers['Accept'] = 'application/octet-stream'
+
+    with requests.get(
+      "{api_url}/repos/{repo_id}/releases/assets/{asset_id}".format(
+        api_url=self.gh.url,
+        repo_id=self.repo_id,
+        asset_id=asset['id'],
+      ),
+      headers=headers,
+      timeout=util.REQUEST_TIMEOUT,
+    ) as r:
+
+      r.raise_for_status()
+
+      with open(join(directory, asset['name']), 'wb') as f:
+        for chunk in r.iter_content():
+          if chunk:
+            f.write(chunk)
+
+
+  def list_releases(self):
+    try:
+      resp = requests.get(
+        "{api_url}/repos/{repo_id}/releases?per_page={results_per_page}".format(
+          api_url=self.gh.url,
+          repo_id=self.repo_id,
+          results_per_page=RESULTS_PER_PAGE,
+        ),
+        headers=self.gh.default_headers(),
+        timeout=util.REQUEST_TIMEOUT,
+      )
+
+      while True:
+        resp.raise_for_status()
+
+        for r in resp.json():
+          yield r
+
+        nextpage = resp.links.get("next", {}).get("url", None)
+        if not nextpage:
+          break
+
+        resp = requests.get(
+          nextpage,
+          headers=self.gh.default_headers(),
+          timeout=util.REQUEST_TIMEOUT,
+        )
+
+    except HTTPError as httpe:
+      if httpe.response.status_code == 404:
+        # A 404 suggests that the repository doesn't exist
+        # so we return an empty list
+        pass
+      else:
+        # propagate everything else
+        raise
+
+
+  def get_release(self, tag):
+    for r in self.list_releases():
+      if r['tag_name'] == tag:
+        return r
+    return None
+
+
+  def create_release(self, tag, revision):
+    with requests.post(
+      "{api_url}/repos/{repo_id}/releases".format(
+        api_url=self.gh.url,
+        repo_id=self.repo_id,
+      ),
+      data=json.dumps(
+        {
+          'tag_name': tag,
+          'target_commitish': revision,
+        }
+      ),
+      headers=self.gh.default_headers(),
+      timeout=util.REQUEST_TIMEOUT,
+    ) as r:
+
+      r.raise_for_status()
+      return r.json()
+
+
+  def upload_asset(self, release, asset_name, filepath):
+    headers=self.gh.default_headers()
+    content_type = mimetypes.guess_type(filepath)[0]
+    headers['Content-Type'] = 'application/octet-stream' if content_type is None else content_type
+
+    with open(filepath, 'rb') as f:
+      with requests.post(
+        release['upload_url'].replace('{?name,label}', '') + "?name={name}&label=dist".format(
+          name=asset_name,
+        ),
+        data=f,
+        headers=headers,
+        timeout=util.REQUEST_TIMEOUT,
+      ) as r:
+
+        r.raise_for_status()
+        return r.json()
+
+
+  def get_or_create_release(self, branch, revision):
+    tag = make_tag(branch, revision)
+    r = self.get_release(tag)
+    if r is None:
+      r = self.create_release(tag, revision)
+    return r
+
+
+  def upload_latest(self, branch, revision, filepath):
+    r = self.get_or_create_release(branch, revision)
+    sha = util.sha1sum(filepath)
+    self.upload_asset(r, 'codeql-bundle-%s-%s.tar.gz' % (util.make_date(), sha), filepath)