staging: Add skeleton of new staging system

maestro/staging/README.md

@@ -0,0 +1,21 @@
+# Maestro Staging
+
+This is new staging system related scripts and supplementary files.
+
+# Principle of operation
+
+The system is based on the following principles:
+
+1. The system is based on the concept of creating staging-snapshot branches in several repositories using pending pull requests that qualify for staging.
+2. System will build and deploy the staging-snapshot branches to the staging environment ( staging instances of kernelci-api, kernelci-pipeline ) which also use staging-prefixed docker images of cross-compile tools.
+3. Based on results maintainers can approve or reject pending pull requests.
+
+# How to use
+
+Run staging.sh
+
+# Files purpose
+
+- staging.sh - main script
+- staging-branch.py - Python script to create staging-snapshot branches, that pull PR, validate if they qualify for staging and create a staging-snapshot branch.
+- users.txt - list of trusted users, PR will be tested only if the author is in this list.

maestro/staging/staging-branch.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+'''
+KernelCI github tool to create staging snapshot
+- Retrieve PRs for a project
+- Apply patches for each PR (if user is allowed)
+- Create a branch staging-snapshot
+- Push branch to origin (if --push is set)
+'''
+
+import argparse
+import os
+import sys
+import requests
+import json
+import logging
+
+'''
+Retrieve current PR list for a project
+'''
+
+ORG="kernelci"
+
+# AFAIK logging lib doesnt add colors? :(
+def print_level(level, text):
+    colorcodes = {
+        'info': '92',
+        'warning': '93',
+        'error': '91',
+        'red': '91',
+        'green': '92',
+        'yellow': '93',
+        'blue': '94',
+        'purple': '95',
+    }
+    print(f"\033[{colorcodes[level]}m{text}\033[0m")
+
+def get_prs(project):
+    url = f"https://api.github.com/repos/{ORG}/{project}/pulls"
+    resp = requests.get(url)
+    resp.raise_for_status()
+    return resp.json()
+
+def shallow_clone(project):
+    if not os.path.exists(project):
+        url = f"https://github.com/{ORG}/{project}"
+        os.system(f"git clone --depth 1 {url}")
+    os.chdir(project)
+    os.system("git checkout origin/main")
+    os.system("git reset --hard origin/main")
+    os.system("git am --abort")
+    os.chdir("..")
+
+def fetch_pr(pr_number, project):
+    url = f"https://github.com/{ORG}/{project}"
+    # use .patch at end of URL to get patch
+    resp = requests.get(f"{url}/pull/{pr_number}.patch")
+    resp.raise_for_status()
+    with open(f"patches/pr_{project}-{pr_number}.patch", "w") as f:
+        f.write(resp.text)
+
+def apply_pr(pr_number, pr_info, project):
+    r = os.system(f"git am -q ../patches/pr_{project}-{pr_number}.patch")
+    if r != 0:
+        print_level('error', f"{pr_info}: Patch failed for PR {pr_number}")
+        os.system("git am --abort")
+    else:
+        print_level('info', f"{pr_info}: Patch applied for PR {pr_number}")
+
+
+def load_users(filename):
+    if not os.path.exists(filename):
+        print_level('error', f"File {filename} not found")
+        sys.exit(1)
+    with open(filename, "r") as f:
+        return f.read().splitlines()
+
+def print_comprehensive(pr, prefix=""):
+    for key, value in pr.items():
+        if isinstance(value, dict):
+            print_comprehensive(value, prefix=f"{prefix}{key}.")
+        else:
+            print(f"{prefix}={key}: {value}")
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--project', help='project', required=True)
+    parser.add_argument('--allowusers', help='allowusers', default='users.txt')
+    parser.add_argument('--push', help='push to origin', action='store_true', default=False)
+    args = parser.parse_args()
+    users = load_users(args.allowusers)
+    if not os.path.exists("patches"):
+        os.mkdir("patches")
+    shallow_clone(args.project)
+    if args.push:
+        os.system(f"git remote set-url --push origin [email protected]:{ORG}/{args.project}.git")
+    prs = get_prs(args.project)
+
+    # sort ascending by PR number
+    prs = sorted(prs, key=lambda pr: pr['number'])
+    for pr in prs:
+        # for debugging
+        #print_comprehensive(pr)
+        pr_info = f"PR {pr['number']} - {pr['state']} - {pr['user']['login']} - " + \
+                  f"{pr['title']}"
+
+        if pr['user']['login'] not in users:
+            print_level('warning', f"{pr_info}: Skipping PR from unauthorized user")
+            continue
+        if pr['draft']:
+            print_level('warning', f"{pr_info}: Skipping PR due to draft status")
+            continue
+        if 'staging-skip' in [label['name'] for label in pr['labels']]:
+            print_level('warning', f"{pr_info}: Skipping PR due to staging-skip label")
+            continue
+        print_level('info', pr_info)
+        fetch_pr(pr['number'], args.project)
+        os.chdir(args.project)
+        apply_pr(pr['number'], pr_info, args.project)
+        os.chdir("..")
+
+    # create branch staging-snapshot
+    os.chdir(args.project)
+    # delete branch if exists
+    os.system("git branch -D staging-snapshot")
+    os.system("git checkout -b staging-snapshot")
+    if args.push:
+        os.system("git push -f origin staging-snapshot")
+    # retrieve last commit id
+    last_commit = os.popen("git log -1 --pretty=%H").read().strip()
+    print_level('info', f"Last commit: {last_commit}")
+    # write as a file projectname.commit
+    os.chdir("..")
+    with open(f"{args.project}.commit", "w") as f:
+        f.write(last_commit)
+
+
+if __name__ == '__main__':
+    main()
+

maestro/staging/staging.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# $(git show --pretty=format:%H -s origin/staging.kernelci.org)
+REPOS=("kernelci-core" "kernelci-pipeline" "kernelci-api")
+
+preparing_repos() {
+    echo "Preparing repos staging snapshots branch"
+    for repo in ${REPOS[@]}; do
+        ./staging-branch.py --project $repo --push
+    done
+}
+
+retrieving_revs() {
+    echo "Retrieving revisions"
+    core_rev=$(cat kernelci-core.commit)
+    pipeline_rev=$(cat kernelci-pipeline.commit)
+    api_rev=$(cat kernelci-api.commit)
+
+    echo "kernelci-core: $core_rev"
+    echo "kernelci-pipeline: $pipeline_rev"
+    echo "kernelci-api: $api_rev"
+}
+
+building_docker() {
+    echo "Building docker images"
+    retrieving_revs
+    if [ -z "$core_rev" ] || [ -z "$pipeline_rev" ] || [ -z "$api_rev" ]; then
+        echo "Error: One or more rev variables are not set"
+        exit 1
+    fi
+    cache_arg=""
+    rev_arg="--build-arg core_rev=$core_rev --build-arg api_rev=$api_rev --build-arg pipeline_rev=$pipeline_rev"
+    px_arg='--prefix=kernelci/staging-'
+    #args="build --push $px_arg $cache_arg $rev_arg"
+    args="build $px_arg $cache_arg $rev_arg"
+    cd kernelci-core
+    # install dependencies
+    pip install -r requirements.txt --break-system-packages
+    ./kci docker $args kernelci
+}
+
+preparing_repos
+building_docker
+
+

maestro/staging/users.txt

@@ -0,0 +1,72 @@
+10ne1
+aistcv
+alexandrasp
+aliceinwire
+andrealmeid
+a-wai
+Bastian-Krause
+BayLibre
+broonie
+cazou
+chaws
+ClangBuiltLinux
+crazoes
+denisyuji
+dzickusrh
+eballetbo
+eds-collabora
+evdenis
+fbezdeka
+gctucker
+glunardi
+Heidifahim
+hardboprobot
+helen-fornazier
+hiwang123
+hthiery
+igaw
+jayaddison-collabora
+JenySadadia
+jeromebrunet
+jluebbe
+kees
+kernelci
+khilman
+kholk
+krisman
+Lakshmipathi
+laura-nao
+lubinszARM
+lufy90
+markfilion
+mattface
+mgalka
+mharyam
+musamaanjum
+mwalle
+mwasilew
+montjoie
+mgrzeschik
+nfraprado
+nkbelin
+nuclearcat
+ojeda
+padovan
+patersonc
+pawiecz
+penvirus
+ribalda
+roxell
+sashalevin
+sbdtu5498
+sjoerdsimons
+sonnydeez
+spbnick
+sre
+suram-nxp
+tomeuv
+touilkhouloud
+VinceHillier
+wangmingyu84
+williamklin
+yurinnick