From b7055119e7a0a926556f3819a02dcaf9f95169ca Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 1 Aug 2024 12:19:42 +0300 Subject: [PATCH] staging: Add skeleton of new staging system This is initial design of new staging system that can run in workflows, but same time can run as standalone script as well. Signed-off-by: Denys Fedoryshchenko --- maestro/staging/README.md | 21 +++++ maestro/staging/staging-branch.py | 139 ++++++++++++++++++++++++++++++ maestro/staging/staging.sh | 44 ++++++++++ maestro/staging/users.txt | 72 ++++++++++++++++ 4 files changed, 276 insertions(+) create mode 100644 maestro/staging/README.md create mode 100755 maestro/staging/staging-branch.py create mode 100755 maestro/staging/staging.sh create mode 100644 maestro/staging/users.txt diff --git a/maestro/staging/README.md b/maestro/staging/README.md new file mode 100644 index 0000000..cba13a7 --- /dev/null +++ b/maestro/staging/README.md @@ -0,0 +1,21 @@ +# Maestro Staging + +This is new staging system related scripts and supplementary files. + +# Principle of operation + +The system is based on the following principles: + +1. The system is based on the concept of creating staging-snapshot branches in several repositories using pending pull requests that qualify for staging. +2. System will build and deploy the staging-snapshot branches to the staging environment ( staging instances of kernelci-api, kernelci-pipeline ) which also use staging-prefixed docker images of cross-compile tools. +3. Based on results maintainers can approve or reject pending pull requests. + +# How to use + +Run staging.sh + +# Files purpose + +- staging.sh - main script +- staging-branch.py - Python script to create staging-snapshot branches, that pull PR, validate if they qualify for staging and create a staging-snapshot branch. +- users.txt - list of trusted users, PR will be tested only if the author is in this list. diff --git a/maestro/staging/staging-branch.py b/maestro/staging/staging-branch.py new file mode 100755 index 0000000..f94234a --- /dev/null +++ b/maestro/staging/staging-branch.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +''' +KernelCI github tool to create staging snapshot +- Retrieve PRs for a project +- Apply patches for each PR (if user is allowed) +- Create a branch staging-snapshot +- Push branch to origin (if --push is set) +''' + +import argparse +import os +import sys +import requests +import json +import logging + +''' +Retrieve current PR list for a project +''' + +ORG="kernelci" + +# AFAIK logging lib doesnt add colors? :( +def print_level(level, text): + colorcodes = { + 'info': '92', + 'warning': '93', + 'error': '91', + 'red': '91', + 'green': '92', + 'yellow': '93', + 'blue': '94', + 'purple': '95', + } + print(f"\033[{colorcodes[level]}m{text}\033[0m") + +def get_prs(project): + url = f"https://api.github.com/repos/{ORG}/{project}/pulls" + resp = requests.get(url) + resp.raise_for_status() + return resp.json() + +def shallow_clone(project): + if not os.path.exists(project): + url = f"https://github.com/{ORG}/{project}" + os.system(f"git clone --depth 1 {url}") + os.chdir(project) + os.system("git checkout origin/main") + os.system("git reset --hard origin/main") + os.system("git am --abort") + os.chdir("..") + +def fetch_pr(pr_number, project): + url = f"https://github.com/{ORG}/{project}" + # use .patch at end of URL to get patch + resp = requests.get(f"{url}/pull/{pr_number}.patch") + resp.raise_for_status() + with open(f"patches/pr_{project}-{pr_number}.patch", "w") as f: + f.write(resp.text) + +def apply_pr(pr_number, pr_info, project): + r = os.system(f"git am -q ../patches/pr_{project}-{pr_number}.patch") + if r != 0: + print_level('error', f"{pr_info}: Patch failed for PR {pr_number}") + os.system("git am --abort") + else: + print_level('info', f"{pr_info}: Patch applied for PR {pr_number}") + + +def load_users(filename): + if not os.path.exists(filename): + print_level('error', f"File {filename} not found") + sys.exit(1) + with open(filename, "r") as f: + return f.read().splitlines() + +def print_comprehensive(pr, prefix=""): + for key, value in pr.items(): + if isinstance(value, dict): + print_comprehensive(value, prefix=f"{prefix}{key}.") + else: + print(f"{prefix}={key}: {value}") + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--project', help='project', required=True) + parser.add_argument('--allowusers', help='allowusers', default='users.txt') + parser.add_argument('--push', help='push to origin', action='store_true', default=False) + args = parser.parse_args() + users = load_users(args.allowusers) + if not os.path.exists("patches"): + os.mkdir("patches") + shallow_clone(args.project) + if args.push: + os.system(f"git remote set-url --push origin git@github.com:{ORG}/{args.project}.git") + prs = get_prs(args.project) + + # sort ascending by PR number + prs = sorted(prs, key=lambda pr: pr['number']) + for pr in prs: + # for debugging + #print_comprehensive(pr) + pr_info = f"PR {pr['number']} - {pr['state']} - {pr['user']['login']} - " + \ + f"{pr['title']}" + + if pr['user']['login'] not in users: + print_level('warning', f"{pr_info}: Skipping PR from unauthorized user") + continue + if pr['draft']: + print_level('warning', f"{pr_info}: Skipping PR due to draft status") + continue + if 'staging-skip' in [label['name'] for label in pr['labels']]: + print_level('warning', f"{pr_info}: Skipping PR due to staging-skip label") + continue + print_level('info', pr_info) + fetch_pr(pr['number'], args.project) + os.chdir(args.project) + apply_pr(pr['number'], pr_info, args.project) + os.chdir("..") + + # create branch staging-snapshot + os.chdir(args.project) + # delete branch if exists + os.system("git branch -D staging-snapshot") + os.system("git checkout -b staging-snapshot") + if args.push: + os.system("git push -f origin staging-snapshot") + # retrieve last commit id + last_commit = os.popen("git log -1 --pretty=%H").read().strip() + print_level('info', f"Last commit: {last_commit}") + # write as a file projectname.commit + os.chdir("..") + with open(f"{args.project}.commit", "w") as f: + f.write(last_commit) + + +if __name__ == '__main__': + main() + diff --git a/maestro/staging/staging.sh b/maestro/staging/staging.sh new file mode 100755 index 0000000..9bd2476 --- /dev/null +++ b/maestro/staging/staging.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# $(git show --pretty=format:%H -s origin/staging.kernelci.org) +REPOS=("kernelci-core" "kernelci-pipeline" "kernelci-api") + +preparing_repos() { + echo "Preparing repos staging snapshots branch" + for repo in ${REPOS[@]}; do + ./staging-branch.py --project $repo --push + done +} + +retrieving_revs() { + echo "Retrieving revisions" + core_rev=$(cat kernelci-core.commit) + pipeline_rev=$(cat kernelci-pipeline.commit) + api_rev=$(cat kernelci-api.commit) + + echo "kernelci-core: $core_rev" + echo "kernelci-pipeline: $pipeline_rev" + echo "kernelci-api: $api_rev" +} + +building_docker() { + echo "Building docker images" + retrieving_revs + if [ -z "$core_rev" ] || [ -z "$pipeline_rev" ] || [ -z "$api_rev" ]; then + echo "Error: One or more rev variables are not set" + exit 1 + fi + cache_arg="" + rev_arg="--build-arg core_rev=$core_rev --build-arg api_rev=$api_rev --build-arg pipeline_rev=$pipeline_rev" + px_arg='--prefix=kernelci/staging-' + #args="build --push $px_arg $cache_arg $rev_arg" + args="build $px_arg $cache_arg $rev_arg" + cd kernelci-core + # install dependencies + pip install -r requirements.txt --break-system-packages + ./kci docker $args kernelci +} + +preparing_repos +building_docker + + diff --git a/maestro/staging/users.txt b/maestro/staging/users.txt new file mode 100644 index 0000000..d8f950e --- /dev/null +++ b/maestro/staging/users.txt @@ -0,0 +1,72 @@ +10ne1 +aistcv +alexandrasp +aliceinwire +andrealmeid +a-wai +Bastian-Krause +BayLibre +broonie +cazou +chaws +ClangBuiltLinux +crazoes +denisyuji +dzickusrh +eballetbo +eds-collabora +evdenis +fbezdeka +gctucker +glunardi +Heidifahim +hardboprobot +helen-fornazier +hiwang123 +hthiery +igaw +jayaddison-collabora +JenySadadia +jeromebrunet +jluebbe +kees +kernelci +khilman +kholk +krisman +Lakshmipathi +laura-nao +lubinszARM +lufy90 +markfilion +mattface +mgalka +mharyam +musamaanjum +mwalle +mwasilew +montjoie +mgrzeschik +nfraprado +nkbelin +nuclearcat +ojeda +padovan +patersonc +pawiecz +penvirus +ribalda +roxell +sashalevin +sbdtu5498 +sjoerdsimons +sonnydeez +spbnick +sre +suram-nxp +tomeuv +touilkhouloud +VinceHillier +wangmingyu84 +williamklin +yurinnick