diff --git a/apps/autotest.py b/apps/autotest.py new file mode 100644 index 0000000..8a70113 --- /dev/null +++ b/apps/autotest.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +""" +Auto update support matrix result lib +""" + +import os +import tempfile +import time +import sys +import importlib +import subprocess +import re + +import pygit2 + +from .testlib.support_matrix_parser import System +from .testlib.template_parser import SysTemplate +from .utils.urlmarker import URL_REGEX + + +def maintain_template_lib(git_url: str, branch="test_template"): + """ + maintain support_matrix template lib + req: cwd under workspace + """ + workspace = os.getcwd() + template_lib = os.path.join(workspace, "template_lib") + if not os.path.exists(template_lib): + os.makedirs(template_lib) + _repo = pygit2.clone_repository( + git_url, template_lib, checkout_branch=branch) + print(f"Template lib is maintained at: {template_lib}") + return template_lib + + +def maintain_result_lib(git_url: str, branch="main"): + """ + maintain support_matrix result lib + req: cwd under workspace + """ + workspace = os.getcwd() + result_lib = os.path.join(workspace, "result_lib") + if not os.path.exists(result_lib): + os.makedirs(result_lib) + _repo = pygit2.clone_repository( + git_url, result_lib, checkout_branch=branch) + print(f"Result lib is maintained at: {result_lib}") + return result_lib + + +def walk_systems(template_lib: str, result_lib: str): + """ + walk through template_lib and update result_lib + """ + boards = filter(lambda x: x in [ + '.github', + 'assets', + '.git', + '.vscode', + '__pycache__', + ], os.listdir(template_lib)) + for board in boards: + template_board_path = os.path.join(template_lib, board) + if not os.path.isdir(template_board_path): + continue + systems = os.listdir(template_board_path) + result_board_path = os.path.join(result_lib, board) + for system in systems: + template_system_path = os.path.join(template_board_path, system) + if not os.path.isdir(template_system_path): + continue + result_system_path = os.path.join(result_board_path, system) + handle_system(template_system_path, result_system_path) + + +def run_script(script: str, template: SysTemplate) -> SysTemplate: + """ + script will always have a function: + default_proc(url: str, work_dir: str, sd: str, username: str, passwd: str) + the output is a res.cast and info.log, under cur dir + """ + sys.path.append(os.path.dirname(script)) + mod = importlib.import_module(os.path.basename(script)[:-3]) + try: + mod.default_proc( + template.url, + os.getcwd(), + "todo!", + template.username, + template.password + ) + with open("info.log", "r", "utf-8") as f: + info = f.read() + template.add_info(info) + # upload the cast to asciinema + cast = subprocess.run( + ["asciinema", "upload", "res.cast"], + capture_output=True, + check=True + ) + cast = cast.stdout.decode("utf-8") + reg = re.compile(URL_REGEX) + link = reg.search(cast).group() + template.add_asciinema(link) + except Exception as e: + print(f"Runtime Error: {e} when running { + script} with system {template.sys}") + template.status = "cfh" + finally: + template.auto_add_img_content(template.image_link) + template.add_username(template.username) + template.add_password(template.password) + template.add_version(template.sys_ver) + current_time = time.strftime("%Y-%m-%d", time.localtime()) + template.last_update = current_time + sys.path.pop() + return template + + +def handle_system(template_system_path: str, result_system_path: str): + res = System(result_system_path) + template = SysTemplate(template_system_path) + + board_name = result_system_path.split('/')[-2] + sys_name = template.sys + + # Currently, use update time to determine whether to update + res_time = int(time.mktime(time.strptime(res.last_update, "%Y-%m-%d"))) + template_time = int(time.mktime( + time.strptime(template.last_update, "%Y-%m-%d"))) + + if template_time <= res_time: + print(f"No need to update {board_name}/{sys_name}") + return + + # for specific board, the script will be written in {board_name}/{sys_name}.py, + # otherwise, use generic script + script_path = os.path.abspath(__file__) + script_path = os.path.dirname(script_path) + + specific_script = os.path.join(script_path, f"{board_name}/{sys_name}.py") + generic_script = os.path.join(script_path, "generic.py") + + if os.path.exists(specific_script): + script = specific_script + else: + script = generic_script + + template = run_script(script, template) + new_content = str(template) + with open(os.path.join(result_system_path, "README.md"), "w", encoding="utf-8") as f: + f.write(new_content) + print(f"Renew {board_name}/{sys_name} result") + +def main(): + # create a temproary workspace from mktemp -d + workspace = tempfile.mkdtemp() + os.chdir(workspace) + print(f"Workspace: {workspace}") + + git_url = "https://github.com/wychlw/support-matrix.git" + + template_lib = maintain_template_lib(git_url) + result_lib = maintain_result_lib(git_url) + + walk_systems(template_lib, result_lib) + +if __name__ == "__main__": + main() diff --git a/apps/testlib/support_matrix_parser.py b/apps/testlib/support_matrix_parser.py new file mode 100755 index 0000000..33fda20 --- /dev/null +++ b/apps/testlib/support_matrix_parser.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +""" +Parse metadata of systems and boards +""" +import os +import yaml +import frontmatter + +class System: + """ + eg: + --- + sys: deepin + sys_ver: 23 + sys_var: null + + status: basic + last_update: 2024-06-21 + --- + """ + sys: str + sys_ver: str | None + sys_var: str | None + status: str + last_update: str + link: str | None + + def strip(self): + """ + dummy for strip the system + """ + return self + + def __str__(self): + return status_map(self.status) + + def __len__(self): + return len(status_map(self.status)) + + def __init_by_file(self, path, base_link=""): + base_name = os.path.basename(path) + self.link = os.path.join(base_link, base_name) + meta_path = os.path.join(path, 'README.md') + if not os.path.exists(meta_path): + raise FileNotFoundError(f"{meta_path} not found") + with open(meta_path, 'r', encoding="utf-8") as file: + post = frontmatter.load(file) + if 'sys' not in post.keys(): + raise FileNotFoundError(f"{meta_path} has no frontmatter") + if post['sys'] == 'revyos': + self.sys = 'debian' + else: + self.sys = post['sys'] + self.sys_ver = post['sys_ver'] + self.sys_var = post['sys_var'] + self.status = post['status'] + self.last_update = post['last_update'] + + def __init__(self, *args, **kwargs): + if len(kwargs) > 0: + self.sys = kwargs['sys'] + self.sys_ver = kwargs['sys_ver'] + self.sys_var = kwargs['sys_var'] + self.status = kwargs['status'] + self.last_update = kwargs['last_update'] + self.link = kwargs['link'] + else: + self.__init_by_file(*args, **kwargs) + + +def status_map(status: str): + """ + map status to pretty string + """ + if status == 'wip': + return 'WIP' + if status == 'cft': + return 'CFT' + if status == 'cfh': + return 'CFH' + if status == 'basic': + return 'Basic' + if status == 'good': + return 'Good' + return status + + +class Board: + """ + a collection of systems and eg: + --- + product: VisionFive 2 + cpu: JH7110 + cpu_core: SiFive U74 + SiFive S7 + SiFive E24 + --- + """ + product: str + cpu: str + link: str + cpu_core: str + systems: list[System] + + def append_system(self, system: System): + """ + append a system to the board + """ + self.systems.append(system) + + def gen_row(self, system_arr: dict[str]): + """ + generate a row of the table + """ + row = [ + self.cpu, + self.cpu_core, + self.link, + self + ] + + na_count = 0 + + for k, _ in system_arr.items(): + for system in self.systems: + if system.sys == k: + row.append(system) + break + else: + row.append('N/A') + na_count += 1 + + if na_count == len(system_arr): + return None + + return row + + def strip(self): + """ + dummy for strip the board + """ + self.product = self.product.strip() + return self + + def __str__(self): + return self.product + + def __len__(self): + return len(self.product) + + def __init__(self, path: str): + base_name = os.path.basename(path) + self.link = base_name + readme_path = os.path.join(path, 'README.md') + if not os.path.exists(readme_path): + raise FileNotFoundError(f"{readme_path} not found") + with open(readme_path, 'r', encoding="utf-8") as file: + post = frontmatter.load(file) + self.product = post['product'] + self.cpu = post['cpu'] + self.cpu_core = post['cpu_core'] + self.systems = [] + + for folder in os.listdir(path): + if os.path.isdir(os.path.join(path, folder)): + try: + system = System(os.path.join(path, folder), self.link) + except FileNotFoundError as e: + global check_success + check_success = False + print(f"Error: {e}") + continue + self.append_system(system) + + if not os.path.exists(os.path.join(path, 'others.yml')): + return + with open(os.path.join(path, 'others.yml'), 'r', encoding="utf-8") as file: + data = yaml.load(file, Loader=yaml.FullLoader) + for i in data: + system = System( + sys=i['sys'], + sys_ver=i['sys_ver'], + sys_var=i['sys_var'], + status=i['status'], + last_update='2000-00-00', + link=None + ) + self.append_system(system) + + +class Systems: + """ + support matrix of systems + """ + linux: dict[str] + bsd: dict[str] + rtos: dict[str] + others: dict[str] + + exclude_dir = [ + '.github', + 'assets', + '.git', + '.vscode', + '__pycache__', + ] + boards: list[Board] + + def __init__(self, path): + meta_path = os.path.join(path, 'assets', 'metadata.yml') + with open(meta_path, 'r', encoding="utf-8") as file: + def mp(x): + res = {} + for l in x: + for i in l.items(): + res[i[0]] = i[1] + return res + data = yaml.load(file, Loader=yaml.FullLoader) + self.linux = mp(data['linux']) + self.bsd = mp(data['bsd']) + self.rtos = mp(data['rtos']) + self.others = mp(data['others']) + self.boards = [] + for folder in os.listdir(path): + if folder in self.exclude_dir: + continue + p = os.path.join(path, folder) + if not os.path.isdir(p): + continue + try: + board = Board(p) + self.boards.append(board) + except FileNotFoundError as e: + global check_success + check_success = False + print(f"Error: {e}") + continue diff --git a/apps/testlib/template_parser.py b/apps/testlib/template_parser.py new file mode 100644 index 0000000..319fd07 --- /dev/null +++ b/apps/testlib/template_parser.py @@ -0,0 +1,133 @@ +""" +Parse metadata for systems +""" +import os +import re +import urllib.parse +import frontmatter + +class SysTemplate: + """ + eg: + --- + sys: bianbu + sys_ver: 2.0rc1 + sys_var: null + + status: basic + last_update: 2024-09-20 + + image_link: https://archive.spacemit.com/image/k1/version/bianbu/v2.0rc1/bianbu-24.04-desktop-k1-v2.0rc1-release-20240909135447.img.zip + username: root + password: bianbu + --- + """ + sys: str + sys_ver: str | None + sys_var: str | None + status: str + last_update: str + image_link: str + username: str + password: str + + markdown_content: str + + def __init_by_file(self, path): + meta_path = os.path.join(path, 'README.md') + if not os.path.exists(meta_path): + raise FileNotFoundError(f"{meta_path} not found") + with open(meta_path, 'r', encoding="utf-8") as file: + post = frontmatter.load(file) + if 'sys' not in post.keys(): + raise FileNotFoundError(f"{meta_path} has no frontmatter") + self.sys = post['sys'] + self.sys_ver = post['sys_ver'] + self.sys_var = post['sys_var'] + self.status = post['status'] + self.last_update = post['last_update'] + self.image_link = post['image_link'] + self.username = post['username'] + self.password = post['password'] + self.markdown_content = post.content + + def __init__(self, *args, **kwargs): + if len(kwargs) > 0: + self.sys = kwargs['sys'] + self.sys_ver = kwargs['sys_ver'] + self.sys_var = kwargs['sys_var'] + self.status = kwargs['status'] + self.last_update = kwargs['last_update'] + self.image_link = kwargs['image_link'] + self.username = kwargs['username'] + self.password = kwargs['password'] + self.markdown_content = "" + else: + self.__init_by_file(args[0]) + def add_asciinema(self, link: str): + """ + eg: [![asciicast](https://asciinema.org/a/sAccZbGletHEuqNUrHYeCZkLa.svg)](https://asciinema.org/a/sAccZbGletHEuqNUrHYeCZkLa) + """ + self.markdown_content.replace("[[asciinema]]", f"[![asciinema]({link}.svg)]({link})") + def add_info(self, info: str): + """ + The output of the script + """ + self.markdown_content.replace("[[info]]", info) + def add_version(self, version: str): + """ + eg: v2.0rc1 + """ + self.markdown_content.replace("[[version]]", version) + def add_image_link(self, link: str): + """ + eg: https://archive + """ + self.markdown_content.replace("[[image_link]]", link) + def add_image_zip(self, link: str): + """ + eg: archive.img.tar.gz + """ + self.markdown_content.replace("[[image_file_zip]]", link) + def add_image_img(self, link: str): + """ + eg: archive.img + """ + self.markdown_content.replace("[[image_file_img]]", link) + def auto_add_img_content(self, link: str): + """ + link is the link to the image file + """ + img_zip_name = urllib.parse.urlparse(link).path.split("/")[-1] + img_name = img_zip_name + if img_name.contains(".img"): + img_name = img_name.split(".img")[0] + ".img" + self.add_image_link(link) + self.add_image_zip(img_zip_name) + self.add_image_img(img_name) + def add_username(self, username: str): + """ + eg: root + """ + self.markdown_content.replace("[[username]]", username) + def add_password(self, password: str): + """ + eg: root + """ + self.markdown_content.replace("[[password]]", password) + def strip_replacement(self) -> str: + """ + replace `[[.*]]` to null + """ + strinfo = re.compile(r"\[\[.*\]\]") + return strinfo.sub("", self.markdown_content) + def __str__(self): + article = frontmatter.Post(self.strip_replacement()) + article.metadata.update({ + "sys": self.sys, + "sys_ver": self.sys_ver, + "sys_var": self.sys_var, + "status": self.status, + "last_update": self.last_update, + }) + return frontmatter.dumps(article) diff --git a/apps/utils/urlmarker.py b/apps/utils/urlmarker.py new file mode 100644 index 0000000..f4df571 --- /dev/null +++ b/apps/utils/urlmarker.py @@ -0,0 +1,6 @@ +""" +the web url matching regex used by markdown +http://daringfireball.net/2010/07/improved_regex_for_matching_urls +https://gist.github.com/gruber/8891611 +""" +URL_REGEX = r"""(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:\'\".,<>?«»“”‘’])|(?:(?