Refactoring test execution framework to support multiple plugins

.flake8

@@ -10,6 +10,7 @@ exclude =
   examples
 per-file-ignores =
     api/api.py:E402
+    api/testrun.py
     api/test/*.py:E402
     db/models/init_db.py:E402,F401
     api/tmttestrun.py:E402,F401

.pre-commit-config.yaml

@@ -9,13 +9,13 @@ repos:
         language: system
       - id: prettier
         name: prettier
-        entry: "npx prettier . --write"
+        entry: "npx prettier . --write --log-level error"
         pass_filenames: false
         always_run: true
         language: system
       - id: eslint
         name: eslint
-        entry: "cd app && npx eslint src"
+        entry: "cd app && npx eslint --quiet src"
         pass_filenames: false
         always_run: true
         language: system
@@ -27,7 +27,7 @@ repos:
         language: system
       - id: tmt
         name: tmt
-        entry: "tmt lint . --exclude test-mr && tmt lint . --disable-check P005"
+        entry: "tmt lint . --disable-check P005"
         pass_filenames: false
         always_run: true
         language: system

Dockerfile-api

@@ -13,12 +13,13 @@
 
 # Set the public ip in the api/api_url.py
 ARG API_PORT=5000
 ARG ADMIN_PASSWORD=admin
 ENV BASIL_ADMIN_PASSWORD=${ADMIN_PASSWORD} BASIL_API_PORT=${API_PORT}
 
 # Init the database and
 # Write permission to db
-RUN mkdir -p /var/tmp && cd /BASIL-API/db/models && \
+RUN mkdir -p /var/tmp && \
+    cd /BASIL-API/db/models && \
     python3 init_db.py && \
     chmod a+rw /BASIL-API/db
 
@@ -29,10 +30,12 @@
 RUN tmt init
 
 # Remove BASIL ADMIN PASSWORD from the environment
 ENV BASIL_ADMIN_PASSWORD=
 
 EXPOSE ${BASIL_API_PORT}
 CMD echo "BASIL_API_PORT: ${BASIL_API_PORT}" && cd api && \
-                       gunicorn --access-logfile /var/tmp/tc-gunicorn-access.log \
-                       --error-logfile /var/tmp/tc-gunicorn-error.log \
-                       --bind 0.0.0.0:${BASIL_API_PORT} api:app 2>&1 | tee /var/tmp/tc-error.log
+                       gunicorn \
+                       --timeout 120 \
+                       --access-logfile /var/tmp/gunicorn-access.log \
+                       --error-logfile /var/tmp/gunicorn-error.log \
+                       --bind 0.0.0.0:${BASIL_API_PORT} api:app 2>&1 | tee /var/tmp/basil-api-error.log

api/spdx_manager.py

@@ -1,20 +1,22 @@
 import datetime
+import hashlib
+import json
+
+from spdx_tools.spdx.model import (Checksum, ChecksumAlgorithm, CreationInfo, Document, File, FileType, Relationship,
+                                   RelationshipType, Snippet)
+from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document
+from spdx_tools.spdx.writer.write_anything import write_file
+
 from db import db_orm
 from db.models.api import ApiModel
+from db.models.api_justification import ApiJustificationModel
 from db.models.api_sw_requirement import ApiSwRequirementModel
-from db.models.api_test_specification import ApiTestSpecificationModel
 from db.models.api_test_case import ApiTestCaseModel
-from db.models.api_justification import ApiJustificationModel
+from db.models.api_test_specification import ApiTestSpecificationModel
 from db.models.sw_requirement_sw_requirement import SwRequirementSwRequirementModel
-from db.models.sw_requirement_test_specification import SwRequirementTestSpecificationModel
 from db.models.sw_requirement_test_case import SwRequirementTestCaseModel
+from db.models.sw_requirement_test_specification import SwRequirementTestSpecificationModel
 from db.models.test_specification_test_case import TestSpecificationTestCaseModel
-import hashlib
-import json
-from spdx_tools.spdx.model import (Document, CreationInfo, Checksum, ChecksumAlgorithm, File,
-                                   FileType, Relationship, RelationshipType, Snippet)
-from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document
-from spdx_tools.spdx.writer.write_anything import write_file
 
 
 class SPDXManager():

api/testrun.py

@@ -0,0 +1,266 @@
+#! /bin/python3
+import argparse
+import os
+import sys
+import traceback
+
+currentdir = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(1, os.path.dirname(currentdir))
+
+from pyaml_env import parse_config
+from sqlalchemy.orm.exc import NoResultFound
+from testrun_github_actions import TestRunnerGithubActionsPlugin
+from testrun_gitlab_ci import TestRunnerGitlabCIPlugin
+from testrun_testing_farm import TestRunnerTestingFarmPlugin
+from testrun_tmt import TestRunnerTmtPlugin
+
+from db import db_orm
+from db.models.api_test_case import ApiTestCaseModel
+from db.models.notification import NotificationModel
+from db.models.sw_requirement_test_case import SwRequirementTestCaseModel
+from db.models.test_run import TestRunModel
+from db.models.test_specification_test_case import TestSpecificationTestCaseModel
+
+
+class TestRunner:
+    """
+    TestRunner class is aimed to read the request from the database
+    and to run the test using the desired plugin.
+    The default plugin is `tmt` implemented at testrun_tmt.py
+    this file provides a class named TestRunnerTmtPlugin that inherit from
+    TestRunnerBasePlugin and is aimed to implement the run() method.
+    The goal of the run() is to execute the test and provide information for the following
+    variables:
+    + log
+    + test_report
+    + test_result
+    + test_status
+
+    TestRunner - Error numbers
+     - 1: Unable to find the Test Run in the db
+     - 2: Test Run has been already triggered
+     - 3: Unable to find the Model of the parent item in the mapping definition
+     - 4: Unable to find the Mapping in the db
+     - 5: The selected plugin is not supported yet
+     - 6: Exceptions
+    """
+    RESULT_FAIL = 'fail'
+    RESULT_PASS = 'pass'
+
+    STATUS_CREATED = 'created'
+    STATUS_ERROR = 'error'
+    STATUS_RUNNING = 'running'
+    STATUS_COMPLETED = 'completed'
+
+    KERNEL_CI = 'KernelCI'
+    GITLAB_CI = 'gitlab_ci'
+    GITHUB_ACTIONS = 'github_actions'
+    TMT = 'tmt'
+    TESTING_FARM = 'testing_farm'
+
+    test_run_plugin_models = {'github_actions': TestRunnerGithubActionsPlugin,
+                              'gitlab_ci': TestRunnerGitlabCIPlugin,
+                              'KernelCI': None,
+                              'tmt': TestRunnerTmtPlugin,
+                              'testing_farm': TestRunnerTestingFarmPlugin}
+
+    runner_plugin = None
+    config = {}
+
+    id = None
+    execution_result = ''
+    execution_return_code = -1
+    test_result = ''
+    test_report = ''
+    ssh_keys_dir = os.path.join(currentdir, 'ssh_keys')  # Same as SSH_KEYS_PATH defined in api.py
+    presets_filepath = os.path.join(currentdir, 'testrun_plugin_presets.yaml')
+
+    dbi = None
+    db_test_run = None
+    db_test_case = None
+    mapping_to_model = None
+    mapping = None
+    DB_NAME = 'basil.db'
+
+    def __del__(self):
+        if self.dbi:
+            self.dbi.engine.dispose()
+
+    def __init__(self, id):
+        self.id = id
+        self.dbi = db_orm.DbInterface(self.DB_NAME)
+
+        # Test Run
+        try:
+            self.db_test_run = self.dbi.session.query(TestRunModel).filter(
+                TestRunModel.id == self.id
+            ).one()
+        except NoResultFound:
+            print("ERROR: Unable to find the Test Run in the db")
+            sys.exit(1)
+
+        if not self.db_test_run.log:
+            self.db_test_run.log = ""
+
+        if self.db_test_run.status != self.STATUS_CREATED:
+            print(f"ERROR: Test Run {id} has been already triggered, current status is `{self.db_test_run.status}`.")
+            sys.exit(2)
+
+        # Test Case
+        if self.db_test_run.mapping_to == ApiTestCaseModel.__tablename__:
+            self.mapping_model = ApiTestCaseModel
+        elif self.db_test_run.mapping_to == SwRequirementTestCaseModel.__tablename__:
+            self.mapping_model = SwRequirementTestCaseModel
+        elif self.db_test_run.mapping_to == TestSpecificationTestCaseModel.__tablename__:
+            self.mapping_model = TestSpecificationTestCaseModel
+        else:
+            # TODO: Update db with the error info
+            print("Unable to find the Model of the parent item in the mapping definition")
+            sys.exit(3)
+
+        try:
+            self.mapping = self.dbi.session.query(self.mapping_model).filter(
+                self.mapping_model.id == self.db_test_run.mapping_id
+            ).one()
+        except BaseException:
+            # TODO: Update db with the error info
+            print("ERROR: Unable to find the Mapping in the db")
+            sys.exit(4)
+
+        db_config = self.db_test_run.test_run_config.as_dict()
+
+        # Load preset configuration or explode the plugin_vars
+        preset = self.db_test_run.test_run_config.plugin_preset
+        if preset:
+            # Init config with preset if required
+            self.load_preset()
+        else:
+            plugin_vars = self.unpack_kv_str(db_config["plugin_vars"])
+            for k, v in plugin_vars.items():
+                db_config[k] = v
+
+        # Override preset values from test run configuration
+        # but for lists, for the ones we append to the existing values
+        db_config["env"] = self.unpack_kv_str(db_config["environment_vars"])
+        db_config["context"] = self.unpack_kv_str(db_config["context_vars"])
+
+        del db_config["plugin_vars"]
+        del db_config["environment_vars"]
+        del db_config["context_vars"]
+
+        for k, v in db_config.items():
+            if isinstance(v, dict):
+                if k in self.config.keys():
+                    pass
+                else:
+                    self.config[k] = {}
+                for kk, vv in v.items():
+                    self.config[k][kk] = vv
+            else:
+                if v:
+                    self.config[k] = v
+
+    def load_preset(self):
+        plugin = self.db_test_run.test_run_config.plugin
+        preset = self.db_test_run.test_run_config.plugin_preset
+
+        if preset:
+            presets = parse_config(self.presets_filepath)
+
+            if plugin in presets.keys():
+                tmp = [x for x in presets[plugin] if x["name"] == preset]
+                if tmp:
+                    # Init the config with the preset
+                    # Values from test_run_config will override preset values
+                    self.config = tmp[0]
+
+    def unpack_kv_str(self, _string):
+        # return a dict from a string formatted as
+        # key1=value1;key2=value2...
+        PAIRS_DIV = ';'
+        KV_DIV = '='
+        ret = {}
+        pairs = _string.split(PAIRS_DIV)
+        for pair in pairs:
+            if KV_DIV in pair:
+                if pair.count(KV_DIV) == 1:
+                    ret[pair.split(KV_DIV)[0].strip()] = pair.split(KV_DIV)[1].strip()
+        return ret
+
+    def pack_str_kv(self, _dict):
+        # return a string formatted as following
+        # key1=value1;key2=value2...
+        # from a flat key values dict
+        ret = ""
+        for k, v in _dict.items():
+            ret += f"{k}={v};"
+        if ret.endswith(";"):
+            ret = ret[:-1]
+        return ret
+
+    def notify(self):
+        # Notification
+        if self.test_result == self.RESULT_PASS:
+            variant = 'success'
+        else:
+            variant = 'danger'
+
+        notification = f'Test Run for Test Case ' \
+                       f'{self.mapping.test_case.title} as part of the sw component ' \
+                       f'{self.db_test_run.api.api}, library {self.db_test_run.api.library} ' \
+                       f'completed with: {self.test_result.upper()}'
+        notifications = NotificationModel(self.db_test_run.api,
+                                          variant,
+                                          f'Test Run for {self.db_test_run.api.api} {self.test_result.upper()}',
+                                          notification,
+                                          '',
+                                          f'/mapping/{self.db_test_run.api.id}')
+        self.dbi.session.add(notifications)
+        self.dbi.session.commit()
+
+    def publish(self):
+        """
+        Update the database with the current version of the TestRunModel instance
+        """
+        self.dbi.session.add(self.db_test_run)
+        self.dbi.session.commit()
+
+    def run(self):
+        # Test Run Plugin
+        try:
+            if self.db_test_run.test_run_config.plugin in self.test_run_plugin_models.keys():
+                self.runner_plugin = self.test_run_plugin_models[
+                    self.db_test_run.test_run_config.plugin](runner=self,
+                                                             currentdir=currentdir)
+                self.runner_plugin.run()
+                self.runner_plugin.collect_artifacts()
+                self.runner_plugin.cleanup()
+                self.publish()
+            else:
+                reason = "\nERROR: The selected plugin is not supported yet"
+                print(reason)
+                self.db_test_run.status = "error"
+                self.db_test_run.log += reason
+                self.publish()
+                sys.exit(5)
+        except Exception:
+            self.db_test_run.status = "error"
+            self.db_test_run.log += f"\nException: {traceback.format_exc()}"
+            self.publish()
+            print(self.db_test_run.log)
+            sys.exit(6)
+
+
+if __name__ == '__main__':
+    """
+    This file is called by the api.py via a terminal and
+    require as argument an id of the TestRun table
+    """
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--id", type=str, help="TODO")
+    args = parser.parse_args()
+
+    tr = TestRunner(id=args.id)
+    tr.run()
+    tr.notify()