twister: support testing multiple toolchain variants

Added integration_toolchains to allow building/testing with multiple
toolchains available in the environment.

This changes the output structure and adds another level in the path
under twister_out signifying the toolchain used. The toolchain used
(variant) is also part of the json output now.

Signed-off-by: Anas Nashif <[email protected]>

scripts/pylib/twister/twisterlib/config_parser.py

@@ -66,6 +66,7 @@ class TwisterConfigParser:
                        "vendor_exclude": {"type": "set"},
                        "extra_sections": {"type": "list", "default": []},
                        "integration_platforms": {"type": "list", "default": []},
+                       "integration_toolchains": {"type": "list", "default": []},
                        "ignore_faults": {"type": "bool", "default": False },
                        "ignore_qemu_crash": {"type": "bool", "default": False },
                        "testcases": {"type": "list", "default": []},

scripts/pylib/twister/twisterlib/package.py

@@ -11,11 +11,12 @@
 
 
 class Artifacts:
-
+    """Package the test artifacts into a tarball."""
     def __init__(self, env):
         self.options = env.options
 
     def make_tarfile(self, output_filename, source_dirs):
+        """Create a tarball from the test artifacts."""
         root = os.path.basename(self.options.outdir)
         with tarfile.open(output_filename, "w:bz2") as tar:
             tar.add(self.options.outdir, recursive=False)
@@ -24,14 +25,21 @@ def make_tarfile(self, output_filename, source_dirs):
                 tar.add(d, arcname=os.path.join(root, f))
 
     def package(self):
+        """Package the test artifacts into a tarball."""
         dirs = []
-        with open(os.path.join(self.options.outdir, "twister.json")) as json_test_plan:
+        with open(
+            os.path.join(self.options.outdir, "twister.json"), encoding='utf-8'
+        ) as json_test_plan:
             jtp = json.load(json_test_plan)
             for t in jtp['testsuites']:
                 if t['status'] != TwisterStatus.FILTER:
                     p = t['platform']
                     normalized  = p.replace("/", "_")
-                    dirs.append(os.path.join(self.options.outdir, normalized, t['name']))
+                    dirs.append(
+                        os.path.join(
+                            self.options.outdir, normalized, t['toolchain'], t['name']
+                        )
+                    )
 
         dirs.extend(
             [

scripts/pylib/twister/twisterlib/reports.py

@@ -357,6 +357,8 @@ def json_report(self, filename, version="NA", platform=None, filters=None):
                 suite["used_rom"] = used_rom
 
             suite['retries'] = instance.retries
+            if instance.toolchain:
+                suite['toolchain'] = instance.toolchain
 
             if instance.dut:
                 suite["dut"] = instance.dut

scripts/pylib/twister/twisterlib/runner.py

@@ -658,6 +658,9 @@ def run_cmake(self, args="", filter_stages=None):
                 '-DCONFIG_COVERAGE=y'
             ])
 
+        if self.instance.toolchain:
+            cmake_args.append(f'-DZEPHYR_TOOLCHAIN_VARIANT={self.instance.toolchain}')
+
         # If needed, run CMake using the package_helper script first, to only run
         # a subset of all cmake modules. This output will be used to filter
         # testcases, and the full CMake configuration will be run for
@@ -830,7 +833,13 @@ def parse_generated(self, filter_stages=None):
             and self.env.options.west_flash is None
         ):
             logger.warning("Sysbuild test will be skipped. West must be used for flashing.")
-            return {os.path.join(self.platform.name, self.testsuite.name): True}
+            return {
+                os.path.join(
+                    self.platform.name,
+                    self.instance.toolchain,
+                    self.testsuite.name
+                ): True
+            }
 
         if self.testsuite and self.testsuite.filter:
             try:
@@ -846,9 +855,21 @@ def parse_generated(self, filter_stages=None):
                 raise se
 
             if not ret:
-                return {os.path.join(self.platform.name, self.testsuite.name): True}
+                return {
+                    os.path.join(
+                        self.platform.name,
+                        self.instance.toolchain,
+                        self.testsuite.name
+                    ): True
+                }
             else:
-                return {os.path.join(self.platform.name, self.testsuite.name): False}
+                return {
+                    os.path.join(
+                        self.platform.name,
+                        self.instance.toolchain,
+                        self.testsuite.name
+                    ): False
+                }
         else:
             self.platform.filter_data = filter_data
             return filter_data
@@ -1548,6 +1569,8 @@ def report_out(self, results):
                      and hasattr(self.instance.handler, 'seed')
                      and self.instance.handler.seed is not None ):
                     more_info += "/seed: " + str(self.options.seed)
+                if instance.toolchain:
+                    more_info += f" <{instance.toolchain}>"
             logger.info(
                 f"{results.done - results.filtered_static:>{total_tests_width}}/{total_to_do}"
                 f" {instance.platform.name:<25} {instance.testsuite.name:<50}"

scripts/pylib/twister/twisterlib/testinstance.py

@@ -49,7 +49,7 @@ class TestInstance:
 
     __test__ = False
 
-    def __init__(self, testsuite, platform, outdir):
+    def __init__(self, testsuite, platform, toolchain, outdir):
 
         self.testsuite: TestSuite = testsuite
         self.platform: Platform = platform
@@ -63,19 +63,23 @@ def __init__(self, testsuite, platform, outdir):
         self.execution_time = 0
         self.build_time = 0
         self.retries = 0
+        self.toolchain = toolchain
 
-        self.name = os.path.join(platform.name, testsuite.name)
+        self.name = os.path.join(platform.name, toolchain, testsuite.name)
         self.dut = None
 
         if testsuite.detailed_test_id:
-            self.build_dir = os.path.join(outdir, platform.normalized_name, testsuite.name)
+            self.build_dir = os.path.join(
+                outdir, platform.normalized_name, self.toolchain, testsuite.name
+            )
         else:
             # if suite is not in zephyr,
             # keep only the part after ".." in reconstructed dir structure
             source_dir_rel = testsuite.source_dir_rel.rsplit(os.pardir+os.path.sep, 1)[-1]
             self.build_dir = os.path.join(
                 outdir,
                 platform.normalized_name,
+                self.toolchain,
                 source_dir_rel,
                 testsuite.name
             )

scripts/pylib/twister/twisterlib/testplan.py

@@ -8,6 +8,7 @@
 import collections
 import copy
 import glob
+import itertools
 import json
 import logging
 import os
@@ -698,11 +699,14 @@ def load_from_file(self, file, filter_platform=None):
                 for ts in jtp.get("testsuites", []):
                     logger.debug(f"loading {ts['name']}...")
                     testsuite = ts["name"]
+                    toolchain = ts["toolchain"]
 
                     platform = self.get_platform(ts["platform"])
                     if filter_platform and platform.name not in filter_platform:
                         continue
-                    instance = TestInstance(self.testsuites[testsuite], platform, self.env.outdir)
+                    instance = TestInstance(
+                        self.testsuites[testsuite], platform, toolchain, self.env.outdir
+                    )
                     if ts.get("run_id"):
                         instance.run_id = ts.get("run_id")
 
@@ -777,7 +781,6 @@ def check_platform(self, platform, platform_list):
 
     def apply_filters(self, **kwargs):
 
-        toolchain = self.env.toolchain
         platform_filter = self.options.platform
         vendor_filter = self.options.vendor
         exclude_platform = self.options.exclude_platform
@@ -890,8 +893,16 @@ def apply_filters(self, **kwargs):
                     )
             # list of instances per testsuite, aka configurations.
             instance_list = []
-            for plat in platform_scope:
-                instance = TestInstance(ts, plat, self.env.outdir)
+            for itoolchain, plat in itertools.product(
+                ts.integration_toolchains or [None], platform_scope
+            ):
+                if itoolchain:
+                    toolchain = itoolchain
+                else:
+                    default_toolchain = "zephyr" if not self.env.toolchain else self.env.toolchain
+                    toolchain = default_toolchain if plat.arch not in ['posix', 'unit'] else "host"
+
+                instance = TestInstance(ts, plat, toolchain, self.env.outdir)
                 instance.run = instance.check_runnable(
                     self.options,
                     self.hwm
@@ -999,9 +1010,7 @@ def apply_filters(self, **kwargs):
                     )
 
                 if not force_toolchain \
-                        and toolchain and (toolchain not in plat.supported_toolchains) \
-                        and "host" not in plat.supported_toolchains \
-                        and ts.type != 'unit':
+                        and toolchain and (toolchain not in plat.supported_toolchains):
                     instance.add_filter(
                         f"Not supported by the toolchain: {toolchain}",
                         Filters.PLATFORM

scripts/schemas/twister/testsuite-schema.yaml

@@ -89,6 +89,11 @@ schema;scenario-schema:
       required: false
       sequence:
         - type: str
+    "integration_toolchains":
+      type: seq
+      required: false
+      sequence:
+        - type: str
     "ignore_faults":
       type: bool
       required: false

scripts/tests/twister/conftest.py

@@ -90,7 +90,7 @@ def instances_fixture(class_testplan, platforms_list, all_testsuites_dict, tmpdi
     platform = class_testplan.get_platform("demo_board_2")
     instance_list = []
     for _, testcase in all_testsuites_dict.items():
-        instance = TestInstance(testcase, platform, class_testplan.outdir)
+        instance = TestInstance(testcase, platform, 'zephyr', class_testplan.outdir)
         instance_list.append(instance)
     class_testplan.add_instances(instance_list)
     return class_testplan.instances

scripts/tests/twister/pytest_integration/test_harness_pytest.py

@@ -22,7 +22,7 @@ def testinstance() -> TestInstance:
     testsuite.sysbuild = False
     platform = Platform()
 
-    testinstance = TestInstance(testsuite, platform, 'outdir')
+    testinstance = TestInstance(testsuite, platform, 'zephyr', 'outdir')
     testinstance.handler = mock.Mock()
     testinstance.handler.options = mock.Mock()
     testinstance.handler.options.verbose = 1

scripts/tests/twister/test_harness.py

@@ -207,7 +207,7 @@ def test_robot_configure(tmp_path):
     outdir.mkdir()
 
     instance = TestInstance(
-        testsuite=mock_testsuite, platform=mock_platform, outdir=outdir
+        testsuite=mock_testsuite, platform=mock_platform, toolchain='zephyr',  outdir=outdir
     )
     instance.testsuite.harness_config = {
         "robot_testsuite": "/path/to/robot/test",
@@ -238,7 +238,7 @@ def test_robot_handle(tmp_path):
     outdir.mkdir()
 
     instance = TestInstance(
-        testsuite=mock_testsuite, platform=mock_platform, outdir=outdir
+        testsuite=mock_testsuite, platform=mock_platform, toolchain='zephyr', outdir=outdir
     )
 
     handler = Robot()
@@ -288,7 +288,7 @@ def test_robot_run_robot_test(tmp_path, caplog, exp_out, returncode, expected_st
     outdir.mkdir()
 
     instance = TestInstance(
-        testsuite=mock_testsuite, platform=mock_platform, outdir=outdir
+        testsuite=mock_testsuite, platform=mock_platform, toolchain='zephyr', outdir=outdir
     )
     instance.build_dir = "build_dir"
 
@@ -342,7 +342,7 @@ def test_console_configure(tmp_path, type, num_patterns):
     outdir.mkdir()
 
     instance = TestInstance(
-        testsuite=mock_testsuite, platform=mock_platform, outdir=outdir
+        testsuite=mock_testsuite, platform=mock_platform, toolchain='zephyr', outdir=outdir
     )
     instance.testsuite.harness_config = {
         "type": type,
@@ -403,7 +403,7 @@ def test_console_handle(
     outdir.mkdir()
 
     instance = TestInstance(
-        testsuite=mock_testsuite, platform=mock_platform, outdir=outdir
+        testsuite=mock_testsuite, platform=mock_platform, toolchain='zephyr', outdir=outdir
     )
 
     console = Console()
@@ -465,7 +465,7 @@ def test_pytest__generate_parameters_for_hardware(tmp_path, pty_value, hardware_
     outdir.mkdir()
 
     instance = TestInstance(
-        testsuite=mock_testsuite, platform=mock_platform, outdir=outdir
+        testsuite=mock_testsuite, platform=mock_platform, toolchain='zephyr', outdir=outdir
     )
 
     handler = mock.Mock()
@@ -563,7 +563,7 @@ def test_pytest_run(tmp_path, caplog):
     outdir.mkdir()
 
     instance = TestInstance(
-        testsuite=mock_testsuite, platform=mock_platform, outdir=outdir
+        testsuite=mock_testsuite, platform=mock_platform, toolchain='zephyr', outdir=outdir
     )
     instance.handler = handler
 
@@ -712,7 +712,7 @@ def test_test_handle(
     outdir = tmp_path / "ztest_out"
     with mock.patch('twisterlib.testsuite.TestSuite.get_unique', return_value="dummy_suite"):
         instance = TestInstance(
-            testsuite=mock_testsuite, platform=mock_platform, outdir=outdir
+            testsuite=mock_testsuite, platform=mock_platform, toolchain='zephyr', outdir=outdir
         )
     instance.handler = mock.Mock(options=mock.Mock(verbose=0), type_str="handler_type")
 
@@ -753,7 +753,7 @@ def gtest(tmp_path):
     outdir.mkdir()
 
     instance = TestInstance(
-        testsuite=mock_testsuite, platform=mock_platform, outdir=outdir
+        testsuite=mock_testsuite, platform=mock_platform, toolchain='zephyr', outdir=outdir
     )
 
     harness = Gtest()