From 824f7e5ea2efd69fecd534af3d98de419c494290 Mon Sep 17 00:00:00 2001 From: Suraj Deore Date: Tue, 3 Dec 2024 16:31:45 +0530 Subject: [PATCH] LOBSTER-Codebeamer tool to use configuration files - Replaced command-line arguments with a YAML-based configuration file for improved usability and maintainability. - Simplifies parameter management and enhances readability. --- CHANGELOG.md | 7 + lobster/tools/codebeamer/codebeamer.py | 215 +++++++++--------- packages/lobster-tool-codebeamer/README.md | 78 ++----- .../test_codebeamer_schema.py | 90 ++++++-- 4 files changed, 215 insertions(+), 175 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0584631f..07b32b7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ ### 0.9.20-dev +* Enhanced Configuration Management: Transitioned from command-line arguments to YAML + configuration files for LOBSTER-Codebeamer tools. + + **Rationale:** Managing numerous parameters via command-line arguments was cumbersome and error-prone for users. + + **Benefits:** Improved configurability, better readability, and simplified management of tool settings. + * `lobster-gtest` accepts XML nodes other than `testcase`, but ignores them. ### 0.9.19 diff --git a/lobster/tools/codebeamer/codebeamer.py b/lobster/tools/codebeamer/codebeamer.py index dfca5949..e9d4b6f7 100755 --- a/lobster/tools/codebeamer/codebeamer.py +++ b/lobster/tools/codebeamer/codebeamer.py @@ -44,6 +44,7 @@ from enum import Enum import json import requests +import yaml from lobster.items import Tracing_Tag, Requirement, Implementation, Activity from lobster.location import Codebeamer_Reference @@ -55,15 +56,26 @@ class SupportedConfigKeys(Enum): - REFS = "refs" - SCHEMA = "schema" + """Helper class to define supported configuration keys.""" + IMPORT_TAGGED = "import_tagged" + IMPORT_QUERY = "import_query" + VERIFY_SSL = "verify_ssl" + PAGE_SIZE = "page_size" + REFS = "refs" + SCHEMA = "schema" + CB_TOKEN = "token" + CB_ROOT = "root" + CB_USER = "user" + CB_PASS = "pass" + TIMEOUT = "timeout" + OUT = "out" @classmethod def as_set(cls) -> set: return {parameter.value for parameter in cls} -def add_refs_refrences(req, flat_values_list): +def add_refs_references(req, flat_values_list): # refs for value in flat_values_list: if value.get("id"): @@ -72,7 +84,7 @@ def add_refs_refrences(req, flat_values_list): map_reference_name_to_function = { - SupportedConfigKeys.REFS.value: add_refs_refrences + SupportedConfigKeys.REFS.value: add_refs_references } @@ -234,12 +246,7 @@ def to_lobster(cb_config, cb_item): else: kind = "codebeamer item" - if "status" in cb_item: - status = cb_item["status"].get("name", None) - else: - status = None - - # TODO: Parse item text + status = cb_item["status"].get("name", None) if "status" in cb_item else None # Get item name. Sometimes items do not have one, in which case we # come up with one. @@ -270,12 +277,11 @@ def to_lobster(cb_config, cb_item): isinstance(cb_item.get(displayed_name), list)) \ else [cb_item.get(displayed_name)] else: - flat_values_list = ( - list(value for custom_field - in cb_item["customFields"] - if custom_field["name"] == displayed_name and - custom_field.get("values") - for value in custom_field["values"])) + flat_values_list = [value for custom_field + in cb_item["customFields"] + if custom_field["name"] == displayed_name and + custom_field.get("values") + for value in custom_field["values"]] if not flat_values_list: continue @@ -373,32 +379,66 @@ def ensure_array_of_strings(instance): return [str(instance)] -def parse_cb_config(file_name): - assert isinstance(file_name, str) - assert os.path.isfile(file_name) +def parse_yaml_config(file_name: str): + """ + Parses a YAML configuration file and returns a validated configuration dictionary. - with open(file_name, "r", encoding='utf-8') as file: - data = json.loads(file.read()) + Args: + file_name (str): Path to the YAML configuration file. - json_config = {REFERENCES: {}} + Returns: + Dict[str, Any]: Parsed and validated configuration. - if TOKEN in data: - json_config["token"] = data.pop(TOKEN) + Raises: + ValueError: If `file_name` is not a string. + FileNotFoundError: If the file does not exist. + KeyError: If required fields are missing or unsupported keys are present. + """ + if not isinstance(file_name, str): + raise ValueError("File name must be a string.") + if not os.path.isfile(file_name): + raise FileNotFoundError(f"File not found: {file_name}") + + DEFAULT_VALUES = { + 'timeout': 30, + 'page_size': 100, + 'verify_ssl': False, + 'schema': 'Requirement', + } - provided_config_keys = set(data.keys()) - schema = data.get("schema", "Requirement").lower() + REQUIRED_FIELDS = {"import_tagged", "import_query"} - if not provided_config_keys.issubset(SupportedConfigKeys.as_set()): - raise KeyError("The provided config keys are not supported! " - "supported keys: '%s'" % - ', '.join(SupportedConfigKeys.as_set())) + with open(file_name, "r", encoding='utf-8') as file: + data = yaml.safe_load(file) or {} + + # Ensure at least one required field is present + if not REQUIRED_FIELDS & data.keys(): + raise KeyError(f"One of the required fields " + f"must be present: {', '.join(REQUIRED_FIELDS)}") + + # Build the configuration dictionary + json_config = { + "references": { + "refs": ensure_array_of_strings(data["refs"]) + } if "refs" in data else {}, + "token": data.pop("token", None), + "base": f"{data.get('root', '')}/cb/api/v3", + } - for key, value in data.items(): - json_config[REFERENCES][key] = ensure_array_of_strings(value) + # Validate supported keys + provided_config_keys = set(data.keys()) + unsupported_keys = provided_config_keys - SupportedConfigKeys.as_set() + if unsupported_keys: + raise KeyError( + f"Unsupported config keys: {', '.join(unsupported_keys)}. " + f"Supported keys are: {', '.join(SupportedConfigKeys.as_set())}." + ) - json_config[key] = ensure_array_of_strings(value) + # Merge with default values + json_config.update({key: data.get(key, DEFAULT_VALUES.get(key)) + for key in SupportedConfigKeys.as_set()}) - json_config["schema"] = schema + print(json_config) return json_config @@ -406,75 +446,35 @@ def parse_cb_config(file_name): def main(): # lobster-trace: codebeamer_req.Dummy_Requirement ap = argparse.ArgumentParser() - - modes = ap.add_mutually_exclusive_group(required=True) - modes.add_argument("--import-tagged", - metavar="LOBSTER_FILE", - default=None) - - modes.add_argument("--import-query", - metavar="CB_QUERY_ID", - default=None) - ap.add_argument("--config", - help=("name of codebeamer " - "config file, supported references: '%s'" % + help=("Path to YAML file with arguments, " + "by default (codebeamer-config.yaml) " + "supported references: '%s'" % ', '.join(SupportedConfigKeys.as_set())), - default=None) - - ap.add_argument("--ignore-ssl-errors", - action="store_true", - default=False, - help="ignore ssl errors and accept any certificate") - - ap.add_argument("--query-size", - type=int, - default=100, - help=("Fetch this many cb items at once (by default 100)," - " reduce if you get too many timeouts.")) - - ap.add_argument("--timeout", - type=int, - default=30, - help="Timeout in s (by default 30) for each REST call.") - - ap.add_argument("--schema", - default='requirement', - help="Specify the output schema" - "(Requirement, Implementation, Activity).") - - ap.add_argument("--cb-root", default=os.environ.get("CB_ROOT", None)) - ap.add_argument("--cb-user", default=os.environ.get("CB_USERNAME", None)) - ap.add_argument("--cb-pass", default=os.environ.get("CB_PASSWORD", None)) - ap.add_argument("--cb-token", default=None) - ap.add_argument("--out", default=None) + default=os.path.join(os.getcwd(), "codebeamer-config.yaml")) + + ap.add_argument("--out", + help=("Name of output file"), + default="codebeamer.lobster") + options = ap.parse_args() mh = Message_Handler() - cb_config = { - 'schema' : options.schema, - "root" : options.cb_root, - "base" : "%s/cb/api/v3" % options.cb_root, - "user" : options.cb_user, - "pass" : options.cb_pass, - "token" : options.cb_token, - "verify_ssl" : not options.ignore_ssl_errors, - "page_size" : options.query_size, - "timeout" : options.timeout, - } + if not os.path.isfile(options.config): + print((f"lobster-codebeamer: Config file '{options.config}' not found.")) + return 1 - if options.config: - if os.path.isfile(options.config): - cb_config.update(parse_cb_config(options.config)) - else: - ap.error("cannot open config file '%s'" % options.config) + cb_config = parse_yaml_config(options.config) + + if cb_config["out"] is None: + cb_config["out"] = options.out if cb_config["root"] is None: - ap.error("please set CB_ROOT or use --cb-root") + sys.exit("lobster-codebeamer: Please set 'root' in the config file") if not cb_config["root"].startswith("https://"): - ap.error("codebeamer root %s must start with https://") + sys.exit(f"Codebeamer root {cb_config['root']} must start with https://") if (cb_config["token"] is None and (cb_config["user"] is None or cb_config["pass"] is None)): @@ -484,23 +484,24 @@ def main(): netrc_config = netrc.netrc() auth = netrc_config.authenticators(cb_config["root"][8:]) if auth is not None: - print("using .netrc login for %s" % cb_config["root"]) + print("Using .netrc login for %s" % cb_config["root"]) cb_config["user"], _, cb_config["pass"] = auth if (cb_config["token"] is None and (cb_config["user"] is None or cb_config["pass"] is None)): - ap.error("please set --cb-token or add your token to the config-file" + sys.exit("lobster-codebeamer: please set --cb-token" + "or add your token to the config-file" "or use --cb-user and --cb-pass") items_to_import = set() - if options.import_tagged: - if not os.path.isfile(options.import_tagged): - ap.error("%s is not a file" % options.import_tagged) + if cb_config.get("import_tagged"): + if not os.path.isfile(cb_config["import_tagged"]): + sys.exit(f"lobster-codebeamer: {cb_config['import_tagged']} is not a file.") items = {} try: lobster_read(mh = mh, - filename = options.import_tagged, + filename = cb_config["import_tagged"], level = "N/A", items = items) except LOBSTER_Error: @@ -520,31 +521,31 @@ def main(): except ValueError: pass - elif options.import_query: + elif cb_config.get("import_query"): try: - query_id = int(options.import_query) + query_id = int(cb_config["import_query"]) except ValueError: - ap.error("query-id must be an integer") + sys.exit("lobster-codebeamer: Query ID must be an integer.") if query_id < 1: - ap.error("query-id must be a positive") + sys.exit("lobster-codebeamer: Query ID must be a positive integer.") try: - if options.import_tagged: + if cb_config.get("import_tagged"): items = import_tagged(mh, cb_config, items_to_import) - elif options.import_query: + elif cb_config.get("import_query"): items = get_query(mh, cb_config, query_id) except LOBSTER_Error: return 1 schema_config = get_schema_config(cb_config) - if options.out is None: + if cb_config["out"] is None: with sys.stdout as fd: lobster_write(fd, schema_config["class"], "lobster_codebeamer", items) else: - with open(options.out, "w", encoding="UTF-8") as fd: + with open(cb_config["out"], "w", encoding="UTF-8") as fd: lobster_write(fd, schema_config["class"], "lobster_codebeamer", items) - print(f"Written {len(items)} requirements to {options.out}") + print(f"Written {len(items)} requirements to {cb_config['out']}") return 0 diff --git a/packages/lobster-tool-codebeamer/README.md b/packages/lobster-tool-codebeamer/README.md index 216a426d..2038f8b0 100644 --- a/packages/lobster-tool-codebeamer/README.md +++ b/packages/lobster-tool-codebeamer/README.md @@ -14,7 +14,9 @@ requirements management tool * `lobster-codebeamer`: Extract requirements from codebeamer. ## Configuration -This tool works with an optional config file. +The tool requires a YAML configuration file to define its settings. +You must provide this file when running the tool to specify parameters to process. + It allows to configure the following features: ### References @@ -23,47 +25,33 @@ Codebeamer items can reference other items through This piece of information can be extracted by the tool, and serialized into the LOBSTER output file. It only needs to know which fields to take into account. -Following the LOBSTER JSON schema, the item references will be added to the +Following the LOBSTER yaml schema, the item references will be added to the `refs` property of the LOBSTER item. Accordingly, the configuration parameter to specify the codebeamer field names -is called `refs`, too. +is called `refs`, too. It can contain a single field name, or a list of field names. Field IDs cannot be used, only the field names. Examples: -```json -{ - "refs" : "cb-fieldname" -} +```yaml + refs : "cb-fieldname" ``` or -```json -{ - "refs" : ["cb-fieldname"] -} +```yaml + refs : ["cb-fieldname"] ``` or -```json -{ - "refs" : ["cb-fieldname1", "cb-fieldname2"] -} +```yaml + refs : ["cb-fieldname1", "cb-fieldname2"] ``` ### Bearer Authentication Token -It is also possible to define the Bearer authentication token in the -configuration file: -```json -{ - "token" : "your-codebeamer-Bearer-token" -} +Define the Bearer authentication token in the configuration file as follows +```yaml + token : "your-codebeamer-Bearer-token" ``` Note: -- The Bearer authentication token can also be provided as a command line - argument (use `--cb-token=your-codebeamer-Bearer-token`). -- If the token is provided in the configuration file and as command line - argument, then the configuration file takes precedence. -- If `--cb-user` or `--cb-pass` is given together with a Bearer token (either - as a command line argument or through the configuration file), then the +- If `--cb-user` or `--cb-pass` is given together with a Bearer token (through the configuration file), then the Bearer authentication method is used, and the username and/or password are ignored. - If neither a token nor a username or password are given, then the tool tries @@ -71,20 +59,16 @@ Note: directory. ### Example Configuration -Here is an example configuration file: -```json -{ +Here is an example of a configuration file: +```yaml "refs" : ["derived from", "satisfied by"], "token" : "SomeTokenStringABC123" -} ``` Is it also possible to define the codebeamer token in this file: -``` -{ +```yaml "refs" : "cb-fieldname", "token" : "your cb-token" -} ``` ### Schema @@ -95,24 +79,14 @@ You can also specify the type of schema for the resulting output file. The suppo If the schema is not specified, the tool will default to Requirement, and the schema lobster-req-trace will be used. -Here is an example configuration file: -```json -{ +Here is an example of a configuration file: +```yaml "schema": "Activity", // Specifies schema "refs": ["cb-fieldname1", "cb-fieldname2"] // Specifies references -} ``` If an invalid schema is provided, the tool will raise an exception. Supported schema values are Activity, Implementation, and Requirement. -## Command-Line Arguments and Configuration - -When running the tool, you can specify the `--schema` flag via the command line or set the `schema` value in -the configuration file. **The configuration file will always take precedence over the command-line argument**. - -- **`--schema`**: The schema to be used (optional, overrides the config file if provided). -- **Configuration file**: If `schema` is defined in the configuration file, it will be used, and the command-line `--schema` argument will be ignored. - ## Usage There are two ways you can use this tool: @@ -123,18 +97,16 @@ There are two ways you can use this tool: 2. Download all requirements generated by a saved codebeamer query (using `--import-query`) -* Configure the 'refs' upstream reference (this argument is optional) -(using `--config`) +* Configure the 'refs' upstream reference (this parameter is optional) * Additionally, you can specify the schema and references: - 1. Specify the schema of trace to be generated (optional, defaults to Requirement) - using the --schema argument, or configure it in the config file. - 2. Configure the 'refs' upstream reference (optional) using --config or - specify directly via command line. + 1. Specify the schema of the trace to be generated (optional, defaults to "Requirement"). You can set it using the + schema configure it in the configuration file. + 2. Configure the 'refs' upstream reference (optional). * For example: - ``` codepython lobster_codebeamer.py --config lobster-codebeamer-config.json --schema "Activity" --references "custom_ref_1``` + ```python lobster_codebeamer.py --config codebeamer-config.yaml``` This command will extract activity traces (lobster-act-trace) with specified references. diff --git a/tests-unit/lobster-codebeamer/test_codebeamer_schema.py b/tests-unit/lobster-codebeamer/test_codebeamer_schema.py index 312bc7b0..57595c81 100644 --- a/tests-unit/lobster-codebeamer/test_codebeamer_schema.py +++ b/tests-unit/lobster-codebeamer/test_codebeamer_schema.py @@ -1,34 +1,41 @@ import unittest +import tempfile +import os +import sys +import yaml + +from contextlib import redirect_stdout +from io import StringIO from lobster.items import Tracing_Tag, Requirement, Implementation, Activity from lobster.location import Codebeamer_Reference from lobster.errors import LOBSTER_Error -from lobster.tools.codebeamer.codebeamer import _create_common_params, _create_lobster_item - -class TestCreateFunctions(unittest.TestCase): - +from lobster.tools.codebeamer.codebeamer import _create_common_params, _create_lobster_item, main, parse_yaml_config +class TestCreateFunctions(unittest.TestCase): def setUp(self): + # Create temporary directory and files for testing + self.temp_dir = tempfile.TemporaryDirectory() + self.config_path = os.path.join(self.temp_dir.name, 'codebeamer-config.yaml') self.root_url = 'http://root_url' self.cb_item_template = { 'version': 1, 'tracker': {'id': 123} } - def generate_cb_item(self, item_id, name): """Generate a codebeamer item dictionary.""" + return { 'id': item_id, **self.cb_item_template, 'name': name } - def generate_common_params(self, namespace, item_name, kind, expected_class): """Generate a test case for common params and lobster item creation.""" cb_item = self.generate_cb_item(1, item_name) common_params = _create_common_params(namespace, cb_item, self.root_url, item_name, kind) - + return { 'common_params': common_params, 'item_name': item_name, @@ -38,7 +45,6 @@ def generate_common_params(self, namespace, item_name, kind, expected_class): 'kind' : kind } - def generate_test_case(self): return [ @@ -47,14 +53,11 @@ def generate_test_case(self): self.generate_common_params('act', 'Activity Item', 'activity', Activity), ] - def test_create_common_params(self): - test_cases = self.generate_test_case() for case in test_cases: with self.subTest(case=case): - self.assertEqual(case['common_params']['tag'].namespace, case['tag'].namespace) self.assertEqual(case['common_params']['tag'].tag, case['tag'].tag) self.assertEqual(case['common_params']['tag'].version, case['tag'].version) @@ -64,11 +67,11 @@ def test_create_common_params(self): self.assertEqual(case['common_params']['location'].version, case['location'].version) self.assertEqual(case['common_params']['location'].name, case['location'].name) self.assertEqual(case['common_params']['kind'], case['kind']) - def test_create_lobster_item(self): # lobster-trace: codebeamer_req.Dummy_Requirement_Unit_Test test_cases = self.generate_test_case() + for case in test_cases: with self.subTest(case=case): lobster_item = _create_lobster_item(case['expected_class'], case['common_params'], case['item_name'], None) @@ -83,6 +86,63 @@ def test_create_lobster_item(self): self.assertEqual(lobster_item.language, 'python') elif case['kind'] == 'activity': self.assertEqual(lobster_item.framework, 'codebeamer') - -if __name__ == '__main__': - unittest.main() \ No newline at end of file + + def tearDown(self): + """ + This method is called after every test case. + We use it to clean up resources like removing temporary files. + """ + # Remove the temporary file created in setUp after the test completes + self.temp_dir.cleanup() + + def test_main_missing_yaml_file(self): + """ + Test the main function with a non-existent YAML file. + This checks if the main function raises a FileNotFoundError when the file is missing. + """ + missing_config_path = os.path.join(self.temp_dir.name, 'missing-config.yaml') + sys.argv = ['codebeamer.py', '--config', missing_config_path] + with StringIO() as stdout, redirect_stdout(stdout): + exit_code = main() + output = stdout.getvalue() + + self.assertEqual(exit_code, 1) + self.assertIn(f"Config file '{missing_config_path}' not found", output) + + def test_main_missing_config_field(self): + missing_field_yaml_path = os.path.join(self.temp_dir.name, 'missing-field.yaml') + with open(missing_field_yaml_path, 'w', encoding='utf-8') as yaml_file: + yaml_file.write(""" + root: https://example.com + schema: Requirement + """) + + sys.argv = ['codebeamer.py', '--config', missing_field_yaml_path] + with self.assertRaises(KeyError) as context: + main() + + self.assertIn('One of the required fields must be present:', str(context.exception)) + + def test_unsupported_config_keys(self): + # Create a YAML file with unsupported keys + unsupported_config = { + "unsupported_key": "value", + "out": "trlc-config.conf", + "import_query":8805855 + } + unsupported_config_path = os.path.join(self.temp_dir.name, "unsupported_config.yaml") + with open(unsupported_config_path, "w", encoding="utf-8") as f: + yaml.dump(unsupported_config, f) + + # Update sys.argv with the unsupported config file + sys.argv = ["codebeamer.py", "--config", unsupported_config_path] + + # Capture stdout + with self.assertRaises(KeyError) as context: + main() + + self.assertIn("Unsupported config keys", str(context.exception)) + +# This block ensures that the tests are run when the script is executed directly +if __name__ == "__main__": + unittest.main()