From 1a0b9d4dae66ce4f81b8aaaa03cb860e2d43fecb Mon Sep 17 00:00:00 2001 From: Florian Schanda Date: Fri, 20 Oct 2023 15:36:49 +0200 Subject: [PATCH] Improve CB tool * Better errors * Tuning for timeouts and query size * Support for unnamed items --- CHANGELOG.md | 12 ++++++ lobster/items.py | 11 +++-- lobster/tools/codebeamer/codebeamer.py | 57 +++++++++++++++++++++----- 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 075fb9b2..ad1c14a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,18 @@ trace to a class, similar to how the module name is included if you trace to a function or method. +* Improve error reporting of `lobster-codebeamer`, you should get way + fewer raw exceptions and instead more helpful messages. + +* Add two new parameters to `lobster-codebeamer` if your codebeamer + instance is painfully slow: `--timeout` to increase the timeout for + each REST query and `--query-size` to limit how many items are + attempted to be fetched at once. + +* Add support for items without a summary. They are now named + something like "Unnamed item 12345". These items will show up as + problematic in the tracing report. + ### 0.9.14 * The `lobster-codebeamer` tool has a new option `--ignore-ssl-errors` diff --git a/lobster/items.py b/lobster/items.py index 8b4de0e1..7fb1774a 100644 --- a/lobster/items.py +++ b/lobster/items.py @@ -133,9 +133,10 @@ def determine_status(self, config, stab): level = config[self.level] - has_up_ref = len(self.ref_up) > 0 - has_just_up = len(self.just_up) > 0 or len(self.just_global) > 0 - has_just_down = len(self.just_down) > 0 or len(self.just_global) > 0 + has_up_ref = len(self.ref_up) > 0 + has_just_up = len(self.just_up) > 0 or len(self.just_global) > 0 + has_just_down = len(self.just_down) > 0 or len(self.just_global) > 0 + has_init_errors = len(self.messages) > 0 # Check up references ok_up = True @@ -174,6 +175,10 @@ def determine_status(self, config, stab): else: self.tracing_status = Tracing_Status.MISSING + # Overwrite status if there are initial errors + if self.tracing_status == Tracing_Status.OK and has_init_errors: + self.tracing_status = Tracing_Status.PARTIAL + def additional_data_from_json(self, level, data, schema_version): assert isinstance(level, str) assert isinstance(data, dict) diff --git a/lobster/tools/codebeamer/codebeamer.py b/lobster/tools/codebeamer/codebeamer.py index 1b941344..f636b02c 100755 --- a/lobster/tools/codebeamer/codebeamer.py +++ b/lobster/tools/codebeamer/codebeamer.py @@ -55,11 +55,23 @@ def query_cb_single(cb_config, url): assert isinstance(cb_config, dict) assert isinstance(url, str) - result = requests.get(url, - auth=(cb_config["user"], - cb_config["pass"]), - timeout=10.0, - verify=cb_config["verify_ssl"]) + try: + result = requests.get(url, + auth=(cb_config["user"], + cb_config["pass"]), + timeout=cb_config["timeout"], + verify=cb_config["verify_ssl"]) + except requests.exceptions.ReadTimeout: + print("Timeout when fetching %s" % url) + print("You can either:") + print("* increase the timeout with --timeout") + print("* decrease the query size with --query-size") + sys.exit(1) + except requests.exceptions.RequestException as err: + print("Could not fetch %s" % url) + print(err) + sys.exit(1) + if result.status_code != 200: print("Could not fetch %s" % url) print("Status = %u" % result.status_code) @@ -111,10 +123,11 @@ def get_query(mh, cb_config, query_id): while total_items is None or len(rv) < total_items: print("Fetching page %u of query..." % page_id) - url = "%s/query/%u/page/%u?pagesize=100" % \ + url = "%s/query/%u/page/%u?pagesize=%u" % \ (cb_config["base"], query_id, - page_id) + page_id, + cb_config["page_size"]) data = query_cb_single(cb_config, url) assert len(data) == 1 data = data["trackerItems"] @@ -161,7 +174,14 @@ def to_lobster(cb_config, cb_item): # TODO: Parse item text - return Requirement( + # Get item name. Sometimes items do not have one, in which case we + # come up with one. + if "name" in cb_item: + item_name = cb_item["name"] + else: + item_name = "Unnamed item %u" % cb_item["id"] + + req = Requirement( tag = Tracing_Tag(namespace = "req", tag = str(cb_item["id"]), version = cb_item["version"]), @@ -169,13 +189,18 @@ def to_lobster(cb_config, cb_item): tracker = cb_item["tracker"]["id"], item = cb_item["id"], version = cb_item["version"], - name = cb_item["name"]), + name = item_name), framework = "codebeamer", kind = kind, - name = cb_item["name"], + name = item_name, text = None, status = status) + if "name" not in cb_item: + req.error("Item lacks a summary text") + + return req + def import_tagged(mh, cb_config, items_to_import): assert isinstance(mh, Message_Handler) @@ -227,6 +252,16 @@ def main(): default=False, help="ignore ssl errors and accept any certificate") + ap.add_argument("--query-size", + type=int, + default=100, + help=("Fetch this many cb items at once (by default 100)," + " reduce if you get too many timeouts.")) + ap.add_argument("--timeout", + type=int, + default=30, + help="Timeout in s (by default 30) for each REST call.") + ap.add_argument("--cb-root", default=os.environ.get("CB_ROOT", None)) ap.add_argument("--cb-user", default=os.environ.get("CB_USERNAME", None)) ap.add_argument("--cb-pass", default=os.environ.get("CB_PASSWORD", None)) @@ -241,6 +276,8 @@ def main(): "user" : options.cb_user, "pass" : options.cb_pass, "verify_ssl" : not options.ignore_ssl_errors, + "page_size" : options.query_size, + "timeout" : options.timeout, } if cb_config["root"] is None: