Improve CB tool

* Better errors
* Tuning for timeouts and query size
* Support for unnamed items

CHANGELOG.md

@@ -12,6 +12,18 @@
   trace to a class, similar to how the module name is included if you
   trace to a function or method.
 
+* Improve error reporting of `lobster-codebeamer`, you should get way
+  fewer raw exceptions and instead more helpful messages.
+
+* Add two new parameters to `lobster-codebeamer` if your codebeamer
+  instance is painfully slow: `--timeout` to increase the timeout for
+  each REST query and `--query-size` to limit how many items are
+  attempted to be fetched at once.
+
+* Add support for items without a summary. They are now named
+  something like "Unnamed item 12345". These items will show up as
+  problematic in the tracing report.
+
 ### 0.9.14
 
 * The `lobster-codebeamer` tool has a new option `--ignore-ssl-errors`

lobster/items.py

@@ -133,9 +133,10 @@ def determine_status(self, config, stab):
 
         level = config[self.level]
 
-        has_up_ref    = len(self.ref_up) > 0
-        has_just_up   = len(self.just_up) > 0 or len(self.just_global) > 0
-        has_just_down = len(self.just_down) > 0 or len(self.just_global) > 0
+        has_up_ref      = len(self.ref_up) > 0
+        has_just_up     = len(self.just_up) > 0 or len(self.just_global) > 0
+        has_just_down   = len(self.just_down) > 0 or len(self.just_global) > 0
+        has_init_errors = len(self.messages) > 0
 
         # Check up references
         ok_up = True
@@ -174,6 +175,10 @@ def determine_status(self, config, stab):
         else:
             self.tracing_status = Tracing_Status.MISSING
 
+        # Overwrite status if there are initial errors
+        if self.tracing_status == Tracing_Status.OK and has_init_errors:
+            self.tracing_status = Tracing_Status.PARTIAL
+
     def additional_data_from_json(self, level, data, schema_version):
         assert isinstance(level, str)
         assert isinstance(data, dict)

lobster/tools/codebeamer/codebeamer.py

@@ -55,11 +55,23 @@ def query_cb_single(cb_config, url):
     assert isinstance(cb_config, dict)
     assert isinstance(url, str)
 
-    result = requests.get(url,
-                          auth=(cb_config["user"],
-                                cb_config["pass"]),
-                          timeout=10.0,
-                          verify=cb_config["verify_ssl"])
+    try:
+        result = requests.get(url,
+                              auth=(cb_config["user"],
+                                    cb_config["pass"]),
+                              timeout=cb_config["timeout"],
+                              verify=cb_config["verify_ssl"])
+    except requests.exceptions.ReadTimeout:
+        print("Timeout when fetching %s" % url)
+        print("You can either:")
+        print("* increase the timeout with --timeout")
+        print("* decrease the query size with --query-size")
+        sys.exit(1)
+    except requests.exceptions.RequestException as err:
+        print("Could not fetch %s" % url)
+        print(err)
+        sys.exit(1)
+
     if result.status_code != 200:
         print("Could not fetch %s" % url)
         print("Status = %u" % result.status_code)
@@ -111,10 +123,11 @@ def get_query(mh, cb_config, query_id):
 
     while total_items is None or len(rv) < total_items:
         print("Fetching page %u of query..." % page_id)
-        url = "%s/query/%u/page/%u?pagesize=100" % \
+        url = "%s/query/%u/page/%u?pagesize=%u" % \
             (cb_config["base"],
              query_id,
-             page_id)
+             page_id,
+             cb_config["page_size"])
         data = query_cb_single(cb_config, url)
         assert len(data) == 1
         data = data["trackerItems"]
@@ -161,21 +174,33 @@ def to_lobster(cb_config, cb_item):
 
     # TODO: Parse item text
 
-    return Requirement(
+    # Get item name. Sometimes items do not have one, in which case we
+    # come up with one.
+    if "name" in cb_item:
+        item_name = cb_item["name"]
+    else:
+        item_name = "Unnamed item %u" % cb_item["id"]
+
+    req = Requirement(
         tag       = Tracing_Tag(namespace = "req",
                                 tag       = str(cb_item["id"]),
                                 version   = cb_item["version"]),
         location  = Codebeamer_Reference(cb_root = cb_config["root"],
                                          tracker = cb_item["tracker"]["id"],
                                          item    = cb_item["id"],
                                          version = cb_item["version"],
-                                         name    = cb_item["name"]),
+                                         name    = item_name),
         framework = "codebeamer",
         kind      = kind,
-        name      = cb_item["name"],
+        name      = item_name,
         text      = None,
         status    = status)
 
+    if "name" not in cb_item:
+        req.error("Item lacks a summary text")
+
+    return req
+
 
 def import_tagged(mh, cb_config, items_to_import):
     assert isinstance(mh, Message_Handler)
@@ -227,6 +252,16 @@ def main():
                     default=False,
                     help="ignore ssl errors and accept any certificate")
 
+    ap.add_argument("--query-size",
+                    type=int,
+                    default=100,
+                    help=("Fetch this many cb items at once (by default 100),"
+                          " reduce if you get too many timeouts."))
+    ap.add_argument("--timeout",
+                    type=int,
+                    default=30,
+                    help="Timeout in s (by default 30) for each REST call.")
+
     ap.add_argument("--cb-root", default=os.environ.get("CB_ROOT", None))
     ap.add_argument("--cb-user", default=os.environ.get("CB_USERNAME", None))
     ap.add_argument("--cb-pass", default=os.environ.get("CB_PASSWORD", None))
@@ -241,6 +276,8 @@ def main():
         "user"       : options.cb_user,
         "pass"       : options.cb_pass,
         "verify_ssl" : not options.ignore_ssl_errors,
+        "page_size"  : options.query_size,
+        "timeout"    : options.timeout,
     }
 
     if cb_config["root"] is None: