Additional totals added to summary log

HDRUK · May 21, 2024 · ddafde6 · ddafde6
1 parent c0883b9
commit ddafde6
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 2 deletions.
diff --git a/carrot/_version.py b/carrot/_version.py
@@ -1 +1 @@
-__version__ = '0.73'
+__version__ = '0.735'
diff --git a/carrot/cli/subcommands/run.py b/carrot/cli/subcommands/run.py
@@ -789,6 +789,12 @@ def mapstream(rules_file, output_dir, write_mode, person_file, omop_ddl_file, om
                                 else:
                                     key = srcfilename + "~" + datacol +"~" + tgtfile + "~" + outrecord[2] + "~"
                                     metrics.increment_key_count(key, "output_count")
+                                    key = srcfilename + "~all~" + tgtfile + "~" + outrecord[2] + "~"
+                                    metrics.increment_key_count(key, "output_count")
+                                    key = "all~all~" + tgtfile + "~" + outrecord[2] + "~"
+                                    metrics.increment_key_count(key, "output_count")
+                                    key = "all~all~all~" + outrecord[2] + "~"
+                                    metrics.increment_key_count(key, "output_count")
                                 fhd[tgtfile].write("\t".join(outrecord) + "\n")
                             else:
                                 key = srcfilename + "~all~" + tgtfile + "~all~"

diff --git a/carrot/tools/metrics.py b/carrot/tools/metrics.py
@@ -114,7 +114,8 @@ def get_mapstream_summary(self):
             if "output_count" in dvalue:
                 output_count = str(dvalue["output_count"])
 
-            if (int(output_count) >= self.log_threshold) or (tablename == "person"):
+            #if (int(output_count) >= self.log_threshold) or (tablename == "person"):
+            if (int(output_count) >= self.log_threshold):
               summary_str += self.dataset_name + "\t" + source + "\t" + fieldname + "\t" + tablename + "\t" + concept_id + "\t" + additional +"\t" + input_count + "\t" + invalid_person_ids + "\t" + invalid_date_fields + "\t" + invalid_source_fields + "\t" + output_count + "\n"
 
         return summary_str