fix: avoid assembly duplication due to duplicate taxonomy ids and add checks for duplicate entity ids (#367) (#368)

Author: hunterckx

catalog/build/intermediate/genomes-from-ncbi.tsv

@@ -109,7 +109,7 @@ def get_species_row(taxon_info, taxonomic_group_sets, taxonomic_levels):
 def get_species_df(taxonomy_ids, taxonomic_group_sets, taxonomic_levels):
   species_info = get_batched_ncbi_results(
     lambda ids: f"https://api.ncbi.nlm.nih.gov/datasets/v2/taxonomy/taxon/{",".join(ids)}/dataset_report",
-    [str(id) for id in taxonomy_ids],
+    [str(id) for id in set(taxonomy_ids)],
     "taxa"
   )
   return pd.DataFrame([get_species_row(info, taxonomic_group_sets, taxonomic_levels) for info in species_info])

catalog/build/py/package/catalog_build/build.py

@@ -2,7 +2,7 @@
 
 setup(
   name="catalog_build",
-  version="1.4.0",
+  version="1.4.1",
   packages=["catalog_build"],
   install_requires=["pandas", "requests", "PyYAML", "BeautifulSoup4", "lxml"],
 )

catalog/build/py/package/setup.py

@@ -6,6 +6,10 @@ import {
   BRCDataCatalogOrganism,
   WorkflowCategory,
 } from "../../../app/apis/catalog/brc-analytics-catalog/common/entities";
+import {
+  getGenomeId,
+  getOrganismId,
+} from "../../../app/apis/catalog/brc-analytics-catalog/common/utils";
 import {
   Organisms as SourceOrganisms,
   Workflow as SourceWorkflow,
@@ -28,6 +32,9 @@ async function buildCatalog(): Promise<void> {
   const organisms = buildOrganisms(genomes);
   const workflows = await buildWorkflows();
 
+  verifyUniqueIds("assembly", genomes, getGenomeId);
+  verifyUniqueIds("organism", organisms, getOrganismId);
+
   console.log("Assemblies:", genomes.length);
   await saveJson("catalog/output/assemblies.json", genomes);
 
@@ -228,6 +235,33 @@ async function saveJson(filePath: string, data: unknown): Promise<void> {
   await fsp.writeFile(filePath, JSON.stringify(data, undefined, 2) + "\n");
 }
 
+/**
+ * Take a list of entities and check for duplicate IDs, as calculated by the given function, and throw an error if there are any.
+ * @param entityName - Name of the entity type, to use in the error message.
+ * @param entities - Array of entities.
+ * @param getId - Function to get an entity's ID.
+ */
+function verifyUniqueIds<T>(
+  entityName: string,
+  entities: T[],
+  getId: (entity: T) => string
+): void {
+  const idCounts = new Map<string, number>();
+  for (const entity of entities) {
+    const id = getId(entity);
+    idCounts.set(id, (idCounts.get(id) ?? 0) + 1);
+  }
+  const duplicateIdEntries = Array.from(idCounts.entries()).filter(
+    ([, count]) => count > 1
+  );
+  if (duplicateIdEntries.length > 0) {
+    const duplicateIds = duplicateIdEntries.map(([id]) => id);
+    throw new Error(
+      `Duplicate ${entityName} IDs found: ${duplicateIds.join(", ")}`
+    );
+  }
+}
+
 function accumulateArrayValue<T>(array: T[] | undefined, value: T): T[] {
   if (!array) return [value];
   if (array.includes(value)) return array;