VespaSync Pool Size

WorldHistoricalGazetteer · Jan 25, 2025 · 580b03d · 580b03d
1 parent 81aeda0
commit 580b03d
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 4 deletions.
diff --git a/vespa/repository/api/config.py b/vespa/repository/api/config.py
@@ -20,6 +20,9 @@ class VespaSyncExtended(VespaSync):
     """
     A subclass of VespaSync that adds the methods from VespaExtended.
     """
+    def __init__(self, app, pool_maxsize=20, **kwargs):
+        # Increase the pool size to 20 (see https://pyvespa.readthedocs.io/en/stable/reference-api.html#vespasync)
+        super().__init__(app, pool_maxsize=pool_maxsize, **kwargs)
 
     def get_existing(self, data_id: str = None, namespace: str = None, schema: str = None) -> dict:
         return self.app.get_existing(data_id=data_id, namespace=namespace, schema=schema)

diff --git a/vespa/repository/api/ingestion/processor.py b/vespa/repository/api/ingestion/processor.py
@@ -75,7 +75,7 @@ def update_existing_place(task):
             schema=schema,
             data_id=document_id,
             fields={
-                "names": existing_names + transformed_document['fields']['names']
+                "names": list(set(existing_names + transformed_document['fields']['names']))
             }
         )
         # logger.info(f"Update response: {response.get('status_code')}: {response}")
@@ -163,7 +163,7 @@ def feed_document(sync_app, namespace, schema, transformed_document, task_id, co
                 schema=schema,
                 data_id=existing_toponym_id,
                 fields={
-                    "places": existing_places + [document_id]
+                    "places": list(set(existing_places + [document_id]))
                 }
             )
         else:

diff --git a/vespa/repository/api/ingestion/triples.py b/vespa/repository/api/ingestion/triples.py
@@ -131,8 +131,9 @@ def feed_triple(task):
                     schema=schema,
                 )
                 if preexisting and schema == "place":
-                    document["fields"]["types"] = preexisting.get("fields").get("types", []) + document.get(
-                        "fields").get("types", [])
+                    document["fields"]["types"] = list(
+                        set(preexisting.get("fields", {}).get("types", []) + document.get("fields", {}).get("types",
+                                                                                                            [])))
 
             # logger.info(f"Updating {schema} {preexisting} with {document}")