(api) Add maintenance function to purge old data

api/main.py

@@ -36,7 +36,7 @@
 from fastapi_users import FastAPIUsers
 from beanie import PydanticObjectId
 from pydantic import BaseModel
 from kernelci.api.models import (
     Node,
     Hierarchy,
     PublishEvent,
@@ -61,6 +61,7 @@
     UserGroup,
 )
 from .metrics import Metrics
+from .maintenance import purge_old_nodes
 
 
 @asynccontextmanager
@@ -1088,6 +1089,17 @@
     return PlainTextResponse(response)
 
 
+@app.get('/maintenance/purge-old-nodes')
+async def purge_old_nodes(current_user: User = Depends(get_current_superuser)):
+    """Purge old nodes from the database
+    This is a maintenance operation and should be performed
+    only by superusers.
+    """
+    metrics.add('http_requests_total', 1)
+    await purge_old_nodes()
+    return "OK"
+
+
 versioned_app = VersionedFastAPI(
         app,
         version_format='{major}',

api/maintenance.py

@@ -0,0 +1,33 @@
+from pymongo import MongoClient
+import datetime
+import os
+
+
+def purge_ids(db, collection, ids):
+    print("Purging", len(ids), "from", collection)
+    db[collection].delete_many({"_id": {"$in": ids}})
+
+
+def connect_to_db():
+    mongo_service = os.environ["MONGO_SERVICE"]
+    if not mongo_service:
+        raise ValueError("MONGO_SERVICE environment variable is not set")
+    client = MongoClient(mongo_service)
+    db = client["kernelci"]
+    return db
+
+
+async def purge_old_nodes(age_days=180):
+    date_end = datetime.datetime.today() - datetime.timedelta(days=age_days)
+    db = connect_to_db()
+    nodes = db["nodes"].find({"created": {"$lt": date_end}})
+    # We need to delete node in chunks of 1000,
+    # to not block the main thread for too long
+    del_batch = []
+    for node in nodes:
+        del_batch.append(node["_id"])
+        if len(del_batch) == 1000:
+            purge_ids(db, "nodes", del_batch)
+            del_batch = []
+    if del_batch:
+        purge_ids(db, "nodes", del_batch)

kube/aks/api.yaml

@@ -19,9 +19,13 @@ spec:
       labels:
         app: api
     spec:
+      initContainers:
+        - name: wait-for-mongo
+          image: busybox
+          command: ["sh", "-c", "until nc -z mongo.kernelci-api.svc.cluster.local 27017; do echo waiting for mongo; sleep 2; done"]
       containers:
         - name: api
-          image: kernelci/api
+          image: kernelci/kernelci:api
           imagePullPolicy: Always
           resources:
             requests: