DH-4724/add paginating to the db

dataherald/api/fastapi.py

@@ -332,12 +332,11 @@ def get_instructions(
         self, db_connection_id: str, page: int = 1, limit: int = 10
     ) -> List[Instruction]:
         instruction_repository = InstructionRepository(self.storage)
-        instructions = instruction_repository.find_by(
-            {"db_connection_id": db_connection_id}
+        return instruction_repository.find_by(
+            {"db_connection_id": db_connection_id},
+            page=page,
+            limit=limit,
         )
-        start_idx = (page - 1) * limit
-        end_idx = start_idx + limit
-        return instructions[start_idx:end_idx]
 
     @override
     def delete_instruction(self, db_connection_id: str, instruction_id: str) -> dict:

dataherald/db/__init__.py

@@ -29,7 +29,7 @@ def find_by_id(self, collection: str, id: str) -> dict:
         pass
 
     @abstractmethod
-    def find(self, collection: str, query: dict, sort: list = None) -> list:
+    def find(self, collection: str, query: dict, sort: list = None, page: int = 0, limit: int = 0) -> list:
         pass
 
     @abstractmethod

dataherald/db/mongo.py

@@ -40,10 +40,14 @@ def find_by_id(self, collection: str, id: str) -> dict:
         return self._data_store[collection].find_one({"_id": ObjectId(id)})
 
     @override
-    def find(self, collection: str, query: dict, sort: list = None) -> list:
+    def find(self, collection: str, query: dict, sort: list = None, page: int = 0, limit: int = 0) -> list:
+        skip_count = (page - 1) * limit
+        cursor = self._data_store[collection].find(query)
         if sort:
-            return self._data_store[collection].find(query).sort(sort)
-        return self._data_store[collection].find(query)
+            cursor = cursor.sort(sort)
+        if page > 0 and limit > 0:
+            cursor = cursor.skip(skip_count).limit(limit)
+        return list(cursor)
 
     @override
     def find_all(self, collection: str) -> list:

dataherald/repositories/instructions.py

@@ -35,8 +35,8 @@ def find_by_id(self, id: str) -> Instruction | None:
             return None
         return Instruction(**row)
 
-    def find_by(self, query: dict) -> list[Instruction]:
-        rows = self.storage.find(DB_COLLECTION, query)
+    def find_by(self, query: dict, page: int = 1, limit: int = 10) -> list[Instruction]:
+        rows = self.storage.find(DB_COLLECTION, query, page=page, limit=limit)
         result = []
         for row in rows:
             obj = Instruction(**row)

dataherald/tests/db/test_db.py

@@ -55,7 +55,7 @@ def find_by_id(self, collection: str, id: str) -> dict:
         return None
 
     @override
-    def find(self, collection: str, query: dict, sort: list = None) -> list:
+    def find(self, collection: str, query: dict, sort: list = None, page: int = 0, limit: int = 0) -> list:
         return []
 
     @override