refactor(backend,taxonomy-editor-frontend): Handle background task on frontend

backend/editor/api.py

@@ -3,8 +3,6 @@
 """
 import logging
 import os
-import shutil
-import tempfile
 
 # Required imports
 # ------------------------------------------------------------------------------------#
@@ -82,7 +80,7 @@ async def shutdown():
     """
     Shutdown database
     """
-    graph_db.shutdown_db()
+    await graph_db.shutdown_db()
 
 
 @app.middleware("http")
@@ -122,6 +120,8 @@ class StatusFilter(str, Enum):
 
     OPEN = "OPEN"
     CLOSED = "CLOSED"
+    LOADING = "LOADING"
+    FAILED = "FAILED"
 
 
 @app.exception_handler(RequestValidationError)
@@ -372,28 +372,35 @@ async def export_to_github(
 
 
 @app.post("/{taxonomy_name}/{branch}/import")
-async def import_from_github(request: Request, branch: str, taxonomy_name: str):
+async def import_from_github(
+    request: Request, branch: str, taxonomy_name: str, background_tasks: BackgroundTasks
+):
     """
     Get taxonomy from Product Opener GitHub repository
     """
     incoming_data = await request.json()
     description = incoming_data["description"]
 
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
+
     if not taxonomy.is_valid_branch_name():
         raise HTTPException(status_code=422, detail="branch_name: Enter a valid branch name!")
     if await taxonomy.does_project_exist():
         raise HTTPException(status_code=409, detail="Project already exists!")
     if not await taxonomy.is_branch_unique():
         raise HTTPException(status_code=409, detail="branch_name: Branch name should be unique!")
 
-    result = await taxonomy.import_from_github(description)
-    return result
+    status = await taxonomy.import_from_github(description, background_tasks)
+    return status
 
 
 @app.post("/{taxonomy_name}/{branch}/upload")
 async def upload_taxonomy(
-    branch: str, taxonomy_name: str, file: UploadFile, description: str = Form(...)
+    branch: str,
+    taxonomy_name: str,
+    file: UploadFile,
+    background_tasks: BackgroundTasks,
+    description: str = Form(...),
 ):
     """
     Upload taxonomy file to be parsed
@@ -406,11 +413,7 @@ async def upload_taxonomy(
     if not await taxonomy.is_branch_unique():
         raise HTTPException(status_code=409, detail="branch_name: Branch name should be unique!")
 
-    with tempfile.TemporaryDirectory(prefix="taxonomy-") as tmpdir:
-        filepath = f"{tmpdir}/{file.filename}"
-        with open(filepath, "wb") as f:
-            shutil.copyfileobj(file.file, f)
-        result = await taxonomy.upload_taxonomy(filepath, description)
+    result = await taxonomy.upload_taxonomy(file, description, background_tasks)
 
     return result
 

backend/editor/entries.py

@@ -1,10 +1,13 @@
 """
 Database helper functions for API
 """
+import asyncio
 import re
+import shutil
 import tempfile
 import urllib.request  # Sending requests
 
+from fastapi import BackgroundTasks, UploadFile
 from openfoodfacts_taxonomy_parser import normalizer  # Normalizing tags
 from openfoodfacts_taxonomy_parser import parser  # Parser for taxonomies
 from openfoodfacts_taxonomy_parser import unparser  # Unparser for taxonomies
@@ -82,59 +85,86 @@ async def create_node(self, label, entry, main_language_code):
         result = await get_current_transaction().run(" ".join(query), params)
         return (await result.data())[0]["n.id"]
 
-    async def parse_taxonomy(self, filename):
+    def parse_taxonomy(self, uploadfile=None):
         """
         Helper function to call the Open Food Facts Python Taxonomy Parser
         """
-        # Close current transaction to use the session variable in parser
-        await get_current_transaction().commit()
-
-        with SyncTransactionCtx() as session:
-            # Create parser object and pass current session to it
-            parser_object = parser.Parser(session)
-            try:
-                # Parse taxonomy with given file name and branch name
-                parser_object(filename, self.branch_name, self.taxonomy_name)
-                return True
-            except Exception as e:
-                raise TaxonomyParsingError() from e
+        if uploadfile is None:  # taxonomy is imported
+            base_url = (
+                "https://raw.githubusercontent.com/openfoodfacts/openfoodfacts-server"
+                "/main/taxonomies/"
+            )
+            filename = f"{self.taxonomy_name}.txt"
+            base_url += filename
+        else:  # taxonomy is uploaded
+            filename = uploadfile.filename
 
-    async def import_from_github(self, description):
-        """
-        Helper function to import a taxonomy from GitHub
-        """
-        base_url = (
-            "https://raw.githubusercontent.com/openfoodfacts/openfoodfacts-server"
-            "/main/taxonomies/"
-        )
-        filename = self.taxonomy_name + ".txt"
-        base_url += filename
         try:
             with tempfile.TemporaryDirectory(prefix="taxonomy-") as tmpdir:
                 # File to save the downloaded taxonomy
                 filepath = f"{tmpdir}/{filename}"
+                if uploadfile is None:
+                    # Downloads and creates taxonomy file in current working directory
+                    urllib.request.urlretrieve(base_url, filepath)
+                else:
+                    with open(filepath, "wb") as f:
+                        shutil.copyfileobj(uploadfile.file, f)
+
+                with SyncTransactionCtx() as session:
+                    # Create parser object and pass current session to it
+                    parser_object = parser.Parser(session)
+                    try:
+                        # Parse taxonomy with given file name and branch name
+                        parser_object(filepath, self.branch_name, self.taxonomy_name)
+                        return True
+                    except Exception as e:
+                        raise TaxonomyParsingError() from e
+        except Exception as e:
+            # async with TransactionCtx():
+            #     await self.set_project_status(status="FAILED")
+            # Call the asynchronous function to update status without awaiting it
+            asyncio.create_task(self.set_project_status_async(status="FAILED"))
+            raise TaxonomyImportError() from e
+
+    async def import_from_github(self, description, background_tasks: BackgroundTasks):
+        """
+        Helper function to import a taxonomy from GitHub
+        """
+        # Close current transaction to use the session variable in parser
+        await get_current_transaction().commit()
 
-                # Downloads and creates taxonomy file in current working directory
-                urllib.request.urlretrieve(base_url, filepath)
+        async with TransactionCtx():
+            await self.create_project(description)  # Creates a "project node" in neo4j
 
-                status = await self.parse_taxonomy(filepath)  # Parse the taxonomy
+        # Add the task to background tasks
+        background_tasks.add_task(self.parse_taxonomy)
+        print("Background task added")
 
-            async with TransactionCtx():
-                await self.create_project(description)  # Creates a "project node" in neo4j
+        # Create a new transaction context and create a "project node" in Neo4j
+        async with TransactionCtx():
+            await self.create_project(description)
 
-            return status
-        except Exception as e:
-            raise TaxonomyImportError() from e
+        return True
 
-    async def upload_taxonomy(self, filepath, description):
+    # async def upload_taxonomy(self, filepath, description,background_tasks: BackgroundTasks):
+    async def upload_taxonomy(
+        self, uploadfile: UploadFile, description, background_tasks: BackgroundTasks
+    ):
         """
         Helper function to upload a taxonomy file and create a project node
         """
+        # Close current transaction to use the session variable in parser
+        await get_current_transaction().commit()
         try:
-            status = await self.parse_taxonomy(filepath)
             async with TransactionCtx():
                 await self.create_project(description)
-            return status
+
+            # Add the task to background tasks
+            background_tasks.add_task(self.parse_taxonomy, uploadfile)
+            print("Background task added")
+            async with TransactionCtx():
+                await self.create_project(description)
+            return True
         except Exception as e:
             raise TaxonomyImportError() from e
 
@@ -244,10 +274,14 @@ async def create_project(self, description):
             "taxonomy_name": self.taxonomy_name,
             "branch_name": self.branch_name,
             "description": description,
-            "status": "OPEN",
+            "status": "LOADING",
         }
         await get_current_transaction().run(query, params)
 
+    async def set_project_status_async(self, status):
+        async with TransactionCtx():
+            await self.set_project_status(status=status)
+
     async def set_project_status(self, status):
         """
         Helper function to update a Taxonomy Editor project status

backend/editor/graph_db.py

@@ -49,11 +49,11 @@ def initialize_db():
     driver = neo4j.AsyncGraphDatabase.driver(uri)
 
 
-def shutdown_db():
+async def shutdown_db():
     """
     Close session and driver of Neo4J database
     """
-    driver.close()
+    await driver.close()
 
 
 def get_current_transaction():
@@ -85,6 +85,6 @@ def SyncTransactionCtx():
     Normally it should be reserved to background tasks
     """
     uri = settings.uri
-    driver = neo4j.GraphDatabase.driver(uri)
-    with driver.session() as _session:
-        yield _session
+    with neo4j.GraphDatabase.driver(uri) as driverSync:
+        with driverSync.session() as _session:
+            yield _session

taxonomy-editor-frontend/src/pages/go-to-project/index.tsx

@@ -29,22 +29,30 @@ const GoToProject = ({ clearNavBarLinks }: Props) => {
   const navigate = useNavigate();
 
   const { data, isPending, isError } = useFetch<ProjectsAPIResponse>(
-    `${API_URL}projects?status=OPEN`
+    `${API_URL}projects`
   );
 
   useEffect(() => {
     let newProjects: ProjectType[] = [];
 
     if (data) {
       const backendProjects = data.map(
-        ({ id, branch_name, taxonomy_name, description, errors_count }) => {
+        ({
+          id,
+          branch_name,
+          taxonomy_name,
+          description,
+          errors_count,
+          status,
+        }) => {
           return {
             id, // needed by MaterialTable as key
             projectName: id,
             taxonomyName: toTitleCase(taxonomy_name),
             branchName: branch_name,
             description: description,
             errors_count: errors_count,
+            status: status,
           };
         }
       );
@@ -117,6 +125,7 @@ const GoToProject = ({ clearNavBarLinks }: Props) => {
                 }
               },
             },
+            { title: "Status", field: "status" },
           ]}
           options={{
             actionsColumnIndex: -1,

taxonomy-editor-frontend/src/pages/root-nodes/index.tsx

@@ -84,7 +84,7 @@ const RootNodes = ({
     );
   }
 
-  if (isPending || !nodes) {
+  if (isPending || !nodes || nodes.length === 0) {
     return (
       <Box
         sx={{
@@ -93,6 +93,12 @@ const RootNodes = ({
         }}
       >
         <CircularProgress />
+        <Typography sx={{ m: 5 }} variant="h6">
+          Taxonomy parsing may take several minutes, depending on the complexity
+          of the taxonomy being imported.
+          <br />
+          Kindly refresh the page to view the updated status of the project.
+        </Typography>
       </Box>
     );
   }

taxonomy-editor-frontend/src/pages/startproject/index.tsx

@@ -42,6 +42,7 @@ const StartProject = ({ clearNavBarLinks }) => {
     const baseUrl = createBaseURL(toSnakeCase(taxonomyName), branchName);
     setLoading(true);
     const dataToBeSent = { description: description };
+    let errorMessage: string = "Unable to import";
 
     fetch(`${baseUrl}import`, {
       method: "POST",
@@ -51,12 +52,13 @@ const StartProject = ({ clearNavBarLinks }) => {
       .then(async (response) => {
         const responseBody = await response.json();
         if (!response.ok) {
-          throw new Error(responseBody?.detail ?? "Unable to import");
+          errorMessage = responseBody?.detail ?? "Unable to import";
+          throw new Error(errorMessage);
         }
         navigate(`/${toSnakeCase(taxonomyName)}/${branchName}/entry`);
       })
       .catch(() => {
-        setErrorMessage("Unable to import");
+        setErrorMessage(errorMessage);
       })
       .finally(() => setLoading(false));
   };