From a749c940faa35f9c979c561cba2d87d5f0339f3f Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Wed, 7 Feb 2024 17:39:09 +0100 Subject: [PATCH 1/4] fix(backend): fix create_node function --- backend/editor/entries.py | 20 +++++++++---------- .../src/pages/root-nodes/index.tsx | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/editor/entries.py b/backend/editor/entries.py index 5c832e11..80885b26 100644 --- a/backend/editor/entries.py +++ b/backend/editor/entries.py @@ -59,38 +59,38 @@ def get_label(self, id): else: return "ENTRY" - async def create_node(self, label, entry, main_language_code): + async def create_node(self, label, id, main_language_code) -> str: """ Helper function used for creating a node with given id and label """ - params = {"id": entry} + params = {"id": id} query = [f"""CREATE (n:{self.project_name}:{label})\n"""] stopwords = await self.get_stopwords_dict() - # Build all basic keys of a node + # Build all basic properties of a node if label == "ENTRY": # Normalizing new canonical tag - language_code, canonical_tag = entry.split(":", 1) + language_code, canonical_tag = id.split(":", 1) normalised_canonical_tag = parser_utils.normalize_text( canonical_tag, main_language_code, stopwords=stopwords ) - # Reconstructing and updation of node ID + # Update node id and add params params["id"] = language_code + ":" + normalised_canonical_tag params["main_language_code"] = main_language_code + params["canonical_tag"] = canonical_tag + params["normalised_canonical_tag"] = normalised_canonical_tag query.append( """ SET n.main_language = $main_language_code """ - ) # Required for only an entry - else: - canonical_tag = "" + ) # Required only for an entry + query.append(f""" SET n.tags_{main_language_code} = [$canonical_tag] """) + query.append(f""" SET n.tags_ids_{main_language_code} = [$normalised_canonical_tag] """) query.append(""" SET n.id = $id """) - query.append(f""" SET n.tags_{main_language_code} = [$canonical_tag] """) query.append(""" SET n.preceding_lines = [] """) query.append(""" RETURN n.id """) - params["canonical_tag"] = canonical_tag result = await get_current_transaction().run(" ".join(query), params) return (await result.data())[0]["n.id"] diff --git a/taxonomy-editor-frontend/src/pages/root-nodes/index.tsx b/taxonomy-editor-frontend/src/pages/root-nodes/index.tsx index 2a250c67..6d02a3db 100644 --- a/taxonomy-editor-frontend/src/pages/root-nodes/index.tsx +++ b/taxonomy-editor-frontend/src/pages/root-nodes/index.tsx @@ -169,7 +169,7 @@ const RootNodes = ({ - Taxonony Name + Taxonomy Name Branch Name From 59a822734b3f743006dfa456b65b455e9698655e Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 8 Feb 2024 12:46:30 +0100 Subject: [PATCH 2/4] refactor to use models and controller --- backend/editor/controllers/node_controller.py | 36 +++++++++++++++++++ backend/editor/entries.py | 30 ++++++---------- backend/editor/models/node_models.py | 13 ++++--- 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/backend/editor/controllers/node_controller.py b/backend/editor/controllers/node_controller.py index b82cc3e9..f09599a3 100644 --- a/backend/editor/controllers/node_controller.py +++ b/backend/editor/controllers/node_controller.py @@ -1,4 +1,7 @@ +from openfoodfacts_taxonomy_parser import utils as parser_utils + from ..graph_db import get_current_transaction +from ..models.node_models import EntryNodeCreate async def delete_project_nodes(project_id: str): @@ -12,3 +15,36 @@ async def delete_project_nodes(project_id: str): DETACH DELETE n """ await get_current_transaction().run(query) + + +async def create_entry_node( + project_id: str, entry_node: EntryNodeCreate, stopwords: dict[str, list[str]] +) -> str: + """ + Creates a new entry node in the database + """ + # Create params dict + params = {"entry_node": {"main_language": entry_node.main_language_code, "preceding_lines": []}} + + # Add tags and normalized tags for each language + for language_code, tags in entry_node.tags.items(): + normalized_tags = [ + parser_utils.normalize_text(tag, language_code, stopwords=stopwords) for tag in tags + ] + params["entry_node"][f"tags_{language_code}"] = tags + params["entry_node"][f"tags_ids_{language_code}"] = normalized_tags + + # Add node id + params["entry_node"]["id"] = ( + entry_node.main_language_code + + ":" + + params["entry_node"][f"tags_ids_{entry_node.main_language_code}"][0] + ) + + query = f""" + CREATE (n:{project_id}:ENTRY $entry_node) + RETURN n.id + """ + + result = await get_current_transaction().run(query, params) + return (await result.data())[0]["n.id"] diff --git a/backend/editor/entries.py b/backend/editor/entries.py index 80885b26..1a622a17 100644 --- a/backend/editor/entries.py +++ b/backend/editor/entries.py @@ -16,6 +16,7 @@ from openfoodfacts_taxonomy_parser import utils as parser_utils from . import settings +from .controllers.node_controller import create_entry_node from .controllers.project_controller import create_project, edit_project, get_project from .exceptions import GithubBranchExistsError # Custom exceptions from .exceptions import ( @@ -30,6 +31,7 @@ TransactionCtx, get_current_transaction, ) +from .models.node_models import EntryNodeCreate from .models.project_models import ProjectCreate, ProjectEdit, ProjectStatus from .utils import file_cleanup @@ -63,34 +65,22 @@ async def create_node(self, label, id, main_language_code) -> str: """ Helper function used for creating a node with given id and label """ - params = {"id": id} - query = [f"""CREATE (n:{self.project_name}:{label})\n"""] stopwords = await self.get_stopwords_dict() - # Build all basic properties of a node if label == "ENTRY": - # Normalizing new canonical tag - language_code, canonical_tag = id.split(":", 1) - normalised_canonical_tag = parser_utils.normalize_text( - canonical_tag, main_language_code, stopwords=stopwords + tags = {main_language_code: [id.split(":", 1)[1]]} + return await create_entry_node( + self.project_name, + EntryNodeCreate(main_language_code=main_language_code, tags=tags), + stopwords, ) - # Update node id and add params - params["id"] = language_code + ":" + normalised_canonical_tag - params["main_language_code"] = main_language_code - params["canonical_tag"] = canonical_tag - params["normalised_canonical_tag"] = normalised_canonical_tag - - query.append( - """ SET n.main_language = $main_language_code """ - ) # Required only for an entry - query.append(f""" SET n.tags_{main_language_code} = [$canonical_tag] """) - query.append(f""" SET n.tags_ids_{main_language_code} = [$normalised_canonical_tag] """) - + # TODO: This part is not used currently as we don't create synonyms and stopwords + params = {"id": id} + query = [f"""CREATE (n:{self.project_name}:{label})\n"""] query.append(""" SET n.id = $id """) query.append(""" SET n.preceding_lines = [] """) query.append(""" RETURN n.id """) - result = await get_current_transaction().run(" ".join(query), params) return (await result.data())[0]["n.id"] diff --git a/backend/editor/models/node_models.py b/backend/editor/models/node_models.py index 80923990..bb6749aa 100644 --- a/backend/editor/models/node_models.py +++ b/backend/editor/models/node_models.py @@ -1,18 +1,17 @@ """ Required pydantic models for API """ -from typing import List - from .base_models import BaseModel -class Marginal(BaseModel): - preceding_lines: List +class Header(BaseModel): + pass -class Header(Marginal): +class Footer(BaseModel): pass -class Footer(Marginal): - pass +class EntryNodeCreate(BaseModel): + main_language_code: str + tags: dict[str, list[str]] # {language_code: [tag1, tag2, ...]} From e6d27522623fe38b74fee199a28ef9e873337497 Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 8 Feb 2024 13:44:45 +0100 Subject: [PATCH 3/4] refactor further --- backend/editor/api.py | 20 +++++-------- backend/editor/controllers/node_controller.py | 23 +++++--------- backend/editor/entries.py | 30 +++++++------------ backend/editor/models/node_models.py | 2 +- .../components/CreateNodeDialogContent.tsx | 11 +++---- 5 files changed, 30 insertions(+), 56 deletions(-) diff --git a/backend/editor/api.py b/backend/editor/api.py index 2ec22a79..099e37b3 100644 --- a/backend/editor/api.py +++ b/backend/editor/api.py @@ -403,26 +403,22 @@ async def upload_taxonomy( return result -@app.post("/{taxonomy_name}/{branch}/nodes") -async def create_node(request: Request, branch: str, taxonomy_name: str): +@app.post("/{taxonomy_name}/{branch}/entry") +async def create_entry_node(request: Request, branch: str, taxonomy_name: str): """ - Creating a new node in a taxonomy + Creating a new entry node in a taxonomy """ taxonomy = TaxonomyGraph(branch, taxonomy_name) incoming_data = await request.json() - id = incoming_data["id"] + name = incoming_data["name"] main_language = incoming_data["main_language"] - if id is None: - raise HTTPException(status_code=400, detail="Invalid id") + if name is None: + raise HTTPException(status_code=400, detail="Invalid node name") if main_language is None: raise HTTPException(status_code=400, detail="Invalid main language code") - label = taxonomy.get_label(id) - normalized_id = await taxonomy.create_node(label, id, main_language) - if label == "ENTRY": - await taxonomy.add_node_to_end(label, normalized_id) - else: - await taxonomy.add_node_to_beginning(label, normalized_id) + normalized_id = await taxonomy.create_entry_node(name, main_language) + await taxonomy.add_node_to_end("ENTRY", normalized_id) @app.post("/{taxonomy_name}/{branch}/entry/{entry}") diff --git a/backend/editor/controllers/node_controller.py b/backend/editor/controllers/node_controller.py index f09599a3..6dc39f59 100644 --- a/backend/editor/controllers/node_controller.py +++ b/backend/editor/controllers/node_controller.py @@ -23,23 +23,14 @@ async def create_entry_node( """ Creates a new entry node in the database """ + name, language_code = entry_node.name, entry_node.main_language_code + normalized_name = parser_utils.normalize_text(name, language_code, stopwords=stopwords) + # Create params dict - params = {"entry_node": {"main_language": entry_node.main_language_code, "preceding_lines": []}} - - # Add tags and normalized tags for each language - for language_code, tags in entry_node.tags.items(): - normalized_tags = [ - parser_utils.normalize_text(tag, language_code, stopwords=stopwords) for tag in tags - ] - params["entry_node"][f"tags_{language_code}"] = tags - params["entry_node"][f"tags_ids_{language_code}"] = normalized_tags - - # Add node id - params["entry_node"]["id"] = ( - entry_node.main_language_code - + ":" - + params["entry_node"][f"tags_ids_{entry_node.main_language_code}"][0] - ) + params = {"entry_node": {"main_language": language_code, "preceding_lines": []}} + params["entry_node"]["id"] = language_code + ":" + normalized_name + params["entry_node"][f"tags_{language_code}"] = [name] + params["entry_node"][f"tags_ids_{language_code}"] = [normalized_name] query = f""" CREATE (n:{project_id}:ENTRY $entry_node) diff --git a/backend/editor/entries.py b/backend/editor/entries.py index 1a622a17..bec50097 100644 --- a/backend/editor/entries.py +++ b/backend/editor/entries.py @@ -61,28 +61,17 @@ def get_label(self, id): else: return "ENTRY" - async def create_node(self, label, id, main_language_code) -> str: + async def create_entry_node(self, name, main_language_code) -> str: """ - Helper function used for creating a node with given id and label + Helper function used to create an entry node with given name and main language """ stopwords = await self.get_stopwords_dict() - if label == "ENTRY": - tags = {main_language_code: [id.split(":", 1)[1]]} - return await create_entry_node( - self.project_name, - EntryNodeCreate(main_language_code=main_language_code, tags=tags), - stopwords, - ) - - # TODO: This part is not used currently as we don't create synonyms and stopwords - params = {"id": id} - query = [f"""CREATE (n:{self.project_name}:{label})\n"""] - query.append(""" SET n.id = $id """) - query.append(""" SET n.preceding_lines = [] """) - query.append(""" RETURN n.id """) - result = await get_current_transaction().run(" ".join(query), params) - return (await result.data())[0]["n.id"] + return await create_entry_node( + self.project_name, + EntryNodeCreate(name=name, main_language_code=main_language_code), + stopwords, + ) async def get_local_taxonomy_file(self, tmpdir: str, uploadfile: UploadFile): filename = uploadfile.filename @@ -335,6 +324,7 @@ async def add_node_to_end(self, label, entry): """ await get_current_transaction().run(query, {"id": entry, "endnodeid": end_node["id"]}) + # UNUSED FOR NOW async def add_node_to_beginning(self, label, entry): """ Helper function which adds an existing node to beginning of taxonomy @@ -599,8 +589,8 @@ async def update_node_children(self, entry, new_children_ids): created_child_ids = [] for child in to_create: - main_language_code = child.split(":", 1)[0] - created_node_id = await self.create_node("ENTRY", child, main_language_code) + main_language_code, child_name = child.split(":", 1) + created_node_id = await self.create_entry_node(child_name, main_language_code) created_child_ids.append(created_node_id) # TODO: We would prefer to add the node just after its parent entry diff --git a/backend/editor/models/node_models.py b/backend/editor/models/node_models.py index bb6749aa..0ec68aa7 100644 --- a/backend/editor/models/node_models.py +++ b/backend/editor/models/node_models.py @@ -13,5 +13,5 @@ class Footer(BaseModel): class EntryNodeCreate(BaseModel): + name: str main_language_code: str - tags: dict[str, list[str]] # {language_code: [tag1, tag2, ...]} diff --git a/taxonomy-editor-frontend/src/components/CreateNodeDialogContent.tsx b/taxonomy-editor-frontend/src/components/CreateNodeDialogContent.tsx index 3a528138..80b4284f 100644 --- a/taxonomy-editor-frontend/src/components/CreateNodeDialogContent.tsx +++ b/taxonomy-editor-frontend/src/components/CreateNodeDialogContent.tsx @@ -30,15 +30,12 @@ const CreateNodeDialogContent = ({ const baseUrl = createBaseURL(taxonomyName, branchName); - const handleAddNode = () => { + const handleAddEntryNode = () => { setIsFailedSubmit(false); - // TODO: Add support for synonyms and stopwords + const data = { name: newNodeName, main_language: newLanguageCode }; - const newNodeID = newLanguageCode + ":" + newNodeName; // Reconstructing node ID - const data = { id: newNodeID, main_language: newLanguageCode }; - - fetch(`${baseUrl}nodes`, { + fetch(`${baseUrl}entry`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(data), @@ -95,7 +92,7 @@ const CreateNodeDialogContent = ({ From e90c2571edbef6f8506487b2ae12cee034419fad Mon Sep 17 00:00:00 2001 From: Charles Perier Date: Thu, 8 Feb 2024 14:36:04 +0100 Subject: [PATCH 4/4] resolve comments --- backend/editor/api.py | 22 ++++++++----------- backend/editor/controllers/node_controller.py | 12 ++++++---- backend/editor/models/node_models.py | 9 ++++++++ .../components/CreateNodeDialogContent.tsx | 2 +- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/backend/editor/api.py b/backend/editor/api.py index 099e37b3..4ccca373 100644 --- a/backend/editor/api.py +++ b/backend/editor/api.py @@ -38,7 +38,7 @@ from .exceptions import GithubBranchExistsError, GithubUploadError # Data model imports -from .models.node_models import Footer, Header +from .models.node_models import EntryNodeCreate, Footer, Header, NodeType from .models.project_models import Project, ProjectEdit, ProjectStatus from .scheduler import scheduler_lifespan @@ -403,22 +403,18 @@ async def upload_taxonomy( return result -@app.post("/{taxonomy_name}/{branch}/entry") -async def create_entry_node(request: Request, branch: str, taxonomy_name: str): +@app.post("/{taxonomy_name}/{branch}/entry", status_code=status.HTTP_201_CREATED) +async def create_entry_node( + branch: str, taxonomy_name: str, new_entry_node: EntryNodeCreate +) -> None: """ Creating a new entry node in a taxonomy """ taxonomy = TaxonomyGraph(branch, taxonomy_name) - incoming_data = await request.json() - name = incoming_data["name"] - main_language = incoming_data["main_language"] - if name is None: - raise HTTPException(status_code=400, detail="Invalid node name") - if main_language is None: - raise HTTPException(status_code=400, detail="Invalid main language code") - - normalized_id = await taxonomy.create_entry_node(name, main_language) - await taxonomy.add_node_to_end("ENTRY", normalized_id) + normalized_id = await taxonomy.create_entry_node( + new_entry_node.name, new_entry_node.main_language_code + ) + await taxonomy.add_node_to_end(NodeType.ENTRY, normalized_id) @app.post("/{taxonomy_name}/{branch}/entry/{entry}") diff --git a/backend/editor/controllers/node_controller.py b/backend/editor/controllers/node_controller.py index 6dc39f59..29d871f6 100644 --- a/backend/editor/controllers/node_controller.py +++ b/backend/editor/controllers/node_controller.py @@ -27,10 +27,14 @@ async def create_entry_node( normalized_name = parser_utils.normalize_text(name, language_code, stopwords=stopwords) # Create params dict - params = {"entry_node": {"main_language": language_code, "preceding_lines": []}} - params["entry_node"]["id"] = language_code + ":" + normalized_name - params["entry_node"][f"tags_{language_code}"] = [name] - params["entry_node"][f"tags_ids_{language_code}"] = [normalized_name] + entry_node_data = { + "main_language": language_code, + "preceding_lines": [], + "id": language_code + ":" + normalized_name, + f"tags_{language_code}": [name], + f"tags_ids_{language_code}": [normalized_name], + } + params = {"entry_node": entry_node_data} query = f""" CREATE (n:{project_id}:ENTRY $entry_node) diff --git a/backend/editor/models/node_models.py b/backend/editor/models/node_models.py index 0ec68aa7..b8b84e3f 100644 --- a/backend/editor/models/node_models.py +++ b/backend/editor/models/node_models.py @@ -1,9 +1,18 @@ """ Required pydantic models for API """ +from enum import Enum + from .base_models import BaseModel +class NodeType(str, Enum): + TEXT = "TEXT" + SYNONYMS = "SYNONYMS" + STOPWORDS = "STOPWORDS" + ENTRY = "ENTRY" + + class Header(BaseModel): pass diff --git a/taxonomy-editor-frontend/src/components/CreateNodeDialogContent.tsx b/taxonomy-editor-frontend/src/components/CreateNodeDialogContent.tsx index 80b4284f..5dcc0c18 100644 --- a/taxonomy-editor-frontend/src/components/CreateNodeDialogContent.tsx +++ b/taxonomy-editor-frontend/src/components/CreateNodeDialogContent.tsx @@ -33,7 +33,7 @@ const CreateNodeDialogContent = ({ const handleAddEntryNode = () => { setIsFailedSubmit(false); - const data = { name: newNodeName, main_language: newLanguageCode }; + const data = { name: newNodeName, main_language_code: newLanguageCode }; fetch(`${baseUrl}entry`, { method: "POST",