From 7180f5ca69e22208c455b1378a9b1652d26346ca Mon Sep 17 00:00:00 2001 From: Alex Garel Date: Thu, 14 Mar 2024 11:06:59 +0100 Subject: [PATCH] feat: support new folders structure (#434) openfoodfacts/openfoodfacts-server@2b2d85f added a new structure to taxonomy folders. This is a quick fix to support it --- backend/editor/api.py | 2 +- backend/editor/entries.py | 24 ++++++++++++------- backend/editor/github_functions.py | 14 ++++++----- backend/editor/utils.py | 9 +++++++ taxonomy-editor-frontend/src/constants.ts | 4 ++-- .../src/pages/startproject/index.tsx | 2 +- 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/backend/editor/api.py b/backend/editor/api.py index be270795..aa4e065b 100644 --- a/backend/editor/api.py +++ b/backend/editor/api.py @@ -374,7 +374,7 @@ async def import_from_github( taxonomy = TaxonomyGraph(branch, taxonomy_name) if not taxonomy.is_valid_branch_name(): - raise HTTPException(status_code=422, detail="branch_name: Enter a valid branch name!") + raise HTTPException(status_code=422, detail="branch_name: Enter a valid branch name!") if await taxonomy.does_project_exist(): raise HTTPException(status_code=409, detail="Project already exists!") if not await taxonomy.is_branch_unique(from_github=True): diff --git a/backend/editor/entries.py b/backend/editor/entries.py index 9b683677..d6c84fa5 100644 --- a/backend/editor/entries.py +++ b/backend/editor/entries.py @@ -2,6 +2,7 @@ Database helper functions for API """ +import logging import re import shutil import tempfile @@ -15,7 +16,7 @@ from openfoodfacts_taxonomy_parser import unparser # Unparser for taxonomies from openfoodfacts_taxonomy_parser import utils as parser_utils -from . import settings +from . import settings, utils from .controllers.node_controller import create_entry_node from .controllers.project_controller import create_project, edit_project, get_project from .exceptions import GithubBranchExistsError # Custom exceptions @@ -33,7 +34,8 @@ ) from .models.node_models import EntryNodeCreate from .models.project_models import ProjectCreate, ProjectEdit, ProjectStatus -from .utils import file_cleanup + +log = logging.getLogger(__name__) async def async_list(async_iterable): @@ -48,6 +50,10 @@ def __init__(self, branch_name, taxonomy_name): self.branch_name = branch_name self.project_name = "p_" + taxonomy_name + "_" + branch_name + @property + def taxonomy_path_in_repository(self): + return utils.taxonomy_path_in_repository(self.taxonomy_name) + def get_label(self, id): """ Helper function for getting the label for a given id @@ -82,13 +88,13 @@ async def get_local_taxonomy_file(self, tmpdir: str, uploadfile: UploadFile): async def get_github_taxonomy_file(self, tmpdir: str): async with TransactionCtx(): - filename = f"{self.taxonomy_name}.txt" - filepath = f"{tmpdir}/{filename}" - base_url = ( - "https://raw.githubusercontent.com/" + settings.repo_uri + "/main/taxonomies/" + filepath = f"{tmpdir}/{self.taxonomy_name}.txt" + target_url = ( + f"https://raw.githubusercontent.com/{settings.repo_uri}" + f"/main/{self.taxonomy_path_in_repository}" ) try: - await run_in_threadpool(urllib.request.urlretrieve, base_url + filename, filepath) + await run_in_threadpool(urllib.request.urlretrieve, target_url, filepath) github_object = GithubOperations(self.taxonomy_name, self.branch_name) commit_sha = (await github_object.get_branch("main")).commit.sha file_sha = await github_object.get_file_sha() @@ -131,6 +137,7 @@ async def get_and_parse_taxonomy(self, uploadfile: UploadFile | None = None): # add an error node so we can display it with errors in the app async with TransactionCtx(): await edit_project(self.project_name, ProjectEdit(status=ProjectStatus.FAILED)) + log.exception(e) raise e async def import_taxonomy( @@ -169,9 +176,10 @@ def dump_taxonomy(self, background_tasks: BackgroundTasks): # Dump taxonomy with given file name and branch name unparser_object(filename, self.branch_name, self.taxonomy_name) # Program file removal in the background - background_tasks.add_task(file_cleanup, filename) + background_tasks.add_task(utils.file_cleanup, filename) return filename except Exception as e: + log.exception(e) raise TaxonomyUnparsingError() from e async def file_export(self, background_tasks: BackgroundTasks): diff --git a/backend/editor/github_functions.py b/backend/editor/github_functions.py index 7470d90e..6c81ba31 100644 --- a/backend/editor/github_functions.py +++ b/backend/editor/github_functions.py @@ -16,7 +16,7 @@ PullRequest, ) -from . import settings +from . import settings, utils class GithubOperations: @@ -26,6 +26,10 @@ def __init__(self, taxonomy_name: str, branch_name: str): self.taxonomy_name = taxonomy_name self.branch_name = branch_name + @property + def taxonomy_path_in_repository(self): + return utils.taxonomy_path_in_repository(self.taxonomy_name) + @cached_property def repo_info(self) -> tuple[str, str]: repo_uri = settings.repo_uri @@ -68,10 +72,9 @@ async def get_file_sha(self) -> str: """ Get the file SHA from the 'main' branch in the "openfoodfacts-server" repo """ - github_filepath = f"taxonomies/{self.taxonomy_name}.txt" file_contents: ContentFile = ( await self.connection.rest.repos.async_get_content( - *self.repo_info, path=github_filepath + *self.repo_info, path=self.taxonomy_path_in_repository ) ).parsed_data @@ -90,8 +93,7 @@ async def update_file(self, filename: str, file_sha: str, author_name) -> FileCo Update the taxonomy txt file edited by user using the Taxonomy Editor """ # Find taxonomy text file to be updated - github_filepath = f"taxonomies/{self.taxonomy_name}.txt" - commit_message = f"Update {self.taxonomy_name}.txt" + commit_message = f"Update {self.taxonomy_path_in_repository}" author = {"name": author_name, "email": "contact@openfoodfacts.org"} try: with open(filename, "r") as f: @@ -103,7 +105,7 @@ async def update_file(self, filename: str, file_sha: str, author_name) -> FileCo return ( await self.connection.rest.repos.async_create_or_update_file_contents( *self.repo_info, - path=github_filepath, + path=self.taxonomy_path_in_repository, message=commit_message, content=base64.b64encode(new_file_contents.encode("utf-8")), sha=file_sha, diff --git a/backend/editor/utils.py b/backend/editor/utils.py index ce6f388f..d0cc2760 100644 --- a/backend/editor/utils.py +++ b/backend/editor/utils.py @@ -12,3 +12,12 @@ def file_cleanup(filepath): os.remove(filepath) except FileNotFoundError: log.warn(f"Taxonomy file {filepath} not found for deletion") + + +def taxonomy_path_in_repository(taxonomy_name): + """Helper function to get the path of a taxonomy in the repository""" + path = taxonomy_name + # hacky for now until we restructure better + if path in ("food_ingredients", "food_categories"): + path = path.replace("_", "/") + return f"taxonomies/{path}.txt" diff --git a/taxonomy-editor-frontend/src/constants.ts b/taxonomy-editor-frontend/src/constants.ts index 8bb14095..1526ad06 100644 --- a/taxonomy-editor-frontend/src/constants.ts +++ b/taxonomy-editor-frontend/src/constants.ts @@ -27,11 +27,11 @@ export const TAXONOMY_NAMES = [ "Additives", "Allergens", "Amino Acids", - "Categories", + "Food Categories", "Data Quality", "Food Groups", "Improvements", - "Ingredients", + "Food Ingredients", "Ingredients Analysis", "Ingredients Processing", "Labels", diff --git a/taxonomy-editor-frontend/src/pages/startproject/index.tsx b/taxonomy-editor-frontend/src/pages/startproject/index.tsx index 552826e1..4917ba88 100644 --- a/taxonomy-editor-frontend/src/pages/startproject/index.tsx +++ b/taxonomy-editor-frontend/src/pages/startproject/index.tsx @@ -32,7 +32,7 @@ export const StartProject = () => { const findDefaultBranchName = useCallback(() => { if (taxonomyName === "" || ownerName === "") return ""; - return `${taxonomyName.toLowerCase()}_${ownerName + return `${toSnakeCase(taxonomyName.toLowerCase())}_${ownerName .replace(" ", "") .toLowerCase()}_${Math.floor(Date.now() / 1000)}`; }, [ownerName, taxonomyName]);