From eddaa69703f10b4063dcd364a3617f649ae2dfcb Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 14:26:50 +0100 Subject: [PATCH 01/14] =?UTF-8?q?Renommer=20le=20champ=20fabricant=20->=20?= =?UTF-8?q?responsable=5Fmise=20sur=20march=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/schemas/schema_declarations.json | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/data/schemas/schema_declarations.json b/data/schemas/schema_declarations.json index 42cff70e..5df31eda 100644 --- a/data/schemas/schema_declarations.json +++ b/data/schemas/schema_declarations.json @@ -68,10 +68,10 @@ "constraints": { "required": "True" }, - "description": "Nom de l'entreprise ayant fabriqué le complément alimentaire", - "example": "", - "name": "nom_fabricant", - "title": "Nom fabricant", + "description": "Nom de l'entreprise responsable de la mise sur le marché du complément alimentaire", + "example": "Compl Corp", + "name": "responsable_mise_sur_marche", + "title": "Responsable de la mise sur le marché", "type": "string" }, { @@ -79,9 +79,10 @@ "pattern": "^[0-9]{14}$", "required": true }, + "description": "Siret de l'entreprise responsable de la mise sur le marché du complément alimentaire", "example": "11007001800012", - "name": "siret_fabricant", - "title": "Siret fabricant", + "name": "siret_responsable_mise_sur_marche", + "title": "Siret responsable de la mise sur le marché", "type": "integer" }, { From 0bcea270fccfe16c7c8e05d8662fe59f5398fd42 Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 14:49:51 +0100 Subject: [PATCH 02/14] =?UTF-8?q?Impl=C3=A9mentation=20des=20micro-organis?= =?UTF-8?q?mes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/serializers/declaration.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api/serializers/declaration.py b/api/serializers/declaration.py index 4f41fb65..aba4f4c3 100644 --- a/api/serializers/declaration.py +++ b/api/serializers/declaration.py @@ -369,7 +369,16 @@ def get_declared_plants(self, obj): ] def get_declared_microorganisms(self, obj): - return {"nom": obj.declared_plants.name} + return [ + { + "genre": declared_microorganism.microorganism.genus, + "espece": declared_microorganism.microorganism.species, + "souche": declared_microorganism.strain, + "quantité_par_djr": declared_microorganism.quantity, + "inactive": not declared_microorganism.activated, + } + for declared_microorganism in obj.declared_microorganisms.all() + ] def get_declared_ingredients(self, obj): return {"nom": obj.declared_plants.name} From 2d32a8c22610d211e8d33d84929c3d9393dcf882 Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 16:33:21 +0100 Subject: [PATCH 03/14] Fix: Ajouter des conditions dans le serializer pour les plantes n ayant pas toutes les infos --- api/serializers/declaration.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/api/serializers/declaration.py b/api/serializers/declaration.py index aba4f4c3..3284d312 100644 --- a/api/serializers/declaration.py +++ b/api/serializers/declaration.py @@ -365,6 +365,12 @@ def get_declared_plants(self, obj): "quantité_par_djr": declared_plant.quantity, "unit": declared_plant.unit, } + if declared_plant.plant + and declared_plant.used_part + and declared_plant.preparation + and declared_plant.quantity + and declared_plant.unit + else {} for declared_plant in obj.declared_plants.all() ] @@ -381,7 +387,7 @@ def get_declared_microorganisms(self, obj): ] def get_declared_ingredients(self, obj): - return {"nom": obj.declared_plants.name} + return [declared_ingredient.microorganism.genus for declared_ingredient in obj.declared_ingredients.all()] def get_declared_substances(self, obj): return {"nom": obj.declared_plants.name} From 2c986d71635c05b5317f224fff019b609e5ff652 Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 16:51:41 +0100 Subject: [PATCH 04/14] Suite implementation plantes, ingredients, substances... --- api/serializers/declaration.py | 23 ++++++++++++++++------- data/etl/extractor.py | 1 - 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/api/serializers/declaration.py b/api/serializers/declaration.py index 3284d312..533102f9 100644 --- a/api/serializers/declaration.py +++ b/api/serializers/declaration.py @@ -313,7 +313,6 @@ class OpenDataDeclarationSerializer(serializers.ModelSerializer): declared_plants = serializers.SerializerMethodField() declared_microorganisms = serializers.SerializerMethodField() - declared_ingredients = serializers.SerializerMethodField() declared_substances = serializers.SerializerMethodField() modification_date = serializers.DateTimeField(format="%Y-%m-%d") @@ -332,7 +331,6 @@ class Meta: "declared_plants", "declared_microorganisms", "declared_substances", - "declared_ingredients", "modification_date", ) read_only_fields = fields @@ -363,7 +361,7 @@ def get_declared_plants(self, obj): "partie": declared_plant.used_part.name, "preparation": declared_plant.preparation.name, "quantité_par_djr": declared_plant.quantity, - "unit": declared_plant.unit, + "unite": declared_plant.unit.name, } if declared_plant.plant and declared_plant.used_part @@ -383,14 +381,25 @@ def get_declared_microorganisms(self, obj): "quantité_par_djr": declared_microorganism.quantity, "inactive": not declared_microorganism.activated, } + if declared_microorganism.microorganism + and declared_microorganism.strain + and declared_microorganism.quantity + and declared_microorganism.activated + else {} for declared_microorganism in obj.declared_microorganisms.all() ] - def get_declared_ingredients(self, obj): - return [declared_ingredient.microorganism.genus for declared_ingredient in obj.declared_ingredients.all()] - def get_declared_substances(self, obj): - return {"nom": obj.declared_plants.name} + return [ + { + "nom": declared_substance.new_name, + "quantité_par_djr": declared_substance.quantity, + "unite": declared_substance.unit.name, + } + if declared_substance.new_name and declared_substance.quantity and declared_substance.unit + else {} + for declared_substance in obj.declared_substances.all() + ] class DeclarationSerializer(serializers.ModelSerializer): diff --git a/data/etl/extractor.py b/data/etl/extractor.py index 70f7ec3c..c804ac7e 100644 --- a/data/etl/extractor.py +++ b/data/etl/extractor.py @@ -108,7 +108,6 @@ def __init__(self): "galenic_formulation": "forme_galenique", "declared_plants": "plantes", "declared_microorganisms": "micro_organismes", - "declared_ingredients": "ingredients_inactifs", "declared_substances": "substances", "modification_date": "date_decision", # Warning : Se baser sur la du snapshot d'autorisation si la plateforme Compl'Alim permet d'editer la déclaration (ex: abandon) } From 5cdb13d16f9a70ee9b2ac1d12c53fe08ae774591 Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 16:55:01 +0100 Subject: [PATCH 05/14] Fix: Siret --- data/etl/extractor.py | 2 +- data/etl/transformer_loader.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/data/etl/extractor.py b/data/etl/extractor.py index c804ac7e..18511af9 100644 --- a/data/etl/extractor.py +++ b/data/etl/extractor.py @@ -103,7 +103,7 @@ def __init__(self): "name": "nom_commercial", "brand": "marque", "gamme": "gamme", - "company": "responsable_mise_sur_marche", + "company": "complet_responsable_mise_sur_marche", "article": "article", "galenic_formulation": "forme_galenique", "declared_plants": "plantes", diff --git a/data/etl/transformer_loader.py b/data/etl/transformer_loader.py index 07cbb4ba..fe65d0b6 100644 --- a/data/etl/transformer_loader.py +++ b/data/etl/transformer_loader.py @@ -45,6 +45,8 @@ def load_dataset(self): class ETL_OPEN_DATA_DECLARATIONS(DECLARATIONS, OPEN_DATA): def transform_dataset(self): self.df = self.df.rename(columns=self.columns_mapper) - self.df["responsable_mise_sur_marche"] = self.df["responsable_mise_sur_marche"].apply(lambda x: x[0]) - self.df["siret_responsable_mise_sur_marche"] = self.df["responsable_mise_sur_marche"].apply(lambda x: x[1]) + self.df["responsable_mise_sur_marche"] = self.df["complet_responsable_mise_sur_marche"].apply(lambda x: x[0]) + self.df["siret_responsable_mise_sur_marche"] = self.df["complet_responsable_mise_sur_marche"].apply( + lambda x: x[1] + ) self.clean_dataset() From ed3d84c49e7049e78d61e98535bba490826ded25 Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 16:59:50 +0100 Subject: [PATCH 06/14] =?UTF-8?q?Refacto=20:=20d=C3=A9placer=20le=20column?= =?UTF-8?q?=20mapper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/etl/extractor.py | 15 --------------- data/etl/transformer_loader.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/data/etl/extractor.py b/data/etl/extractor.py index 18511af9..b9571722 100644 --- a/data/etl/extractor.py +++ b/data/etl/extractor.py @@ -97,21 +97,6 @@ def __init__(self): self.df = None self.columns = [i["name"] for i in self.schema["fields"]] - self.columns_mapper = { - "id": "id", - "status": "decision", - "name": "nom_commercial", - "brand": "marque", - "gamme": "gamme", - "company": "complet_responsable_mise_sur_marche", - "article": "article", - "galenic_formulation": "forme_galenique", - "declared_plants": "plantes", - "declared_microorganisms": "micro_organismes", - "declared_substances": "substances", - "modification_date": "date_decision", # Warning : Se baser sur la du snapshot d'autorisation si la plateforme Compl'Alim permet d'editer la déclaration (ex: abandon) - } - def extract_dataset(self): open_data_view = OpenDataDeclarationsListView() queryset = open_data_view.get_queryset() diff --git a/data/etl/transformer_loader.py b/data/etl/transformer_loader.py index fe65d0b6..2f892db7 100644 --- a/data/etl/transformer_loader.py +++ b/data/etl/transformer_loader.py @@ -43,6 +43,23 @@ def load_dataset(self): class ETL_OPEN_DATA_DECLARATIONS(DECLARATIONS, OPEN_DATA): + def __init__(self): + super().__init__() + self.columns_mapper = { + "id": "id", + "status": "decision", + "name": "nom_commercial", + "brand": "marque", + "gamme": "gamme", + "company": "complet_responsable_mise_sur_marche", + "article": "article", + "galenic_formulation": "forme_galenique", + "declared_plants": "plantes", + "declared_microorganisms": "micro_organismes", + "declared_substances": "substances", + "modification_date": "date_decision", # Warning : Se baser sur la du snapshot d'autorisation si la plateforme Compl'Alim permet d'editer la déclaration (ex: abandon) + } + def transform_dataset(self): self.df = self.df.rename(columns=self.columns_mapper) self.df["responsable_mise_sur_marche"] = self.df["complet_responsable_mise_sur_marche"].apply(lambda x: x[0]) From b1663691a967c62c78244e9758fc487b86c5a9dc Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 17:00:47 +0100 Subject: [PATCH 07/14] AJout initialisation self.columns --- data/etl/extractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data/etl/extractor.py b/data/etl/extractor.py index b9571722..191125b0 100644 --- a/data/etl/extractor.py +++ b/data/etl/extractor.py @@ -30,6 +30,7 @@ def __init__(self): self.schema = None self.schema_url = "" self.dataset_name = "" + self.columns = [] def get_schema(self): return self.schema From 76393b432b090cc0fdc3fdf050e4f40910f6c5c5 Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 17:04:24 +0100 Subject: [PATCH 08/14] =?UTF-8?q?Ajout=20champ=20dose=20journaliere=20reco?= =?UTF-8?q?mmand=C3=A9es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/serializers/declaration.py | 1 + data/etl/transformer_loader.py | 1 + 2 files changed, 2 insertions(+) diff --git a/api/serializers/declaration.py b/api/serializers/declaration.py index 533102f9..2344691b 100644 --- a/api/serializers/declaration.py +++ b/api/serializers/declaration.py @@ -328,6 +328,7 @@ class Meta: "gamme", "article", "galenic_formulation", + "daily_recommended_dose", "declared_plants", "declared_microorganisms", "declared_substances", diff --git a/data/etl/transformer_loader.py b/data/etl/transformer_loader.py index 2f892db7..9c9a52dc 100644 --- a/data/etl/transformer_loader.py +++ b/data/etl/transformer_loader.py @@ -54,6 +54,7 @@ def __init__(self): "company": "complet_responsable_mise_sur_marche", "article": "article", "galenic_formulation": "forme_galenique", + "quantity": "dose_journaliere", "declared_plants": "plantes", "declared_microorganisms": "micro_organismes", "declared_substances": "substances", From f34dd896398d9fa2119b8d52faa5c9040d711585 Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 17:07:22 +0100 Subject: [PATCH 09/14] Ajout champs mises_en_garde et mode_emploi --- api/serializers/declaration.py | 2 ++ data/etl/transformer_loader.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/api/serializers/declaration.py b/api/serializers/declaration.py index 2344691b..44ffba14 100644 --- a/api/serializers/declaration.py +++ b/api/serializers/declaration.py @@ -329,6 +329,8 @@ class Meta: "article", "galenic_formulation", "daily_recommended_dose", + "instructions", + "warning", "declared_plants", "declared_microorganisms", "declared_substances", diff --git a/data/etl/transformer_loader.py b/data/etl/transformer_loader.py index 9c9a52dc..28dfbe63 100644 --- a/data/etl/transformer_loader.py +++ b/data/etl/transformer_loader.py @@ -55,6 +55,8 @@ def __init__(self): "article": "article", "galenic_formulation": "forme_galenique", "quantity": "dose_journaliere", + "instructions": "mode_emploi", + "warning": "mises_en_garde", "declared_plants": "plantes", "declared_microorganisms": "micro_organismes", "declared_substances": "substances", From 7990a5a150f634d6bda182915e459cb50cb4bd5a Mon Sep 17 00:00:00 2001 From: qloridant Date: Fri, 29 Nov 2024 17:30:25 +0100 Subject: [PATCH 10/14] Filtrer sur les colonnes du schema --- data/etl/extractor.py | 7 +++++++ data/etl/transformer_loader.py | 9 +++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/data/etl/extractor.py b/data/etl/extractor.py index 191125b0..4e9d04d8 100644 --- a/data/etl/extractor.py +++ b/data/etl/extractor.py @@ -52,6 +52,13 @@ def filter_dataframe_with_schema_cols(self): def clean_dataset(self): self.df = self.df.loc[:, ~self.df.columns.duplicated()] + ## Code temporaire en attendant d'avoir tous les champs du schéma + columns_to_keep = [] + for col in self.columns: + if col in self.df.columns: + columns_to_keep.append(col) + # --------------------------------------------------- + self.df = self.df[columns_to_keep] self.filter_dataframe_with_schema_cols() def is_valid(self) -> bool: diff --git a/data/etl/transformer_loader.py b/data/etl/transformer_loader.py index 28dfbe63..113c7d2b 100644 --- a/data/etl/transformer_loader.py +++ b/data/etl/transformer_loader.py @@ -51,10 +51,9 @@ def __init__(self): "name": "nom_commercial", "brand": "marque", "gamme": "gamme", - "company": "complet_responsable_mise_sur_marche", "article": "article", "galenic_formulation": "forme_galenique", - "quantity": "dose_journaliere", + "daily_recommended_dose": "dose_journaliere", "instructions": "mode_emploi", "warning": "mises_en_garde", "declared_plants": "plantes", @@ -65,8 +64,6 @@ def __init__(self): def transform_dataset(self): self.df = self.df.rename(columns=self.columns_mapper) - self.df["responsable_mise_sur_marche"] = self.df["complet_responsable_mise_sur_marche"].apply(lambda x: x[0]) - self.df["siret_responsable_mise_sur_marche"] = self.df["complet_responsable_mise_sur_marche"].apply( - lambda x: x[1] - ) + self.df["responsable_mise_sur_marche"] = self.df["company"].apply(lambda x: x[0]) + self.df["siret_responsable_mise_sur_marche"] = self.df["company"].apply(lambda x: x[1]) self.clean_dataset() From 9dc521a4c016ef31f53d16872e1746617fe9eab2 Mon Sep 17 00:00:00 2001 From: qloridant Date: Sun, 1 Dec 2024 19:41:42 +0100 Subject: [PATCH 11/14] Refacto: Utuilisation du filter() pour garder que les plantes actives --- api/serializers/declaration.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/api/serializers/declaration.py b/api/serializers/declaration.py index 44ffba14..3846a354 100644 --- a/api/serializers/declaration.py +++ b/api/serializers/declaration.py @@ -366,13 +366,7 @@ def get_declared_plants(self, obj): "quantité_par_djr": declared_plant.quantity, "unite": declared_plant.unit.name, } - if declared_plant.plant - and declared_plant.used_part - and declared_plant.preparation - and declared_plant.quantity - and declared_plant.unit - else {} - for declared_plant in obj.declared_plants.all() + for declared_plant in obj.declared_plants.filter(active=True) ] def get_declared_microorganisms(self, obj): From 49fe56267f49dd661e5ee1ec3a3cb176920b5c6f Mon Sep 17 00:00:00 2001 From: qloridant Date: Sun, 1 Dec 2024 19:44:34 +0100 Subject: [PATCH 12/14] =?UTF-8?q?Permetter=20aux=20micro-organismes=20inac?= =?UTF-8?q?tiv=C3=A9s=20d=20appara=C3=AEtre?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/serializers/declaration.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/api/serializers/declaration.py b/api/serializers/declaration.py index 3846a354..fe2f691e 100644 --- a/api/serializers/declaration.py +++ b/api/serializers/declaration.py @@ -369,20 +369,17 @@ def get_declared_plants(self, obj): for declared_plant in obj.declared_plants.filter(active=True) ] - def get_declared_microorganisms(self, obj): + def get_declared_ingredients(self, obj): return [ { "genre": declared_microorganism.microorganism.genus, "espece": declared_microorganism.microorganism.species, - "souche": declared_microorganism.strain, - "quantité_par_djr": declared_microorganism.quantity, + "souche": declared_microorganism.strain + if declared_microorganism.strain + else None, # elle est normalement obligatoire mais quelques entrées ont pu être rentrées avant le required + "quantité_par_djr": declared_microorganism.quantity if declared_microorganism.activated else None, "inactive": not declared_microorganism.activated, } - if declared_microorganism.microorganism - and declared_microorganism.strain - and declared_microorganism.quantity - and declared_microorganism.activated - else {} for declared_microorganism in obj.declared_microorganisms.all() ] From a054fb2ad915059875c0554f7373b1e496d6525e Mon Sep 17 00:00:00 2001 From: qloridant Date: Sun, 1 Dec 2024 19:45:40 +0100 Subject: [PATCH 13/14] Dans substance, remplacer new_name par name --- api/serializers/declaration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/serializers/declaration.py b/api/serializers/declaration.py index fe2f691e..ae8bb6ff 100644 --- a/api/serializers/declaration.py +++ b/api/serializers/declaration.py @@ -386,11 +386,11 @@ def get_declared_ingredients(self, obj): def get_declared_substances(self, obj): return [ { - "nom": declared_substance.new_name, + "nom": declared_substance.name, "quantité_par_djr": declared_substance.quantity, "unite": declared_substance.unit.name, } - if declared_substance.new_name and declared_substance.quantity and declared_substance.unit + if declared_substance.name and declared_substance.quantity and declared_substance.unit else {} for declared_substance in obj.declared_substances.all() ] From 7af92c92a7a5687769c56e3c64887edaec4601a2 Mon Sep 17 00:00:00 2001 From: qloridant Date: Sun, 1 Dec 2024 19:51:01 +0100 Subject: [PATCH 14/14] Fix: remplacer declared_substance.name par declared_substance.substance.name --- api/serializers/declaration.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/serializers/declaration.py b/api/serializers/declaration.py index ae8bb6ff..8f03e892 100644 --- a/api/serializers/declaration.py +++ b/api/serializers/declaration.py @@ -366,10 +366,12 @@ def get_declared_plants(self, obj): "quantité_par_djr": declared_plant.quantity, "unite": declared_plant.unit.name, } + if declared_plant.unit + else {} for declared_plant in obj.declared_plants.filter(active=True) ] - def get_declared_ingredients(self, obj): + def get_declared_microorganisms(self, obj): return [ { "genre": declared_microorganism.microorganism.genus, @@ -386,11 +388,11 @@ def get_declared_ingredients(self, obj): def get_declared_substances(self, obj): return [ { - "nom": declared_substance.name, + "nom": declared_substance.substance.name, "quantité_par_djr": declared_substance.quantity, "unite": declared_substance.unit.name, } - if declared_substance.name and declared_substance.quantity and declared_substance.unit + if declared_substance.substance.name and declared_substance.quantity and declared_substance.unit else {} for declared_substance in obj.declared_substances.all() ]