Import de l'historique des entreprises

api/tests/test_company.py

@@ -190,7 +190,7 @@ def test_create_company_with_siret_ok(self):
         self.login()
         company = CompanyWithSiretFactory.build(social_name="Too Good To Leave")  # créé en mémoire, pas en DB
         companies_count = Company.objects.count()
-        for key in ["_state", "id", "vat"]:  # retire les champs à ne pas fournir dans le payload
+        for key in ["_state", "id", "vat", "siccrf_id"]:  # retire les champs à ne pas fournir dans le payload
             company.__dict__.pop(key, None)
         response = self.post(self.url(), company.__dict__)
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
@@ -201,7 +201,7 @@ def test_create_company_with_vat_ok(self):
         self.login()
         company = CompanyWithVatFactory.build(social_name="Too Bad To Join")  # créé en mémoire, pas en DB
         companies_count = Company.objects.count()
-        for key in ["_state", "id", "siret"]:  # retire les champs à ne pas fournir dans le payload
+        for key in ["_state", "id", "siret", "siccrf_id"]:  # retire les champs à ne pas fournir dans le payload
             company.__dict__.pop(key, None)
         response = self.post(self.url(), company.__dict__)
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)

data/etl/teleicare_history/extractor.py

@@ -0,0 +1,48 @@
+import logging
+
+from data.models import Company, IcaEtablissement
+
+logger = logging.getLogger(__name__)
+
+
+def match_companies_on_siret_or_vat():
+    """
+    Le matching pourrait aussi être fait sur
+    * Q(social_name__icontains=etab.etab_raison_sociale)
+    * Q(commercial_name__icontains=etab.etab_enseigne)
+    * Q(email__icontains=etab.etab_courriel)
+    * Q(phone_number__icontains=etab.etab_telephone)
+    Mais il serait moins précis.
+    """
+    nb_vat_match = 0
+    nb_siret_match = 0
+    for etab in IcaEtablissement.objects.all():
+        if etab.etab_siret is not None:
+            siret_matching = Company.objects.filter(siret=etab.etab_siret)
+            # seulement 2 options possible pour len(siret_matching) sont 0 et 1 car il y a une contrainte d'unicité sur le champ Company.siret
+            if len(siret_matching) == 1:
+                if siret_matching[0].siccrf_id is not None and etab.etab_ident != siret_matching[0].siccrf_id:
+                    logger.error(
+                        "Plusieurs Etablissement provenant de Teleicare ont le même SIRET, ce qui rend le matching avec une Company Compl'Alim incertain."
+                    )
+                else:
+                    nb_siret_match += 1
+                    siret_matching[0].siccrf_id = etab.etab_ident
+                    siret_matching[0].save()
+
+        elif etab.etab_numero_tva_intra is not None:
+            vat_matching = Company.objects.filter(vat=etab.etab_numero_tva_intra)
+            # seulement 2 options possible pour len(vat_matching) sont 0 et 1 car il y a une contrainte d'unicité sur le champ Company.vat
+            if len(vat_matching) == 1:
+                if vat_matching[0].siccrf_id is not None and etab.etab_ident != vat_matching[0].siccrf_id:
+                    logger.error(
+                        "Plusieurs Etablissement provenant de Teleicare ont le même n° TVA, ce qui rend le matching avec une Company Compl'Alim incertain."
+                    )
+                else:
+                    nb_vat_match += 1
+                    vat_matching[0].siccrf_id = etab.etab_ident
+                    vat_matching[0].save()
+
+    logger.info(
+        f"{nb_vat_match} + {nb_siret_match} entreprises réconcilliées sur {len(IcaEtablissement.objects.all())}"
+    )

data/etl/teleicare_history/sql/company_declaration_table_creation.sql

@@ -0,0 +1,108 @@
+-- Le but de ce fichier est de permettre la recréation des tables Teleicare correspondant
+-- au modèle de Declaration Compl'Alim
+
+
+-- int int smallint -> integer
+-- vachar -> text
+-- bit -> boolean
+-- datetime -> text (pour une conversion ensuite)
+
+CREATE TABLE ICA_ETABLISSEMENT (
+    ETAB_IDENT integer PRIMARY KEY,
+    COG_IDENT integer NULL,
+    ETAB_IDENT_PARENT integer NULL,
+    PAYS_IDENT integer NOT NULL,
+    ETAB_SIRET text NULL,
+    ETAB_NUMERO_TVA_INTRA text NULL,
+    ETAB_RAISON_SOCIALE text NOT NULL,
+    ETAB_ENSEIGNE text NULL,
+    ETAB_ADRE_VILLE text NULL,
+    ETAB_ADRE_CP text NULL,
+    ETAB_ADRE_VOIE text NULL,
+    ETAB_ADRE_COMP text NULL,
+    ETAB_ADRE_COMP2 text NULL,
+    ETAB_ADRE_DIST text NULL,
+    ETAB_TELEPHONE text NULL,
+    ETAB_FAX text NULL,
+    ETAB_COURRIEL text NULL,
+    ETAB_SITE_INTERNET text NULL,
+    ETAB_ICA_FACONNIER boolean NULL,
+    ETAB_ICA_FABRICANT boolean NULL,
+    ETAB_ICA_CONSEIL boolean NULL,
+    ETAB_ICA_IMPORTATEUR boolean NULL,
+    ETAB_ICA_INTRODUCTEUR boolean NULL,
+    ETAB_ICA_DISTRIBUTEUR boolean NULL,
+    ETAB_ICA_ENSEIGNE text NULL,
+    ETAB_ADRE_REGION text NULL,
+    ETAB_DT_AJOUT_IDENT_PARENT text NULL,
+    ETAB_NUM_ADH_TELE_PROC text NULL,
+    ETAB_COMMENTAIRE_IDENT_PARENT text NULL,
+    ETAB_NOM_DOMAINE text NULL,
+    ETAB_DATE_ADHESION text NULL,
+    ETAB_NB_COMPTE_AUTORISE integer NOT NULL
+);
+
+
+CREATE TABLE ICA_COMPLEMENTALIMENTAIRE (
+    CPLALIM_IDENT integer PRIMARY KEY,
+    FRMGAL_IDENT integer NULL,
+    ETAB_IDENT integer NOT NULL,
+    CPLALIM_MARQUE text NULL,
+    CPLALIM_GAMME text NULL,
+    CPLALIM_NOM text NOT NULL,
+    DCLENCOURS_GOUT_AROME_PARFUM text NULL,
+    CPLALIM_FORME_GALENIQUE_AUTRE text NULL
+);
+
+
+CREATE TABLE ICA_DECLARATION (
+    DCL_IDENT integer PRIMARY KEY,
+    CPLALIM_IDENT integer NOT NULL,
+    TYDCL_IDENT integer NOT NULL,
+    ETAB_IDENT integer NULL,
+    ETAB_IDENT_RMM_DECLARANT integer NOT NULL,
+    DCL_DATE text NOT NULL,
+    DCL_SAISIE_ADMINISTRATION boolean NOT NULL,
+    DCL_ANNEE integer NOT NULL,
+    DCL_MOIS integer NOT NULL,
+    DCL_NUMERO integer NOT NULL,
+    DCL_DATE_FIN_COMMERCIALISATION text NULL
+);
+
+
+CREATE TABLE ICA_VERSIONDECLARATION (
+    VRSDECL_IDENT integer PRIMARY KEY,
+    AG_IDENT integer NULL,
+    TYPVRS_IDENT integer NOT NULL,
+    UNT_IDENT integer NULL,
+    PAYS_IDENT_ADRE integer NULL,
+    ETAB_IDENT integer NULL,
+    EX_IDENT integer NOT NULL,
+    PAYS_IDENT_PAYS_DE_REFERENCE integer NULL,
+    DCL_IDENT integer NOT NULL,
+    STATTDCL_IDENT integer NULL,
+    STADCL_IDENT integer NULL,
+    VRSDECL_NUMERO integer NOT NULL,
+    VRSDECL_COMMENTAIRES text NULL,
+    VRSDECL_MISE_EN_GARDE text NULL,
+    VRSDECL_DURABILITE integer NULL,
+    VRSDECL_MODE_EMPLOI text NULL,
+    VRSDECL_DJR text NULL,
+    VRSDECL_CONDITIONNEMENT text NULL,
+    VRSDECL_POIDS_UC float NULL,
+    VRSDECL_FORME_GALENIQUE_AUTRE text NULL,
+    VRSDECL_DATE_LIMITE_REPONSE_PRO text NULL,
+    VRSDECL_OBSERVATIONS_AC text NULL,
+    VRSDECL_OBSERVATIONS_PRO text NULL,
+    VRSDECL_MODE_JSON boolean NOT NULL,
+    VRSDECL_NUMERO_DOSSIEL text NULL,
+    VRSDECL_MODE_SANS_VERIF boolean NOT NULL,
+    VRSDECL_ADRE_VILLE text NULL,
+    VRSDECL_ADRE_CP text NULL,
+    VRSDECL_ADRE_VOIE text NULL,
+    VRSDECL_ADRE_COMP text NULL,
+    VRSDECL_ADRE_COMP2 text NULL,
+    VRSDECL_ADRE_DIST text NULL,
+    VRSDECL_ADRE_REGION text NULL,
+    VRSDECL_ADRE_RAISON_SOCIALE text NULL
+)

data/etl/csv_importer.py → data/etl/teleicare_ingredients/csv_importer.py

@@ -12,8 +12,10 @@
     ManyToManyField,
 )
 
+from data.etl.exceptions import CSVFileError
+
 # Import the model
-from ..models import (
+from data.models import (
     Condition,
     Effect,
     GalenicFormulation,
@@ -33,7 +35,7 @@
     SubstanceSynonym,
     SubstanceUnit,
 )
-from .exceptions import CSVFileError
+
 from .utils import get_update_or_create_related_object, pre_import_treatments, update_or_create_object
 
 logger = logging.getLogger(__name__)

data/etl/post_load_transformation.py → data/etl/teleicare_ingredients/post_load_transformation.py

@@ -3,7 +3,8 @@
 from django.db import transaction
 from django.db.models import Q, TextField, Transform
 
-from ..models import Ingredient, IngredientStatus, Substance
+from data.models import Ingredient, IngredientStatus, Substance
+
 from .utils import update_or_create_object
 
 

data/factories/teleicare_history/__init__.py

@@ -0,0 +1,27 @@
+import random
+import string
+
+import factory
+import faker
+from phonenumber_field.phonenumber import PhoneNumber
+
+from data.choices import CountryChoices
+from data.models.teleicare_history.ica_etablissement import IcaEtablissement
+from data.utils.string_utils import make_random_str
+from data.factories.company import _make_siret, _make_vat, _make_phone_number
+
+
+class EtablissementFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = IcaEtablissement
+
+    etab_ident = factory.Sequence(lambda n: n + 1)
+    etab_raison_sociale = factory.Faker("company", locale="FR")
+    etab_enseigne = factory.Faker("company", locale="FR")
+    etab_siret = factory.LazyFunction(_make_siret)
+    etab_numero_tva_intra = factory.LazyFunction(_make_siret)
+    pays_ident = factory.Faker("pyint", min_value=0, max_value=200)
+    etab_nb_compte_autorise = factory.Faker("pyint", min_value=0, max_value=5)
+    # contact
+    etab_telephone = factory.LazyFunction(_make_phone_number)
+    etab_courriel = factory.Faker("email", locale="FR")

data/management/commands/load_ingredients.py

@@ -6,9 +6,9 @@
 from django.db import transaction
 from django.utils import timezone
 
-from data.etl.csv_importer import import_csv_from_filepath
 from data.etl.exceptions import CSVFileError
-from data.etl.post_load_transformation import deduplicate_substances_ingredients
+from data.etl.teleicare_ingredients.csv_importer import import_csv_from_filepath
+from data.etl.teleicare_ingredients.post_load_transformation import deduplicate_substances_ingredients
 from data.models.plant import Part
 
 logger = logging.getLogger(__name__)

data/migrations/0109_icacomplementalimentaire_icadeclaration_and_more.py

@@ -0,0 +1,134 @@
+# Generated by Django 5.1.4 on 2024-12-20 09:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0108_merge_20241205_1521'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='IcaComplementAlimentaire',
+            fields=[
+                ('cplalim_ident', models.IntegerField(primary_key=True, serialize=False)),
+                ('frmgal_ident', models.IntegerField(blank=True, null=True)),
+                ('cplalim_marque', models.TextField(blank=True, null=True)),
+                ('cplalim_gamme', models.TextField(blank=True, null=True)),
+                ('cplalim_nom', models.TextField()),
+                ('dclencours_gout_arome_parfum', models.TextField(blank=True, null=True)),
+                ('cplalim_forme_galenique_autre', models.TextField(blank=True, null=True)),
+            ],
+            options={
+                'db_table': 'ica_complementalimentaire',
+                'managed': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='IcaDeclaration',
+            fields=[
+                ('dcl_ident', models.IntegerField(primary_key=True, serialize=False)),
+                ('tydcl_ident', models.IntegerField()),
+                ('etab_ident_rmm_declarant', models.IntegerField()),
+                ('dcl_date', models.TextField()),
+                ('dcl_saisie_administration', models.BooleanField()),
+                ('dcl_annee', models.IntegerField()),
+                ('dcl_mois', models.IntegerField()),
+                ('dcl_numero', models.IntegerField()),
+                ('dcl_date_fin_commercialisation', models.TextField(blank=True, null=True)),
+            ],
+            options={
+                'db_table': 'ica_declaration',
+                'managed': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='IcaEtablissement',
+            fields=[
+                ('etab_ident', models.IntegerField(primary_key=True, serialize=False)),
+                ('cog_ident', models.IntegerField(blank=True, null=True)),
+                ('etab_ident_parent', models.IntegerField(blank=True, null=True)),
+                ('pays_ident', models.IntegerField()),
+                ('etab_siret', models.TextField(blank=True, null=True)),
+                ('etab_numero_tva_intra', models.TextField(blank=True, null=True)),
+                ('etab_raison_sociale', models.TextField()),
+                ('etab_enseigne', models.TextField(blank=True, null=True)),
+                ('etab_adre_ville', models.TextField(blank=True, null=True)),
+                ('etab_adre_cp', models.TextField(blank=True, null=True)),
+                ('etab_adre_voie', models.TextField(blank=True, null=True)),
+                ('etab_adre_comp', models.TextField(blank=True, null=True)),
+                ('etab_adre_comp2', models.TextField(blank=True, null=True)),
+                ('etab_adre_dist', models.TextField(blank=True, null=True)),
+                ('etab_telephone', models.TextField(blank=True, null=True)),
+                ('etab_fax', models.TextField(blank=True, null=True)),
+                ('etab_courriel', models.TextField(blank=True, null=True)),
+                ('etab_site_internet', models.TextField(blank=True, null=True)),
+                ('etab_ica_faconnier', models.BooleanField(blank=True, null=True)),
+                ('etab_ica_fabricant', models.BooleanField(blank=True, null=True)),
+                ('etab_ica_conseil', models.BooleanField(blank=True, null=True)),
+                ('etab_ica_importateur', models.BooleanField(blank=True, null=True)),
+                ('etab_ica_introducteur', models.BooleanField(blank=True, null=True)),
+                ('etab_ica_distributeur', models.BooleanField(blank=True, null=True)),
+                ('etab_ica_enseigne', models.TextField(blank=True, null=True)),
+                ('etab_adre_region', models.TextField(blank=True, null=True)),
+                ('etab_dt_ajout_ident_parent', models.TextField(blank=True, null=True)),
+                ('etab_num_adh_tele_proc', models.TextField(blank=True, null=True)),
+                ('etab_commentaire_ident_parent', models.TextField(blank=True, null=True)),
+                ('etab_nom_domaine', models.TextField(blank=True, null=True)),
+                ('etab_date_adhesion', models.TextField(blank=True, null=True)),
+                ('etab_nb_compte_autorise', models.IntegerField()),
+            ],
+            options={
+                'db_table': 'ica_etablissement',
+                'managed': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='IcaVersionDeclaration',
+            fields=[
+                ('vrsdecl_ident', models.IntegerField(primary_key=True, serialize=False)),
+                ('ag_ident', models.IntegerField(blank=True, null=True)),
+                ('typvrs_ident', models.IntegerField()),
+                ('unt_ident', models.IntegerField(blank=True, null=True)),
+                ('pays_ident_adre', models.IntegerField(blank=True, null=True)),
+                ('ex_ident', models.IntegerField()),
+                ('pays_ident_pays_de_reference', models.IntegerField(blank=True, null=True)),
+                ('stattdcl_ident', models.IntegerField(blank=True, null=True)),
+                ('stadcl_ident', models.IntegerField(blank=True, null=True)),
+                ('vrsdecl_numero', models.IntegerField()),
+                ('vrsdecl_commentaires', models.TextField(blank=True, null=True)),
+                ('vrsdecl_mise_en_garde', models.TextField(blank=True, null=True)),
+                ('vrsdecl_durabilite', models.IntegerField(blank=True, null=True)),
+                ('vrsdecl_mode_emploi', models.TextField(blank=True, null=True)),
+                ('vrsdecl_djr', models.TextField(blank=True, null=True)),
+                ('vrsdecl_conditionnement', models.TextField(blank=True, null=True)),
+                ('vrsdecl_poids_uc', models.FloatField(blank=True, null=True)),
+                ('vrsdecl_forme_galenique_autre', models.TextField(blank=True, null=True)),
+                ('vrsdecl_date_limite_reponse_pro', models.TextField(blank=True, null=True)),
+                ('vrsdecl_observations_ac', models.TextField(blank=True, null=True)),
+                ('vrsdecl_observations_pro', models.TextField(blank=True, null=True)),
+                ('vrsdecl_mode_json', models.BooleanField()),
+                ('vrsdecl_numero_dossiel', models.TextField(blank=True, null=True)),
+                ('vrsdecl_mode_sans_verif', models.BooleanField()),
+                ('vrsdecl_adre_ville', models.TextField(blank=True, null=True)),
+                ('vrsdecl_adre_cp', models.TextField(blank=True, null=True)),
+                ('vrsdecl_adre_voie', models.TextField(blank=True, null=True)),
+                ('vrsdecl_adre_comp', models.TextField(blank=True, null=True)),
+                ('vrsdecl_adre_comp2', models.TextField(blank=True, null=True)),
+                ('vrsdecl_adre_dist', models.TextField(blank=True, null=True)),
+                ('vrsdecl_adre_region', models.TextField(blank=True, null=True)),
+                ('vrsdecl_adre_raison_sociale', models.TextField(blank=True, null=True)),
+            ],
+            options={
+                'db_table': 'ica_versiondeclaration',
+                'managed': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='company',
+            name='siccrf_id',
+            field=models.IntegerField(blank=True, db_index=True, editable=False, null=True, unique=True, verbose_name='id dans les tables et tables relationnelles SICCRF'),
+        ),
+    ]

data/models/__init__.py

@@ -31,6 +31,8 @@
 )
 from .solicitation import SupervisionClaim, CompanyAccessClaim, CollaborationInvitation
 from .snapshot import Snapshot
+from .teleicare_history.ica_declaration import IcaComplementAlimentaire, IcaDeclaration, IcaVersionDeclaration
+from .teleicare_history.ica_etablissement import IcaEtablissement
 
 ELEMENT_MODELS = [
     Ingredient,