diff --git a/andes_migrate/biometrie_mollusque.py b/andes_migrate/biometrie_mollusque.py new file mode 100644 index 0000000..9c9146d --- /dev/null +++ b/andes_migrate/biometrie_mollusque.py @@ -0,0 +1,218 @@ +import logging +import numpy as np + +from andes_migrate.capture_mollusque import CaptureMollusque +from andes_migrate.table_peche_sentinelle import TablePecheSentinelle +from andes_migrate.decorators import ( + NotAndes, + Computed, + HardCoded, + Deprecated, + AndesCodeLookup, + tag, + log_results, + validate_string, + validate_int, +) + +# logging.basicConfig(level=logging.INFO) + + +class BiometrieMollusque(TablePecheSentinelle): + """ + Object model representing the BIOMETRIE_MOLLUSQUE table + + WARNING: only good for getting the weight of whelk eggs, other features (actual biometry) are not implemented + """ + + def __init__(self, capture: CaptureMollusque, *args, no_moll_init=0, **kwargs): + super().__init__(*args, ref=capture.reference_data, **kwargs) + + self.capture: CaptureMollusque = capture + + self.table_name = "BIOMETRIE_MOLLUSQUE" + self.andes_db = capture.andes_db + self.data = {} + self._init_rows() + self.no_moll = no_moll_init + + def _init_rows(self): + """Initialisation method + This queries the Andes DB and creates a list of row entries to be added to the current table + + After running this methods initialises the following attribute: + self._row_list + self._row_idx (hopefully to self._row_idx=0) + + self._row_list will be populated with the associated Andes catch ids for the current set + self._row_idx will start at 0 + For this class, the row will contain a tuple: (specimen_id, length, basket_id) + + """ + + # HACK! only choose baskets that are WHELK EGGs size-class (shared_models_sizeclass.code=3) + # 3 -> oeufs buccin + target_size_class = 3 + + # DOUBLE HACK! only choose whelkes species (aphia_id=138878) + target_aphia_id = 138878 + + query = ( + "SELECT basket_wt_kg " + "FROM ecosystem_survey_catch " + "LEFT JOIN ecosystem_survey_basket " + "ON ecosystem_survey_catch.id=ecosystem_survey_basket.catch_id " + "LEFT JOIN shared_models_species " + "ON ecosystem_survey_catch.species_id = shared_models_species.id " + f"WHERE ecosystem_survey_catch.id={self.capture._get_current_row_pk()} " + f"AND shared_models_species.aphia_id = {target_aphia_id} " + f"AND ecosystem_survey_basket.size_class = {target_size_class} " + ) + result = self.andes_db.execute_query(query) + + # a list of all the catch pk's (need to unpack a bit) + self._row_list = result + self._row_idx = 0 + + def populate_data(self): + """Populate data: run all getters""" + self.data["COD_SOURCE_INFO"] = self.get_cod_source_info() + self.data["NO_RELEVE"] = self.get_no_releve() + self.data["COD_NBPC"] = self.get_cod_nbpc() + self.data["IDENT_NO_TRAIT"] = self.get_ident_no_trait() + self.data["COD_ENG_GEN"] = self.get_cod_eng_gen() + self.data["COD_TYP_PANIER"] = self.get_cod_typ_panier() + self.data["NO_ENGIN"] = self.get_no_engin() + self.data["COD_ESP_GEN"] = self.get_cod_esp_gen() + self.data['NO_MOLLUSQUE'] = self.get_no_mollusque() + + self.data['COD_SEXE'] = self.get_cod_sexe() + self.data['VOLUME_GONADE'] = self.get_volume_gonade() + self.data['VOLUME_GONADE_P'] = self.get_volume_gonade_p() + self.data['NO_CHARGEMENT'] = self.get_no_chargement() + + def get_cod_esp_gen(self) -> int: + """COD_ESP_GEN INTEGER / NUMBER(5,0) + Identification de l'espèce capturée tel que défini dans la table ESPECE_GENERAL + + Extrait de la capture ::func:`~andes_migrate.capture_mollusque.CapturenMollusque.get_cod_esp_gen` + + """ + return self.capture.get_cod_esp_gen() + + def get_cod_eng_gen(self) -> int: + """COD_ENG_GEN INTEGER / NUMBER(5,0) + Identification de l'engin de pêche utilisé tel que défini dans la table ENGIN_GENERAL + + Extrait de la capture ::func:`~andes_migrate.capture_mollusque.CapturenMollusque.get_cod_eng_gen` + + """ + return self.capture.get_cod_eng_gen() + + def get_cod_source_info(self) -> int: + """COD_SOURCE_INFO INTEGER / NUMBER(5,0) + Identification de la source d'information tel que défini dans la table SOURCE_INFO + + Extrait de la capture ::func:`~andes_migrate.capture_mollusque.CaptureMollusque.get_cod_source_info` + + """ + return self.capture.get_cod_source_info() + + def get_no_releve(self) -> int: + """NO_RELEVE INTEGER / NUMBER(5,0) + Numéro séquentiel du relevé + + Extrait de la capture ::func:`~andes_migrate.capture_mollusque.CAptureMollusque.get_no_releve` + """ + return self.capture.get_no_releve() + + def get_ident_no_trait(self) -> int: + """IDENT_NO_TRAIT INTEGER / NUMBER(5,0) + Numéro séquentiel d'identification du trait + + Extrait de la capture ::func:`~andes_migrate.capture_mollusque.CaptureMollusque.get_ident_no_trait` + """ + return self.capture.get_ident_no_trait() + + def get_cod_typ_panier(self) -> int: + """COD_TYP_PANIER INTEGER / NUMBER(5,0) + Identification du type de panier utilisé tel que défini dans la table TYPE_PANIER + + Extrait de la capture ::func:`~andes_migrate.capture_mollusque.CaptureMollusque.get_cod_type_panier` + + """ + + return self.capture.get_cod_typ_panier() + + def get_cod_nbpc(self) -> str: + """COD_NBPC VARCHAR(6) / VARCHAR2(6) + Numéro du navire utilisé pour réaliser le relevé tel que défini dans la table NAVIRE + + Extrait de la capture ::func:`~andes_migrate.capture_mollusque.CaptureMollusque.get_cod_nbpc` + + """ + return self.capture.get_cod_nbpc() + + def get_no_engin(self) -> int: + """NO_ENGIN INTEGER/ NUMBER(5,0) + Numéro identifiant l'engin utilisé tel que défini dans la table NO_ENGIN + + Extrait de la capture ::func:`~andes_migrate.capture_mollusque.CaptureMollusque.get_no_engin` + + """ + + return self.capture.get_no_engin() + + @tag(HardCoded) + @validate_int() + def get_cod_sexe(self) -> int: + """COD_SEXE INTEGER / NUMBER(5,0) + Sexe de l''organisme tel que décrit dans la table SEXE + + Hard-coded: This function always returns 9 + 9 -> inconnu / indéterminé + """ + + return self._hard_coded_result(9) + + @tag(HardCoded) + def get_volume_gonade(self) -> float|None: + """VOLUME_GONADE FLOAT / NUMBER + Volume occupé par les gonades unité ml + + Hard-coded: This function always returns None + + """ + return self._hard_coded_result(None) + + @tag(HardCoded) + def get_volume_gonade_p(self) -> float|None: + """VOLUME_GONADE_P FLOAT / NUMBER + Nombre de chiffre après la décimale pour la précision d'affichage associée à "Volume_Gonade" + + Hard-coded: This function always returns None + + """ + return self._hard_coded_result(None) + + @validate_int() + @log_results + def get_no_mollusque(self) -> int : + """NO_MOLLUSQUE INTEGER / NUMBER(5,0) + Numéro séquentiel attribué à l'individus mesuré + + """ + adjusted_row_idx = self._row_idx-1 + + return self.no_moll + adjusted_row_idx + # raise NotImplementedError + + @log_results + def get_no_chargement(self) -> float|None: + """NO_CHARGEMENT DOUBLE / NUMBER + Numéro de l'activité de chargement de données dans la base Oracle + + Extrait de la capture ::func:`~andes_migrate.capture_mollusque.CaptureMollusque.get_no_engin` + + """ + return self.capture.get_no_chargement() diff --git a/andes_migrate/capture_mollusque.py b/andes_migrate/capture_mollusque.py index bd4ddfe..dc73ad5 100644 --- a/andes_migrate/capture_mollusque.py +++ b/andes_migrate/capture_mollusque.py @@ -53,9 +53,19 @@ def _init_rows(self): # "ORDER BY ecosystem_survey_catch.id ASC;" # ) - # HACK! only choose scallops aphia IDs + # HACK! only choose aphia IDs for commercial target species placopecten_magellanicus = 156972 chlamys_islandica = 140692 + buccinum_undatum = 138878 + if 'pétoncle'==self.engin.trait.proj.espece: + species_filter = f"AND (shared_models_species.aphia_id = {placopecten_magellanicus} OR shared_models_species.aphia_id = {chlamys_islandica}) " + elif 'buccin'==self.engin.trait.proj.espece: + # NEED other bucinums, glacial, teranovae etc + species_filter = f"AND (shared_models_species.aphia_id = {buccinum_undatum}) " + else: + print("Problem filtering species, need aphia-ids for extraction") + raise ValueError + # DOUBLE HACK! only choose catches that contain baskets that are NOT an NA size-class (shared_models_sizeclass.code=0) # some NA baskets can still exist in a catch that has a non-NA basket @@ -80,7 +90,7 @@ def _init_rows(self): f"AND NOT shared_models_sizeclass.code={size_class_code_NA} " f"AND NOT shared_models_sizeclass.code={size_class_code_biodiversity} " f"AND ecosystem_survey_catch.set_id={self.engin.trait._get_current_row_pk()} " - f"AND (shared_models_species.aphia_id = {placopecten_magellanicus} OR shared_models_species.aphia_id = {chlamys_islandica}) " + f"{species_filter} " "ORDER BY ecosystem_survey_catch.id ASC " ) diff --git a/andes_migrate/poids_biometrie.py b/andes_migrate/poids_biometrie.py new file mode 100644 index 0000000..6b68a55 --- /dev/null +++ b/andes_migrate/poids_biometrie.py @@ -0,0 +1,227 @@ +import logging +import numpy as np + +from andes_migrate.biometrie_mollusque import BiometrieMollusque +from andes_migrate.table_peche_sentinelle import TablePecheSentinelle +from andes_migrate.decorators import ( + NotAndes, + Computed, + HardCoded, + Deprecated, + AndesCodeLookup, + tag, + log_results, + validate_string, + validate_int, +) + +# logging.basicConfig(level=logging.INFO) + + +class PoidsBiometrie(TablePecheSentinelle): + """ + Object model representing the POIDS_BIOMETRIE table + + WARNING: only good for getting the weight of whelk eggs, other features (actual biometry) are not implemented + """ + + def __init__(self, biometrie: BiometrieMollusque, *args, **kwargs): + super().__init__(*args, ref=biometrie.reference_data, **kwargs) + + self.biometrie: BiometrieMollusque = biometrie + + self.table_name = "POIDS_BIOMETRIE" + self.andes_db = biometrie.andes_db + self.data = {} + self._init_rows() + + def _init_rows(self): + """Initialisation method + This queries the Andes DB and creates a list of row entries to be added to the current table + + After running this methods initialises the following attribute: + self._row_list + self._row_idx (hopefully to self._row_idx=0) + + self._row_list will be populated with the associated Andes catch ids for the current set + self._row_idx will start at 0 + For this class, the row will contain a tuple: (specimen_id, length, basket_id) + + """ + + # HACK! only choose baskets that are WHELK EGGs size-class (shared_models_sizeclass.code=3) + # 3 -> oeufs buccin + target_size_class = 3 + + # DOUBLE HACK! only choose whelkes species (aphia_id=138878) + target_aphia_id = 138878 + + query = ( + "SELECT basket_wt_kg " + "FROM ecosystem_survey_catch " + "LEFT JOIN ecosystem_survey_basket " + "ON ecosystem_survey_catch.id=ecosystem_survey_basket.catch_id " + "LEFT JOIN shared_models_species " + "ON ecosystem_survey_catch.species_id = shared_models_species.id " + f"WHERE ecosystem_survey_catch.id={self.biometrie.capture._get_current_row_pk()} " + f"AND shared_models_species.aphia_id = {target_aphia_id} " + f"AND ecosystem_survey_basket.size_class = {target_size_class} " + ) + result = self.andes_db.execute_query(query) + + # a list of all the catch pk's (need to unpack a bit) + # a list of all the catch pk's (need to unpack a bit) + self._row_list = [basket_wt_kg[0] for basket_wt_kg in result] + + self._row_list = result + self._row_idx = 0 + + def get_current_weight(self): + if self._row_idx is not None and self._row_list: + # need to adjust becuse the iterator is already looking forward.. + adjusted_row_idx = self._row_idx-1 + return self._row_list[adjusted_row_idx][0] + else: + raise ValueError + + def populate_data(self): + """Populate data: run all getters""" + self.data["COD_ESP_GEN"] = self.get_cod_esp_gen() + self.data["IDENT_NO_TRAIT"] = self.get_ident_no_trait() + self.data["NO_RELEVE"] = self.get_no_releve() + self.data["COD_SOURCE_INFO"] = self.get_cod_source_info() + self.data["COD_NBPC"] = self.get_cod_nbpc() + self.data["COD_ENG_GEN"] = self.get_cod_eng_gen() + self.data["COD_TYP_PANIER"] = self.get_cod_typ_panier() + self.data["NO_ENGIN"] = self.get_no_engin() + + self.data['NO_MOLLUSQUE'] = self.get_no_mollusque() + + self.data['COD_TYP_PDS'] = self.get_cod_typ_pds() + self.data['VALEUR_PDS_MOLL'] = self.get_valeur_pds_moll() + self.data['VALEUR_PDS_MOLL_P'] = self.get_valeur_pds_moll_p() + self.data['NO_CHARGEMENT'] = self.get_no_chargement() + + def get_cod_esp_gen(self) -> int: + """COD_ESP_GEN INTEGER / NUMBER(5,0) + Identification de l'espèce capturée tel que défini dans la table ESPECE_GENERAL + + Extrait de la capture ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_cod_esp_gen` + + """ + return self.biometrie.get_cod_esp_gen() + + def get_cod_eng_gen(self) -> int: + """COD_ENG_GEN INTEGER / NUMBER(5,0) + Identification de l'engin de pêche utilisé tel que défini dans la table ENGIN_GENERAL + + Extrait de la capture ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_cod_eng_gen` + + """ + return self.biometrie.get_cod_eng_gen() + + def get_cod_source_info(self) -> int: + """COD_SOURCE_INFO INTEGER / NUMBER(5,0) + Identification de la source d'information tel que défini dans la table SOURCE_INFO + + Extrait de la capture ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_cod_source_info` + + """ + return self.biometrie.get_cod_source_info() + + def get_no_releve(self) -> int: + """NO_RELEVE INTEGER / NUMBER(5,0) + Numéro séquentiel du relevé + + Extrait de la capture ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_no_releve` + """ + return self.biometrie.get_no_releve() + + def get_ident_no_trait(self) -> int: + """IDENT_NO_TRAIT INTEGER / NUMBER(5,0) + Numéro séquentiel d'identification du trait + + Extrait de la capture ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_ident_no_trait` + """ + return self.biometrie.get_ident_no_trait() + + def get_cod_typ_panier(self) -> int: + """COD_TYP_PANIER INTEGER / NUMBER(5,0) + Identification du type de panier utilisé tel que défini dans la table TYPE_PANIER + + Extrait de la capture ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_cod_type_panier` + + """ + + return self.biometrie.get_cod_typ_panier() + + def get_cod_nbpc(self) -> str: + """COD_NBPC VARCHAR(6) / VARCHAR2(6) + Numéro du navire utilisé pour réaliser le relevé tel que défini dans la table NAVIRE + + Extrait de la capture ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_cod_nbpc` + + """ + return self.biometrie.get_cod_nbpc() + + def get_no_engin(self) -> int: + """NO_ENGIN INTEGER/ NUMBER(5,0) + Numéro identifiant l'engin utilisé tel que défini dans la table NO_ENGIN + + Extrait de la capture ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_no_engin` + + """ + + return self.biometrie.get_no_engin() + + @tag(HardCoded) + @validate_int() + def get_cod_typ_pds(self) -> int: + """ COD_TYP_PDS INTEGER / NUMBER(5,0) + Identification du type de poids évalué tel que défini dans la table TYPE_POIDS + + Hard-coded: This function always returns 1 + 1 -> poids_total_frais + """ + + return self._hard_coded_result(1) + + def get_valeur_pds_moll(self) -> float|None: + """ VALEUR_PDS FLOAT / NUMBER + Valeur associée au type de poids évalué, unité gramme + + """ + return self.get_current_weight()*1000 + + @tag(HardCoded) + def get_valeur_pds_moll_p(self) -> float|None: + """ VALEUR_PDS_P FLOAT / NUMBER + Nombre de chiffre après la décimale pour la précision d'affichage associée à "Valeur_Pds_Moll" + + Hard-coded: This function always returns None + + """ + return self._hard_coded_result(0.1) + + @validate_int() + @log_results + def get_no_mollusque(self) -> int : + """NO_MOLLUSQUE INTEGER / NUMBER(5,0) + Numéro séquentiel attribué à l'individus mesuré + + Extrait de la biometrie ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_no_mollusque` + + """ + + return self.biometrie.get_no_mollusque() + # raise NotImplementedError + + @log_results + def get_no_chargement(self) -> float|None: + """NO_CHARGEMENT DOUBLE / NUMBER + Numéro de l'activité de chargement de données dans la base Oracle + + Extrait de la biometrie ::func:`~andes_migrate.biometrie_mollusque.BiometrieMollusque.get_no_chargement` + + """ + return self.biometrie.get_no_chargement() diff --git a/andes_migrate/ref_data/strat_buccin_hcn.py b/andes_migrate/ref_data/strat_buccin_hcn.py index e17a5e6..3fbdaa0 100644 --- a/andes_migrate/ref_data/strat_buccin_hcn.py +++ b/andes_migrate/ref_data/strat_buccin_hcn.py @@ -23,7 +23,7 @@ stations_pao = [] -with open(f"{dir_path}/stations_PaO.csv", 'r') as fp: +with open(f"{dir_path}/stations_PAO.csv", 'r') as fp: csv_data = csv.reader(fp, delimiter=",", skipinitialspace=True) next(csv_data, None) # skip the headers for r in csv_data: diff --git a/andes_migrate/table_peche_sentinelle.py b/andes_migrate/table_peche_sentinelle.py index 3959a20..ed759f5 100644 --- a/andes_migrate/table_peche_sentinelle.py +++ b/andes_migrate/table_peche_sentinelle.py @@ -44,7 +44,7 @@ def __iter__(self): def _init_rows(self): """Initialisation method - This queries the Andes DB and creaters a list of row entries to be added to the current table + This queries the Andes DB and creates a list of row entries to be added to the current table After running this methods initialises the following attribute: self._row_list @@ -84,8 +84,9 @@ def __next__(self): self.populate_data() try: self.write_row() - except Exception: - print("problem with", self._row_idx, ) + except Exception as exc: + print("problem with", self._row_idx, exc) + raise exc return self.data else: raise StopIteration @@ -160,6 +161,10 @@ def write_row(self): except DataError as exc: self.logger.error("Could not execute statement: %s", statement) raise exc + except Exception as exc: + self.logger.error("Could not execute statement: %s", statement) + raise exc + # print(statement) # self.output_cur.commit() diff --git a/andes_migrate/trait_mollusque.py b/andes_migrate/trait_mollusque.py index f3e3f0e..356c786 100644 --- a/andes_migrate/trait_mollusque.py +++ b/andes_migrate/trait_mollusque.py @@ -316,9 +316,9 @@ def get_cod_strate(self) -> int | None: from andes_migrate.ref_data.strat_buccin_hcn import strat_dict if int(station_name) in strat_dict["Forestville"]: - secteur = "For." + secteur = "FOR" elif int(station_name) in strat_dict["Pointe-aux-Outardes"]: - secteur = "PaO" + secteur = "PAO" elif int(station_name) in strat_dict["Baie-Comeau"]: secteur = "BC" else: @@ -982,6 +982,8 @@ def get_rem_trait_moll(self) -> str | None: # only because it currupts de .dat files, to_return = to_return.replace('\n', ' ') to_return = to_return.replace('\r', ' ') + # need to escape single quotes because it corrupts the SQL statements + to_return = to_return.replace('\'', '\'\'') return to_return @validate_int(not_null=False) diff --git a/make_access_IML-2024022.py b/make_access_IML-2024022.py index c85f0b0..53ec7cc 100644 --- a/make_access_IML-2024022.py +++ b/make_access_IML-2024022.py @@ -8,6 +8,9 @@ from andes_migrate.trait_mollusque import TraitMollusque from andes_migrate.engin_mollusque import EnginMollusque from andes_migrate.freq_long_mollusque import FreqLongMollusque +from andes_migrate.biometrie_mollusque import BiometrieMollusque +from andes_migrate.poids_biometrie import PoidsBiometrie + from andes_migrate.andes_helper import AndesHelper @@ -43,26 +46,36 @@ for t in trait: trait_n = trait_n +1 - no_moll = 1 - print(f"Trait: ", t) + no_moll_freq_long = 1 + no_moll_biometrie = 1 + if trait_n>0: + print(f"Trait: ", t) engin = EnginMollusque(trait, output_cur) for e in engin: # print(f"Engin: ", e) capture = CaptureMollusque(engin, output_cur) for c in capture: - print(f"Capture: ", c) + # print(f"Capture: ", c) - freq = FreqLongMollusque(capture, output_cur, no_moll_init=no_moll) + freq = FreqLongMollusque(capture, output_cur, no_moll_init=no_moll_freq_long) for f in freq: # print(f"FreqLong: ", f) # if (c['COD_ESP_GEN'] == 48 or c['COD_ESP_GEN'] == 50) : - no_moll += 1 + no_moll_freq_long += 1 + + biometrie = BiometrieMollusque(capture, output_cur, no_moll_init=no_moll_biometrie) + for b in biometrie: + print(f"biometrie: ", b) + no_moll_biometrie += 1 + + poids = PoidsBiometrie(biometrie, output_cur) + for p in poids: + pass # monolithic commit if no errors are found output_cur.commit() -