From 99ed700b9fcb2ae9b00bcd8eeacb2cd79a460152 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Wed, 24 Jul 2024 12:39:06 +0200 Subject: [PATCH 01/20] Added pages --- .gitignore | 1 + README.md | 4 ++++ data_bridges_knots/client.py | 3 +-- data_bridges_knots/helpers.py | 6 +++++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index f37ec52..b0ba7f9 100644 --- a/.gitignore +++ b/.gitignore @@ -169,5 +169,6 @@ data_bridges_api_config.yaml *.Rproj *.yaml sandbox.py +sandbox.ipynb *.csv .vscode \ No newline at end of file diff --git a/README.md b/README.md index 805868e..c6fbcf0 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,10 @@ survey_data = client.get_household_survey(survey_id=CONGO_CFSVA["dataset"], acce # get XLSForm data questionnaire = client.get_household_questionnaire(CONGO_CFSVA["questionnaire"]) +# converts numeric columns into ints +numeric_columns = [ 'FCSDairy', 'FCSDairy_SRf', 'FCSFat', 'FCSFat_SRf', 'FCSFruit', 'FCSFruit_SRf', 'FCSNFruiOrg', 'FCSNPrEggs', 'FCSNPrFish', 'FCSNPrMeatF', 'FCSNPrMeatO', 'FCSNVegGre', 'FCSNVegOrg', 'FCSPr', 'FCSPr_SRf', 'FCSPulse', 'FCSPulse_SRf', 'FCSStap', 'FCSStap_SRf', 'FCSSugar', 'FCSSugar_SRf', 'FCSVeg', 'FCSVeg_SRf', 'HHSize', 'HHSize01F', 'HHSize01M', 'HHSize', 'HHSize01F', 'HHSize01M', 'HHSize1217F', 'HHSize1217M', 'HHSize1859F', 'HHSize1859M', 'HHSize24F', 'HHSize24M', 'HHSize511F', 'HHSize511M', 'HHSize60AboveF', 'HHSize60AboveM', 'RESPAge' ] +df = as_numeric(survey_data, numeric_columns) + ``` ## Contributing diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 8767305..4fc36a9 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -68,7 +68,7 @@ def setup_configuration_and_authentication(self, yaml_config_path): ) return configuration - def get_household_survey(self, survey_id, access_type, page_size=600): + def get_household_survey(self, survey_id, access_type, page_size=600, page = 0): """Retrieves survey data using the specified configuration and access type. Args: @@ -84,7 +84,6 @@ def get_household_survey(self, survey_id, access_type, page_size=600): responses = [] total_items = 1 max_item = 0 - page = 0 while total_items > max_item: page += 1 diff --git a/data_bridges_knots/helpers.py b/data_bridges_knots/helpers.py index c028173..a676128 100644 --- a/data_bridges_knots/helpers.py +++ b/data_bridges_knots/helpers.py @@ -58,8 +58,12 @@ def map_value_labels(survey_data, questionnaire): return survey_data_value_labels def as_numeric(df, col_list): + for col in col_list: - df[col] = pd.to_numeric(df[col], errors='ignore').fillna(9999).astype('int64') + try: + df[col] = pd.to_numeric(df[col], errors='ignore').fillna(9999).astype('int64') + except ValueError: + continue return df From a87bd82254da136b7562f995e00281bfd70ca284 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Thu, 12 Sep 2024 17:12:26 +0200 Subject: [PATCH 02/20] feature: commodities API endpoints + udpate ROADMAP --- .gitignore | 5 +- ROADMAP.md | 72 ++++++------------- data_bridges_knots/client.py | 129 ++++++++++++++++++++++++++++++++++- 3 files changed, 152 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index f37ec52..688c217 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,7 @@ data_bridges_api_config.yaml .Rhistory *.Rproj *.yaml -sandbox.py +sandbox.* *.csv -.vscode \ No newline at end of file +.vscode +data_bridges_knots/DataBridgesKnots + Client.code-workspace \ No newline at end of file diff --git a/ROADMAP.md b/ROADMAP.md index dc82881..8a18cc3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,62 +1,32 @@ -# Roadmap for DataBridgesUtils +# Roadmap for DataBridgesKnots -This document outlines the planned features and improvements for the `DataBridgesUtils` package, which provides a wrapper for the WFP Data Bridges API. +This document outlines the planned features and improvements for the `DataBridgesKnots` package, which provides a wrapper for the WFP Data Bridges API. -## Upcoming Release: 0.1.0 (DEV) +## Upcoming Release: 0.2.0 (DataBridges API v5.0) -### Wrapper Endpoints +### New Features +- [X] Endpoints: Commodities +- [ ] Endpoints: Commodity units +- [ ] Endpoints: Economic data +- [ ] Endpoints: Markets +- [ ] Endpoints: Markets Prices +- [ ] Endpoints: Rpme +- [ ] Endpoints: Surveys +- [ ] Endpoints: XlsForms +- [ ] STATA support: Output dta with value and variable labels +- [ ] R support: Example file for household and market data -The following endpoints will be added or improved in the upcoming release: - -- [X] Exchange rates -- [X] Food security (IPC) -- [X] GORP (Global Operational Response Plan) -- [X] Market prices -- [X] Surveys -- [X] XSL Forms - -### Wrapper Endpoints -- [X] Get variable labels -- [X] Get value labels -- [ ] Output dta with value and variable labels - -### Examples and Documentation - -- [X] Provide an example file in Python demonstrating the usage of available endpoints -- [X] Provide an example file in STATA demonstrating the usage of available endpoints - - [ ] Test -- [ ] Provide an example file in R demonstrating the usage of available endpoints - - [ ] Test -- [X] Update the README file with Python usage examples -- [X] Add documentation for STATA users in the README file - -### Repository Setup - -- [X] Create a GitHub repository for the `DataBridgesUtils` package -- [X] Configure optional dependencies in the project files -- [X] Set up the package installation process - -## Improvements (1.1.0) - -## Bug fixing +### Bug Fixes - [ ] DPO change for GORP - [ ] DPO change for XLSForm - [ ] Fix optional dependencies for STATA - [ ] Handle SSL certificate error -## Wrapper points -- [ ] Economic data -- [ ] Commodities -- [ ] Commodity units -- [ ] Markets -- [ ] RPME (Resource Planning and Monitoring Environment) - -## Future Releases (2.0.0) - -- [ ] Improve error handling and logging -- [ ] Add unit tests and integration tests -- [ ] Enhance documentation and provide more usage examples -- [ ] Optimize performance and improve code efficiency -- [ ] Implement additional features based on user feedback and requirements +## Future Releases (1.0.0 and beyond) +- Testing: Unit testing +- Testing: GitHub Actions linting and testing +- Testing: Improve error handling and logging +- Documentation: Enhance documentation and provide more usage examples +- Refactoring: Optimize performance and improve code efficiency Please note that this roadmap is subject to change, and the priorities may be adjusted based on the project's needs and available resources. diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 8767305..74e8109 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -329,7 +329,134 @@ def get_choice_list(self, xls_form_id): choiceList = pd.json_normalize(questionnaire['choiceList']).dropna() choices = choiceList.explode('choices') return choices + + def get_commodities_list(self, country_code=None, commodity_name=None, commodity_id=0, page=1, format='json'): + """ + Retrieves the detailed list of commodities available in the DataBridges platform. + + Args: + country_code (str, optional): The code to identify the country. It can be an ISO-3166 Alpha 3 code or the VAM internal admin0code. + commodity_name (str, optional): The name, even partial and case insensitive, of a commodity. + commodity_id (int, optional): The exact ID of a commodity. Defaults to 0. + page (int, optional): Page number for paged results. Defaults to 1. + format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + + Returns: + pandas.DataFrame: A DataFrame containing the retrieved commodity data. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.CommoditiesApi(api_client) + env = self.env + + try: + api_response = api_instance.commodities_list_get( + country_code=country_code, + commodity_name=commodity_name, + commodity_id=commodity_id, + page=page, + format=format, + env=env + ) + logger.info("Successfully retrieved commodities list") + + # Convert the response to a DataFrame + if hasattr(api_response, 'items'): + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + else: + df = pd.DataFrame([api_response.to_dict()]) + + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling CommoditiesApi->commodities_list_get: {e}") + raise + + def get_commodity_units_conversion_list(self, country_code=None, commodity_id=0, from_unit_id=0, to_unit_id=0, page=1, format='json'): + """ + Retrieves conversion factors to Kilogram or Litres for each convertible unit of measure. + + Args: + country_code (str, optional): The code to identify the country. It can be an ISO-3166 Alpha 3 code or the VAM internal admin0code. + commodity_id (int, optional): The exact ID of a Commodity, as found in /Commodities/List. Defaults to 0. + from_unit_id (int, optional): The exact ID of the original unit of measure of the price of a commodity. Defaults to 0. + to_unit_id (int, optional): The exact ID of the converted unit of measure of the price of a commodity. Defaults to 0. + page (int, optional): Page number for paged results. Defaults to 1. + format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + + Returns: + pandas.DataFrame: A DataFrame containing the retrieved conversion factors. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.CommodityUnitsApi(api_client) + env = self.env + + try: + api_response = api_instance.commodity_units_conversion_list_get( + country_code=country_code, + commodity_id=commodity_id, + from_unit_id=from_unit_id, + to_unit_id=to_unit_id, + page=page, + format=format, + env=env + ) + logger.info("Successfully retrieved commodity units conversion list") + + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling CommodityUnitsApi->commodity_units_conversion_list_get: {e}") + raise + + def get_commodity_units_list(self, country_code=None, commodity_unit_name=None, commodity_unit_id=0, page=1, format='json'): + """ + Retrieves the detailed list of the unit of measure available in DataBridges platform. + + Args: + country_code (str, optional): The code to identify the country. It can be an ISO-3166 Alpha 3 code or the VAM internal admin0code. + commodity_unit_name (str, optional): The name, even partial and case insensitive, of a commodity unit. + commodity_unit_id (int, optional): The exact ID of a commodity unit. Defaults to 0. + page (int, optional): Page number for paged results. Defaults to 1. + format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + + Returns: + pandas.DataFrame: A DataFrame containing the retrieved commodity units data. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.CommodityUnitsApi(api_client) + env = self.env + + try: + api_response = api_instance.commodity_units_list_get( + country_code=country_code, + commodity_unit_name=commodity_unit_name, + commodity_unit_id=commodity_unit_id, + page=page, + format=format, + env=env + ) + logger.info("Successfully retrieved commodity units list") + + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling CommodityUnitsApi->commodity_units_list_get: {e}") + raise + + if __name__ == "__main__": - pass + import yaml + # FOR TESTING + CONFIG_PATH = r"data_bridges_api_config.yaml" + + client = DataBridgesShapes(CONFIG_PATH) + + commodities_list = client.get_commodities_list(country_code='ETH', commodity_name='wheat', page=1, format='json') + print(commodities_list) \ No newline at end of file From 90f77588da8a4ed70394c83ff142c0d4167e548e Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Thu, 12 Sep 2024 17:24:00 +0200 Subject: [PATCH 03/20] docs: updated README example --- README.md | 20 +++----- ROADMAP.md | 3 +- data_bridges_api_config_sample.yaml | 1 + data_bridges_knots/client.py | 71 ++++++++++++++--------------- pyproject.toml | 2 +- setup.py | 12 ++--- 6 files changed, 51 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 805868e..85843a8 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,12 @@ pip install git+https://github.com/WFP-VAM/DataBridgesKnots.git ## Configuration 1. Create a ```data_bridges_api_config.yaml``` in the main folder you're running your core from. 2. The structure of the file is: - ``` + ```yaml NAME: '' VERSION : '' KEY: '' SECRET: '' + DATA_BRIDGES_API_KEY = '' SCOPES: - '' - '' @@ -29,26 +30,17 @@ pip install git+https://github.com/WFP-VAM/DataBridgesKnots.git 3. External users can reach out to [wfp.vaminfo@wfp.org](mailto:wfp.vaminfo@wfp.org) for support on getting the API credentials. ### Python -Run the following code to extract household survey data. +Run the following code to extract commoditiy data. ```python from data_bridges_knots import DataBridgesShapes -CONFIG_PATH = r"data_bridges_api_config.yaml" - -client = DataBridgesShapes(CONFIG_PATH) +CREDENTIALS = r"data_bridges_api_config.yaml" -#%% XSLForm definition and Household dataset +client = DataBridgesShapes(CREDENTIALS) -CONGO_CFSVA = { - 'questionnaire': 1509, - 'dataset': 3094 -} -# get household survey data -survey_data = client.get_household_survey(survey_id=CONGO_CFSVA["dataset"], access_type='full') -# get XLSForm data -questionnaire = client.get_household_questionnaire(CONGO_CFSVA["questionnaire"]) +commodities_list = client.get_commodities_list(country_code='ETH', commodity_name='wheat', page=1, format='json') ``` diff --git a/ROADMAP.md b/ROADMAP.md index 8a18cc3..2f24a98 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,8 +5,9 @@ This document outlines the planned features and improvements for the `DataBridge ## Upcoming Release: 0.2.0 (DataBridges API v5.0) ### New Features +- [ ] Update setup.py and pyproject.toml to include DataBridges API v5.0 - [X] Endpoints: Commodities -- [ ] Endpoints: Commodity units +- [X] Endpoints: Commodity units - [ ] Endpoints: Economic data - [ ] Endpoints: Markets - [ ] Endpoints: Markets Prices diff --git a/data_bridges_api_config_sample.yaml b/data_bridges_api_config_sample.yaml index 3cb1aab..022efd3 100644 --- a/data_bridges_api_config_sample.yaml +++ b/data_bridges_api_config_sample.yaml @@ -2,6 +2,7 @@ NAME: '' VERSION : '' KEY: '' SECRET: '' +DATA_BRIDGES_API_KEY = '' SCOPES: - '' - '' \ No newline at end of file diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 74e8109..c496fe6 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -373,43 +373,43 @@ def get_commodities_list(self, country_code=None, commodity_name=None, commodity raise def get_commodity_units_conversion_list(self, country_code=None, commodity_id=0, from_unit_id=0, to_unit_id=0, page=1, format='json'): - """ - Retrieves conversion factors to Kilogram or Litres for each convertible unit of measure. - - Args: - country_code (str, optional): The code to identify the country. It can be an ISO-3166 Alpha 3 code or the VAM internal admin0code. - commodity_id (int, optional): The exact ID of a Commodity, as found in /Commodities/List. Defaults to 0. - from_unit_id (int, optional): The exact ID of the original unit of measure of the price of a commodity. Defaults to 0. - to_unit_id (int, optional): The exact ID of the converted unit of measure of the price of a commodity. Defaults to 0. - page (int, optional): Page number for paged results. Defaults to 1. - format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + """ + Retrieves conversion factors to Kilogram or Litres for each convertible unit of measure. - Returns: - pandas.DataFrame: A DataFrame containing the retrieved conversion factors. - """ - with data_bridges_client.ApiClient(self.configuration) as api_client: - api_instance = data_bridges_client.CommodityUnitsApi(api_client) - env = self.env + Args: + country_code (str, optional): The code to identify the country. It can be an ISO-3166 Alpha 3 code or the VAM internal admin0code. + commodity_id (int, optional): The exact ID of a Commodity, as found in /Commodities/List. Defaults to 0. + from_unit_id (int, optional): The exact ID of the original unit of measure of the price of a commodity. Defaults to 0. + to_unit_id (int, optional): The exact ID of the converted unit of measure of the price of a commodity. Defaults to 0. + page (int, optional): Page number for paged results. Defaults to 1. + format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. - try: - api_response = api_instance.commodity_units_conversion_list_get( - country_code=country_code, - commodity_id=commodity_id, - from_unit_id=from_unit_id, - to_unit_id=to_unit_id, - page=page, - format=format, - env=env - ) - logger.info("Successfully retrieved commodity units conversion list") - - df = pd.DataFrame([item.to_dict() for item in api_response.items]) - df = df.replace({np.nan: None}) - return df + Returns: + pandas.DataFrame: A DataFrame containing the retrieved conversion factors. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.CommodityUnitsApi(api_client) + env = self.env + + try: + api_response = api_instance.commodity_units_conversion_list_get( + country_code=country_code, + commodity_id=commodity_id, + from_unit_id=from_unit_id, + to_unit_id=to_unit_id, + page=page, + format=format, + env=env + ) + logger.info("Successfully retrieved commodity units conversion list") + + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df - except ApiException as e: - logger.error(f"Exception when calling CommodityUnitsApi->commodity_units_conversion_list_get: {e}") - raise + except ApiException as e: + logger.error(f"Exception when calling CommodityUnitsApi->commodity_units_conversion_list_get: {e}") + raise def get_commodity_units_list(self, country_code=None, commodity_unit_name=None, commodity_unit_id=0, page=1, format='json'): """ @@ -458,5 +458,4 @@ def get_commodity_units_list(self, country_code=None, commodity_unit_name=None, client = DataBridgesShapes(CONFIG_PATH) - commodities_list = client.get_commodities_list(country_code='ETH', commodity_name='wheat', page=1, format='json') - print(commodities_list) \ No newline at end of file + commodities_list = client.get_commodities_list(country_code='ETH', commodity_name='wheat', page=1, format='json') \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 0bc7a20..a6f642d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "data_bridges_knots" -version = "0.1.0" +version = "0.2.0" authors = [{ name = "Alessandra Gherardelli", email = "alessandra.gherardelli@wfp.org" }, {name = "Valerio Giuffrida", email = "valerio.giuffrida@wfp.org"}] description = "Wrapper for Data Bridges API client" readme = "README.md" diff --git a/setup.py b/setup.py index 02460a1..ffdcb71 100644 --- a/setup.py +++ b/setup.py @@ -29,11 +29,11 @@ 'pip-tools', 'pytest', ], - 'data-bridges-utils': [], - 'data-bridges-utils-STATA': [ - 'stata-setup', - 'pystata', - ], - 'data-bridges-utils-R': [], + # 'data-bridges-utils': [], + # 'data-bridges-utils-STATA': [ + # 'stata-setup', + # 'pystata', + # ], + # 'data-bridges-utils-R': [], }, ) From 077cde95e2ab900107207f7db7e96d6f8f46e6ae Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Thu, 12 Sep 2024 17:32:19 +0200 Subject: [PATCH 04/20] docs: added Data Bridges API key from creedential config file --- ROADMAP.md | 1 + data_bridges_api_config_sample.yaml | 12 +-- data_bridges_knots/client.py | 109 +++++++++++++++------------- 3 files changed, 64 insertions(+), 58 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 2f24a98..f94981d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -14,6 +14,7 @@ This document outlines the planned features and improvements for the `DataBridge - [ ] Endpoints: Rpme - [ ] Endpoints: Surveys - [ ] Endpoints: XlsForms +- [ ] Endpoints: Household Survey - [ ] STATA support: Output dta with value and variable labels - [ ] R support: Example file for household and market data diff --git a/data_bridges_api_config_sample.yaml b/data_bridges_api_config_sample.yaml index 022efd3..971d347 100644 --- a/data_bridges_api_config_sample.yaml +++ b/data_bridges_api_config_sample.yaml @@ -1,8 +1,8 @@ -NAME: '' -VERSION : '' -KEY: '' -SECRET: '' -DATA_BRIDGES_API_KEY = '' -SCOPES: +NAME: 'name-of-the-application' +VERSION : '5.0.0' +KEY: '' # WFP API Gateway API Key +SECRET: '' # WFP API Gateway API Secret +DATA_BRIDGES_API_KEY = '' # WFP DataBridges API Key (required for household data) +SCOPES: # WFP DataBridges API Scopes (required for restricted endpoints) - '' - '' \ No newline at end of file diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index c496fe6..5ba3757 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -57,6 +57,7 @@ def setup_configuration_and_authentication(self, yaml_config_path): secret = databridges_config['SECRET'] scopes = databridges_config['SCOPES'] version = databridges_config['VERSION'] + data_bridges_api_key = databridges_config['DATA_BRIDGES_API_KEY'] uri = "https://api.wfp.org/vam-data-bridges/" host = str(uri + version) @@ -343,73 +344,73 @@ def get_commodities_list(self, country_code=None, commodity_name=None, commodity Returns: pandas.DataFrame: A DataFrame containing the retrieved commodity data. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.CommoditiesApi(api_client) + env = self.env + + try: + api_response = api_instance.commodities_list_get( + country_code=country_code, + commodity_name=commodity_name, + commodity_id=commodity_id, + page=page, + format=format, + env=env + ) + logger.info("Successfully retrieved commodities list") + + # Convert the response to a DataFrame + if hasattr(api_response, 'items'): + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + else: + df = pd.DataFrame([api_response.to_dict()]) + + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling CommoditiesApi->commodities_list_get: {e}") + raise + + def get_commodity_units_conversion_list(self, country_code=None, commodity_id=0, from_unit_id=0, to_unit_id=0, page=1, format='json'): + """ + Retrieves conversion factors to Kilogram or Litres for each convertible unit of measure. + + Args: + country_code (str, optional): The code to identify the country. It can be an ISO-3166 Alpha 3 code or the VAM internal admin0code. + commodity_id (int, optional): The exact ID of a Commodity, as found in /Commodities/List. Defaults to 0. + from_unit_id (int, optional): The exact ID of the original unit of measure of the price of a commodity. Defaults to 0. + to_unit_id (int, optional): The exact ID of the converted unit of measure of the price of a commodity. Defaults to 0. + page (int, optional): Page number for paged results. Defaults to 1. + format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + + Returns: + pandas.DataFrame: A DataFrame containing the retrieved conversion factors. """ with data_bridges_client.ApiClient(self.configuration) as api_client: - api_instance = data_bridges_client.CommoditiesApi(api_client) + api_instance = data_bridges_client.CommodityUnitsApi(api_client) env = self.env try: - api_response = api_instance.commodities_list_get( + api_response = api_instance.commodity_units_conversion_list_get( country_code=country_code, - commodity_name=commodity_name, commodity_id=commodity_id, + from_unit_id=from_unit_id, + to_unit_id=to_unit_id, page=page, format=format, env=env ) - logger.info("Successfully retrieved commodities list") - - # Convert the response to a DataFrame - if hasattr(api_response, 'items'): - df = pd.DataFrame([item.to_dict() for item in api_response.items]) - else: - df = pd.DataFrame([api_response.to_dict()]) + logger.info("Successfully retrieved commodity units conversion list") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) df = df.replace({np.nan: None}) return df except ApiException as e: - logger.error(f"Exception when calling CommoditiesApi->commodities_list_get: {e}") - raise - - def get_commodity_units_conversion_list(self, country_code=None, commodity_id=0, from_unit_id=0, to_unit_id=0, page=1, format='json'): - """ - Retrieves conversion factors to Kilogram or Litres for each convertible unit of measure. - - Args: - country_code (str, optional): The code to identify the country. It can be an ISO-3166 Alpha 3 code or the VAM internal admin0code. - commodity_id (int, optional): The exact ID of a Commodity, as found in /Commodities/List. Defaults to 0. - from_unit_id (int, optional): The exact ID of the original unit of measure of the price of a commodity. Defaults to 0. - to_unit_id (int, optional): The exact ID of the converted unit of measure of the price of a commodity. Defaults to 0. - page (int, optional): Page number for paged results. Defaults to 1. - format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. - - Returns: - pandas.DataFrame: A DataFrame containing the retrieved conversion factors. - """ - with data_bridges_client.ApiClient(self.configuration) as api_client: - api_instance = data_bridges_client.CommodityUnitsApi(api_client) - env = self.env - - try: - api_response = api_instance.commodity_units_conversion_list_get( - country_code=country_code, - commodity_id=commodity_id, - from_unit_id=from_unit_id, - to_unit_id=to_unit_id, - page=page, - format=format, - env=env - ) - logger.info("Successfully retrieved commodity units conversion list") - - df = pd.DataFrame([item.to_dict() for item in api_response.items]) - df = df.replace({np.nan: None}) - return df - - except ApiException as e: - logger.error(f"Exception when calling CommodityUnitsApi->commodity_units_conversion_list_get: {e}") - raise + logger.error(f"Exception when calling CommodityUnitsApi->commodity_units_conversion_list_get: {e}") + raise def get_commodity_units_list(self, country_code=None, commodity_unit_name=None, commodity_unit_id=0, page=1, format='json'): """ @@ -458,4 +459,8 @@ def get_commodity_units_list(self, country_code=None, commodity_unit_name=None, client = DataBridgesShapes(CONFIG_PATH) - commodities_list = client.get_commodities_list(country_code='ETH', commodity_name='wheat', page=1, format='json') \ No newline at end of file + commodity_units_list = client.get_commodity_units_list(country_code="TZA", commodity_unit_name="Kg", page=1, format='json') + comodity_unit_conversion_list = client.get_commodity_units_conversion_list(country_code="TZA", commodity_id=1, from_unit_id=1, to_unit_id=2, page=1, format='json') + + print(commodity_units_list) + print(comodity_unit_conversion_list) \ No newline at end of file From ed98601a16996486f36cd82d7ae012a92bb440c3 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Fri, 13 Sep 2024 11:33:49 +0200 Subject: [PATCH 05/20] feature: add currency_list, usd_indirect_quotation, economic_indicator list endpoints --- README.md | 12 ++ data_bridges_api_config_sample.yaml | 2 +- data_bridges_knots/client.py | 212 ++++++++++++++++++++++++---- pyproject.toml | 5 +- setup.py | 20 ++- 5 files changed, 211 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 85843a8..f162663 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,18 @@ You can install the `data_bridges_knots` package using `pip` and the Git reposit pip install git+https://github.com/WFP-VAM/DataBridgesKnots.git ``` +STATA and R users will also need the appropriate optional dependencies to use this package in their respective software. To install the package with these dependencies, use the following command: + +***STATA users*** +``` +pip install git+https://github.com/WFP-VAM/DataBridgesKnots.git#egg=data_bridges_knots[STATA] +``` + +***R users*** +``` +pip install git+https://github.com/WFP-VAM/DataBridgesKnots.git#egg=data_bridges_knots[R] +``` + ## Configuration 1. Create a ```data_bridges_api_config.yaml``` in the main folder you're running your core from. 2. The structure of the file is: diff --git a/data_bridges_api_config_sample.yaml b/data_bridges_api_config_sample.yaml index 971d347..08a6859 100644 --- a/data_bridges_api_config_sample.yaml +++ b/data_bridges_api_config_sample.yaml @@ -2,7 +2,7 @@ NAME: 'name-of-the-application' VERSION : '5.0.0' KEY: '' # WFP API Gateway API Key SECRET: '' # WFP API Gateway API Secret -DATA_BRIDGES_API_KEY = '' # WFP DataBridges API Key (required for household data) +DATABRIDGES_API_KEY = '' # WFP DataBridges API Key (required for household data) SCOPES: # WFP DataBridges API Scopes (required for restricted endpoints) - '' - '' \ No newline at end of file diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 5ba3757..55e6ade 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -57,7 +57,7 @@ def setup_configuration_and_authentication(self, yaml_config_path): secret = databridges_config['SECRET'] scopes = databridges_config['SCOPES'] version = databridges_config['VERSION'] - data_bridges_api_key = databridges_config['DATA_BRIDGES_API_KEY'] + data_bridges_api_key = databridges_config['DATABRIDGES_API_KEY'] uri = "https://api.wfp.org/vam-data-bridges/" host = str(uri + version) @@ -345,35 +345,35 @@ def get_commodities_list(self, country_code=None, commodity_name=None, commodity Returns: pandas.DataFrame: A DataFrame containing the retrieved commodity data. """ - with data_bridges_client.ApiClient(self.configuration) as api_client: - api_instance = data_bridges_client.CommoditiesApi(api_client) - env = self.env + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.CommoditiesApi(api_client) + env = self.env - try: - api_response = api_instance.commodities_list_get( - country_code=country_code, - commodity_name=commodity_name, - commodity_id=commodity_id, - page=page, - format=format, - env=env - ) - logger.info("Successfully retrieved commodities list") - - # Convert the response to a DataFrame - if hasattr(api_response, 'items'): - df = pd.DataFrame([item.to_dict() for item in api_response.items]) - else: - df = pd.DataFrame([api_response.to_dict()]) - - df = df.replace({np.nan: None}) - return df + try: + api_response = api_instance.commodities_list_get( + country_code=country_code, + commodity_name=commodity_name, + commodity_id=commodity_id, + page=page, + format=format, + env=env + ) + logger.info("Successfully retrieved commodities list") + + # Convert the response to a DataFrame + if hasattr(api_response, 'items'): + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + else: + df = pd.DataFrame([api_response.to_dict()]) + + df = df.replace({np.nan: None}) + return df - except ApiException as e: - logger.error(f"Exception when calling CommoditiesApi->commodities_list_get: {e}") - raise + except ApiException as e: + logger.error(f"Exception when calling CommoditiesApi->commodities_list_get: {e}") + raise - def get_commodity_units_conversion_list(self, country_code=None, commodity_id=0, from_unit_id=0, to_unit_id=0, page=1, format='json'): + def get_commodity_units_conversion_list(self, country_code=None, commodity_id=0, from_unit_id=0, to_unit_id=0, page=1, format='json'): """ Retrieves conversion factors to Kilogram or Litres for each convertible unit of measure. @@ -449,6 +449,151 @@ def get_commodity_units_list(self, country_code=None, commodity_unit_name=None, logger.error(f"Exception when calling CommodityUnitsApi->commodity_units_list_get: {e}") raise + def get_currency_list(self, country_code=None, currency_name=None, currency_id=0, page=1, format='json'): + """ + Returns the list of currencies available in the internal VAM database, with Currency 3-letter code, matching with ISO 4217. + + Args: + country_code (str, optional): The code to identify the country. It can be an ISO-3166 Alpha 3 code or the VAM internal admin0code. + currency_name (str, optional): Currency 3-letter code, matching with ISO 4217. + currency_id (int, optional): Unique code to identify the currency in internal VAM currencies. Defaults to 0. + page (int, optional): Page number for paged results. Defaults to 1. + format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + + Returns: + pandas.DataFrame: A DataFrame containing the retrieved currency data. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.CurrencyApi(api_client) + env = self.env + + try: + api_response = api_instance.currency_list_get( + country_code=country_code, + currency_name=currency_name, + currency_id=currency_id, + page=page, + format=format, + env=env + ) + logger.info("Successfully retrieved currency list") + + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling CurrencyApi->currency_list_get: {e}") + raise + + def get_usd_indirect_quotation(self, country_iso3='', currency_name='', page=1, format='json'): + """ + Returns the value of the Exchange rates from Trading Economics, for official rates, and DataViz for unofficial rates. + + Args: + country_iso3 (str, optional): The code to identify the country. Must be a ISO-3166 Alpha 3 code. Defaults to ''. + currency_name (str, optional): The ISO3 code for the currency, based on ISO4217. Defaults to ''. + page (int, optional): Page number for paged results. Defaults to 1. + format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + + Returns: + pandas.DataFrame: A DataFrame containing the retrieved exchange rate data. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.CurrencyApi(api_client) + env = self.env + + try: + api_response = api_instance.currency_usd_indirect_quotation_get( + country_iso3=country_iso3, + currency_name=currency_name, + page=page, + format=format, + env=env + ) + logger.info("Successfully retrieved USD indirect quotation data") + + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling CurrencyApi->currency_usd_indirect_quotation_get: {e}") + raise + + def get_economic_indicator_list(self, page=1, indicator_name='', iso3='', format='json'): + """ + Returns the lists of indicators for which Vulnerability Analysis and Mapping - Economic and Market Analysis Unit has redistribution licensing from Trading Economics. + + Args: + page (int, optional): Page number for paged results. Defaults to 1. + indicator_name (str, optional): Unique indicator name. Defaults to ''. + iso3 (str, optional): The code to identify the country. Must be a ISO-3166 Alpha 3 code. Defaults to ''. + format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + + Returns: + pandas.DataFrame: A DataFrame containing the retrieved economic indicator data. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.EconomicDataApi(api_client) + env = self.env + + try: + api_response = api_instance.economic_data_indicator_list_get( + page=page, + indicator_name=indicator_name, + iso3=iso3, + format=format, + env=env + ) + logger.info("Successfully retrieved economic indicator list") + + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling EconomicDataApi->economic_data_indicator_list_get: {e}") + raise + + def get_economic_data(self, indicator_name, page=1, iso3='', start_date=None, end_date=None, format='json'): + """ + Returns the time series of values for different indicators. + + Args: + indicator_name (str): Name of the indicator as found in /EconomicData/IndicatorList. + page (int, optional): Page number for paged results. Defaults to 1. + iso3 (str, optional): The code to identify the country. Must be a ISO-3166 Alpha 3 code. Defaults to ''. + start_date (datetime, optional): Starting date for the range in which data was collected. + end_date (datetime, optional): Ending date for the range in which data was collected. + format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + + Returns: + pandas.DataFrame: A DataFrame containing the retrieved economic data. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.EconomicDataApi(api_client) + env = self.env + + try: + api_response = api_instance.economic_data_indicator_name_get( + indicator_name=indicator_name, + page=page, + iso3=iso3, + start_date=start_date, + end_date=end_date, + format=format, + env=env + ) + logger.info(f"Successfully retrieved economic data for indicator: {indicator_name}") + + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling EconomicDataApi->economic_data_indicator_name_get: {e}") + raise @@ -462,5 +607,14 @@ def get_commodity_units_list(self, country_code=None, commodity_unit_name=None, commodity_units_list = client.get_commodity_units_list(country_code="TZA", commodity_unit_name="Kg", page=1, format='json') comodity_unit_conversion_list = client.get_commodity_units_conversion_list(country_code="TZA", commodity_id=1, from_unit_id=1, to_unit_id=2, page=1, format='json') - print(commodity_units_list) - print(comodity_unit_conversion_list) \ No newline at end of file + currency_list = client.get_currency_list(country_code="TZA", currency_name="TZS", currency_id=0, page=1, format='json') + usd_indirect_quotation = client.get_usd_indirect_quotation(country_iso3="TZA", currency_name="TZS", page=1, format='json') + + # BUG: 404 error + economic_indicator_list = client.get_economic_indicator_list(page=1, indicator_name='', iso3='', format='json') + # BUG: 404 error + economic_data = client.get_economic_data(indicator_name='', page=1, iso3='', start_date=None, end_date=None, format='json') + print(economic_indicator_list) + + + diff --git a/pyproject.toml b/pyproject.toml index a6f642d..5723f6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,9 +23,8 @@ dependencies = [ [project.optional-dependencies] dev = ["black", "bumpver", "isort", "pip-tools", "pytest"] -data-bridges-utils = [] -data-bridges-utils-STATA = ["stata-setup", "pystata"] -data-bridges-utils-R = [] +STATA = ["stata-setup", "pystata"] +R = [] [tool.setuptools] packages = ["data_bridges_knots"] diff --git a/setup.py b/setup.py index ffdcb71..d64b85a 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='data_bridges_knots', - version='0.1.0', + version='0.2.0', description='Wrapper for Data Bridges API client', long_description=long_description, long_description_content_type='text/markdown', @@ -21,6 +21,13 @@ keywords=['VAM', 'WFP', 'data'], packages=find_packages(exclude=['tests', 'tests.*']), python_requires='>=3.9', + install_requires=[ + 'PyYAML', + 'pandas>=2', + 'pystata', + 'stata-setup', + 'data-bridges-client @ git+https://github.com/WFP-VAM/DataBridgesAPI.git', + ], extras_require={ 'dev': [ 'black', @@ -29,11 +36,10 @@ 'pip-tools', 'pytest', ], - # 'data-bridges-utils': [], - # 'data-bridges-utils-STATA': [ - # 'stata-setup', - # 'pystata', - # ], - # 'data-bridges-utils-R': [], + 'STATA': [ + 'stata-setup', + 'pystata', + ], + 'R': [], }, ) From dc21311000f0aff51431b1da2f60d12209d1f709 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Thu, 19 Sep 2024 16:31:21 +0200 Subject: [PATCH 06/20] feature: add MarketsApi endpoints --- README.md | 2 +- ROADMAP.md | 20 +++--- data_bridges_knots/client.py | 118 ++++++++++++++++++++++------------- 3 files changed, 88 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index f162663..b55fabb 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ pip install git+https://github.com/WFP-VAM/DataBridgesKnots.git#egg=data_bridges ***R users*** ``` -pip install git+https://github.com/WFP-VAM/DataBridgesKnots.git#egg=data_bridges_knots[R] +pip install git+https://github.com/WFP-VAM/DataBridgesusKnots.git#egg=data_bridges_knots[R] ``` ## Configuration diff --git a/ROADMAP.md b/ROADMAP.md index f94981d..1c814b3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,15 +6,17 @@ This document outlines the planned features and improvements for the `DataBridge ### New Features - [ ] Update setup.py and pyproject.toml to include DataBridges API v5.0 -- [X] Endpoints: Commodities -- [X] Endpoints: Commodity units -- [ ] Endpoints: Economic data -- [ ] Endpoints: Markets -- [ ] Endpoints: Markets Prices -- [ ] Endpoints: Rpme -- [ ] Endpoints: Surveys -- [ ] Endpoints: XlsForms -- [ ] Endpoints: Household Survey +- [X] Endpoints: CommoditiesApis +- [X] Endpoints: CurrencyApi +- [X] Endpoints: EconomicDataApi +- [X] Endpoints: MarketsApi +- [ ] Endpoints: FoodSecurityApi +- [ ] Endpoints: GorpApi +- [ ] Endpoints: MarketPricesAPi +- [ ] Endpoints: IncubationApi +- [ ] Endpoints: RpmeApi +- [ ] Endpoints: SurveysApi +- [ ] Endpoints: XlsFormsApi - [ ] STATA support: Output dta with value and variable labels - [ ] R support: Example file for household and market data diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 55e6ade..4697ea9 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -521,6 +521,7 @@ def get_usd_indirect_quotation(self, country_iso3='', currency_name='', page=1, logger.error(f"Exception when calling CurrencyApi->currency_usd_indirect_quotation_get: {e}") raise + # FIXME: JSON response def get_economic_indicator_list(self, page=1, indicator_name='', iso3='', format='json'): """ Returns the lists of indicators for which Vulnerability Analysis and Mapping - Economic and Market Analysis Unit has redistribution licensing from Trading Economics. @@ -535,68 +536,98 @@ def get_economic_indicator_list(self, page=1, indicator_name='', iso3='', format pandas.DataFrame: A DataFrame containing the retrieved economic indicator data. """ with data_bridges_client.ApiClient(self.configuration) as api_client: + # Create an instance of the API class api_instance = data_bridges_client.EconomicDataApi(api_client) - env = self.env try: - api_response = api_instance.economic_data_indicator_list_get( - page=page, - indicator_name=indicator_name, - iso3=iso3, - format=format, - env=env - ) - logger.info("Successfully retrieved economic indicator list") - + # Returns the lists of indicators. + api_response = api_instance.economic_data_indicator_list_get(page=page, indicator_name=indicator_name, iso3=iso3, format=format, env=self.env) + print("The response of EconomicDataApi->economic_data_indicator_list_get:\n") + return api_response + except Exception as e: + print("Exception when calling EconomicDataApi->economic_data_indicator_list_get: %s\n" % e) + + # BUG: Unsupported content type: 'application/geo+json + def get_market_geojson_list(self, adm0code=None): + # Enter a context with an instance of the API client + with data_bridges_client.ApiClient(self.configuration) as api_client: + # Create an instance of the API class + api_instance = data_bridges_client.MarketsApi(api_client) + + try: + # Provide a list of geo referenced markets in a specific country + api_response = api_instance.markets_geo_json_list_get(adm0code=adm0code, env=self.env) + print("The response of MarketsApi->markets_geo_json_list_get:\n") + return api_response + except Exception as e: + print("Exception when calling MarketsApi->markets_geo_json_list_get: %s\n" % e) + + def get_markets_list(self, country_code=None, page=1): + # Enter a context with an instance of the API client + with data_bridges_client.ApiClient(self.configuration) as api_client: + # Create an instance of the API class + api_instance = data_bridges_client.MarketsApi(api_client) + format = 'json' # str | Output format: [JSON|CSV] Json is the default value (optional) (default to 'json') + env = self.env # str | Environment. * `prod` - api.vam.wfp.org * `dev` - dev.api.vam.wfp.org (optional) + + try: + # Get a complete list of markets in a country + api_response = api_instance.markets_list_get(country_code=country_code, page=page, format=format, env=env) + print("The response of MarketsApi->markets_list_get:\n") df = pd.DataFrame([item.to_dict() for item in api_response.items]) df = df.replace({np.nan: None}) return df + except Exception as e: + print("Exception when calling MarketsApi->markets_list_get: %s\n" % e) + - except ApiException as e: - logger.error(f"Exception when calling EconomicDataApi->economic_data_indicator_list_get: {e}") - raise + # FIXME: JSON response + def get_markets_as_csv(self, adm0code=None, local_names=False): + with data_bridges_client.ApiClient(self.configuration) as api_client: + # Create an instance of the API class + api_instance = data_bridges_client.MarketsApi(api_client) + local_names = False # bool | If true the name of markets and regions will be localized if available (optional) (default to False) + + try: + # Get a complete list of markets in a country + api_response = api_instance.markets_markets_as_csv_get(adm0code=adm0code, local_names=local_names, env=self.env) + logger.info("The response of MarketsApi->markets_markets_as_csv_get:\n") + return api_response + except Exception as e: + logger.error("Exception when calling MarketsApi->markets_markets_as_csv_get: %s\n" % e) + + def get_nearby_markets(self): + pass - def get_economic_data(self, indicator_name, page=1, iso3='', start_date=None, end_date=None, format='json'): + def get_nearby_markets(self, adm0code=None, lat=None, lng=None): """ - Returns the time series of values for different indicators. + Find markets near a given location by longitude and latitude within a 15Km distance. Args: - indicator_name (str): Name of the indicator as found in /EconomicData/IndicatorList. - page (int, optional): Page number for paged results. Defaults to 1. - iso3 (str, optional): The code to identify the country. Must be a ISO-3166 Alpha 3 code. Defaults to ''. - start_date (datetime, optional): Starting date for the range in which data was collected. - end_date (datetime, optional): Ending date for the range in which data was collected. - format (str, optional): Output format: 'json' or 'csv'. Defaults to 'json'. + adm0code (int, optional): Code for the country as retrieved from https://api.vam.wfp.org/geodata/CountriesInRegion. + lat (float, optional): Latitude of the point that will be used to search for existing nearby markets. + lng (float, optional): Longitude of the point that will be used to search for existing nearby markets. Returns: - pandas.DataFrame: A DataFrame containing the retrieved economic data. + pandas.DataFrame: A DataFrame containing the nearby markets data. """ with data_bridges_client.ApiClient(self.configuration) as api_client: - api_instance = data_bridges_client.EconomicDataApi(api_client) + api_instance = data_bridges_client.MarketsApi(api_client) env = self.env try: - api_response = api_instance.economic_data_indicator_name_get( - indicator_name=indicator_name, - page=page, - iso3=iso3, - start_date=start_date, - end_date=end_date, - format=format, - env=env - ) - logger.info(f"Successfully retrieved economic data for indicator: {indicator_name}") - - df = pd.DataFrame([item.to_dict() for item in api_response.items]) + api_response = api_instance.markets_nearby_markets_get(adm0code=adm0code, lat=lat, lng=lng, env=env) + logger.info("Successfully retrieved nearby markets") + df = pd.DataFrame([item.to_dict() for item in api_response]) df = df.replace({np.nan: None}) return df - except ApiException as e: - logger.error(f"Exception when calling EconomicDataApi->economic_data_indicator_name_get: {e}") + logger.error(f"Exception when calling MarketsApi->markets_nearby_markets_get: {e}") raise + if __name__ == "__main__": import yaml # FOR TESTING @@ -610,11 +641,14 @@ def get_economic_data(self, indicator_name, page=1, iso3='', start_date=None, en currency_list = client.get_currency_list(country_code="TZA", currency_name="TZS", currency_id=0, page=1, format='json') usd_indirect_quotation = client.get_usd_indirect_quotation(country_iso3="TZA", currency_name="TZS", page=1, format='json') - # BUG: 404 error + # FIXME: JSON Response + printing instead of logging economic_indicator_list = client.get_economic_indicator_list(page=1, indicator_name='', iso3='', format='json') - # BUG: 404 error - economic_data = client.get_economic_data(indicator_name='', page=1, iso3='', start_date=None, end_date=None, format='json') - print(economic_indicator_list) - + # BUG: Error: Unsoppoerted content type application/geo+json + # markets_geo_json = client.get_market_geojson_list(adm0code=56) + + markets_list = client.get_markets_list(country_code="TZA") + + markets_csv = client.get_markets_as_csv(adm0code=4, local_names=False) + nearby_markets = client.get_nearby_markets(adm0code=56) From fd8608dfb5bd8d5bd93ccc8e15ac510afa8ae599 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Thu, 19 Sep 2024 17:07:24 +0200 Subject: [PATCH 07/20] feature: udpate FoodSecurityApi endpoints --- .gitignore | 2 +- ROADMAP.md | 2 ++ data_bridges_knots/client.py | 38 +++++++++++++++++++----------------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 688c217..8d4eb74 100644 --- a/.gitignore +++ b/.gitignore @@ -171,4 +171,4 @@ data_bridges_api_config.yaml sandbox.* *.csv .vscode -data_bridges_knots/DataBridgesKnots + Client.code-workspace \ No newline at end of file +*.code-workspace \ No newline at end of file diff --git a/ROADMAP.md b/ROADMAP.md index 1c814b3..7c2411e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -23,6 +23,8 @@ This document outlines the planned features and improvements for the `DataBridge ### Bug Fixes - [ ] DPO change for GORP - [ ] DPO change for XLSForm +- [ ] Markets GeoJSON response +- [ ] JSON to DataFrame response - [ ] Fix optional dependencies for STATA - [ ] Handle SSL certificate error diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 4697ea9..e59461a 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -34,7 +34,6 @@ class DataBridgesShapes: pandas.DataFrame: A DataFrame containing the retrieved survey data. """ - def __init__(self, yaml_config_path, env='prod'): self.configuration = self.setup_configuration_and_authentication(yaml_config_path) self.env = env @@ -202,7 +201,6 @@ def get_exchange_rates(self, country_iso3, page_size=1000): df = pd.DataFrame(responses) df = df.replace({np.nan: None}) return df - def get_gorp(self, data_type, page=None): """ @@ -252,36 +250,39 @@ def get_gorp(self, data_type, page=None): df = df.replace({np.nan: None}) return df - def get_food_security(self, country_iso3=None, year=None, page=None, env='prod'): + def get_food_security_list(self, iso3=None, year=None, page=1): """ Retrieves food security data from the Data Bridges API. Args: - country_iso3 (str, optional): The ISO3 code of the country to retrieve data for. Defaults to None. - year (int, optional): The year to retrieve data for. Defaults to None. - page (int, optional): The page number for paginated results. Defaults to None. - env (str, optional): The environment to use. Can be 'prod' or 'dev'. Defaults to 'prod'. + iso3 (str, optional): The country ISO3 code. + year (int, optional): The year to retrieve data for. + page (int, optional): Page number for paged results. Defaults to 1. Returns: - The requested food security data. + pandas.DataFrame: A DataFrame containing the retrieved food security data. """ - responses =[] with data_bridges_client.ApiClient(self.configuration) as api_client: food_security_api_instance = data_bridges_client.FoodSecurityApi(api_client) + env = self.env try: - food_security_data = food_security_api_instance.food_security_list_get( - iso3=country_iso3, + api_response = food_security_api_instance.food_security_list_get( + iso3=iso3, year=year, page=page, env=env ) - except data_bridges_client.ApiException as e: - logger.error(f"Exception when calling Food Security data: {e}") + logger.info("Successfully retrieved food security data") + + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling FoodSecurityApi->food_security_list_get: {e}") raise - - responses.extend(item.to_dict() for item in food_security_data.items) - return pd.DataFrame(responses) + def get_household_questionnaire(self, xls_form_id, env='prod', page_size=200): """ @@ -596,8 +597,6 @@ def get_markets_as_csv(self, adm0code=None, local_names=False): except Exception as e: logger.error("Exception when calling MarketsApi->markets_markets_as_csv_get: %s\n" % e) - def get_nearby_markets(self): - pass def get_nearby_markets(self, adm0code=None, lat=None, lng=None): """ @@ -652,3 +651,6 @@ def get_nearby_markets(self, adm0code=None, lat=None, lng=None): markets_csv = client.get_markets_as_csv(adm0code=4, local_names=False) nearby_markets = client.get_nearby_markets(adm0code=56) + + get_food_security_list = client.get_food_security_list() + print(get_food_security_list) \ No newline at end of file From 52f012514f8646ef24732d7eec86676148ccba81 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Thu, 19 Sep 2024 17:28:34 +0200 Subject: [PATCH 08/20] feature: update GorpApi endpoints --- ROADMAP.md | 9 ++- data_bridges_knots/client.py | 123 ++++++++++++++++------------------- 2 files changed, 62 insertions(+), 70 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 7c2411e..1c21bdb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -10,21 +10,24 @@ This document outlines the planned features and improvements for the `DataBridge - [X] Endpoints: CurrencyApi - [X] Endpoints: EconomicDataApi - [X] Endpoints: MarketsApi -- [ ] Endpoints: FoodSecurityApi +- [X] Endpoints: FoodSecurityApi - [ ] Endpoints: GorpApi - [ ] Endpoints: MarketPricesAPi - [ ] Endpoints: IncubationApi - [ ] Endpoints: RpmeApi - [ ] Endpoints: SurveysApi - [ ] Endpoints: XlsFormsApi -- [ ] STATA support: Output dta with value and variable labels -- [ ] R support: Example file for household and market data ### Bug Fixes - [ ] DPO change for GORP - [ ] DPO change for XLSForm - [ ] Markets GeoJSON response - [ ] JSON to DataFrame response + +## Future Release +### v0.3.0 (DataBridges API v.0) +- [ ] STATA support +- [ ] R example files - [ ] Fix optional dependencies for STATA - [ ] Handle SSL certificate error diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index e59461a..d833234 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -8,7 +8,6 @@ from data_bridges_client.token import WfpApiToken import data_bridges_client - logname = "data_bridges_api_calls.log" logging.basicConfig(filename=logname, filemode='a', @@ -18,21 +17,7 @@ logger = logging.getLogger(__name__) - - class DataBridgesShapes: - """ - Retrieves survey data using the specified configuration and access type. - - Args: - survey_id (str): The ID of the survey to retrieve. - access_type (str): The type of access to use for retrieving the survey data. - Can be one of '', 'full', 'draft', 'official', or 'public'. - page_size (int, optional): The number of items to retrieve per page. Defaults to 200. - - Returns: - pandas.DataFrame: A DataFrame containing the retrieved survey data. - """ def __init__(self, yaml_config_path, env='prod'): self.configuration = self.setup_configuration_and_authentication(yaml_config_path) @@ -201,54 +186,7 @@ def get_exchange_rates(self, country_iso3, page_size=1000): df = pd.DataFrame(responses) df = df.replace({np.nan: None}) return df - - def get_gorp(self, data_type, page=None): - """ - Retrieves data from the Global Operational Response Plan (GORP) API. - Args: - data_type (str): The type of GORP data to retrieve. Can be one of 'country_latest', 'global_latest', 'latest', 'list', or 'regional_latest'. - page (int, optional): The page number for paginated results. Defaults to None. - env (str, optional): The environment to use. Can be 'prod' or 'dev'. Defaults to 'prod'. - - Returns: - The requested GORP data. - """ - with data_bridges_client.ApiClient(self.configuration) as api_client: - gorp_api_instance = data_bridges_client.GorpApi(api_client) - env = self.env - - responses = [] - - try: - if data_type == 'country_latest': - gorp_data = gorp_api_instance.gorp_country_latest_get(env=env) - elif data_type == 'global_latest': - gorp_data = gorp_api_instance.gorp_global_latest_get(env=env) - elif data_type == 'latest': - - gorp_data = gorp_api_instance.gorp_latest_get(page=page, env=env) - elif data_type == 'list': - gorp_data = gorp_api_instance.gorp_list_get(page=page, env=env) - elif data_type == 'regional_latest': - gorp_data = gorp_api_instance.gorp_regional_latest_get(env=env) - else: - raise ValueError(f"Invalid data_type: {data_type}") - except ApiException as e: - logger.error("Exception when calling Exchange rates data->household_full_data_get: %s\n", e) - raise - - if "GorpGlobalApiDto" in gorp_data.__doc__: - responses.extend(item for item in gorp_data) - else: - try: - responses.extend(item.to_dict() for item in gorp_data.items) - except AttributeError: - responses.extend(item.to_dict() for item in gorp_data) - - df = pd.DataFrame(responses) - df = df.replace({np.nan: None}) - return df def get_food_security_list(self, iso3=None, year=None, page=1): """ @@ -283,7 +221,6 @@ def get_food_security_list(self, iso3=None, year=None, page=1): logger.error(f"Exception when calling FoodSecurityApi->food_security_list_get: {e}") raise - def get_household_questionnaire(self, xls_form_id, env='prod', page_size=200): """ This function fetches questionnaire data for a given form ID from the Data Bridges API. @@ -580,12 +517,10 @@ def get_markets_list(self, country_code=None, page=1): return df except Exception as e: print("Exception when calling MarketsApi->markets_list_get: %s\n" % e) - # FIXME: JSON response def get_markets_as_csv(self, adm0code=None, local_names=False): with data_bridges_client.ApiClient(self.configuration) as api_client: - # Create an instance of the API class api_instance = data_bridges_client.MarketsApi(api_client) local_names = False # bool | If true the name of markets and regions will be localized if available (optional) (default to False) @@ -597,7 +532,6 @@ def get_markets_as_csv(self, adm0code=None, local_names=False): except Exception as e: logger.error("Exception when calling MarketsApi->markets_markets_as_csv_get: %s\n" % e) - def get_nearby_markets(self, adm0code=None, lat=None, lng=None): """ Find markets near a given location by longitude and latitude within a 15Km distance. @@ -623,9 +557,50 @@ def get_nearby_markets(self, adm0code=None, lat=None, lng=None): except ApiException as e: logger.error(f"Exception when calling MarketsApi->markets_nearby_markets_get: {e}") raise + def get_gorp(self, data_type, page=None): + """ + Retrieves data from the Global Operational Response Plan (GORP) API. + Args: + data_type (str): The type of GORP data to retrieve. Can be one of 'country_latest', 'global_latest', 'latest', 'list', or 'regional_latest'. + page (int, optional): The page number for paginated results. Required for 'latest' and 'list' data types. + Returns: + pandas.DataFrame: A DataFrame containing the requested GORP data. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + gorp_api_instance = data_bridges_client.GorpApi(api_client) + env = self.env + try: + if data_type == 'country_latest': + gorp_data = gorp_api_instance.gorp_country_latest_get(env=env) + elif data_type == 'global_latest': + gorp_data = gorp_api_instance.gorp_global_latest_get(env=env) + elif data_type == 'latest': + gorp_data = gorp_api_instance.gorp_latest_get(page=page, env=env) + elif data_type == 'list': + gorp_data = gorp_api_instance.gorp_list_get(page=page, env=env) + elif data_type == 'regional_latest': + gorp_data = gorp_api_instance.gorp_regional_latest_get(env=env) + else: + raise ValueError(f"Invalid data_type: {data_type}") + + logger.info(f"Successfully retrieved GORP data for type: {data_type}") + + if isinstance(gorp_data, list): + df = pd.DataFrame([item.to_dict() for item in gorp_data]) + elif hasattr(gorp_data, 'items'): + df = pd.DataFrame([item.to_dict() for item in gorp_data.items]) + else: + df = pd.DataFrame([gorp_data.to_dict()]) + + df = df.replace({np.nan: None}) + return df + + except ApiException as e: + logger.error(f"Exception when calling GorpApi->{data_type}: {e}") + raise if __name__ == "__main__": import yaml @@ -653,4 +628,18 @@ def get_nearby_markets(self, adm0code=None, lat=None, lng=None): nearby_markets = client.get_nearby_markets(adm0code=56) get_food_security_list = client.get_food_security_list() - print(get_food_security_list) \ No newline at end of file + + # Get country latest data + country_latest_df = client.get_gorp('country_latest') + + # Get global latest data + global_latest_df = client.get_gorp('global_latest') + + # Get latest data (paginated) + latest_df = client.get_gorp('latest', page=1) + + # Get full list data (paginated) + list_df = client.get_gorp('list', page=1) + + # Get regional latest data + regional_latest_df = client.get_gorp('regional_latest') \ No newline at end of file From 021ae33cef91b5eecca1c5ca8ef72c056dcae2ae Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Thu, 19 Sep 2024 17:45:49 +0200 Subject: [PATCH 09/20] feature: update IncubationApi endpoints --- ROADMAP.md | 3 +- data_bridges_knots/client.py | 373 ++++++++++++++++++++++++++--------- 2 files changed, 277 insertions(+), 99 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 1c21bdb..754fae8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -11,7 +11,7 @@ This document outlines the planned features and improvements for the `DataBridge - [X] Endpoints: EconomicDataApi - [X] Endpoints: MarketsApi - [X] Endpoints: FoodSecurityApi -- [ ] Endpoints: GorpApi +- [X] Endpoints: GorpApi - [ ] Endpoints: MarketPricesAPi - [ ] Endpoints: IncubationApi - [ ] Endpoints: RpmeApi @@ -19,7 +19,6 @@ This document outlines the planned features and improvements for the `DataBridge - [ ] Endpoints: XlsFormsApi ### Bug Fixes -- [ ] DPO change for GORP - [ ] DPO change for XLSForm - [ ] Markets GeoJSON response - [ ] JSON to DataFrame response diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index d833234..b7efe52 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -53,56 +53,56 @@ def setup_configuration_and_authentication(self, yaml_config_path): ) return configuration - def get_household_survey(self, survey_id, access_type, page_size=600): - """Retrieves survey data using the specified configuration and access type. - - Args: - survey_id (str): The ID of the survey to retrieve. - access_type (str): The type of access to use for retrieving the survey data. - Can be one of '', 'full', 'draft', 'official', or 'public'. - page_size (int, optional): The number of items to retrieve per page. Defaults to 200. - - Returns: - pandas.DataFrame: A DataFrame containing the retrieved survey data. - """ - - responses = [] - total_items = 1 - max_item = 0 - page = 0 - - while total_items > max_item: - page += 1 - with data_bridges_client.ApiClient(self.configuration) as api_client: - api_instance = data_bridges_client.IncubationApi(api_client) - env = self.env - - try: - logger.info(f"Calling get_household_survey for survey {survey_id}") - # Select appropriate API call based on access_type - api_survey = { - '': api_instance.household_public_base_data_get, - 'full': api_instance.household_full_data_get, - 'draft': api_instance.household_draft_internal_base_data_get, - 'official': api_instance.household_official_use_base_data_get, - 'public': api_instance.household_public_base_data_get - }.get(access_type)(survey_id=survey_id, page=page, page_size=page_size, env=env) - - logger.info("Fetching page %s", page) - logger.info("Items: %s", len(api_survey.items)) - responses.extend(api_survey.items) - total_items = api_survey.total_items - max_item = len(api_survey.items) + max_item - time.sleep(1) - - except ApiException as e: - logger.error("Exception when calling Household data-> %s%s\n", access_type, e) - raise - - df = pd.DataFrame(responses) - - - return df + # def get_household_survey(self, survey_id, access_type, page_size=600): + # """Retrieves survey data using the specified configuration and access type. + + # Args: + # survey_id (str): The ID of the survey to retrieve. + # access_type (str): The type of access to use for retrieving the survey data. + # Can be one of '', 'full', 'draft', 'official', or 'public'. + # page_size (int, optional): The number of items to retrieve per page. Defaults to 200. + + # Returns: + # pandas.DataFrame: A DataFrame containing the retrieved survey data. + # """ + + # responses = [] + # total_items = 1 + # max_item = 0 + # page = 0 + + # while total_items > max_item: + # page += 1 + # with data_bridges_client.ApiClient(self.configuration) as api_client: + # api_instance = data_bridges_client.IncubationApi(api_client) + # env = self.env + + # try: + # logger.info(f"Calling get_household_survey for survey {survey_id}") + # # Select appropriate API call based on access_type + # api_survey = { + # '': api_instance.household_public_base_data_get, + # 'full': api_instance.household_full_data_get, + # 'draft': api_instance.household_draft_internal_base_data_get, + # 'official': api_instance.household_official_use_base_data_get, + # 'public': api_instance.household_public_base_data_get + # }.get(access_type)(survey_id=survey_id, page=page, page_size=page_size, env=env) + + # logger.info("Fetching page %s", page) + # logger.info("Items: %s", len(api_survey.items)) + # responses.extend(api_survey.items) + # total_items = api_survey.total_items + # max_item = len(api_survey.items) + max_item + # time.sleep(1) + + # except ApiException as e: + # logger.error("Exception when calling Household data-> %s%s\n", access_type, e) + # raise + + # df = pd.DataFrame(responses) + + + # return df def get_prices(self, country_iso3, survey_date, page_size=1000): """ @@ -221,53 +221,7 @@ def get_food_security_list(self, iso3=None, year=None, page=1): logger.error(f"Exception when calling FoodSecurityApi->food_security_list_get: {e}") raise - def get_household_questionnaire(self, xls_form_id, env='prod', page_size=200): - """ - This function fetches questionnaire data for a given form ID from the Data Bridges API. - - Args: - form_id (int): The ID of the questionnaire form to retrieve data for. - page_size (int, optional): The maximum number of items to retrieve per API call. Defaults to 200. - - Returns: - pandas.DataFrame: A DataFrame containing the fetched questionnaire data. - Raises: - ApiException: If an error occurs while calling the Data Bridges API. - """ - - page = 0 - with data_bridges_client.ApiClient(self.configuration) as api_client: - api_instance = data_bridges_client.IncubationApi(api_client) - env = self.env - responses = [] - try: - # Select appropriate API call based on access_type - api_survey = api_instance.xls_forms_definition_get(xls_form_id=xls_form_id, env=env) - page += 1 - try: - logger.info(f"Fetching page {page} from XLSForm definition") - responses.extend(item.to_dict() for item in api_survey.items) - except AttributeError: - responses.extend(item.to_dict() for item in api_survey) - time.sleep(1) - - except ApiException as e: - logger.error("Exception when calling Household questionnaire-> %s%s\n", xls_form_id, e) - raise - - df = pd.DataFrame(responses) - df = df.replace({np.nan: None}) - - questionnaire = pd.DataFrame(list(df.fields)[0]) - self.data[xls_form_id] = questionnaire - return questionnaire - - def get_choice_list(self, xls_form_id): - questionnaire = self.data[xls_form_id] - choiceList = pd.json_normalize(questionnaire['choiceList']).dropna() - choices = choiceList.explode('choices') - return choices def get_commodities_list(self, country_code=None, commodity_name=None, commodity_id=0, page=1, format='json'): """ @@ -557,6 +511,7 @@ def get_nearby_markets(self, adm0code=None, lat=None, lng=None): except ApiException as e: logger.error(f"Exception when calling MarketsApi->markets_nearby_markets_get: {e}") raise + def get_gorp(self, data_type, page=None): """ Retrieves data from the Global Operational Response Plan (GORP) API. @@ -602,6 +557,205 @@ def get_gorp(self, data_type, page=None): logger.error(f"Exception when calling GorpApi->{data_type}: {e}") raise + def get_household_survey(self, survey_id, access_type, page_size=600): + """Retrieves survey data using the specified configuration and access type. + + Args: + survey_id (str): The ID of the survey to retrieve. + access_type (str): The type of access to use for retrieving the survey data. + Can be one of 'full', 'draft', 'official', or 'public'. + page_size (int, optional): The number of items to retrieve per page. Defaults to 600. + + Returns: + pandas.DataFrame: A DataFrame containing the retrieved survey data. + """ + responses = [] + total_items = 1 + max_item = 0 + page = 0 + + while total_items > max_item: + page += 1 + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.IncubationApi(api_client) + env = self.env + + try: + logger.info(f"Calling get_household_survey for survey {survey_id}") + # Select appropriate API call based on access_type + api_survey = { + 'full': api_instance.household_full_data_get, + 'draft': api_instance.household_draft_internal_base_data_get, + 'official': api_instance.household_official_use_base_data_get, + 'public': api_instance.household_public_base_data_get + }.get(access_type)(survey_id=survey_id, page=page, page_size=page_size, env=env) + + logger.info(f"Fetching page {page}") + logger.info(f"Items: {len(api_survey.items)}") + responses.extend(api_survey.items) + total_items = api_survey.total_items + max_item = len(api_survey.items) + max_item + time.sleep(1) + + except ApiException as e: + logger.error(f"Exception when calling Household data-> {access_type}{e}") + raise + + df = pd.DataFrame(responses) + return df + + def get_household_surveys(self, adm0_code=0, page=1, start_date=None, end_date=None, survey_id=None): + """ + Retrieve Survey IDs, their corresponding XLS Form IDs, and Base XLS Form of all household surveys conducted in a country. + + Args: + adm0_code (int, optional): Code for the country. Defaults to 0. + page (int, optional): Page number for paged results. Defaults to 1. + start_date (datetime, optional): Starting date for the range in which data was collected. + end_date (datetime, optional): Ending date for the range in which data was collected. + survey_id (int, optional): Specific survey ID to retrieve. + + Returns: + pandas.DataFrame: A DataFrame containing the survey information. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.IncubationApi(api_client) + env = self.env + + try: + api_response = api_instance.household_surveys_get( + adm0_code=adm0_code, + page=page, + start_date=start_date, + end_date=end_date, + survey_id=survey_id, + env=env + ) + logger.info("Successfully retrieved household surveys") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling IncubationApi->household_surveys_get: {e}") + raise + + def get_household_questionnaire(self, xls_form_id): + """ + Get a complete set of XLS Form definitions of a given XLS Form ID. + + Args: + xls_form_id (int): The ID of the questionnaire form to retrieve data for. + + Returns: + pandas.DataFrame: A DataFrame containing the fetched questionnaire data. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.IncubationApi(api_client) + env = self.env + + try: + api_response = api_instance.xls_forms_definition_get(xls_form_id=xls_form_id, env=env) + logger.info(f"Successfully retrieved XLS Form definition for ID: {xls_form_id}") + df = pd.DataFrame([item.to_dict() for item in api_response]) + df = df.replace({np.nan: None}) + self.data[xls_form_id] = df + return df + except ApiException as e: + logger.error(f"Exception when calling IncubationApi->xls_forms_definition_get: {e}") + raise + + def get_aims_analysis_rounds(self, adm0_code): + """ + Download all analysis rounds for AIMS (Asset Impact Monitoring System) data. + + Args: + adm0_code (int): The country adm0Code. + + Returns: + bytes: The downloaded data as bytes. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.IncubationApi(api_client) + env = self.env + + try: + api_response = api_instance.aims_download_all_analysis_rounds_get(adm0_code=adm0_code, env=env) + logger.info(f"Successfully downloaded AIMS analysis rounds for adm0Code: {adm0_code}") + return api_response + except ApiException as e: + logger.error(f"Exception when calling IncubationApi->aims_download_all_analysis_rounds_get: {e}") + raise + + def get_aims_polygon_files(self, adm0_code): + """ + Download polygon files for Landscape Impact Assessment (LIA) assets. + + Args: + adm0_code (int): The country adm0Code. + + Returns: + bytes: The downloaded polygon files as bytes. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.IncubationApi(api_client) + env = self.env + + try: + api_response = api_instance.aims_download_polygon_files_get(adm0_code=adm0_code, env=env) + logger.info(f"Successfully downloaded AIMS polygon files for adm0Code: {adm0_code}") + return api_response + except ApiException as e: + logger.error(f"Exception when calling IncubationApi->aims_download_polygon_files_get: {e}") + raise + + # def get_household_questionnaire(self, xls_form_id, env='prod', page_size=200): + # """ + # This function fetches questionnaire data for a given form ID from the Data Bridges API. + + # Args: + # form_id (int): The ID of the questionnaire form to retrieve data for. + # page_size (int, optional): The maximum number of items to retrieve per API call. Defaults to 200. + + # Returns: + # pandas.DataFrame: A DataFrame containing the fetched questionnaire data. + + # Raises: + # ApiException: If an error occurs while calling the Data Bridges API. + # """ + + # page = 0 + # with data_bridges_client.ApiClient(self.configuration) as api_client: + # api_instance = data_bridges_client.IncubationApi(api_client) + # env = self.env + # responses = [] + # try: + # # Select appropriate API call based on access_type + # api_survey = api_instance.xls_forms_definition_get(xls_form_id=xls_form_id, env=env) + # page += 1 + # try: + # logger.info(f"Fetching page {page} from XLSForm definition") + # responses.extend(item.to_dict() for item in api_survey.items) + # except AttributeError: + # responses.extend(item.to_dict() for item in api_survey) + # time.sleep(1) + + # except ApiException as e: + # logger.error("Exception when calling Household questionnaire-> %s%s\n", xls_form_id, e) + # raise + + # df = pd.DataFrame(responses) + # df = df.replace({np.nan: None}) + + # questionnaire = pd.DataFrame(list(df.fields)[0]) + # self.data[xls_form_id] = questionnaire + # return questionnaire + + # def get_choice_list(self, xls_form_id): + # questionnaire = self.data[xls_form_id] + # choiceList = pd.json_normalize(questionnaire['choiceList']).dropna() + # choices = choiceList.explode('choices') + # return choices + if __name__ == "__main__": import yaml # FOR TESTING @@ -642,4 +796,29 @@ def get_gorp(self, data_type, page=None): list_df = client.get_gorp('list', page=1) # Get regional latest data - regional_latest_df = client.get_gorp('regional_latest') \ No newline at end of file + regional_latest_df = client.get_gorp('regional_latest') + + # Get household survey data + survey_data = client.get_household_survey(survey_id="123456", access_type="public", page_size=1000) + print("Household Survey Data:") + print(survey_data.head()) + + # Get list of household surveys + surveys_list = client.get_household_surveys(adm0_code=123, page=1, start_date="2023-01-01", end_date="2023-12-31") + print("\nHousehold Surveys List:") + print(surveys_list.head()) + + # Get household questionnaire + questionnaire = client.get_household_questionnaire(xls_form_id=789) + print("\nHousehold Questionnaire:") + print(questionnaire.head()) + + # # Get AIMS analysis rounds + # aims_data = client.get_aims_analysis_rounds(adm0_code=456) + # print("\nAIMS Analysis Rounds Data:") + # print(f"Downloaded {len(aims_data)} bytes") + + # # Get AIMS polygon files + # polygon_files = client.get_aims_polygon_files(adm0_code=789) + # print("\nAIMS Polygon Files:") + # print(f"Downloaded {len(polygon_files)} bytes") \ No newline at end of file From b22666468033947a73243a0f20320912900c0fb6 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Fri, 20 Sep 2024 11:13:56 +0200 Subject: [PATCH 10/20] docs: update example.py with markets endpoint --- data_bridges_knots/client.py | 33 +++++------------------ examples/example.py | 51 ++++++++++++++++++++++++++++++++++++ examples/example_python.py | 50 ----------------------------------- 3 files changed, 57 insertions(+), 77 deletions(-) create mode 100644 examples/example.py delete mode 100644 examples/example_python.py diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index b7efe52..07e1026 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -763,43 +763,22 @@ def get_aims_polygon_files(self, adm0_code): client = DataBridgesShapes(CONFIG_PATH) - commodity_units_list = client.get_commodity_units_list(country_code="TZA", commodity_unit_name="Kg", page=1, format='json') - comodity_unit_conversion_list = client.get_commodity_units_conversion_list(country_code="TZA", commodity_id=1, from_unit_id=1, to_unit_id=2, page=1, format='json') - currency_list = client.get_currency_list(country_code="TZA", currency_name="TZS", currency_id=0, page=1, format='json') - usd_indirect_quotation = client.get_usd_indirect_quotation(country_iso3="TZA", currency_name="TZS", page=1, format='json') - # FIXME: JSON Response + printing instead of logging economic_indicator_list = client.get_economic_indicator_list(page=1, indicator_name='', iso3='', format='json') # BUG: Error: Unsoppoerted content type application/geo+json # markets_geo_json = client.get_market_geojson_list(adm0code=56) - markets_list = client.get_markets_list(country_code="TZA") - - markets_csv = client.get_markets_as_csv(adm0code=4, local_names=False) - - nearby_markets = client.get_nearby_markets(adm0code=56) - - get_food_security_list = client.get_food_security_list() - - # Get country latest data - country_latest_df = client.get_gorp('country_latest') - - # Get global latest data - global_latest_df = client.get_gorp('global_latest') - - # Get latest data (paginated) - latest_df = client.get_gorp('latest', page=1) - - # Get full list data (paginated) - list_df = client.get_gorp('list', page=1) - # Get regional latest data - regional_latest_df = client.get_gorp('regional_latest') + CONGO_CFSVA = { + 'questionnaire': 1509, + 'dataset': 3094 + } + # TODO: Test the following functions # Get household survey data - survey_data = client.get_household_survey(survey_id="123456", access_type="public", page_size=1000) + survey_data = client.get_household_survey(survey_id=3094, access_type="full") print("Household Survey Data:") print(survey_data.head()) diff --git a/examples/example.py b/examples/example.py new file mode 100644 index 0000000..771f9f4 --- /dev/null +++ b/examples/example.py @@ -0,0 +1,51 @@ +""" +Reads Household Data from Data Bridges. The script uses the DataBridgesShapes class from the data_bridges_knots module to interact with the Data Bridges API and retrieve various datasets, including: +- Household survey data +- GORP (Global Operational Response Plan) data +- Exchange rates and prices for Afghanistan +- IPC and equivalent food security Data + +The script demonstrates how to use the DataBridgesShapes class to fetch these datasets and print the first few rows of the resulting pandas DataFrames. +""" + +from data_bridges_knots import DataBridgesShapes + +CONFIG_PATH = r"data_bridges_api_config.yaml" + +client = DataBridgesShapes(CONFIG_PATH) + +# COMMODITY DATA +commodity_units_list = client.get_commodity_units_list(country_code="TZA", commodity_unit_name="Kg", page=1, format='json') +comodity_unit_conversion_list = client.get_commodity_units_conversion_list(country_code="TZA", commodity_id=1, from_unit_id=1, to_unit_id=2, page=1, format='json') + +currency_list = client.get_currency_list(country_code="TZA", currency_name="TZS", currency_id=0, page=1, format='json') +usd_indirect_quotation = client.get_usd_indirect_quotation(country_iso3="TZA", currency_name="TZS", page=1, format='json') + +## MARKETS DATA +# Get a complete list of markets in a country +markets_list = client.get_markets_list(country_code="TZA") + +# Get a complete list of markets in a country +markets_csv = client.get_markets_as_csv(adm0code=4, local_names=False) + +# Get markets near a given location by longitude and latitude within a 15Km distance +nearby_markets = client.get_nearby_markets(adm0code=56) + +### FOOD SECURITY DATA +# Get food security data (IPC and equivalent food insecurity numbers) +get_food_security_list = client.get_food_security_list() + +# Get country-level latest data from the Global Operation Response Plan (GOPR) +country_latest_df = client.get_gorp('country_latest') # no data currently uploaded + +# Get global latest data from the Global Operation Response Plan (GOPR) +global_latest_df = client.get_gorp('global_latest') + +# Get latest data (paginated) from the Global Operation Response Plan (GOPR) +latest_df = client.get_gorp('latest', page=1) + +# Get full list data (paginated from the Global Operation Response Plan (GOPR) +list_df = client.get_gorp('list', page=1) + +# Get regional latest data +regional_latest_df = client.get_gorp('regional_latest') \ No newline at end of file diff --git a/examples/example_python.py b/examples/example_python.py deleted file mode 100644 index b1f074f..0000000 --- a/examples/example_python.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -Reads Household Data from Data Bridges. The script uses the DataBridgesShapes class from the data_bridges_knots module to interact with the Data Bridges API and retrieve various datasets, including: -- Household survey data -- GORP (Global Operational Response Plan) data -- Exchange rates and prices for Afghanistan -- IPC and equivalent food security Data - -The script demonstrates how to use the DataBridgesShapes class to fetch these datasets and print the first few rows of the resulting pandas DataFrames. -""" -#%% - -from data_bridges_knots import DataBridgesShapes -from data_bridges_knots.load_stata import load_stata - -CONFIG_PATH = r"data_bridges_api_config.yaml" - -client = DataBridgesShapes(CONFIG_PATH) - -#%% XSLForm definition and Household dataset - -CONGO_CFSVA = { - 'questionnaire': 1509, - 'dataset': 3094 -} -# get household survey data -survey_data = client.get_household_survey(survey_id=CONGO_CFSVA["dataset"], access_type='full') -# get XLSForm data -questionnaire = client.get_household_questionnaire(CONGO_CFSVA["questionnaire"]) - -# Map the categories to survey_data -mapped_survey_data = map_value_labels(survey_data, questionnaire) - - -#%% GORP data -# Get GORP data -latest_data = client.get_gorp('latest') -list_data = client.get_gorp('list') -regional_latest_data = client.get_gorp('regional_latest') -global_latest_data = client.get_gorp('global_latest') -# #% - -#%% Market data -exchage_rates = client.get_exchange_rates('AFG') -prices = client.get_prices('AFG', '2022-01-01') - - -#%% IPC equivalent -food_security = client.get_food_security() -afg_food_security = client.get_food_security("AFG", 2024) - From b80ec1df87ccf40849ac860c8089853a16579379 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Fri, 20 Sep 2024 11:30:38 +0200 Subject: [PATCH 11/20] docs: update README with examples --- README.md | 12 +++++------- examples/example.py | 1 + 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8613fe8..154ede4 100644 --- a/README.md +++ b/README.md @@ -48,17 +48,15 @@ Run the following code to extract commoditiy data. from data_bridges_knots import DataBridgesShapes -CREDENTIALS = r"data_bridges_api_config.yaml" +CONFIG_PATH = r"data_bridges_api_config.yaml" -client = DataBridgesShapes(CREDENTIALS) +client = DataBridgesShapes(CONFIG_PATH) -commodities_list = client.get_commodities_list(country_code='ETH', commodity_name='wheat', page=1, format='json') - -# converts numeric columns into ints -numeric_columns = [ 'FCSDairy', 'FCSDairy_SRf', 'FCSFat', 'FCSFat_SRf', 'FCSFruit', 'FCSFruit_SRf', 'FCSNFruiOrg', 'FCSNPrEggs', 'FCSNPrFish', 'FCSNPrMeatF', 'FCSNPrMeatO', 'FCSNVegGre', 'FCSNVegOrg', 'FCSPr', 'FCSPr_SRf', 'FCSPulse', 'FCSPulse_SRf', 'FCSStap', 'FCSStap_SRf', 'FCSSugar', 'FCSSugar_SRf', 'FCSVeg', 'FCSVeg_SRf', 'HHSize', 'HHSize01F', 'HHSize01M', 'HHSize', 'HHSize01F', 'HHSize01M', 'HHSize1217F', 'HHSize1217M', 'HHSize1859F', 'HHSize1859M', 'HHSize24F', 'HHSize24M', 'HHSize511F', 'HHSize511M', 'HHSize60AboveF', 'HHSize60AboveM', 'RESPAge' ] -df = as_numeric(survey_data, numeric_columns) +# COMMODITY DATA +commodity_units_list = client.get_commodity_units_list(country_code="TZA", commodity_unit_name="Kg", page=1, format='json') ``` +Additional examples are in the [examples](https://github.com/WFP-VAM/DataBridgesKnots/tree/main/examples) folder. ## Contributing Contributions are welcome! Please open an issue or submit a pull request if you have any improvements or bug fixes. diff --git a/examples/example.py b/examples/example.py index 771f9f4..5d06c07 100644 --- a/examples/example.py +++ b/examples/example.py @@ -20,6 +20,7 @@ currency_list = client.get_currency_list(country_code="TZA", currency_name="TZS", currency_id=0, page=1, format='json') usd_indirect_quotation = client.get_usd_indirect_quotation(country_iso3="TZA", currency_name="TZS", page=1, format='json') +print(commodity_units_list) ## MARKETS DATA # Get a complete list of markets in a country From ba9821fa0a4cd3ccc90cb6081128ab3ba61afb70 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Fri, 20 Sep 2024 11:34:14 +0200 Subject: [PATCH 12/20] docs: update README with installation instructions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 154ede4..45ebaa2 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,12 @@ This Python module allows you to get data from the WFP Data Bridges API, includi ## Installation -> NB This is the dev version of the data_bridges_knots and API client package, it is frequently updated yet not stable. +> NB This is the development version of the data_bridges_knots and API client package, it is frequently updated yet not stable. You can install the `data_bridges_knots` package using `pip` and the Git repository URL: ``` -pip install git+https://github.com/WFP-VAM/DataBridgesKnots.git +pip install --force-reinstall git+https://github.com/WFP-VAM/DataBridgesKnots.git@dev” ``` STATA and R users will also need the appropriate optional dependencies to use this package in their respective software. To install the package with these dependencies, use the following command: From bf22dab897823d93e01afd0430dca0ed2621ac12 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Fri, 20 Sep 2024 11:35:06 +0200 Subject: [PATCH 13/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 45ebaa2..b6dc309 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This Python module allows you to get data from the WFP Data Bridges API, includi You can install the `data_bridges_knots` package using `pip` and the Git repository URL: ``` -pip install --force-reinstall git+https://github.com/WFP-VAM/DataBridgesKnots.git@dev” +pip install --force-reinstall git+https://github.com/WFP-VAM/DataBridgesKnots.git@dev ``` STATA and R users will also need the appropriate optional dependencies to use this package in their respective software. To install the package with these dependencies, use the following command: From 7d80f009e97060ac9606bc5839723fea15211723 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Fri, 20 Sep 2024 15:18:56 +0200 Subject: [PATCH 14/20] feature: update get_household_survey function with DataBridges API key for restricted endpoints --- data_bridges_knots/client.py | 135 +++++++---------------------------- examples/example.py | 5 +- 2 files changed, 28 insertions(+), 112 deletions(-) diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 07e1026..0048c63 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -21,6 +21,7 @@ class DataBridgesShapes: def __init__(self, yaml_config_path, env='prod'): self.configuration = self.setup_configuration_and_authentication(yaml_config_path) + self.data_bridges_api_key = self.setup_databridges_configuration(yaml_config_path) self.env = env self.data = {} @@ -41,7 +42,6 @@ def setup_configuration_and_authentication(self, yaml_config_path): secret = databridges_config['SECRET'] scopes = databridges_config['SCOPES'] version = databridges_config['VERSION'] - data_bridges_api_key = databridges_config['DATABRIDGES_API_KEY'] uri = "https://api.wfp.org/vam-data-bridges/" host = str(uri + version) @@ -52,57 +52,13 @@ def setup_configuration_and_authentication(self, yaml_config_path): host=host, access_token=token.refresh(scopes=scopes) ) return configuration - - # def get_household_survey(self, survey_id, access_type, page_size=600): - # """Retrieves survey data using the specified configuration and access type. - - # Args: - # survey_id (str): The ID of the survey to retrieve. - # access_type (str): The type of access to use for retrieving the survey data. - # Can be one of '', 'full', 'draft', 'official', or 'public'. - # page_size (int, optional): The number of items to retrieve per page. Defaults to 200. - - # Returns: - # pandas.DataFrame: A DataFrame containing the retrieved survey data. - # """ - - # responses = [] - # total_items = 1 - # max_item = 0 - # page = 0 - - # while total_items > max_item: - # page += 1 - # with data_bridges_client.ApiClient(self.configuration) as api_client: - # api_instance = data_bridges_client.IncubationApi(api_client) - # env = self.env - - # try: - # logger.info(f"Calling get_household_survey for survey {survey_id}") - # # Select appropriate API call based on access_type - # api_survey = { - # '': api_instance.household_public_base_data_get, - # 'full': api_instance.household_full_data_get, - # 'draft': api_instance.household_draft_internal_base_data_get, - # 'official': api_instance.household_official_use_base_data_get, - # 'public': api_instance.household_public_base_data_get - # }.get(access_type)(survey_id=survey_id, page=page, page_size=page_size, env=env) - - # logger.info("Fetching page %s", page) - # logger.info("Items: %s", len(api_survey.items)) - # responses.extend(api_survey.items) - # total_items = api_survey.total_items - # max_item = len(api_survey.items) + max_item - # time.sleep(1) - - # except ApiException as e: - # logger.error("Exception when calling Household data-> %s%s\n", access_type, e) - # raise - - # df = pd.DataFrame(responses) - - - # return df + + def setup_databridges_configuration(self, yaml_config_path): + """Loads configuration from a YAML file and sets up authentication.""" + with open(yaml_config_path, "r") as yamlfile: + data_bridges_api_key = yaml.load(yamlfile, Loader=yaml.FullLoader) + + return data_bridges_api_key['DATABRIDGES_API_KEY'] def get_prices(self, country_iso3, survey_date, page_size=1000): """ @@ -581,14 +537,19 @@ def get_household_survey(self, survey_id, access_type, page_size=600): env = self.env try: - logger.info(f"Calling get_household_survey for survey {survey_id}") + logger.info(f"Calling get_household_survey for survey {survey_id}") # Select appropriate API call based on access_type - api_survey = { + api_call = { 'full': api_instance.household_full_data_get, 'draft': api_instance.household_draft_internal_base_data_get, 'official': api_instance.household_official_use_base_data_get, 'public': api_instance.household_public_base_data_get - }.get(access_type)(survey_id=survey_id, page=page, page_size=page_size, env=env) + }.get(access_type) + + if access_type in ['full', 'draft']: + api_survey = api_call(self.data_bridges_api_key, survey_id=survey_id, page=page, page_size=page_size, env=env) + else: + api_survey = api_call(survey_id=survey_id, page=page, page_size=page_size, env=env) logger.info(f"Fetching page {page}") logger.info(f"Items: {len(api_survey.items)}") @@ -604,6 +565,7 @@ def get_household_survey(self, survey_id, access_type, page_size=600): df = pd.DataFrame(responses) return df + def get_household_surveys(self, adm0_code=0, page=1, start_date=None, end_date=None, survey_id=None): """ Retrieve Survey IDs, their corresponding XLS Form IDs, and Base XLS Form of all household surveys conducted in a country. @@ -708,48 +670,6 @@ def get_aims_polygon_files(self, adm0_code): logger.error(f"Exception when calling IncubationApi->aims_download_polygon_files_get: {e}") raise - # def get_household_questionnaire(self, xls_form_id, env='prod', page_size=200): - # """ - # This function fetches questionnaire data for a given form ID from the Data Bridges API. - - # Args: - # form_id (int): The ID of the questionnaire form to retrieve data for. - # page_size (int, optional): The maximum number of items to retrieve per API call. Defaults to 200. - - # Returns: - # pandas.DataFrame: A DataFrame containing the fetched questionnaire data. - - # Raises: - # ApiException: If an error occurs while calling the Data Bridges API. - # """ - - # page = 0 - # with data_bridges_client.ApiClient(self.configuration) as api_client: - # api_instance = data_bridges_client.IncubationApi(api_client) - # env = self.env - # responses = [] - # try: - # # Select appropriate API call based on access_type - # api_survey = api_instance.xls_forms_definition_get(xls_form_id=xls_form_id, env=env) - # page += 1 - # try: - # logger.info(f"Fetching page {page} from XLSForm definition") - # responses.extend(item.to_dict() for item in api_survey.items) - # except AttributeError: - # responses.extend(item.to_dict() for item in api_survey) - # time.sleep(1) - - # except ApiException as e: - # logger.error("Exception when calling Household questionnaire-> %s%s\n", xls_form_id, e) - # raise - - # df = pd.DataFrame(responses) - # df = df.replace({np.nan: None}) - - # questionnaire = pd.DataFrame(list(df.fields)[0]) - # self.data[xls_form_id] = questionnaire - # return questionnaire - # def get_choice_list(self, xls_form_id): # questionnaire = self.data[xls_form_id] # choiceList = pd.json_normalize(questionnaire['choiceList']).dropna() @@ -763,32 +683,25 @@ def get_aims_polygon_files(self, adm0_code): client = DataBridgesShapes(CONFIG_PATH) - # FIXME: JSON Response + printing instead of logging - economic_indicator_list = client.get_economic_indicator_list(page=1, indicator_name='', iso3='', format='json') + # economic_indicator_list = client.get_economic_indicator_list(page=1, indicator_name='', iso3='', format='json') # BUG: Error: Unsoppoerted content type application/geo+json # markets_geo_json = client.get_market_geojson_list(adm0code=56) - - - CONGO_CFSVA = { - 'questionnaire': 1509, - 'dataset': 3094 - } # TODO: Test the following functions # Get household survey data - survey_data = client.get_household_survey(survey_id=3094, access_type="full") + survey_data = client.get_household_survey(survey_id=3094, access_type="official") print("Household Survey Data:") print(survey_data.head()) - # Get list of household surveys - surveys_list = client.get_household_surveys(adm0_code=123, page=1, start_date="2023-01-01", end_date="2023-12-31") - print("\nHousehold Surveys List:") - print(surveys_list.head()) + # # # Get list of household surveys + # # surveys_list = client.get_household_surveys(adm0_code=123, page=1, start_date="2023-01-01", end_date="2023-12-31") + # # print("\nHousehold Surveys List:") + # # print(surveys_list.head()) # Get household questionnaire - questionnaire = client.get_household_questionnaire(xls_form_id=789) + questionnaire = client.get_household_questionnaire(xls_form_id=1509) print("\nHousehold Questionnaire:") print(questionnaire.head()) diff --git a/examples/example.py b/examples/example.py index 771f9f4..4485254 100644 --- a/examples/example.py +++ b/examples/example.py @@ -48,4 +48,7 @@ list_df = client.get_gorp('list', page=1) # Get regional latest data -regional_latest_df = client.get_gorp('regional_latest') \ No newline at end of file +regional_latest_df = client.get_gorp('regional_latest') + +### HOUSEHOLD DATA +survey_data = client.get_household_survey(survey_id=3094, access_type="full") \ No newline at end of file From 056588c96d85d043d013a830b3d9563dc67503a1 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Fri, 20 Sep 2024 17:07:26 +0200 Subject: [PATCH 15/20] docs: add examples for household survey and questionnaire --- ROADMAP.md | 1 + data_bridges_knots/client.py | 14 +++++--------- examples/example.py | 9 +++++++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 754fae8..5029f76 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -22,6 +22,7 @@ This document outlines the planned features and improvements for the `DataBridge - [ ] DPO change for XLSForm - [ ] Markets GeoJSON response - [ ] JSON to DataFrame response +- [ ] Market list JSON and CSV ## Future Release ### v0.3.0 (DataBridges API v.0) diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 0048c63..a4c439d 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -689,22 +689,18 @@ def get_aims_polygon_files(self, adm0_code): # BUG: Error: Unsoppoerted content type application/geo+json # markets_geo_json = client.get_market_geojson_list(adm0code=56) - # TODO: Test the following functions # Get household survey data - survey_data = client.get_household_survey(survey_id=3094, access_type="official") - print("Household Survey Data:") - print(survey_data.head()) + survey_data = client.get_household_survey(survey_id=3094, access_type="full") + + # Get household questionnaire + questionnaire = client.get_household_questionnaire(xls_form_id=1509) + # TODO: Test the following functions # # # Get list of household surveys # # surveys_list = client.get_household_surveys(adm0_code=123, page=1, start_date="2023-01-01", end_date="2023-12-31") # # print("\nHousehold Surveys List:") # # print(surveys_list.head()) - # Get household questionnaire - questionnaire = client.get_household_questionnaire(xls_form_id=1509) - print("\nHousehold Questionnaire:") - print(questionnaire.head()) - # # Get AIMS analysis rounds # aims_data = client.get_aims_analysis_rounds(adm0_code=456) # print("\nAIMS Analysis Rounds Data:") diff --git a/examples/example.py b/examples/example.py index 4485254..59debd3 100644 --- a/examples/example.py +++ b/examples/example.py @@ -32,9 +32,10 @@ nearby_markets = client.get_nearby_markets(adm0code=56) ### FOOD SECURITY DATA -# Get food security data (IPC and equivalent food insecurity numbers) +# Get IPC and equivalent food insecurity numbers for all countries get_food_security_list = client.get_food_security_list() +### Global Operation Response Plan (GOPR) # Get country-level latest data from the Global Operation Response Plan (GOPR) country_latest_df = client.get_gorp('country_latest') # no data currently uploaded @@ -51,4 +52,8 @@ regional_latest_df = client.get_gorp('regional_latest') ### HOUSEHOLD DATA -survey_data = client.get_household_survey(survey_id=3094, access_type="full") \ No newline at end of file +# Get survey data for a specific survey +survey_data = client.get_household_survey(survey_id=3094, access_type="official") + +# Get survey questionnaire for a specific survey +questionnaire = client.get_household_questionnaire(xls_form_id=1509) \ No newline at end of file From 15a7c1df572a1003a0ec74e718a4f6c96ac49f3b Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Mon, 23 Sep 2024 11:53:15 +0200 Subject: [PATCH 16/20] feat: IncubationApi Household Surveys endpoints --- ROADMAP.md | 4 ++-- data_bridges_knots/client.py | 22 ++++++---------------- examples/example.py | 6 +++++- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 5029f76..1191393 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -12,8 +12,8 @@ This document outlines the planned features and improvements for the `DataBridge - [X] Endpoints: MarketsApi - [X] Endpoints: FoodSecurityApi - [X] Endpoints: GorpApi -- [ ] Endpoints: MarketPricesAPi -- [ ] Endpoints: IncubationApi +- [X] Endpoints: MarketPricesAPi +- [X] Endpoints: IncubationApi - [ ] Endpoints: RpmeApi - [ ] Endpoints: SurveysApi - [ ] Endpoints: XlsFormsApi diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index a4c439d..97e8301 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -566,7 +566,7 @@ def get_household_survey(self, survey_id, access_type, page_size=600): return df - def get_household_surveys(self, adm0_code=0, page=1, start_date=None, end_date=None, survey_id=None): + def get_household_surveys(self, adm0_code=None, page=1, start_date=None, end_date=None, survey_id=None): """ Retrieve Survey IDs, their corresponding XLS Form IDs, and Base XLS Form of all household surveys conducted in a country. @@ -686,27 +686,17 @@ def get_aims_polygon_files(self, adm0_code): # FIXME: JSON Response + printing instead of logging # economic_indicator_list = client.get_economic_indicator_list(page=1, indicator_name='', iso3='', format='json') - # BUG: Error: Unsoppoerted content type application/geo+json + # BUG: Error: Unsupported content type application/geo+json # markets_geo_json = client.get_market_geojson_list(adm0code=56) - # Get household survey data - survey_data = client.get_household_survey(survey_id=3094, access_type="full") - - # Get household questionnaire - questionnaire = client.get_household_questionnaire(xls_form_id=1509) - - # TODO: Test the following functions - # # # Get list of household surveys - # # surveys_list = client.get_household_surveys(adm0_code=123, page=1, start_date="2023-01-01", end_date="2023-12-31") - # # print("\nHousehold Surveys List:") - # # print(surveys_list.head()) - + # FIXME: Get scopes for AIMS then test the following function # # Get AIMS analysis rounds - # aims_data = client.get_aims_analysis_rounds(adm0_code=456) + # aims_data = client.get_aims_analysis_rounds(adm0_code=1) # print("\nAIMS Analysis Rounds Data:") # print(f"Downloaded {len(aims_data)} bytes") + # FIXME: Get scopes for AIMS then test the following function # # Get AIMS polygon files - # polygon_files = client.get_aims_polygon_files(adm0_code=789) + # polygon_files = client.get_aims_polygon_files(adm0_code=1) # print("\nAIMS Polygon Files:") # print(f"Downloaded {len(polygon_files)} bytes") \ No newline at end of file diff --git a/examples/example.py b/examples/example.py index ab1bcf7..fc1cfe5 100644 --- a/examples/example.py +++ b/examples/example.py @@ -53,8 +53,12 @@ regional_latest_df = client.get_gorp('regional_latest') ### HOUSEHOLD DATA +# Get list of household surveys available +surveys_list = client.get_household_surveys() + # Get survey data for a specific survey survey_data = client.get_household_survey(survey_id=3094, access_type="official") # Get survey questionnaire for a specific survey -questionnaire = client.get_household_questionnaire(xls_form_id=1509) \ No newline at end of file +questionnaire = client.get_household_questionnaire(xls_form_id=1509) + From 9db9424930126ffbb938531b846a251f58a6f1d4 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Mon, 23 Sep 2024 13:54:41 +0200 Subject: [PATCH 17/20] fix: get_household_questionnaire and get_choice_list functions to retrieve correct information --- data_bridges_knots/client.py | 121 +++++++++++++++++++++++++++++++---- examples/example.py | 7 ++ 2 files changed, 116 insertions(+), 12 deletions(-) diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 97e8301..48d5b23 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -23,7 +23,7 @@ def __init__(self, yaml_config_path, env='prod'): self.configuration = self.setup_configuration_and_authentication(yaml_config_path) self.data_bridges_api_key = self.setup_databridges_configuration(yaml_config_path) self.env = env - self.data = {} + self.xlsform = None def __repr__(self): return "DataBridgesShapes(yamlpath='%s')" % self.configuration.host @@ -601,7 +601,7 @@ def get_household_surveys(self, adm0_code=None, page=1, start_date=None, end_dat logger.error(f"Exception when calling IncubationApi->household_surveys_get: {e}") raise - def get_household_questionnaire(self, xls_form_id): + def get_household_xslform_definition(self, xls_form_id): """ Get a complete set of XLS Form definitions of a given XLS Form ID. @@ -618,14 +618,28 @@ def get_household_questionnaire(self, xls_form_id): try: api_response = api_instance.xls_forms_definition_get(xls_form_id=xls_form_id, env=env) logger.info(f"Successfully retrieved XLS Form definition for ID: {xls_form_id}") - df = pd.DataFrame([item.to_dict() for item in api_response]) - df = df.replace({np.nan: None}) - self.data[xls_form_id] = df - return df + self.xlsform = pd.DataFrame([item.to_dict() for item in api_response]) + return self.xlsform + except ApiException as e: logger.error(f"Exception when calling IncubationApi->xls_forms_definition_get: {e}") raise + def get_household_questionnaire(self, xls_form_id): + if self.xlsform is None: + self.xlsform = self.get_household_xslform_definition(xls_form_id) + return pd.DataFrame(list(self.xlsform.fields)[0]) + + def get_choice_list(self, xls_form_id): + questionnaire = self.get_household_questionnaire(xls_form_id) + + choiceList = pd.json_normalize(questionnaire['choiceList']).dropna() + choices = choiceList.explode('choices') + choices['value'] = choices['choices'].apply(lambda x: x['name']) + choices['label'] = choices['choices'].apply(lambda x: x['label']) + return choices[['name', 'value', 'label']] + + def get_aims_analysis_rounds(self, adm0_code): """ Download all analysis rounds for AIMS (Asset Impact Monitoring System) data. @@ -670,11 +684,90 @@ def get_aims_polygon_files(self, adm0_code): logger.error(f"Exception when calling IncubationApi->aims_download_polygon_files_get: {e}") raise - # def get_choice_list(self, xls_form_id): - # questionnaire = self.data[xls_form_id] - # choiceList = pd.json_normalize(questionnaire['choiceList']).dropna() - # choices = choiceList.explode('choices') - # return choices + + + def get_mfi_surveys_base_data(self, survey_id=None, page=1, page_size=20): + """ + Get data that includes the core Market Functionality Index (MFI) fields only by Survey ID. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.SurveysApi(api_client) + env = self.env + + try: + api_response = api_instance.m_fi_surveys_base_data_get( + survey_id=survey_id, page=page, page_size=page_size, env=env + ) + logger.info("Successfully retrieved MFI surveys base data") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling SurveysApi->m_fi_surveys_base_data_get: {e}") + raise + + def get_mfi_surveys_full_data(self, survey_id=None, format='json', page=1, page_size=20): + """ + Get a full dataset that includes all the fields included in the survey in addition to the core Market Functionality Index (MFI) fields by Survey ID. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.SurveysApi(api_client) + env = self.env + + try: + api_response = api_instance.m_fi_surveys_full_data_get( + survey_id=survey_id, format=format, page=page, page_size=page_size, env=env + ) + logger.info("Successfully retrieved MFI surveys full data") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling SurveysApi->m_fi_surveys_full_data_get: {e}") + raise + + def get_mfi_surveys(self, adm0_code=0, page=1, start_date=None, end_date=None): + """ + Retrieve Survey IDs, their corresponding XLS Form IDs, and Base XLS Form of all MFI surveys conducted in a country. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.SurveysApi(api_client) + env = self.env + + try: + api_response = api_instance.m_fi_surveys_get( + adm0_code=adm0_code, page=page, start_date=start_date, end_date=end_date, env=env + ) + logger.info("Successfully retrieved MFI surveys list") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling SurveysApi->m_fi_surveys_get: {e}") + raise + + def get_mfi_surveys_processed_data(self, survey_id=None, page=1, page_size=20, format='json', start_date=None, end_date=None, adm0_codes=None, market_id=None, survey_type=None): + """ + Get MFI processed data in long format. + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.SurveysApi(api_client) + env = self.env + + try: + api_response = api_instance.m_fi_surveys_processed_data_get( + survey_id=survey_id, page=page, page_size=page_size, format=format, + start_date=start_date, end_date=end_date, adm0_codes=adm0_codes, + market_id=market_id, survey_type=survey_type, env=env + ) + logger.info("Successfully retrieved MFI surveys processed data") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling SurveysApi->m_fi_surveys_processed_data_get: {e}") + raise + if __name__ == "__main__": import yaml @@ -699,4 +792,8 @@ def get_aims_polygon_files(self, adm0_code): # # Get AIMS polygon files # polygon_files = client.get_aims_polygon_files(adm0_code=1) # print("\nAIMS Polygon Files:") - # print(f"Downloaded {len(polygon_files)} bytes") \ No newline at end of file + # print(f"Downloaded {len(polygon_files)} bytes") + + + print(choices) + diff --git a/examples/example.py b/examples/example.py index fc1cfe5..f2b2cd1 100644 --- a/examples/example.py +++ b/examples/example.py @@ -59,6 +59,13 @@ # Get survey data for a specific survey survey_data = client.get_household_survey(survey_id=3094, access_type="official") +# Get XLSForm definition for a specific survey +xlsform = client.get_household_xslform_definition(xls_form_id=1509) + # Get survey questionnaire for a specific survey questionnaire = client.get_household_questionnaire(xls_form_id=1509) +# Get choice list for a specific survey +choices = client.get_choice_list(xls_form_id=1509) + + From 05d4d060c96f6a2cca9da09ae340a038f7cf7648 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Mon, 23 Sep 2024 14:28:15 +0200 Subject: [PATCH 18/20] feature: add MFI endpoints and examples --- data_bridges_knots/client.py | 21 +++++++++------------ examples/example.py | 30 +++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 48d5b23..95547ba 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -639,7 +639,7 @@ def get_choice_list(self, xls_form_id): choices['label'] = choices['choices'].apply(lambda x: x['label']) return choices[['name', 'value', 'label']] - + # FIXME: Get scopes for AIMS then test the following function def get_aims_analysis_rounds(self, adm0_code): """ Download all analysis rounds for AIMS (Asset Impact Monitoring System) data. @@ -662,6 +662,7 @@ def get_aims_analysis_rounds(self, adm0_code): logger.error(f"Exception when calling IncubationApi->aims_download_all_analysis_rounds_get: {e}") raise + # FIXME: Get scopes for AIMS then test the following function def get_aims_polygon_files(self, adm0_code): """ Download polygon files for Landscape Impact Assessment (LIA) assets. @@ -684,8 +685,6 @@ def get_aims_polygon_files(self, adm0_code): logger.error(f"Exception when calling IncubationApi->aims_download_polygon_files_get: {e}") raise - - def get_mfi_surveys_base_data(self, survey_id=None, page=1, page_size=20): """ Get data that includes the core Market Functionality Index (MFI) fields only by Survey ID. @@ -699,28 +698,26 @@ def get_mfi_surveys_base_data(self, survey_id=None, page=1, page_size=20): survey_id=survey_id, page=page, page_size=page_size, env=env ) logger.info("Successfully retrieved MFI surveys base data") - df = pd.DataFrame([item.to_dict() for item in api_response.items]) - df = df.replace({np.nan: None}) + + df = pd.DataFrame(api_response.items) return df except ApiException as e: logger.error(f"Exception when calling SurveysApi->m_fi_surveys_base_data_get: {e}") raise - def get_mfi_surveys_full_data(self, survey_id=None, format='json', page=1, page_size=20): + def get_mfi_surveys_full_data(self, survey_id=None, page=1, page_size=20): """ Get a full dataset that includes all the fields included in the survey in addition to the core Market Functionality Index (MFI) fields by Survey ID. """ with data_bridges_client.ApiClient(self.configuration) as api_client: api_instance = data_bridges_client.SurveysApi(api_client) env = self.env - try: api_response = api_instance.m_fi_surveys_full_data_get( - survey_id=survey_id, format=format, page=page, page_size=page_size, env=env + survey_id=survey_id, format='json', page=page, page_size=page_size, env=env ) logger.info("Successfully retrieved MFI surveys full data") - df = pd.DataFrame([item.to_dict() for item in api_response.items]) - df = df.replace({np.nan: None}) + df = pd.DataFrame(api_response.items) return df except ApiException as e: logger.error(f"Exception when calling SurveysApi->m_fi_surveys_full_data_get: {e}") @@ -769,6 +766,8 @@ def get_mfi_surveys_processed_data(self, survey_id=None, page=1, page_size=20, f raise + + if __name__ == "__main__": import yaml # FOR TESTING @@ -795,5 +794,3 @@ def get_mfi_surveys_processed_data(self, survey_id=None, page=1, page_size=20, f # print(f"Downloaded {len(polygon_files)} bytes") - print(choices) - diff --git a/examples/example.py b/examples/example.py index f2b2cd1..ec1b3d5 100644 --- a/examples/example.py +++ b/examples/example.py @@ -14,15 +14,21 @@ client = DataBridgesShapes(CONFIG_PATH) -# COMMODITY DATA +#%% COMMODITY DATA +# Get commodity unit list for a country commodity_units_list = client.get_commodity_units_list(country_code="TZA", commodity_unit_name="Kg", page=1, format='json') + +# Get commodity unit conversion list for a country comodity_unit_conversion_list = client.get_commodity_units_conversion_list(country_code="TZA", commodity_id=1, from_unit_id=1, to_unit_id=2, page=1, format='json') +#%% CURRENCTY DATA +# Get currency list currency_list = client.get_currency_list(country_code="TZA", currency_name="TZS", currency_id=0, page=1, format='json') + +# Get USD indirect quotation for a country usd_indirect_quotation = client.get_usd_indirect_quotation(country_iso3="TZA", currency_name="TZS", page=1, format='json') -print(commodity_units_list) -## MARKETS DATA +#%% MARKETS DATA # Get a complete list of markets in a country markets_list = client.get_markets_list(country_code="TZA") @@ -32,11 +38,24 @@ # Get markets near a given location by longitude and latitude within a 15Km distance nearby_markets = client.get_nearby_markets(adm0code=56) -### FOOD SECURITY DATA +#%% MARKET FUNCTIONALITY INDEX +# Get the MFI surveys for a given country +get_mfi_surveys = client.get_mfi_surveys(adm0_code=1) + +# Get the MFI functionality index for a given country (standardized data) +get_mfi_surveys_base_data = client.get_mfi_surveys_base_data(survey_id=3673) + +# Get the MFI functionality index for a given country (full data) +get_mfi_surveys_full_data = client.get_mfi_surveys_full_data(survey_id=3673) + +# Get the MFI functionality index for a given country (processed data) +get_mfi_surveys_processed_data = client.get_mfi_surveys_processed_data(survey_id=3673) + +#%% FOOD SECURITY DATA # Get IPC and equivalent food insecurity numbers for all countries get_food_security_list = client.get_food_security_list() -### Global Operation Response Plan (GOPR) +### GLOBAL OPERATION RESPONSE PLAN (GOPR) # Get country-level latest data from the Global Operation Response Plan (GOPR) country_latest_df = client.get_gorp('country_latest') # no data currently uploaded @@ -69,3 +88,4 @@ choices = client.get_choice_list(xls_form_id=1509) + From 9ef72fca28096ff7e3588011d8df040e5fa25e6f Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Mon, 23 Sep 2024 15:44:19 +0200 Subject: [PATCH 19/20] feat: add RPME endpoints --- ROADMAP.md | 2 +- data_bridges_knots/client.py | 135 +++++++++++++++++++++++++++++++++++ examples/example.py | 22 ++---- 3 files changed, 142 insertions(+), 17 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 1191393..672c6a5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -14,7 +14,7 @@ This document outlines the planned features and improvements for the `DataBridge - [X] Endpoints: GorpApi - [X] Endpoints: MarketPricesAPi - [X] Endpoints: IncubationApi -- [ ] Endpoints: RpmeApi +- [X] Endpoints: RpmeApi - [ ] Endpoints: SurveysApi - [ ] Endpoints: XlsFormsApi diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 95547ba..9095572 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -765,8 +765,118 @@ def get_mfi_surveys_processed_data(self, survey_id=None, page=1, page_size=20, f logger.error(f"Exception when calling SurveysApi->m_fi_surveys_processed_data_get: {e}") raise + # TODO: Get the scope and test these functions + def get_rpme_base_data(self, survey_id=None, page=1, page_size=20): + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.RpmeApi(api_client) + env = self.env + + try: + api_response = api_instance.rpme_base_data_get(survey_id=survey_id, page=page, page_size=page_size, env=env) + logger.info("Successfully retrieved RPME base data") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling RpmeApi->rpme_base_data_get: {e}") + raise + # TODO: Get the scope and test these functions + def get_rpme_full_data(self, survey_id=None, format='json', page=1, page_size=20): + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.RpmeApi(api_client) + env = self.env + try: + api_response = api_instance.rpme_full_data_get(survey_id=survey_id, format=format, page=page, page_size=page_size, env=env) + logger.info("Successfully retrieved RPME full data") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling RpmeApi->rpme_full_data_get: {e}") + raise + # TODO: Get the scope and test these functions + def get_rpme_output_values(self, page=1, adm0_code=None, survey_id=None, shop_id=None, market_id=None, adm0_code_dots=''): + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.RpmeApi(api_client) + env = self.env + + try: + api_response = api_instance.rpme_output_values_get(page=page, adm0_code=adm0_code, survey_id=survey_id, + shop_id=shop_id, market_id=market_id, adm0_code_dots=adm0_code_dots, env=env) + logger.info("Successfully retrieved RPME output values") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling RpmeApi->rpme_output_values_get: {e}") + raise + # TODO: Get the scope and test these functions + def get_rpme_surveys(self, adm0_code=0, page=1, start_date=None, end_date=None): + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.RpmeApi(api_client) + env = self.env + + try: + api_response = api_instance.rpme_surveys_get(adm0_code=adm0_code, page=page, start_date=start_date, end_date=end_date, env=env) + logger.info("Successfully retrieved RPME surveys") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling RpmeApi->rpme_surveys_get: {e}") + raise + # TODO: Get the scope and test these functions + def get_rpme_variables(self, page=1): + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.RpmeApi(api_client) + env = self.env + + try: + api_response = api_instance.rpme_variables_get(page=page, env=env) + logger.info("Successfully retrieved RPME variables") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling RpmeApi->rpme_variables_get: {e}") + raise + + # TODO: Get the scope and test these functions + def get_rpme_xls_forms(self, adm0_code=0, page=1, start_date=None, end_date=None): + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.RpmeApi(api_client) + env = self.env + + try: + api_response = api_instance.rpme_xls_forms_get(adm0_code=adm0_code, page=page, start_date=start_date, end_date=end_date, env=env) + logger.info("Successfully retrieved RPME XLS forms") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling RpmeApi->rpme_xls_forms_get: {e}") + raise + + # Add this function to the DataBridgesShapes class + + def get_mfi_xls_forms(self, adm0_code=0, page=1, start_date=None, end_date=None): + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.XlsFormsApi(api_client) + env = self.env + + try: + api_response = api_instance.m_fi_xls_forms_get(adm0_code=adm0_code, page=page, start_date=start_date, end_date=end_date, env=env) + logger.info("Successfully retrieved MFI XLS forms") + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + return df + except ApiException as e: + logger.error(f"Exception when calling XlsFormsApi->m_fi_xls_forms_get: {e}") + raise + + if __name__ == "__main__": import yaml @@ -793,4 +903,29 @@ def get_mfi_surveys_processed_data(self, survey_id=None, page=1, page_size=20, f # print("\nAIMS Polygon Files:") # print(f"Downloaded {len(polygon_files)} bytes") + # FIXME: Get scopes for RPME then test the following function + # Get RPME base data + # rpme_base_data = client.get_rpme_base_data(survey_id=123, page=1, page_size=50) + + # FIXME: Get scopes for RPME then test the following function + # # Get RPME full data + # rpme_full_data = client.get_rpme_full_data(survey_id=123, format='json', page=1, page_size=50) + + # FIXME: Get scopes for RPME then test the following function + # # Get RPME output values + # rpme_output_values = client.get_rpme_output_values(page=1, adm0_code=456, survey_id=123) + + # FIXME: Get scopes for RPME then test the following function + # # Get RPME surveys + # rpme_surveys = client.get_rpme_surveys(adm0_code=456, page=1, start_date='2023-01-01', end_date='2023-12-31') + + # FIXME: Get scopes for RPME then test the following function + # # Get RPME variables + # rpme_variables = client.get_rpme_variables(page=1) + + # FIXME: Get scopes for RPME then test the following function + # # Get RPME XLS forms + # rpme_xls_forms = client.get_rpme_xls_forms(adm0_code=456, page=1, start_date='2023-01-01', end_date='2023-12-31') + + mfi_xls_forms = client.get_mfi_xls_forms(page=1, start_date='2023-01-01', end_date='2023-12-31') diff --git a/examples/example.py b/examples/example.py index ec1b3d5..9e5d800 100644 --- a/examples/example.py +++ b/examples/example.py @@ -1,13 +1,3 @@ -""" -Reads Household Data from Data Bridges. The script uses the DataBridgesShapes class from the data_bridges_knots module to interact with the Data Bridges API and retrieve various datasets, including: -- Household survey data -- GORP (Global Operational Response Plan) data -- Exchange rates and prices for Afghanistan -- IPC and equivalent food security Data - -The script demonstrates how to use the DataBridgesShapes class to fetch these datasets and print the first few rows of the resulting pandas DataFrames. -""" - from data_bridges_knots import DataBridgesShapes CONFIG_PATH = r"data_bridges_api_config.yaml" @@ -51,11 +41,14 @@ # Get the MFI functionality index for a given country (processed data) get_mfi_surveys_processed_data = client.get_mfi_surveys_processed_data(survey_id=3673) +# Get MFI XLSForm information +mfi_xls_forms = client.get_mfi_xls_forms(page=1, start_date='2023-01-01', end_date='2023-12-31') + #%% FOOD SECURITY DATA # Get IPC and equivalent food insecurity numbers for all countries get_food_security_list = client.get_food_security_list() -### GLOBAL OPERATION RESPONSE PLAN (GOPR) +#%% GLOBAL OPERATION RESPONSE PLAN (GOPR) # Get country-level latest data from the Global Operation Response Plan (GOPR) country_latest_df = client.get_gorp('country_latest') # no data currently uploaded @@ -71,7 +64,7 @@ # Get regional latest data regional_latest_df = client.get_gorp('regional_latest') -### HOUSEHOLD DATA +#%% HOUSEHOLD ASSESSMENT & MONITORING DATA # Get list of household surveys available surveys_list = client.get_household_surveys() @@ -85,7 +78,4 @@ questionnaire = client.get_household_questionnaire(xls_form_id=1509) # Get choice list for a specific survey -choices = client.get_choice_list(xls_form_id=1509) - - - +choices = client.get_choice_list(xls_form_id=1509) \ No newline at end of file From ac7c86897f331c479c49f0b718289b377ff78444 Mon Sep 17 00:00:00 2001 From: Alessandra Gherardelli Date: Fri, 29 Nov 2024 14:55:28 +0100 Subject: [PATCH 20/20] Added get_mfi_xls_forms_detailed --- ROADMAP.md | 19 +++++--- data_bridges_knots/client.py | 90 +++++++++++++++++++----------------- 2 files changed, 60 insertions(+), 49 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 672c6a5..399e649 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ This document outlines the planned features and improvements for the `DataBridgesKnots` package, which provides a wrapper for the WFP Data Bridges API. -## Upcoming Release: 0.2.0 (DataBridges API v5.0) +## Upcoming Release: 1.0.0 (DataBridges API v5.0) ### New Features - [ ] Update setup.py and pyproject.toml to include DataBridges API v5.0 @@ -25,17 +25,22 @@ This document outlines the planned features and improvements for the `DataBridge - [ ] Market list JSON and CSV ## Future Release -### v0.3.0 (DataBridges API v.0) -- [ ] STATA support +### v1.x.1 +- [ ] Documentation: Enhance documentation and provide more usage examples +- [ ] GitHub Actions linting and testing +- [ ] Test AIMS and RPME endpoints + +### v1.1.0 - [ ] R example files -- [ ] Fix optional dependencies for STATA - [ ] Handle SSL certificate error -## Future Releases (1.0.0 and beyond) +### v1.2.0 +- [ ] STATA support +- [ ] Fix optional dependencies for STATA + +## Future Releases (2.0.0 and beyond) - Testing: Unit testing -- Testing: GitHub Actions linting and testing - Testing: Improve error handling and logging -- Documentation: Enhance documentation and provide more usage examples - Refactoring: Optimize performance and improve code efficiency Please note that this roadmap is subject to change, and the priorities may be adjusted based on the project's needs and available resources. diff --git a/data_bridges_knots/client.py b/data_bridges_knots/client.py index 9095572..ee981ab 100644 --- a/data_bridges_knots/client.py +++ b/data_bridges_knots/client.py @@ -876,6 +876,47 @@ def get_mfi_xls_forms(self, adm0_code=0, page=1, start_date=None, end_date=None) logger.error(f"Exception when calling XlsFormsApi->m_fi_xls_forms_get: {e}") raise + def get_mfi_xls_forms_detailed(self, adm0_code=0, page=1, start_date=None, end_date=None): + """ + Get a complete list of XLS Forms uploaded on the MFI Data Bridge in a given period of data collection. + + Args: + adm0_code (int): Code for the country. Defaults to 0. + page (int): Page number for paged results. Defaults to 1. + start_date (str): Starting date for data collection range (YYYY-MM-DD format) + end_date (str): Ending date for data collection range (YYYY-MM-DD format) + + Returns: + pandas.DataFrame: DataFrame containing XLS Forms data + """ + with data_bridges_client.ApiClient(self.configuration) as api_client: + api_instance = data_bridges_client.XlsFormsApi(api_client) + env = self.env + + try: + api_response = api_instance.m_fi_xls_forms_get( + adm0_code=adm0_code, + page=page, + start_date=start_date, + end_date=end_date, + env=env + ) + logger.info("Successfully retrieved detailed MFI XLS forms") + + # Convert response items to DataFrame + df = pd.DataFrame([item.to_dict() for item in api_response.items]) + df = df.replace({np.nan: None}) + + # Add total items count as DataFrame attribute + df.total_items = api_response.total_items + + return df + + except ApiException as e: + logger.error(f"Exception when calling XlsFormsApi->m_fi_xls_forms_get: {e}") + raise + + if __name__ == "__main__": @@ -885,47 +926,12 @@ def get_mfi_xls_forms(self, adm0_code=0, page=1, start_date=None, end_date=None) client = DataBridgesShapes(CONFIG_PATH) - # FIXME: JSON Response + printing instead of logging - # economic_indicator_list = client.get_economic_indicator_list(page=1, indicator_name='', iso3='', format='json') - - # BUG: Error: Unsupported content type application/geo+json - # markets_geo_json = client.get_market_geojson_list(adm0code=56) - - # FIXME: Get scopes for AIMS then test the following function - # # Get AIMS analysis rounds - # aims_data = client.get_aims_analysis_rounds(adm0_code=1) - # print("\nAIMS Analysis Rounds Data:") - # print(f"Downloaded {len(aims_data)} bytes") - - # FIXME: Get scopes for AIMS then test the following function - # # Get AIMS polygon files - # polygon_files = client.get_aims_polygon_files(adm0_code=1) - # print("\nAIMS Polygon Files:") - # print(f"Downloaded {len(polygon_files)} bytes") - - # FIXME: Get scopes for RPME then test the following function - # Get RPME base data - # rpme_base_data = client.get_rpme_base_data(survey_id=123, page=1, page_size=50) - - # FIXME: Get scopes for RPME then test the following function - # # Get RPME full data - # rpme_full_data = client.get_rpme_full_data(survey_id=123, format='json', page=1, page_size=50) - - # FIXME: Get scopes for RPME then test the following function - # # Get RPME output values - # rpme_output_values = client.get_rpme_output_values(page=1, adm0_code=456, survey_id=123) - - # FIXME: Get scopes for RPME then test the following function - # # Get RPME surveys - # rpme_surveys = client.get_rpme_surveys(adm0_code=456, page=1, start_date='2023-01-01', end_date='2023-12-31') - - # FIXME: Get scopes for RPME then test the following function - # # Get RPME variables - # rpme_variables = client.get_rpme_variables(page=1) - - # FIXME: Get scopes for RPME then test the following function - # # Get RPME XLS forms - # rpme_xls_forms = client.get_rpme_xls_forms(adm0_code=456, page=1, start_date='2023-01-01', end_date='2023-12-31') - mfi_xls_forms = client.get_mfi_xls_forms(page=1, start_date='2023-01-01', end_date='2023-12-31') + xls_forms = client.get_mfi_xls_forms_detailed( + adm0_code=0, + page=1, + start_date='2023-01-01', + end_date='2023-12-31' +) + print(xls_forms) \ No newline at end of file