From 05a361939d7ca130d3880573d3510c343062d7e2 Mon Sep 17 00:00:00 2001 From: chrwm Date: Mon, 5 Dec 2022 20:36:45 +0100 Subject: [PATCH 01/15] Improve logging of md upload #31 --- oem2orm/oep_oedialect_oem2orm.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index 44e9b08..f6185ca 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -107,6 +107,7 @@ def create_tables(db: DB, tables: List[sa.Table]): :return: none """ for table in tables: + logging.info(f"Working on table: {table}") if not db.engine.dialect.has_schema(db.engine, table.schema): error_msg = f'The provided database schema: "{table.schema}" does not exist. Please use an existing schema' logging.info(error_msg) @@ -314,7 +315,7 @@ def collect_tables_from_oem(db: DB, oem_folder_path): def load_json(filepath): - logging.info("reading %s" % filepath) + logging.info("Reading metadata: %s" % filepath) with open(filepath, "rb") as f: return json.load(f) @@ -373,12 +374,12 @@ def api_updateMdOnTable(metadata, token=None): schema = getTableSchemaNameFromOEM(metadata)[0] table = getTableSchemaNameFromOEM(metadata)[1] - logging.info("UPDATE METADATA") + logging.info(f"Update metadata: {table}") api_action = setupApiAction(schema, table, token) resp = requests.post(api_action.dest_url, json=metadata, headers=api_action.headers) if resp.status_code == 200: - logging.info(" ok.") - logging.info(api_action.dest_url) + logging.info(f"METADATA SUCCESSFULLY UPDATED: {table}") + logging.info(f"Link to updated metadata on OEP: {api_action.dest_url}") else: error_msg = resp.json() logging.info(error_msg) From 125ce5782e605e5cab0e9b039d9062b3cb847288 Mon Sep 17 00:00:00 2001 From: chrwm Date: Mon, 5 Dec 2022 20:38:07 +0100 Subject: [PATCH 02/15] Improve err msg if table doesn't exist #31 --- oem2orm/oep_oedialect_oem2orm.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index f6185ca..601d00d 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -43,7 +43,8 @@ def setup_logger(): Easy logging setup depending on user input. Provides a logger for INFO level logging. :return: logging.INFO or none """ - logger_level = input("Display logging information[Yes] or [No]:") + # logger_level = input("Display logging information[Yes] or [No]:") + logger_level = "yes" if re.fullmatch("[Yy]es", logger_level): print("logging activated") return logging.basicConfig( @@ -381,11 +382,19 @@ def api_updateMdOnTable(metadata, token=None): logging.info(f"METADATA SUCCESSFULLY UPDATED: {table}") logging.info(f"Link to updated metadata on OEP: {api_action.dest_url}") else: - error_msg = resp.json() - logging.info(error_msg) - logging.info("HTTP status code: ") - logging.info(resp.status_code) - raise MetadataError(f"Uploading of metadata failed. Response from OEP: {error_msg}") + oep_err_msg_header = re.search("

(.*)

", resp.text).group(1) + err_message = ( + f"Uploading of metadata failed. HTTPS response from OEP: {resp.status_code}, Message: {oep_err_msg_header}, URL: {resp.url}" + "" + ) + if not resp.text.startswith("{"): + raise MetadataError( + f"The response text doesn't seem to be a json: {resp.text[:40]}..... \n\n Please check " + f"that the table is already created on the OEP! \n {err_message}" + ) + else: + # in case a json is returned and there is a json error + raise MetadataError(err_message, resp.json()) def api_downloadMd(schema, table, token=None): From 5b171e926564785e3ebbddada108c92df7315aad Mon Sep 17 00:00:00 2001 From: chrwm Date: Tue, 13 Dec 2022 19:44:31 +0100 Subject: [PATCH 03/15] Activate logging by default #31 I don't see why logging shouldn't be the default. Non-developers will appreciate it to see what is going on in the programme, and so will developers in doubt when debugging. If devs want to deactivate logging they can set the parameter to `No`. This appears more functional than inputting this information or writing a custom workaround to avoid user-input with each execution. --- oem2orm/oep_oedialect_oem2orm.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index 601d00d..b1aa399 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -38,15 +38,14 @@ class MetadataError(Exception): pass -def setup_logger(): +def setup_logger(logger_level: str = "Yes"): """ Easy logging setup depending on user input. Provides a logger for INFO level logging. :return: logging.INFO or none """ - # logger_level = input("Display logging information[Yes] or [No]:") - logger_level = "yes" + if re.fullmatch("[Yy]es", logger_level): - print("logging activated") + print("Logging activated") return logging.basicConfig( format="%(levelname)s:%(message)s", level=logging.INFO ) From 624df67de4130e7db2f8729f66fc689aa20e3683 Mon Sep 17 00:00:00 2001 From: chrwm Date: Wed, 4 Jan 2023 17:06:03 +0100 Subject: [PATCH 04/15] Improve err msg for wrong schema #31 --- oem2orm/oep_oedialect_oem2orm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index b1aa399..3a038f0 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -109,7 +109,8 @@ def create_tables(db: DB, tables: List[sa.Table]): for table in tables: logging.info(f"Working on table: {table}") if not db.engine.dialect.has_schema(db.engine, table.schema): - error_msg = f'The provided database schema: "{table.schema}" does not exist. Please use an existing schema' + error_msg = f'The provided database schema: "{table.schema}" does not exist. Please use an existing ' \ + f'schema from the `name` column from: https://openenergy-platform.org/dataedit/schemas' logging.info(error_msg) raise DatabaseError(error_msg) else: @@ -269,7 +270,7 @@ def check_oep_api_schema_whitelist(oem_schema): return True else: logging.info( - "The OEP-API does not allow to write un-reviewed data to another schema then model_draft or sandbox" + "The OEP-API does not allow to write un-reviewed data to another schema then 'model_draft' or 'sandbox'" ) return False From 1772885d4787bab2192a47e9eeda99fe4ad18fcd Mon Sep 17 00:00:00 2001 From: chrwm Date: Wed, 4 Jan 2023 17:12:13 +0100 Subject: [PATCH 05/15] Update CHANGELOG.md #31 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15e0c0b..d86a4ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ Template: ### Added ### Changed - +- Improve logging and error messages (table creation, data upload, metadata registration)[#38] ### Removed ______________________________________________________________________ From 72fae2b092ba30b2091a581b259a4187871487d8 Mon Sep 17 00:00:00 2001 From: chrwm Date: Thu, 5 Jan 2023 18:02:22 +0100 Subject: [PATCH 06/15] Improve logging #31 --- oem2orm/oep_oedialect_oem2orm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index 3a038f0..0a16f37 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -375,7 +375,7 @@ def api_updateMdOnTable(metadata, token=None): schema = getTableSchemaNameFromOEM(metadata)[0] table = getTableSchemaNameFromOEM(metadata)[1] - logging.info(f"Update metadata: {table}") + logging.info(f"Update metadata on table: {table}") api_action = setupApiAction(schema, table, token) resp = requests.post(api_action.dest_url, json=metadata, headers=api_action.headers) if resp.status_code == 200: From 002be12fd73c613d10a0fb0cf2e87ceda4406461 Mon Sep 17 00:00:00 2001 From: chrwm <54852694+chrwm@users.noreply.github.com> Date: Tue, 5 Mar 2024 15:37:57 +0100 Subject: [PATCH 07/15] Improve error message printing --- oem2orm/oep_oedialect_oem2orm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index 0a16f37..ed0bba9 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -119,7 +119,7 @@ def create_tables(db: DB, tables: List[sa.Table]): table.create(checkfirst=True) logging.info(f"Created table {table.name}") except oedialect.engine.ConnectionException as ce: - error_msg = f'Error when uploading table "{table.name}".' + error_msg = f'Error when uploading table "{table.name}". Reason: {ce}.' logging.error(error_msg) raise DatabaseError(error_msg) from ce except sa.exc.ProgrammingError as pe: From f2e67cfd7da4d6ceb898a95a9d99863d9cebdb19 Mon Sep 17 00:00:00 2001 From: Hendrik Huyskens Date: Tue, 5 Mar 2024 16:16:58 +0100 Subject: [PATCH 08/15] Fix error if field type is None --- CHANGELOG.md | 2 ++ oem2orm/oep_oedialect_oem2orm.py | 2 +- oem2orm/postgresql_types.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d86a4ce..b6d0ed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ Template: ### Changed - Improve logging and error messages (table creation, data upload, metadata registration)[#38] +- Fix error if field type is None + ### Removed ______________________________________________________________________ diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index ed0bba9..6b22f21 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -205,7 +205,7 @@ def create_tables_from_metadata_file( # Get column type: try: column_type = TYPES[field["type"]] - except KeyError: + except (KeyError, ValueError): raise MetadataError( "Unknown column type", field, field["type"], metadata_file ) diff --git a/oem2orm/postgresql_types.py b/oem2orm/postgresql_types.py index 78c00af..fca5dcd 100644 --- a/oem2orm/postgresql_types.py +++ b/oem2orm/postgresql_types.py @@ -40,6 +40,8 @@ class DatabaseTypes: } def __getitem__(self, item): + if item is None: + raise ValueError("Field type is 'None'.") if item.split(" ")[-1] == "array": db_type = self.types[item[:-6]] return psql.ARRAY(db_type) From 4b0219ae5186274460d4ea4bf7197a5e05b3cb23 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Mon, 15 Apr 2024 16:43:13 +0200 Subject: [PATCH 09/15] update oep url in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50e3ec0..e23f72b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ see [Pipx-Documentation](https://pypa.github.io/pipx/) for further information. ## Usage: -This tool is part of the open-energy-metadata (OEM) integration into the [OEP](https://openenergy-platform.org/). +This tool is part of the open-energy-metadata (OEM) integration into the [OEP](https://openenergyplatform.org/). To use this tool with the OEP API you need to be signed up to the OEP since you need to provide an API-Token. From 60015c1748e51f51ad2ce7bf477825eda6fbcd5a Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Mon, 15 Apr 2024 16:43:48 +0200 Subject: [PATCH 10/15] add settings.py module to store the oep host url and variations --- oem2orm/settings.py | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 oem2orm/settings.py diff --git a/oem2orm/settings.py b/oem2orm/settings.py new file mode 100644 index 0000000..0154bfc --- /dev/null +++ b/oem2orm/settings.py @@ -0,0 +1,2 @@ +OEP_URL = "openenergy-platform.org" +OEP_API_URL = "https://openenergy-platform.org/api/v0/" From ab3f39d0ce1c3ec72fa53d9a90860e2ee9e39f3b Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Mon, 15 Apr 2024 16:44:25 +0200 Subject: [PATCH 11/15] use oep url info from settings module and remove hardcoded urls --- oem2orm/oep_oedialect_oem2orm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/oem2orm/oep_oedialect_oem2orm.py b/oem2orm/oep_oedialect_oem2orm.py index 6b22f21..ac58353 100644 --- a/oem2orm/oep_oedialect_oem2orm.py +++ b/oem2orm/oep_oedialect_oem2orm.py @@ -19,6 +19,7 @@ from oem2orm.postgresql_types import TYPES from oem2orm.oep_compliance import run_metadata_checks +from oem2orm.settings import OEP_URL, OEP_API_URL # prepare connection string to connect via oep API CONNECTION_STRING = "{engine}://{user}:{token}@{host}" @@ -53,7 +54,7 @@ def setup_logger(logger_level: str = "Yes"): pass -def setup_db_connection(engine="postgresql+oedialect", host="openenergy-platform.org"): +def setup_db_connection(engine="postgresql+oedialect", host=OEP_URL): """ Create SQLAlchemy connection to Database API with Username and Token. Default is the OEP RESTful-API. @@ -81,9 +82,8 @@ def setup_db_connection(engine="postgresql+oedialect", host="openenergy-platform def setupApiAction(schema, table, token=None): API_ACTION = namedtuple("API_Action", ["dest_url", "headers"]) - OEP_URL = "https://openenergy-platform.org" - url = OEP_URL + "/api/v0/schema/{schema}/tables/{table}/meta/".format( + url = OEP_API_URL + "schema/{schema}/tables/{table}/meta/".format( schema=schema, table=table ) @@ -110,7 +110,7 @@ def create_tables(db: DB, tables: List[sa.Table]): logging.info(f"Working on table: {table}") if not db.engine.dialect.has_schema(db.engine, table.schema): error_msg = f'The provided database schema: "{table.schema}" does not exist. Please use an existing ' \ - f'schema from the `name` column from: https://openenergy-platform.org/dataedit/schemas' + f'schema from the `name` column from: {OEP_URL}/dataedit/schemas' logging.info(error_msg) raise DatabaseError(error_msg) else: From 6a01a507fb3944475f735b894e5f07c1f8077a45 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Mon, 15 Apr 2024 16:44:40 +0200 Subject: [PATCH 12/15] fix oedialect version to latest pypi release --- oem2orm/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oem2orm/requirements.txt b/oem2orm/requirements.txt index aecc812..b1db214 100644 --- a/oem2orm/requirements.txt +++ b/oem2orm/requirements.txt @@ -2,7 +2,7 @@ sqlalchemy psycopg2-binary jmespath GeoAlchemy2 -oedialect +oedialect==0.1.1 geopandas pandas omi \ No newline at end of file From a8fd46c0ca5867a1faa07b77e92668e8fc203e99 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Mon, 15 Apr 2024 16:46:34 +0200 Subject: [PATCH 13/15] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6d0ed0..63d63c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,10 +29,12 @@ Template: ## [Unreleased] - 20XX-MM-DD ### Added +- New settings module [#44](https://github.com/OpenEnergyPlatform/oem2orm/pull/44) ### Changed - Improve logging and error messages (table creation, data upload, metadata registration)[#38] - Fix error if field type is None +- Update oep host url [#44](https://github.com/OpenEnergyPlatform/oem2orm/pull/44) ### Removed From b21844a417fbd90239a37d4e9f828d39723b907d Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Mon, 15 Apr 2024 16:49:45 +0200 Subject: [PATCH 14/15] bump version --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4b5aefa..436d026 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="oem2orm", - version="0.3.2", + version="0.3.3", author="henhuy, jh-RLI", author_email="Hendrik.Huyskens@rl-institut.de", description="SQLAlchemy module to generate ORM, read from data model (oedatamodel) in open-energy-metadata JSON format", @@ -28,7 +28,7 @@ "Operating System :: OS Independent", ], python_requires='>=3.6', - install_requires=['sqlalchemy==1.3.14', 'oedialect', 'requests', 'jmespath', 'omi', 'click'], # Optional + install_requires=['sqlalchemy==1.3.14', 'oedialect==1.1', 'requests', 'jmespath', 'omi', 'click'], # Optional project_urls={ # Optional 'Bug Reports': 'https://github.com/OpenEnergyPlatform/oem2orm/issues', 'Source': 'https://github.com/OpenEnergyPlatform/oem2orm/tree/develop/oem2orm', From b28b0b951e344de380b716a4d20f17692c7e4a87 Mon Sep 17 00:00:00 2001 From: Jonas Huber Date: Mon, 15 Apr 2024 16:50:00 +0200 Subject: [PATCH 15/15] add release changelog --- CHANGELOG.md | 67 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63d63c1..d44f251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # Changelog + All notable changes to this project will be documented in this file. The format is inpired from [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) @@ -26,123 +27,163 @@ Template: ### Removed - basic description [#PR/#Issue/#Commit] ``` + ## [Unreleased] - 20XX-MM-DD ### Added + +- + +### Changed + +- + +### Removed + +______________________________________________________________________ + +## [v0.3.3] - 2024-04-15 + +### Added + - New settings module [#44](https://github.com/OpenEnergyPlatform/oem2orm/pull/44) ### Changed -- Improve logging and error messages (table creation, data upload, metadata registration)[#38] + +- Improve logging and error messages [table creation, data upload, metadata registration](#38) - Fix error if field type is None - Update oep host url [#44](https://github.com/OpenEnergyPlatform/oem2orm/pull/44) -### Removed - ______________________________________________________________________ + ## [0.3.2] - 2022-11-29 ### Added + - token can be passed as parameter (to support usage in APIs) ### Changed + - Error is raised if schema does not exist - Metadata compilance checks now inculue optinal jsonschema validation for metadata (PR#32) - MetadataError is thrown if uploading metadata to OEP fails ______________________________________________________________________ + ## [0.3.1] - 2022-10-24 ### Changed + - fix module not installed error after pip install and import of oem2orm PR(#26) ______________________________________________________________________ + ## [0.3.0] - 2022-10-24 ### Added + - Option to create tables from OEM JSON instead of file - New module to check if metadata is oep compliant. Can check (omi's 1 parse 2 compile) oemetadata v1.5 and v1.4 (PR#23) - Add PYPI release workflow to automate python package releases for pypi test and pypi official (PR#22) ### Changed + - omi_validateMD was outdated and now runs the new oep compliance checks. ______________________________________________________________________ + ## [0.2.7] - 2021-03-11 ### Added + - enable console usage [PR#8] -- new package requirement "omi" -- support for sqlachemy "numeric" data type +- new package requirement "omi" +- support for sqlachemy "numeric" data type ### Changed + - fix missing dependency that made pip install fail [ISSUE#1;PR#8] ### Removed -- console script, remove table delete + +- console script, remove table delete ______________________________________________________________________ + ## [0.2.6] - 2020-09-23 ### Added + - support for datatypes "hstore" and "decimal" - provide new example files that work with oem2orm ______________________________________________________________________ + ## [0.2.5] - 2020-08-06 ### Changed + - fix installation error caused by jmespath package dependency ______________________________________________________________________ + ## [0.2.4] - 2020-07-20 ### Added + - Support to setup the OEP API-URL - Metadata Up- and download are supported - Save downloaded metadata to file - Validate metadata using OMI parser v1.4.0 ### Changed + - change functions names ______________________________________________________________________ + ## [0.2.3] - 2020-06-02 ### Added + - provide a minimal working example as jupyter notebook tutorial -- New OEP-API related functions: Prepare the oemetadata string to send to api +- New OEP-API related functions: Prepare the oemetadata string to send to api - Simple User Input function to set the OEP-API-Token ### Changed + - Update README - include OEP public schema (whitelist) check - Spatial types from Geoalchemy2 do not set a spatial_index anymore ______________________________________________________________________ + ## [0.2.2] - 2020-06-02 ### Added + - new function: setting up a logger ### Changed + - add missing input parameter - extended description in changelog -- Fix logging +- Fix logging ______________________________________________________________________ + ## [0.2.0] - 2020-05-27 ### Added + - new function: delete tables from DB now possible -- new function: select the oem data folder +- new function: select the oem data folder - new function: tables are collected and ordered by fk (increase usability) ### Changed + - added docstrings ### Removed -- the user is no longer required to use a for loop in the main function to collect tables - - - +- the user is no longer required to use a for loop in the main function to collect tables