From 5d0578baf3004bafc00ae11ae817a2e5dd191c46 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Thu, 7 Nov 2024 19:07:23 +0530 Subject: [PATCH 1/4] feat: mssql database support --- .../insights_data_source_v3/connectors/mssql.py | 15 +++++++++++++++ .../insights_data_source_v3.json | 4 ++-- .../insights_data_source_v3.py | 13 ++++++------- pyproject.toml | 3 ++- 4 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 insights/insights/doctype/insights_data_source_v3/connectors/mssql.py diff --git a/insights/insights/doctype/insights_data_source_v3/connectors/mssql.py b/insights/insights/doctype/insights_data_source_v3/connectors/mssql.py new file mode 100644 index 00000000..8c914326 --- /dev/null +++ b/insights/insights/doctype/insights_data_source_v3/connectors/mssql.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt +from urllib.parse import quote_plus + + +def get_mssql_connection_string(data_source): + password = data_source.get_password(raise_exception=False) + password = quote_plus(password) if password else "" + connection_string = ( + f"mssql://{data_source.username}:{password}" + f"@{data_source.host}:{data_source.port}/{data_source.database_name}" + ) + if data_source.use_ssl: + pass + return connection_string diff --git a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.json b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.json index f75ebd99..63a10803 100644 --- a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.json +++ b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.json @@ -41,7 +41,7 @@ "fieldname": "database_type", "fieldtype": "Select", "label": "Database Type", - "options": "MariaDB\nPostgreSQL\nSQLite\nDuckDB\nBigQuery" + "options": "MariaDB\nPostgreSQL\nSQLite\nDuckDB\nBigQuery\nMSSQL" }, { "fieldname": "host", @@ -120,7 +120,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2024-11-03 17:10:59.295243", + "modified": "2024-11-07 19:06:14.814805", "modified_by": "Administrator", "module": "Insights", "name": "Insights Data Source v3", diff --git a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py index 25b09545..f8dabd3b 100644 --- a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py +++ b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py @@ -10,12 +10,6 @@ from frappe.model.document import Document from ibis import BaseBackend -from insights.insights.doctype.insights_data_source_v3.connectors.bigquery import ( - get_bigquery_connection, -) -from insights.insights.doctype.insights_data_source_v3.data_warehouse import ( - WAREHOUSE_DB_NAME, -) from insights.insights.doctype.insights_table_link_v3.insights_table_link_v3 import ( InsightsTableLinkv3, ) @@ -23,6 +17,7 @@ InsightsTablev3, ) +from .connectors.bigquery import get_bigquery_connection from .connectors.duckdb import get_duckdb_connection_string from .connectors.frappe_db import ( get_frappedb_connection_string, @@ -31,8 +26,10 @@ is_frappe_db, ) from .connectors.mariadb import get_mariadb_connection_string +from .connectors.mssql import get_mssql_connection_string from .connectors.postgresql import get_postgres_connection_string from .connectors.sqlite import get_sqlite_connection_string +from .data_warehouse import WAREHOUSE_DB_NAME class InsightsDataSourceDocument: @@ -154,7 +151,7 @@ class InsightsDataSourcev3(InsightsDataSourceDocument, Document): connection_string: DF.Text | None database_name: DF.Data | None database_type: DF.Literal[ - "MariaDB", "PostgreSQL", "SQLite", "DuckDB", "BigQuery" + "MariaDB", "PostgreSQL", "SQLite", "DuckDB", "BigQuery", "MSSQL" ] host: DF.Data | None is_frappe_db: DF.Check @@ -201,6 +198,8 @@ def _get_connection_string(self): return get_mariadb_connection_string(self) if self.database_type == "PostgreSQL": return get_postgres_connection_string(self) + if self.database_type == "MSSQL": + return get_mssql_connection_string(self) frappe.throw(f"Unsupported database type: {self.database_type}") diff --git a/pyproject.toml b/pyproject.toml index 1263a167..1e91c2b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,9 +15,10 @@ dependencies = [ "ibis-framework==9.5.0", "ibis-framework[duckdb]", "ibis-framework[mysql]", + "ibis-framework[postgres]", "ibis-framework[sqlite]", "ibis-framework[bigquery]", - # "ibis-framework[postgres]", + "ibis-framework[mssql]" ] [build-system] From 7af37e5381fea5fa9390f6477e827d6c63c43bfe Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Sat, 23 Nov 2024 13:47:06 +0530 Subject: [PATCH 2/4] fix: consider all other data types as string --- flake8 | 5 +++-- .../insights/doctype/insights_data_source_v3/ibis_utils.py | 6 +----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/flake8 b/flake8 index 080259d4..86b39c11 100644 --- a/flake8 +++ b/flake8 @@ -61,6 +61,7 @@ ignore = W391, W503, W504, - E731 + E731, + C420 -max-line-length = 88 \ No newline at end of file +max-line-length = 88 diff --git a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py index fd71768b..7758ba0a 100644 --- a/insights/insights/doctype/insights_data_source_v3/ibis_utils.py +++ b/insights/insights/doctype/insights_data_source_v3/ibis_utils.py @@ -606,11 +606,7 @@ def to_insights_type(dtype: DataType): return "Date" if dtype.is_time(): return "Time" - if dtype.is_boolean(): - return "Boolean" - if dtype.is_uuid(): - return "UUID" - frappe.throw(f"Cannot infer data type for: {dtype}") + return "String" def cache_results(sql, result: pd.DataFrame, cache_expiry=3600): From 94ef17cb405c275e917ad32b6e40c32452188fb4 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Sat, 23 Nov 2024 13:47:57 +0530 Subject: [PATCH 3/4] fix: cannot import tables --- .../insights_table_v3/insights_table_v3.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/insights/insights/doctype/insights_table_v3/insights_table_v3.py b/insights/insights/doctype/insights_table_v3/insights_table_v3.py index a199213e..8306e0c2 100644 --- a/insights/insights/doctype/insights_table_v3/insights_table_v3.py +++ b/insights/insights/doctype/insights_table_v3/insights_table_v3.py @@ -32,21 +32,6 @@ def autoname(self): @staticmethod def bulk_create(data_source: str, tables: list[str]): - tables = [] - for table in tables: - tables.append( - [ - get_table_name(data_source, table), - data_source, - table, - table, - frappe.utils.now(), - frappe.utils.now(), - frappe.session.user, - frappe.session.user, - ] - ) - frappe.db.bulk_insert( "Insights Table v3", [ @@ -59,7 +44,19 @@ def bulk_create(data_source: str, tables: list[str]): "modified_by", "owner", ], - tables, + [ + [ + get_table_name(data_source, table), + data_source, + table, + table, + frappe.utils.now(), + frappe.utils.now(), + frappe.session.user, + frappe.session.user, + ] + for table in tables + ], ignore_duplicates=True, ) From d10bf8b80be550338978d3a3360250fcdfbc9eed Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Sat, 23 Nov 2024 13:48:47 +0530 Subject: [PATCH 4/4] feat: set odbc driver for mssql connection --- .../connectors/mssql.py | 28 +++++++++++++------ .../insights_data_source_v3.py | 7 +++-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/insights/insights/doctype/insights_data_source_v3/connectors/mssql.py b/insights/insights/doctype/insights_data_source_v3/connectors/mssql.py index 8c914326..f425e071 100644 --- a/insights/insights/doctype/insights_data_source_v3/connectors/mssql.py +++ b/insights/insights/doctype/insights_data_source_v3/connectors/mssql.py @@ -1,15 +1,25 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from urllib.parse import quote_plus +import frappe +import ibis + + +def get_mssql_connection(data_source): + if not frappe.conf.get("mssql_odbc_driver"): + frappe.throw( + "MSSQL ODBC driver path not configured. Please set it in common_site_config.json" + " under 'mssql_odbc_driver' key." + " Eg. 'mssql_odbc_driver': '/usr/local/lib/libtdsodbc.so' or 'FreeTDS'" + ) -def get_mssql_connection_string(data_source): password = data_source.get_password(raise_exception=False) - password = quote_plus(password) if password else "" - connection_string = ( - f"mssql://{data_source.username}:{password}" - f"@{data_source.host}:{data_source.port}/{data_source.database_name}" + + return ibis.mssql.connect( + host=data_source.host, + port=int(data_source.port) or 1433, + user=data_source.username, + password=password, + database=data_source.database_name, + driver=frappe.conf.get("mssql_odbc_driver"), ) - if data_source.use_ssl: - pass - return connection_string diff --git a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py index 4936c6ea..93e73162 100644 --- a/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py +++ b/insights/insights/doctype/insights_data_source_v3/insights_data_source_v3.py @@ -26,7 +26,7 @@ is_frappe_db, ) from .connectors.mariadb import get_mariadb_connection_string -from .connectors.mssql import get_mssql_connection_string +from .connectors.mssql import get_mssql_connection from .connectors.postgresql import get_postgres_connection_string from .connectors.sqlite import get_sqlite_connection_string from .data_warehouse import WAREHOUSE_DB_NAME @@ -187,6 +187,8 @@ def _get_db_connection(self) -> BaseBackend: return get_bigquery_connection(self) if self.database_type == "DuckDB": return get_duckdb_connection(self) + if self.database_type == "MSSQL": + return get_mssql_connection(self) connection_string = self._get_connection_string() return ibis.connect(connection_string) @@ -202,8 +204,6 @@ def _get_connection_string(self): return get_mariadb_connection_string(self) if self.database_type == "PostgreSQL": return get_postgres_connection_string(self) - if self.database_type == "MSSQL": - return get_mssql_connection_string(self) frappe.throw(f"Unsupported database type: {self.database_type}") @@ -223,6 +223,7 @@ def test_connection(self, raise_exception=False): db.list_tables(database=self.get_quoted_db_name()) return True except Exception as e: + frappe.msgprint(str(e), title="Error", indicator="red") frappe.log_error("Testing Data Source connection failed", e) if raise_exception: raise e