diff --git a/etlhelper/db_helper_factory.py b/etlhelper/db_helper_factory.py index 2551526..63c73cc 100644 --- a/etlhelper/db_helper_factory.py +++ b/etlhelper/db_helper_factory.py @@ -7,6 +7,7 @@ from etlhelper.db_helpers.postgres import PostgresDbHelper from etlhelper.db_helpers.mssql import MSSQLDbHelper from etlhelper.db_helpers.sqlite import SQLiteDbHelper +from etlhelper.db_helpers.informix import InformixDbHelper from etlhelper.exceptions import ETLHelperHelperError @@ -72,3 +73,5 @@ def from_dbtype(self, dbtype): MSSQLDbHelper) DB_HELPER_FACTORY.register_helper('SQLITE', "", SQLiteDbHelper) +DB_HELPER_FACTORY.register_helper('INFORMIX', "", + InformixDbHelper) diff --git a/etlhelper/db_helpers/informix.py b/etlhelper/db_helpers/informix.py new file mode 100644 index 0000000..3150c52 --- /dev/null +++ b/etlhelper/db_helpers/informix.py @@ -0,0 +1,79 @@ +""" +Database helper for Postgres +""" +from contextlib import contextmanager + +from etlhelper.db_helpers.db_helper import DbHelper + + +class InformixDbHelper(DbHelper): + """ + Postgres db helper class + """ + def __init__(self): + super().__init__() + try: + import ibm_db_dbi + self.sql_exceptions = (ibm_db_dbi.ProgrammingError) + self._connect_func = ibm_db_dbi.connect + self.connect_exceptions = (ibm_db_dbi.OperationalError) + self.required_params = {'hostname', 'port', 'database', 'uid'} + except ImportError: + print("The Informix Python libraries could not be found.\n" + "Run: python -m pip install ibm_db") + + def get_connection_string(self, db_params, password_variable): + """ + Return a connection string + :param db_params: DbParams + :param password: str, password + :return: str + """ + # Prepare connection string + password = self.get_password(password_variable) + return f'database={db_params.database};hostname={db_params.hostname};' \ + f'port={db_params.port};protocol=tcpip;uid={db_params.uid};pwd={password}' + + def get_sqlalchemy_connection_string(self, db_params, password_variable): + """ + Returns connection string for sql alchemy + """ + password = self.get_password(password_variable) + return (f'postgresql://{db_params.user}:{password}@' + f'{db_params.host}:{db_params.port}/{db_params.dbname}') + + @staticmethod + def executemany(cursor, query, chunk): + """ + Call execute_batch method for PostGres. + + :param cursor: Open database cursor. + :param query: str, SQL query + :param chunk: list, Rows of parameters. + """ + # Here we use execute_batch to send multiple inserts to db at once. + # This is faster than execute_many() because it results in fewer db + # calls. execute_values() or preparing single statement with + # mogrify() were not used because resulting input statement is less + # clear and selective formatting of inputs for spatial vs non-spatial + # tables adds significant code complexity. + # See following for background: + # https://github.com/psycopg/psycopg2/issues/491#issuecomment-276551038 + # https://www.compose.com/articles/formatted-sql-in-python-with-psycopgs-mogrify/ + from psycopg2.extras import execute_batch + + execute_batch(cursor, query, chunk, page_size=len(chunk)) + + + @staticmethod + @contextmanager + def cursor(conn): + """ + Return a cursor on current connection. This implementation allows + SQLite cursor to be used as context manager as with other db types. + """ + try: + cursor = conn.cursor() + yield cursor + finally: + cursor.close() diff --git a/requirements.txt b/requirements.txt index 942c22d..74718d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,72 @@ -flake8 -ipdb -ipython -pytest -pytest-cov -Sphinx -sphinxcontrib-applehelp -sphinxcontrib-devhelp -sphinxcontrib-htmlhelp -sphinxcontrib-jsmath -sphinxcontrib-qthelp -sphinxcontrib-serializinghtml -sphinx_rtd_theme -versioneer -cx-oracle -pyodbc -psycopg2-binary -twine +alabaster==0.7.12 +attrs==19.3.0 +Babel==2.8.0 +backcall==0.1.0 +bleach==3.1.0 +certifi==2019.11.28 +cffi==1.13.2 +chardet==3.0.4 +coverage==5.0.3 +cryptography==2.8 +cx-Oracle==7.3.0 +decorator==4.4.1 +docutils==0.16 +entrypoints==0.3 +flake8==3.7.9 +ibm-db==3.0.1 +idna==2.8 +imagesize==1.2.0 +importlib-metadata==1.5.0 +ipdb==0.12.3 +ipython==7.12.0 +ipython-genutils==0.2.0 +jedi==0.16.0 +jeepney==0.4.2 +Jinja2==2.11.1 +keyring==21.1.0 +MarkupSafe==1.1.1 +mccabe==0.6.1 +more-itertools==8.2.0 +packaging==20.1 +parso==0.6.1 +pexpect==4.8.0 +pickleshare==0.7.5 +pkginfo==1.5.0.1 +pluggy==0.13.1 +prompt-toolkit==3.0.3 +psycopg2-binary==2.8.4 +ptyprocess==0.6.0 +py==1.8.1 +pycodestyle==2.5.0 +pycparser==2.19 +pyflakes==2.1.1 +Pygments==2.5.2 +pyodbc==4.0.28 +pyparsing==2.4.6 +pytest==5.3.5 +pytest-cov==2.8.1 +pytest-sugar==0.9.2 +pytz==2019.3 +readme-renderer==24.0 +requests==2.22.0 +requests-toolbelt==0.9.1 +SecretStorage==3.1.2 +six==1.14.0 +snowballstemmer==2.0.0 +Sphinx==2.3.1 +sphinx-rtd-theme==0.4.3 +sphinxcontrib-applehelp==1.0.1 +sphinxcontrib-devhelp==1.0.1 +sphinxcontrib-htmlhelp==1.0.2 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.2 +sphinxcontrib-serializinghtml==1.1.3 +termcolor==1.1.0 +tqdm==4.42.1 +traitlets==4.3.3 +twine==3.1.1 +urllib3==1.25.8 +versioneer==0.18 +wcwidth==0.1.8 +webencodings==0.5.1 +zipp==2.1.0 diff --git a/test/unit/test_db_helpers.py b/test/unit/test_db_helpers.py index d12299b..8b6f457 100644 --- a/test/unit/test_db_helpers.py +++ b/test/unit/test_db_helpers.py @@ -1,11 +1,12 @@ """Unit tests for db_helpers module.""" -from unittest.mock import Mock -import pytest +import pyodbc import sqlite3 +from unittest.mock import Mock import cx_Oracle -import pyodbc +import ibm_db_dbi import psycopg2 +import pytest from etlhelper import DbParams from etlhelper.db_helper_factory import DB_HELPER_FACTORY @@ -26,6 +27,8 @@ SQLITEDB = DbParams(dbtype='SQLITE', filename='/myfile.db') +INFORMIXDB = DbParams(dbtype='INFORMIX', database='testDb', hostname='localhost', port='1111', protocol='tcpip',uid='user' ) + def test_oracle_sql_exceptions(): helper = OracleDbHelper() @@ -43,7 +46,8 @@ def test_oracle_connect_exceptions(): 'DRIVER=test driver;SERVER=tcp:server;PORT=1521;DATABASE=testdb;UID=testuser;PWD=mypassword'), # NOQA (POSTGRESDB, psycopg2, 'host=server port=1521 dbname=testdb user=testuser password=mypassword'), - (SQLITEDB, sqlite3, '/myfile.db') + (SQLITEDB, sqlite3, '/myfile.db'), + (INFORMIXDB, ibm_db_dbi, 'database=testDb;hostname=localhost;port=1111;protocol=tcpip;uid=user;pwd=mypassword') ]) def test_connect(monkeypatch, db_params, driver, expected): # Arrange