SQLAlchemy 2.0.36 operational

crate_anon/anonymise/anonymise.py

@@ -280,18 +280,14 @@ def insert(records_: List[Dict[str, Any]]) -> None:
     commit_destdb()
 
     # 4. Index -- no, hang on, it's a primary key already
-    #
-    # log.debug("... creating index on temporary table")
-    # index = Index('_temptable_idx', temptable.columns[pkfield])
-    # index.create(destengine)
 
     # 5. DELETE FROM desttable
     #    WHERE destpk NOT IN (SELECT srcpk FROM temptable)
     log.debug("... deleting from destination where appropriate")
     query = dest_table.delete().where(
         ~column(pkddr.dest_field).in_(select(temptable.columns[pkfield]))
     )
-    destengine.execute(query)
+    destsession.execute(query)
     commit_destdb()
 
     # 6. Drop temporary table
@@ -1406,9 +1402,6 @@ def process_table(
         # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         # Insert values into database
         # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-        # Previously (prior to 2025-01-05 and prior to SQLAlchemy 2):
-        #       q = sqla_table.insert_on_duplicate().values(destvalues)
-        # See comments in insert_with_upsert_if_supported().
         q = insert_with_upsert_if_supported(
             table=sqla_table, values=destvalues, session=session
         )
@@ -1645,7 +1638,7 @@ def insert(records_: List[Dict[str, Any]]) -> None:
         query = dest_table.delete().where(
             column(ridfield).in_(select(temptable.columns[pkfield]))
         )
-        destengine.execute(query)
+        destsession.execute(query)
         commit_destdb()
 
     log.debug(start + ": 6. dropping temporary table")

crate_anon/anonymise/config.py

@@ -976,9 +976,7 @@ def get_database(
         self._dest_bytes_written = 0
         self._echo = False
 
-    def get_destdb_engine_outside_transaction(
-        self, encoding: str = "utf-8"
-    ) -> Engine:
+    def get_destdb_engine_outside_transaction(self) -> Engine:
         """
         Get a standalone SQLAlchemy Engine for the destination database, and
         configure itself so transactions aren't used (equivalently:
@@ -988,16 +986,12 @@ def get_destdb_engine_outside_transaction(
         See
         https://github.com/mkleehammer/pyodbc/wiki/Database-Transaction-Management
 
-        Args:
-            encoding: passed to the SQLAlchemy :func:`create_engine` function
-
         Returns:
             the Engine
         """
         url = self._destination_database_url
         return create_engine(
             url,
-            encoding=encoding,
             echo=self._echo,
             connect_args={"autocommit": True},  # for pyodbc
             future=True,

crate_anon/anonymise/dbholder.py

@@ -59,7 +59,6 @@ def __init__(
         with_session: bool = False,
         with_conn: bool = True,
         reflect: bool = True,
-        encoding: str = "utf-8",
         echo: bool = False,
     ) -> None:
         """
@@ -70,14 +69,11 @@ def __init__(
             with_session: create an SQLAlchemy Session?
             with_conn: create an SQLAlchemy connection (via an Engine)?
             reflect: read the database structure (when required)?
-            encoding: passed to SQLAlchemy's :func:`create_engine`
             echo: passed to SQLAlchemy's :func:`create_engine`
         """
         self.name = name
         self.srccfg = srccfg
-        self.engine = create_engine(
-            url, encoding=encoding, echo=echo, future=True
-        )
+        self.engine = create_engine(url, echo=echo, future=True)
         self.conn = None  # type: Optional[Connection]
         self.session = None  # type: Optional[Session]
         self._reflect_on_request = reflect
@@ -102,7 +98,9 @@ def create_session(self) -> None:
         Creates a database session, if not created to begin with.
         """
         if not self.session:
-            self.session = sessionmaker(bind=self.engine)()  # for ORM
+            self.session = sessionmaker(
+                bind=self.engine, future=True
+            )()  # type: Session
 
     def _reflect(self) -> None:
         """

crate_anon/anonymise/make_demo_database.py

@@ -59,8 +59,6 @@
 from sqlalchemy.orm.session import sessionmaker
 from sqlalchemy.sql import text
 
-from crate_anon.anonymise.constants import CHARSET
-
 from crate_anon.testing import Base
 from crate_anon.testing.factories import (
     DemoFilenameDocFactory,
@@ -79,6 +77,7 @@
 
 REPORT_EVERY = 50
 
+
 # =============================================================================
 # Randomness
 # =============================================================================
@@ -115,8 +114,8 @@ def mk_demo_database(
 
     log.info("Opening database.")
     log.debug(f"URL: {url}")
-    engine = create_engine(url, echo=echo, encoding=CHARSET, future=True)
-    session = sessionmaker(bind=engine)()
+    engine = create_engine(url, echo=echo, future=True)
+    session = sessionmaker(bind=engine, future=True)()
 
     # 2. Create tables
 

crate_anon/anonymise/researcher_report.py

@@ -207,7 +207,7 @@ def safe_db_url_if_selected(self) -> str:
         Sanitised version of the database URL, or a blank string if not
         enabled.
         """
-        if not self.show_url:
+        if not self.show_url or not self.db_url:
             return ""
         url_obj = make_url(self.db_url)  # type: URL
         return repr(url_obj)

crate_anon/anonymise/templates/researcher_report/report.html

@@ -36,7 +36,7 @@ <h1>{{ db_name }}</h1>
         </ul>
 
         <p class="info">
-            Key:
+            Abbreviations and terms (* indicates terms used by CRATE):
         </p>
         <ul class="info">
             <li>
@@ -56,7 +56,7 @@ <h1>{{ db_name }}</h1>
                 FK: foreign key, a cross-reference to a PK in another table.
             </li>
             <li>
-                MRID: master patient research identifier.
+                MRID (*): master patient research identifier.
             </li>
             <li>
                 NLP: natural language processing, in which computers read free
@@ -77,12 +77,12 @@ <h1>{{ db_name }}</h1>
                 defining and querying databases.
             </li>
             <li>
-                RID: patient research identifier.
+                RID (*): patient research identifier.
             </li>
             <li>
-                TRID: transient patient research identifier; this can be useful
-                for fast cross-referencing within SQL queries, but will change
-                when the database is rebuilt.
+                TRID (*): transient patient research identifier; this can be
+                useful for fast cross-referencing within SQL queries, but will
+                change when the database is rebuilt.
             </li>
             <li>
                 URL: uniform resource locator.

crate_anon/anonymise/tests/anonymise_tests.py

@@ -33,7 +33,10 @@
 from typing import Any, Dict, Generator, List, Tuple
 from unittest import mock
 
-from cardinal_pythonlib.sqlalchemy.schema import mssql_table_has_ft_index
+from cardinal_pythonlib.sqlalchemy.schema import (
+    execute_ddl,
+    mssql_table_has_ft_index,
+)
 import factory
 import pytest
 from sqlalchemy import (
@@ -303,11 +306,12 @@ def test_full_text_index_created_with_mysql(self) -> None:
         self.assertEqual(indexes["surname"]["type"], "FULLTEXT")
 
     def _drop_mysql_full_text_indexes(self) -> None:
-        sql = "DROP INDEX _idxft_forename ON anon_patient"
-        self.engine.execute(sql)
-
-        sql = "DROP INDEX _idxft_surname ON anon_patient"
-        self.engine.execute(sql)
+        execute_ddl(
+            self.engine, sql="DROP INDEX _idxft_forename ON anon_patient"
+        )
+        execute_ddl(
+            self.engine, sql="DROP INDEX _idxft_surname ON anon_patient"
+        )
 
     def _get_mysql_anon_patient_table_full_text_indexes(
         self,
@@ -350,7 +354,6 @@ def engine_outside_transaction(self) -> None:
         if self._engine_outside_transaction is None:
             self._engine_outside_transaction = create_engine(
                 self.engine.url,
-                encoding="utf-8",
                 connect_args={"autocommit": True},  # for pyodbc
                 future=True,
             )

crate_anon/common/sql.py

@@ -57,7 +57,10 @@
 )
 from cardinal_pythonlib.sqlalchemy.core_query import count_star
 from cardinal_pythonlib.sqlalchemy.dialect import SqlaDialectName
-from cardinal_pythonlib.sqlalchemy.schema import column_creation_ddl
+from cardinal_pythonlib.sqlalchemy.schema import (
+    column_creation_ddl,
+    execute_ddl,
+)
 from cardinal_pythonlib.timing import MultiTimerContext, timer
 from pyparsing import ParseResults
 from sqlalchemy import inspect
@@ -1014,9 +1017,9 @@ def set_print_not_execute(print_not_execute: bool) -> None:
     _global_print_not_execute_sql = print_not_execute
 
 
-def execute(engine: Engine, sql: str) -> None:
+def _exec_ddl(engine: Engine, sql: str) -> None:
     """
-    Executes SQL.
+    Executes SQL as DDL.
 
     Whether we act or just print is conditional on previous calls to
     :func:`set_print_not_execute`.
@@ -1030,7 +1033,7 @@ def execute(engine: Engine, sql: str) -> None:
         print(format_sql_for_print(sql) + "\n;")
         # extra \n in case the SQL ends in a comment
     else:
-        engine.execute(sql)
+        execute_ddl(engine, sql=sql)
 
 
 def add_columns(engine: Engine, table: Table, columns: List[Column]) -> None:
@@ -1084,7 +1087,7 @@ def add_columns(engine: Engine, table: Table, columns: List[Column]) -> None:
     for column_def in column_defs:
         log.info(f"Table {table.name!r}: adding column {column_def!r}")
         sql = f"ALTER TABLE {table.name} ADD {column_def}"
-        execute(engine, sql)
+        _exec_ddl(engine, sql)
 
 
 def drop_columns(
@@ -1115,12 +1118,11 @@ def drop_columns(
             )
         else:
             log.info(f"Table {table.name!r}: dropping column {name!r}")
-            sql = f"ALTER TABLE {table.name} DROP COLUMN {name}"
             # SQL Server:
             #   http://www.techonthenet.com/sql_server/tables/alter_table.php
             # MySQL:
             #   http://dev.mysql.com/doc/refman/5.7/en/alter-table.html
-            execute(engine, sql)
+            _exec_ddl(engine, f"ALTER TABLE {table.name} DROP COLUMN {name}")
 
 
 def add_indexes(
@@ -1151,12 +1153,12 @@ def add_indexes(
                 f"Table {table.name!r}: adding index {index_name!r} on "
                 f"column {column!r}"
             )
-            execute(
+            _exec_ddl(
                 engine,
                 f"""
-                CREATE{" UNIQUE" if i.unique else ""} INDEX {index_name}
-                    ON {table.name} ({column})
-            """,
+                    CREATE{" UNIQUE" if i.unique else ""} INDEX {index_name}
+                        ON {table.name} ({column})
+                """,
             )
         else:
             log.debug(
@@ -1196,7 +1198,7 @@ def drop_indexes(
                 sql = f"DROP INDEX {table.name}.{index_name}"
             else:
                 assert False, f"Unknown dialect: {engine.dialect.name}"
-            execute(engine, sql)
+            _exec_ddl(engine, sql)
 
 
 def get_table_names(
@@ -1353,7 +1355,7 @@ def create_view(engine: Engine, viewname: str, select_sql: str) -> None:
         drop_view(engine, viewname, quiet=True)
         sql = f"CREATE VIEW {viewname} AS {select_sql}"
     log.info(f"Creating view: {viewname!r}")
-    execute(engine, sql)
+    _exec_ddl(engine, sql)
 
 
 def assert_view_has_same_num_rows(
@@ -1406,8 +1408,7 @@ def drop_view(engine: Engine, viewname: str, quiet: bool = False) -> None:
     else:
         if not quiet:
             log.info(f"Dropping view: {viewname!r}")
-        sql = f"DROP VIEW {viewname}"
-        execute(engine, sql)
+        _exec_ddl(engine, f"DROP VIEW {viewname}")
 
 
 def get_column_fk_description(c: Column) -> str:

crate_anon/conftest.py

@@ -259,7 +259,7 @@ def dbsession(
     # begin the nested transaction
     transaction = connection.begin()
     # use the connection with the already started transaction
-    session = Session(bind=connection)
+    session = Session(bind=connection, future=True)
 
     yield session