Fix a PDF-reading change in ResearcherReportTests

crate_anon/anonymise/anonymise.py

@@ -40,6 +40,9 @@
 
 from cardinal_pythonlib.datetimefunc import get_now_utc_pendulum
 from cardinal_pythonlib.sqlalchemy.core_query import count_star, exists_plain
+from cardinal_pythonlib.sqlalchemy.insert_on_duplicate import (
+    insert_with_upsert_if_supported,
+)
 from cardinal_pythonlib.sqlalchemy.schema import (
     add_index,
     get_column_names,
@@ -74,7 +77,6 @@
 )
 from crate_anon.common.parallel import is_my_job_by_hash, is_my_job_by_int
 from crate_anon.common.sql import matches_tabledef
-from crate_anon.common.sqlalchemy import insert_with_upsert_if_supported
 
 log = logging.getLogger(__name__)
 

crate_anon/anonymise/tests/researcher_report_tests.py

@@ -117,20 +117,27 @@ def index_of_list_substring(items: List[str], substr: str) -> int:
             patient_found = False
             note_found = False
             for page in reader.pages:
-                lines = page.extract_text().splitlines()
+                lines = page.extract_text().replace("\t", " ").splitlines()
+                # Sometimes spaces come back as tabs; fix that.
+
                 rows_index = index_of_list_substring(
                     lines,
                     "Number of rows in this table:",
                 )
+                # The label text here is from
+                # crate_anon/anonymise/templates/researcher_report/table.html.
+
+                if rows_index < 0:
+                    continue
 
-                if rows_index > 0:
-                    num_rows = int(lines[rows_index + 1])
+                num_rows = int(lines[rows_index + 1])
+                table_name = lines[0]
 
-                if lines[0] == "patient":
+                if table_name == "patient":
                     patient_found = True
                     self.assertEqual(num_rows, self.num_patients)
 
-                elif lines[0] == "note":
+                elif table_name == "note":
                     note_found = True
                     self.assertEqual(
                         num_rows, self.num_patients * self.notes_per_patient

crate_anon/common/tests/upsert_test_1.sql

@@ -1,15 +0,0 @@
--- For MySQL:
-
-USE crud;  -- pre-existing database
-
-CREATE TABLE ut (a INTEGER PRIMARY KEY, b INTEGER);
-
-INSERT INTO ut (a, b) VALUES (1, 101);  -- OK
-INSERT INTO ut (a, b) VALUES (2, 102);  -- OK
-
-INSERT INTO ut (a, b) VALUES (1, 101);  -- fails; duplicate key
-
-INSERT INTO ut (a, b) VALUES (1, 101) ON DUPLICATE KEY UPDATE a = 1, b = 103;
-    -- succeeds and changes only one row
-
-SELECT * FROM ut;

crate_anon/preprocess/preprocess_pcmis.py

@@ -860,9 +860,7 @@ def process_table(
         # ---------------------------------------------------------------------
         # SQL Server requires Table-bound columns in order to generate DDL:
         if adding_crate_pk:
-            crate_pk_col = make_bigint_autoincrement_column(
-                CRATE_COL_PK, engine.dialect
-            )
+            crate_pk_col = make_bigint_autoincrement_column(CRATE_COL_PK)
             table.append_column(crate_pk_col)
             add_columns(engine, table, [crate_pk_col])
         ensure_columns_present(

crate_anon/preprocess/preprocess_rio.py

@@ -584,9 +584,7 @@ def process_patient_table(
         # ... can't do NOT NULL; need to populate it
         required_cols.append(rio_pk)
     else:  # RCEP type, or no PK in RiO
-        crate_pk_col = make_bigint_autoincrement_column(
-            CRATE_COL_PK, engine.dialect
-        )
+        crate_pk_col = make_bigint_autoincrement_column(CRATE_COL_PK)
         # ... autopopulates
     crate_rio_number_col = Column(
         CRATE_COL_RIO_NUMBER, BigInteger, nullable=True
@@ -693,9 +691,7 @@ def process_nonpatient_table(
     if other_pk_col:  # table has a primary key already
         crate_pk_col = Column(CRATE_COL_PK, BigInteger, nullable=True)
     else:
-        crate_pk_col = make_bigint_autoincrement_column(
-            CRATE_COL_PK, engine.dialect
-        )
+        crate_pk_col = make_bigint_autoincrement_column(CRATE_COL_PK)
     table.append_column(crate_pk_col)  # must be Table-bound, as above
     add_columns(engine, table, [crate_pk_col])
     if not configoptions.print_sql_only:

crate_anon/preprocess/preprocess_systmone.py

@@ -266,9 +266,7 @@ def preprocess_systmone(
 
         # Create step #1
         if not drop_danger_drop and table_needs_pk:
-            crate_pk_col = make_bigint_autoincrement_column(
-                CRATE_COL_PK, engine.dialect
-            )
+            crate_pk_col = make_bigint_autoincrement_column(CRATE_COL_PK)
             # SQL Server requires Table-bound columns in order to generate DDL:
             table.append_column(crate_pk_col)
             add_columns(engine, table, [crate_pk_col])

devnotes/2025_01_sqlalchemy2.txt

@@ -156,6 +156,5 @@ Migration to 2.0 Step Seven - Test against a SQLAlchemy 2.0 Release
 
 ~~~
 
-todo: CURRENTLY ON: cardinal_pythonlib / pytest -k orm_query_tests --log-cli-level INFO
-
-todo: *** also # noqa \n
+todo: CURRENTLY ON:
+    STEP THREE TESTS

setup.py

@@ -69,7 +69,7 @@
     "appdirs==1.4.4",  # where to store some temporary data
     "arrow==0.15.7",  # [pin exact version from cardinal_pythonlib]
     "beautifulsoup4==4.9.1",  # [pin exact version from cardinal_pythonlib]
-    "cardinal_pythonlib==1.1.28",  # RNC libraries
+    "cardinal_pythonlib==2.0.0",  # RNC libraries
     "cairosvg==2.7.0",  # work with SVG files
     "celery==5.2.7",  # back-end scheduling
     "chardet==3.0.4",  # character encoding detection for cardinal_pythonlib