(PPS-107): Feat/single table driver (#376)

* Skeleton code for Single Table Indexd * single table alchemy * Parameterized app fixture to run single table for all existing tests Implemented migrate_db for single_table_alchemy Implemented session for single_table_alchemy * add to alchemy * fixing things * pushing some stuff * Fix tests (partial) * Fix unit tests * Fix urls endpoints unit tests * migration script * Add async code * add asyncpg * fix async * remove worker * Async migration * migrate no bulk inserts * async migration remove + PR comments * Updated sync migration * pr review changes * fix migration * undo select queries * Add unit tests for migration + fix alias * remove comment + password * Add doc string * remove unused imports * Resolve * fix args * fix count * add raise and error log * Change error message + add inserts to try block * fix error message * add backoff + raise error * poetry + lock * lock * Bumb version * Fix snyk * append test res * update test * fix cmd * append test * Fix coverage * Fix stuff from comments * Add how-to * fix bin setting file * Add to settings * fix things * minor fixes --------- Co-authored-by: Kyle Burton <[email protected]> Co-authored-by: Pauline Ribeyre <[email protected]>
uc-cdis · Sep 10, 2024 · 8ff50b9 · 8ff50b9
1 parent 3f69200
commit 8ff50b9
Show file tree

Hide file tree

Showing 31 changed files with 3,607 additions and 978 deletions.
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -7,6 +7,9 @@
     {
       "name": "AWSKeyDetector"
     },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
     {
       "name": "Base64HighEntropyString",
       "limit": 4.5
@@ -17,9 +20,15 @@
     {
       "name": "CloudantDetector"
     },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
     {
       "name": "HexHighEntropyString",
-      "limit": 3
+      "limit": 3.0
     },
     {
       "name": "IbmCloudIamDetector"
@@ -37,15 +46,24 @@
     {
       "name": "MailchimpDetector"
     },
+    {
+      "name": "NpmDetector"
+    },
     {
       "name": "PrivateKeyDetector"
     },
+    {
+      "name": "SendGridDetector"
+    },
     {
       "name": "SlackDetector"
     },
     {
       "name": "SoftlayerDetector"
     },
+    {
+      "name": "SquareOAuthDetector"
+    },
     {
       "name": "StripeDetector"
     },
@@ -57,10 +75,6 @@
     {
       "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
     },
-    {
-      "path": "detect_secrets.filters.common.is_baseline_file",
-      "filename": ".secrets.baseline"
-    },
     {
       "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
       "min_level": 2
@@ -215,6 +229,22 @@
         "line_number": 125
       }
     ],
+    "docs/migration_to_single_table_indexd.md": [
+      {
+        "type": "Secret Keyword",
+        "filename": "docs/migration_to_single_table_indexd.md",
+        "hashed_secret": "ed8883d3d8dc2e3c7fc5d14cc2e8830dd27e8f96",
+        "is_verified": false,
+        "line_number": 26
+      },
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "docs/migration_to_single_table_indexd.md",
+        "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
+        "is_verified": false,
+        "line_number": 57
+      }
+    ],
     "migrations/versions/15f2e9345ade_create_tables.py": [
       {
         "type": "Hex High Entropy String",
@@ -239,7 +269,7 @@
         "filename": "tests/default_test_settings.py",
         "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
         "is_verified": false,
-        "line_number": 40
+        "line_number": 39
       }
     ],
     "tests/postgres/migrations/test_15f2e9345ade_create_tables.py": [
@@ -266,7 +296,7 @@
         "filename": "tests/postgres/migrations/test_legacy_schema_migration.py",
         "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0",
         "is_verified": false,
-        "line_number": 88
+        "line_number": 91
       }
     ],
     "tests/test_aliases_endpoints.py": [
@@ -291,28 +321,28 @@
         "filename": "tests/test_bundles.py",
         "hashed_secret": "fd66f51cba49640055a05a6173764b5f0241c63e",
         "is_verified": false,
-        "line_number": 137
+        "line_number": 143
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_bundles.py",
         "hashed_secret": "168762db39e35d49d630689f2ff453b5813a9255",
         "is_verified": false,
-        "line_number": 152
+        "line_number": 160
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_bundles.py",
         "hashed_secret": "c5f0378cf93d896ecc394150943f13afa16ba766",
         "is_verified": false,
-        "line_number": 174
+        "line_number": 184
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_bundles.py",
         "hashed_secret": "a2ca8b84f631b40d866b8e376d077da3527b1fe4",
         "is_verified": false,
-        "line_number": 177
+        "line_number": 187
       }
     ],
     "tests/test_client.py": [
@@ -328,70 +358,70 @@
         "filename": "tests/test_client.py",
         "hashed_secret": "15a6d8daad1278efcaadc0d6e3d1dd2d9ebbc262",
         "is_verified": false,
-        "line_number": 1084
+        "line_number": 1121
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "1b0d1a618b5c213dd792bbc3aa96ffa6bc370ef3",
         "is_verified": false,
-        "line_number": 1300
+        "line_number": 1345
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "1170ace44158ff189902ff44597efef121623353",
         "is_verified": false,
-        "line_number": 1731
+        "line_number": 1792
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "ff9c79b737b3ea7386618cc9437d3fb0a772182b",
         "is_verified": false,
-        "line_number": 2408
+        "line_number": 2507
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "c8176f1e75e62e15dabaa4087fb7194451c8f6d2",
         "is_verified": false,
-        "line_number": 2411
+        "line_number": 2510
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "d5198f8eddb1cbeb437899cd99e5ee97ab8531b4",
         "is_verified": false,
-        "line_number": 2411
+        "line_number": 2510
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "02dc196562514eaa3e2feac1f441ccf6ad81e09d",
         "is_verified": false,
-        "line_number": 2415
+        "line_number": 2514
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "f1cb2d91a95165a2ab909eadd9f7b65f312c7e2d",
         "is_verified": false,
-        "line_number": 2416
+        "line_number": 2515
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "58db546de03270b55a4c889a5c5e6296b29fef25",
         "is_verified": false,
-        "line_number": 2417
+        "line_number": 2516
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "b6c0bd08fde409c18760f32bef8705191840c402",
         "is_verified": false,
-        "line_number": 2418
+        "line_number": 2517
       }
     ],
     "tests/test_deprecated_aliases_endpoints.py": [
@@ -409,9 +439,9 @@
         "filename": "tests/test_drs.py",
         "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0",
         "is_verified": false,
-        "line_number": 38
+        "line_number": 39
       }
     ]
   },
-  "generated_at": "2023-09-27T23:03:38Z"
+  "generated_at": "2024-09-09T17:22:44Z"
 }
diff --git a/bin/indexd_settings.py b/bin/indexd_settings.py
@@ -4,6 +4,7 @@
 from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver
 from indexd.alias.drivers.alchemy import SQLAlchemyAliasDriver
 from indexd.auth.drivers.alchemy import SQLAlchemyAuthDriver
+from indexd.index.drivers.single_table_alchemy import SingleTableSQLAlchemyIndexDriver
 
 
 APP_NAME = "indexd"
@@ -24,6 +25,8 @@ def load_json(file_name):
 
 CONFIG["JSONIFY_PRETTYPRINT_REGULAR"] = False
 
+USE_SINGLE_TABLE = False
+
 dist = environ.get("DIST", None)
 if dist:
     CONFIG["DIST"] = json.loads(dist)
@@ -32,18 +35,33 @@ def load_json(file_name):
 if drs_service_info:
     CONFIG["DRS_SERVICE_INFO"] = json.loads(drs_service_info)
 
-CONFIG["INDEX"] = {
-    "driver": SQLAlchemyIndexDriver(
-        "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format(
-            usr=usr,
-            psw=psw,
-            pghost=pghost,
-            pgport=pgport,
-            db=db,
+if USE_SINGLE_TABLE is True:
+    CONFIG["INDEX"] = {
+        "driver": SingleTableSQLAlchemyIndexDriver(
+            "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format(
+                usr=usr,
+                psw=psw,
+                pghost=pghost,
+                pgport=pgport,
+                db=db,
+            ),
+            index_config=index_config,
         ),
-        index_config=index_config,
-    ),
-}
+    }
+else:
+    CONFIG["INDEX"] = {
+        "driver": SQLAlchemyIndexDriver(
+            "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format(
+                usr=usr,
+                psw=psw,
+                pghost=pghost,
+                pgport=pgport,
+                db=db,
+            ),
+            index_config=index_config,
+        ),
+    }
+
 
 CONFIG["ALIAS"] = {
     "driver": SQLAlchemyAliasDriver(
@@ -68,4 +86,4 @@ def load_json(file_name):
     arborist="http://localhost/",
 )
 
-settings = {"config": CONFIG, "auth": AUTH}
+settings = {"config": CONFIG, "auth": AUTH, "use_single_table": USE_SINGLE_TABLE}