feat(robot-server): split database into v5 and v6

robot-server/robot_server/persistence/_files_and_directories.py

@@ -3,5 +3,6 @@
 
 DECK_CONFIGURATION_FILE: Final = "deck_configuration.json"
 PROTOCOLS_DIRECTORY: Final = "protocols"
+DATA_FILES_DIRECTORY: Final = "data_files"
 DB_FILE: Final = "robot_server.db"
-LATEST_VERSION_DIRECTORY: Final = "5.1"
+LATEST_VERSION_DIRECTORY: Final = "6"

robot-server/robot_server/persistence/_migrations/v4_to_v5.py

@@ -4,128 +4,50 @@
 
 - Adds a new "protocol_kind" column to protocols table
 - Adds a new "data_files" table
-- Removes the "run_time_parameter_values_and_defaults" column of analysis_table and
-  creates a separate analysis_primitive_type_rtp_table instead. The migration does not
-  port the data from run_time_parameter_values_and_defaults into analysis_primitive_type_rtp_table.
-  The consequence of which is that any checks for previous matching analysis (for protocols with RTPs only)
-  will fail and a new analysis will be triggered. This new analysis will then save its
-  RTP data to the new table. RTP data belonging to previous analyses will still be available
-  as part of the completed analysis blob.
 """
 
 from pathlib import Path
 from contextlib import ExitStack
+import shutil
+from typing import Any
 
 import sqlalchemy
 
-from ..database import sql_engine_ctx, sqlite_rowid
-from ..tables import schema_5, schema_4
+from ..database import sql_engine_ctx
+from ..tables import schema_5
 from .._folder_migrator import Migration
 
-from ._util import copy_rows_unmodified, copy_if_exists, copytree_if_exists
-from .._files_and_directories import (
-    DECK_CONFIGURATION_FILE,
-    PROTOCOLS_DIRECTORY,
-    DB_FILE,
-)
+_DB_FILE = "robot_server.db"
 
 
 class Migration4to5(Migration):  # noqa: D101
     def migrate(self, source_dir: Path, dest_dir: Path) -> None:
         """Migrate the persistence directory from schema 4 to 5."""
-        # Copy over unmodified directories and files to new version
-        copy_if_exists(
-            source_dir / DECK_CONFIGURATION_FILE, dest_dir / DECK_CONFIGURATION_FILE
-        )
-        copytree_if_exists(
-            source_dir / PROTOCOLS_DIRECTORY, dest_dir / PROTOCOLS_DIRECTORY
-        )
-
-        source_db_file = source_dir / DB_FILE
-        dest_db_file = dest_dir / DB_FILE
+        # Copy over all existing directories and files to new version
+        for item in source_dir.iterdir():
+            if item.is_dir():
+                shutil.copytree(src=item, dst=dest_dir / item.name)
+            else:
+                shutil.copy(src=item, dst=dest_dir / item.name)
+        dest_db_file = dest_dir / _DB_FILE
 
         # Append the new column to existing protocols in v4 database
         with ExitStack() as exit_stack:
-            source_engine = exit_stack.enter_context(sql_engine_ctx(source_db_file))
-
             dest_engine = exit_stack.enter_context(sql_engine_ctx(dest_db_file))
             schema_5.metadata.create_all(dest_engine)
 
-            source_transaction = exit_stack.enter_context(source_engine.begin())
-            dest_transaction = exit_stack.enter_context(dest_engine.begin())
-
-            _migrate_db_with_changes(source_transaction, dest_transaction)
-
-
-def _migrate_db_with_changes(
-    source_transaction: sqlalchemy.engine.Connection,
-    dest_transaction: sqlalchemy.engine.Connection,
-) -> None:
-    _migrate_protocol_table_with_new_protocol_kind_col(
-        source_transaction,
-        dest_transaction,
-    )
-    _migrate_analysis_table_excluding_rtp_defaults_and_vals(
-        source_transaction,
-        dest_transaction,
-    )
-    copy_rows_unmodified(
-        schema_4.run_table,
-        schema_5.run_table,
-        source_transaction,
-        dest_transaction,
-        order_by_rowid=True,
-    )
-    copy_rows_unmodified(
-        schema_4.action_table,
-        schema_5.action_table,
-        source_transaction,
-        dest_transaction,
-        order_by_rowid=True,
-    )
-    copy_rows_unmodified(
-        schema_4.run_command_table,
-        schema_5.run_command_table,
-        source_transaction,
-        dest_transaction,
-        order_by_rowid=True,
-    )
-
-
-def _migrate_protocol_table_with_new_protocol_kind_col(
-    source_transaction: sqlalchemy.engine.Connection,
-    dest_transaction: sqlalchemy.engine.Connection,
-) -> None:
-    """Add a new 'protocol_kind' column to protocols table."""
-    select_old_protocols = sqlalchemy.select(schema_4.protocol_table).order_by(
-        sqlite_rowid
-    )
-    insert_new_protocols = sqlalchemy.insert(schema_5.protocol_table)
-    for old_row in source_transaction.execute(select_old_protocols).all():
-        dest_transaction.execute(
-            insert_new_protocols,
-            id=old_row.id,
-            created_at=old_row.created_at,
-            protocol_key=old_row.protocol_key,
-            protocol_kind=None,
-        )
-
-
-def _migrate_analysis_table_excluding_rtp_defaults_and_vals(
-    source_transaction: sqlalchemy.engine.Connection,
-    dest_transaction: sqlalchemy.engine.Connection,
-) -> None:
-    """Remove run_time_parameter_values_and_defaults column from analysis_table."""
-    select_old_analyses = sqlalchemy.select(schema_4.analysis_table).order_by(
-        sqlite_rowid
-    )
-    insert_new_analyses = sqlalchemy.insert(schema_5.analysis_table)
-    for old_row in source_transaction.execute(select_old_analyses).all():
-        dest_transaction.execute(
-            insert_new_analyses,
-            id=old_row.id,
-            protocol_id=old_row.protocol_id,
-            analyzer_version=old_row.analyzer_version,
-            completed_analysis=old_row.completed_analysis,
-            # run_time_parameter_values_and_defaults column is omitted
-        )
+            def add_column(
+                engine: sqlalchemy.engine.Engine,
+                table_name: str,
+                column: Any,
+            ) -> None:
+                column_type = column.type.compile(engine.dialect)
+                engine.execute(
+                    f"ALTER TABLE {table_name} ADD COLUMN {column.key} {column_type}"
+                )
+
+            add_column(
+                dest_engine,
+                schema_5.protocol_table.name,
+                schema_5.protocol_table.c.protocol_kind,
+            )

robot-server/robot_server/persistence/_migrations/v5_to_v6.py

@@ -0,0 +1,152 @@
+"""Migrate the persistence directory from schema 5 to 6.
+
+Summary of changes from schema 5:
+
+- Removes the "run_time_parameter_values_and_defaults" column of analysis_table
+- Adds a separate analysis_primitive_type_rtp_table to store fully validated primitive
+  run time parameters.
+  - NOTE: V5 to V6 migration does not port the data from run_time_parameter_values_and_defaults
+          into analysis_primitive_type_rtp_table. The consequence of which is that
+          any checks for previous matching analysis (for protocols with RTPs only)
+          will fail and a new analysis will be triggered. This new analysis will then
+          save its RTP data to the new table. RTP data belonging to previous analyses
+          will still be available as part of the completed analysis blob.
+- Adds a new analysis_csv_rtp_table to store the CSV parameters' file IDs used in analysis
+- Adds a new run_csv_rtp_table to store the CSV parameters' file IDs used in runs
+- Converts protocol.protocol_kind to a constrained string (a SQL "enum"), makes it
+  non-nullable (NULL was semantically equivalent to "standard"), and adds an index.
+"""
+
+from pathlib import Path
+from contextlib import ExitStack
+
+import sqlalchemy
+
+from ..database import sql_engine_ctx, sqlite_rowid
+from ..tables import schema_5, schema_6
+from .._folder_migrator import Migration
+
+from ._util import copy_rows_unmodified, copy_if_exists, copytree_if_exists
+from .._files_and_directories import (
+    DECK_CONFIGURATION_FILE,
+    PROTOCOLS_DIRECTORY,
+    DATA_FILES_DIRECTORY,
+    DB_FILE,
+)
+
+
+class Migration5to6(Migration):  # noqa: D101
+    def migrate(self, source_dir: Path, dest_dir: Path) -> None:
+        """Migrate the persistence directory from schema 5 to 6."""
+        # Copy over unmodified directories and files to new version
+        copy_if_exists(
+            source_dir / DECK_CONFIGURATION_FILE, dest_dir / DECK_CONFIGURATION_FILE
+        )
+        copytree_if_exists(
+            source_dir / PROTOCOLS_DIRECTORY, dest_dir / PROTOCOLS_DIRECTORY
+        )
+        copytree_if_exists(
+            source_dir / DATA_FILES_DIRECTORY, dest_dir / DATA_FILES_DIRECTORY
+        )
+
+        source_db_file = source_dir / DB_FILE
+        dest_db_file = dest_dir / DB_FILE
+
+        # Append the new column to existing protocols in v4 database
+        with ExitStack() as exit_stack:
+            source_engine = exit_stack.enter_context(sql_engine_ctx(source_db_file))
+
+            dest_engine = exit_stack.enter_context(sql_engine_ctx(dest_db_file))
+            schema_6.metadata.create_all(dest_engine)
+
+            source_transaction = exit_stack.enter_context(source_engine.begin())
+            dest_transaction = exit_stack.enter_context(dest_engine.begin())
+
+            _migrate_db_with_changes(source_transaction, dest_transaction)
+
+
+def _migrate_db_with_changes(
+    source_transaction: sqlalchemy.engine.Connection,
+    dest_transaction: sqlalchemy.engine.Connection,
+) -> None:
+    copy_rows_unmodified(
+        schema_5.data_files_table,
+        schema_6.data_files_table,
+        source_transaction,
+        dest_transaction,
+        order_by_rowid=True,
+    )
+    _migrate_protocol_table_with_new_protocol_kind_col(
+        source_transaction,
+        dest_transaction,
+    )
+    _migrate_analysis_table_excluding_rtp_defaults_and_vals(
+        source_transaction,
+        dest_transaction,
+    )
+    copy_rows_unmodified(
+        schema_5.run_table,
+        schema_6.run_table,
+        source_transaction,
+        dest_transaction,
+        order_by_rowid=True,
+    )
+    copy_rows_unmodified(
+        schema_5.action_table,
+        schema_6.action_table,
+        source_transaction,
+        dest_transaction,
+        order_by_rowid=True,
+    )
+    copy_rows_unmodified(
+        schema_5.run_command_table,
+        schema_6.run_command_table,
+        source_transaction,
+        dest_transaction,
+        order_by_rowid=True,
+    )
+
+
+def _migrate_protocol_table_with_new_protocol_kind_col(
+    source_transaction: sqlalchemy.engine.Connection,
+    dest_transaction: sqlalchemy.engine.Connection,
+) -> None:
+    """Add a new 'protocol_kind' column to protocols table."""
+    select_old_protocols = sqlalchemy.select(schema_5.protocol_table).order_by(
+        sqlite_rowid
+    )
+    insert_new_protocol = sqlalchemy.insert(schema_6.protocol_table)
+    for old_row in source_transaction.execute(select_old_protocols).all():
+        new_protocol_kind = (
+            # Account for old_row.protocol_kind being NULL.
+            schema_6.ProtocolKindSQLEnum.QUICK_TRANSFER
+            if old_row.protocol_kind == "quick-transfer"
+            else schema_6.ProtocolKindSQLEnum.STANDARD
+        )
+        dest_transaction.execute(
+            insert_new_protocol,
+            id=old_row.id,
+            created_at=old_row.created_at,
+            protocol_key=old_row.protocol_key,
+            protocol_kind=new_protocol_kind,
+        )
+
+
+def _migrate_analysis_table_excluding_rtp_defaults_and_vals(
+    source_transaction: sqlalchemy.engine.Connection,
+    dest_transaction: sqlalchemy.engine.Connection,
+) -> None:
+    """Remove run_time_parameter_values_and_defaults column from analysis_table."""
+    select_old_analyses = sqlalchemy.select(schema_5.analysis_table).order_by(
+        sqlite_rowid
+    )
+    insert_new_analyses = sqlalchemy.insert(schema_6.analysis_table)
+    for old_row in source_transaction.execute(select_old_analyses).all():
+        dest_transaction.execute(
+            insert_new_analyses,
+            id=old_row.id,
+            protocol_id=old_row.protocol_id,
+            analyzer_version=old_row.analyzer_version,
+            completed_analysis=old_row.completed_analysis,
+            # run_time_parameter_values_and_defaults column is omitted
+        )

robot-server/robot_server/persistence/persistence_directory.py

@@ -11,7 +11,7 @@
 from anyio import Path as AsyncPath, to_thread
 
 from ._folder_migrator import MigrationOrchestrator
-from ._migrations import up_to_3, v3_to_v4, v4_to_v5
+from ._migrations import up_to_3, v3_to_v4, v4_to_v5, v5_to_v6
 from . import LATEST_VERSION_DIRECTORY
 
 _TEMP_PERSISTENCE_DIR_PREFIX: Final = "opentrons-robot-server-"
@@ -50,7 +50,8 @@ async def prepare_active_subdirectory(prepared_root: Path) -> Path:
         migrations=[
             up_to_3.MigrationUpTo3(subdirectory="3"),
             v3_to_v4.Migration3to4(subdirectory="4"),
-            v4_to_v5.Migration4to5(subdirectory=LATEST_VERSION_DIRECTORY),
+            v4_to_v5.Migration4to5(subdirectory="5"),
+            v5_to_v6.Migration5to6(subdirectory=LATEST_VERSION_DIRECTORY),
         ],
         temp_file_prefix="temp-",
     )

robot-server/robot_server/persistence/tables/__init__.py

@@ -1,7 +1,7 @@
 """SQL database schemas."""
 
 # Re-export the latest schema.
-from .schema_5 import (
+from .schema_6 import (
     metadata,
     protocol_table,
     analysis_table,
@@ -12,6 +12,7 @@
     action_table,
     data_files_table,
     PrimitiveParamSQLEnum,
+    ProtocolKindSQLEnum,
 )
 
 
@@ -26,4 +27,5 @@
     "action_table",
     "data_files_table",
     "PrimitiveParamSQLEnum",
+    "ProtocolKindSQLEnum",
 ]