fix(backend): add foreign key to delete preprocessed data (#3259)

Co-authored-by: Anna (Anya) Parker <[email protected]> Co-authored-by: Cornelius Roemer <[email protected]>
loculus-project · Nov 23, 2024 · 2bcaf48 · 2bcaf48
1 parent a223017
commit 2bcaf48
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 11 deletions.
diff --git a/backend/docs/db/schema.sql b/backend/docs/db/schema.sql
@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 15.8 (Debian 15.8-1.pgdg120+1)
--- Dumped by pg_dump version 16.4 (Debian 16.4-1.pgdg120+2)
+-- Dumped from database version 15.10 (Debian 15.10-1.pgdg120+1)
+-- Dumped by pg_dump version 16.6 (Debian 16.6-1.pgdg120+1)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -811,6 +811,14 @@ ALTER TABLE ONLY public.sequence_entries
     ADD CONSTRAINT sequence_entries_group_id_fkey FOREIGN KEY (group_id) REFERENCES public.groups_table(group_id);
 
 
+--
+-- Name: sequence_entries_preprocessed_data sequence_entries_preprocessed_data_accession_version_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres
+--
+
+ALTER TABLE ONLY public.sequence_entries_preprocessed_data
+    ADD CONSTRAINT sequence_entries_preprocessed_data_accession_version_fkey FOREIGN KEY (accession, version) REFERENCES public.sequence_entries(accession, version) ON UPDATE CASCADE ON DELETE CASCADE;
+
+
 --
 -- Name: user_groups_table user_groups_table_group_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres
 --

diff --git a/.../main/resources/db/migration/V1.6__add_sequence_entries_preprocessed_data_foreign_key.sql b/.../main/resources/db/migration/V1.6__add_sequence_entries_preprocessed_data_foreign_key.sql
@@ -0,0 +1,13 @@
+delete from sequence_entries_preprocessed_data sepd
+where
+    not exists(
+        select
+        from sequence_entries se
+        where
+            se.accession = sepd.accession
+            and se.version = sepd.version
+    );
+
+alter table sequence_entries_preprocessed_data
+add constraint sequence_entries_preprocessed_data_accession_version_fkey foreign key (accession, version)
+    references sequence_entries on update cascade on delete cascade;
diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt
@@ -154,7 +154,8 @@ class EndpointTestExtension :
     }
 
     override fun beforeEach(context: ExtensionContext) {
-        postgres.execInContainer(
+        log.debug("Clearing database")
+        val result = postgres.execInContainer(
             "psql",
             "-U",
             postgres.username,
@@ -163,6 +164,11 @@ class EndpointTestExtension :
             "-c",
             clearDatabaseStatement(),
         )
+        if (result.exitCode != 0) {
+            throw RuntimeException(
+                "Database clearing failed with exit code ${result.exitCode}. Stderr: ${result.stderr}",
+            )
+        }
     }
 
     override fun testPlanExecutionFinished(testPlan: TestPlan) {
@@ -175,13 +181,15 @@ class EndpointTestExtension :
 }
 
 private fun clearDatabaseStatement(): String = """
-        truncate table $GROUPS_TABLE_NAME cascade;
-        update $CURRENT_PROCESSING_PIPELINE_TABLE_NAME set version = 1, started_using_at = now();
-        truncate table $SEQUENCE_ENTRIES_TABLE_NAME;
-        truncate table $SEQUENCE_ENTRIES_PREPROCESSED_DATA_TABLE_NAME;
+        truncate table 
+            $GROUPS_TABLE_NAME,
+            $SEQUENCE_ENTRIES_TABLE_NAME,
+            $SEQUENCE_ENTRIES_PREPROCESSED_DATA_TABLE_NAME,
+            $USER_GROUPS_TABLE_NAME,
+            $METADATA_UPLOAD_AUX_TABLE_NAME,
+            $SEQUENCE_UPLOAD_AUX_TABLE_NAME,
+            $DATA_USE_TERMS_TABLE_NAME 
+            cascade;
         alter sequence $ACCESSION_SEQUENCE_NAME restart with 1;
-        truncate table $USER_GROUPS_TABLE_NAME;
-        truncate $METADATA_UPLOAD_AUX_TABLE_NAME;
-        truncate $SEQUENCE_UPLOAD_AUX_TABLE_NAME;
-        truncate table $DATA_USE_TERMS_TABLE_NAME cascade;
+        update $CURRENT_PROCESSING_PIPELINE_TABLE_NAME set version = 1, started_using_at = now();
     """
diff --git a/...src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt b/...src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt
@@ -13,6 +13,7 @@ import org.hamcrest.Matchers.not
 import org.junit.jupiter.api.Test
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DataUseTermsChangeRequest
+import org.loculus.backend.api.DeleteSequenceScope
 import org.loculus.backend.api.Status
 import org.loculus.backend.config.BackendSpringProperty
 import org.loculus.backend.controller.DEFAULT_USER_NAME
@@ -26,6 +27,7 @@ import org.loculus.backend.controller.submission.SubmissionControllerClient
 import org.loculus.backend.controller.submission.SubmissionConvenienceClient
 import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES
 import org.loculus.backend.controller.withAuth
+import org.loculus.backend.service.submission.SubmissionDatabaseService
 import org.loculus.backend.service.submission.UseNewerProcessingPipelineVersionTask
 import org.loculus.backend.utils.DateProvider
 import org.springframework.beans.factory.annotation.Autowired
@@ -40,6 +42,7 @@ class DeleteAllSequenceDataEndpointTest(
     @Autowired private val submissionControllerClient: SubmissionControllerClient,
     @Autowired private val dataUseTermsClient: DataUseTermsControllerClient,
     @Autowired private val useNewerProcessingPipelineVersionTask: UseNewerProcessingPipelineVersionTask,
+    @Autowired val submissionDatabaseService: SubmissionDatabaseService,
     @Autowired private val mockMvc: MockMvc,
 ) {
     @Test
@@ -142,6 +145,27 @@ class DeleteAllSequenceDataEndpointTest(
         assertThat(extractedDataAfterDeletion, hasSize(NUMBER_OF_SEQUENCES))
     }
 
+    @Test
+    fun `GIVEN preprocessing pipeline version 1 WHEN some sequences deleted THEN can update pipeline to version 2`() {
+        val accessionVersions = submissionConvenienceClient.prepareDataTo(Status.PROCESSED)
+
+        val accessionFirst = accessionVersions.first()
+
+        submissionControllerClient.deleteSequenceEntries(
+            scope = DeleteSequenceScope.ALL,
+            accessionVersionsFilter = listOf(accessionFirst),
+            jwt = jwtForSuperUser,
+        )
+
+        val extractedDataVersion2 = submissionConvenienceClient.extractUnprocessedData(pipelineVersion = 2)
+        val processedDataVersion2 = extractedDataVersion2
+            .map { PreparedProcessedData.successfullyProcessed(accession = it.accession, version = it.version) }
+        submissionConvenienceClient.submitProcessedData(processedDataVersion2, pipelineVersion = 2)
+
+        val canUpdate = submissionDatabaseService.useNewerProcessingPipelineIfPossible()
+        assertThat("An update to v2 should be possible", canUpdate, `is`(2L))
+    }
+
     @Test
     fun `GIVEN I deleted all sequences WHEN submitting again THEN new sequences are present`() {
         submissionConvenienceClient.prepareDataTo(Status.APPROVED_FOR_RELEASE)