From 2bcaf4830501c23be20dde368973510adb205bc4 Mon Sep 17 00:00:00 2001 From: Chaoran Chen Date: Sat, 23 Nov 2024 06:59:54 +0100 Subject: [PATCH] fix(backend): add foreign key to delete preprocessed data (#3259) Co-authored-by: Anna (Anya) Parker <50943381+anna-parker@users.noreply.github.com> Co-authored-by: Cornelius Roemer --- backend/docs/db/schema.sql | 12 +++++++-- ..._entries_preprocessed_data_foreign_key.sql | 13 ++++++++++ .../controller/EndpointTestExtension.kt | 26 ++++++++++++------- .../DeleteAllSequenceDataEndpointTest.kt | 24 +++++++++++++++++ 4 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 backend/src/main/resources/db/migration/V1.6__add_sequence_entries_preprocessed_data_foreign_key.sql diff --git a/backend/docs/db/schema.sql b/backend/docs/db/schema.sql index b01219f76..426a7f41d 100644 --- a/backend/docs/db/schema.sql +++ b/backend/docs/db/schema.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 15.8 (Debian 15.8-1.pgdg120+1) --- Dumped by pg_dump version 16.4 (Debian 16.4-1.pgdg120+2) +-- Dumped from database version 15.10 (Debian 15.10-1.pgdg120+1) +-- Dumped by pg_dump version 16.6 (Debian 16.6-1.pgdg120+1) SET statement_timeout = 0; SET lock_timeout = 0; @@ -811,6 +811,14 @@ ALTER TABLE ONLY public.sequence_entries ADD CONSTRAINT sequence_entries_group_id_fkey FOREIGN KEY (group_id) REFERENCES public.groups_table(group_id); +-- +-- Name: sequence_entries_preprocessed_data sequence_entries_preprocessed_data_accession_version_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY public.sequence_entries_preprocessed_data + ADD CONSTRAINT sequence_entries_preprocessed_data_accession_version_fkey FOREIGN KEY (accession, version) REFERENCES public.sequence_entries(accession, version) ON UPDATE CASCADE ON DELETE CASCADE; + + -- -- Name: user_groups_table user_groups_table_group_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres -- diff --git a/backend/src/main/resources/db/migration/V1.6__add_sequence_entries_preprocessed_data_foreign_key.sql b/backend/src/main/resources/db/migration/V1.6__add_sequence_entries_preprocessed_data_foreign_key.sql new file mode 100644 index 000000000..870182073 --- /dev/null +++ b/backend/src/main/resources/db/migration/V1.6__add_sequence_entries_preprocessed_data_foreign_key.sql @@ -0,0 +1,13 @@ +delete from sequence_entries_preprocessed_data sepd +where + not exists( + select + from sequence_entries se + where + se.accession = sepd.accession + and se.version = sepd.version + ); + +alter table sequence_entries_preprocessed_data +add constraint sequence_entries_preprocessed_data_accession_version_fkey foreign key (accession, version) + references sequence_entries on update cascade on delete cascade; diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt index 4270cebb0..5a5833b74 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/EndpointTestExtension.kt @@ -154,7 +154,8 @@ class EndpointTestExtension : } override fun beforeEach(context: ExtensionContext) { - postgres.execInContainer( + log.debug("Clearing database") + val result = postgres.execInContainer( "psql", "-U", postgres.username, @@ -163,6 +164,11 @@ class EndpointTestExtension : "-c", clearDatabaseStatement(), ) + if (result.exitCode != 0) { + throw RuntimeException( + "Database clearing failed with exit code ${result.exitCode}. Stderr: ${result.stderr}", + ) + } } override fun testPlanExecutionFinished(testPlan: TestPlan) { @@ -175,13 +181,15 @@ class EndpointTestExtension : } private fun clearDatabaseStatement(): String = """ - truncate table $GROUPS_TABLE_NAME cascade; - update $CURRENT_PROCESSING_PIPELINE_TABLE_NAME set version = 1, started_using_at = now(); - truncate table $SEQUENCE_ENTRIES_TABLE_NAME; - truncate table $SEQUENCE_ENTRIES_PREPROCESSED_DATA_TABLE_NAME; + truncate table + $GROUPS_TABLE_NAME, + $SEQUENCE_ENTRIES_TABLE_NAME, + $SEQUENCE_ENTRIES_PREPROCESSED_DATA_TABLE_NAME, + $USER_GROUPS_TABLE_NAME, + $METADATA_UPLOAD_AUX_TABLE_NAME, + $SEQUENCE_UPLOAD_AUX_TABLE_NAME, + $DATA_USE_TERMS_TABLE_NAME + cascade; alter sequence $ACCESSION_SEQUENCE_NAME restart with 1; - truncate table $USER_GROUPS_TABLE_NAME; - truncate $METADATA_UPLOAD_AUX_TABLE_NAME; - truncate $SEQUENCE_UPLOAD_AUX_TABLE_NAME; - truncate table $DATA_USE_TERMS_TABLE_NAME cascade; + update $CURRENT_PROCESSING_PIPELINE_TABLE_NAME set version = 1, started_using_at = now(); """ diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt index a96a89c90..1644751f0 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt @@ -13,6 +13,7 @@ import org.hamcrest.Matchers.not import org.junit.jupiter.api.Test import org.loculus.backend.api.DataUseTerms import org.loculus.backend.api.DataUseTermsChangeRequest +import org.loculus.backend.api.DeleteSequenceScope import org.loculus.backend.api.Status import org.loculus.backend.config.BackendSpringProperty import org.loculus.backend.controller.DEFAULT_USER_NAME @@ -26,6 +27,7 @@ import org.loculus.backend.controller.submission.SubmissionControllerClient import org.loculus.backend.controller.submission.SubmissionConvenienceClient import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES import org.loculus.backend.controller.withAuth +import org.loculus.backend.service.submission.SubmissionDatabaseService import org.loculus.backend.service.submission.UseNewerProcessingPipelineVersionTask import org.loculus.backend.utils.DateProvider import org.springframework.beans.factory.annotation.Autowired @@ -40,6 +42,7 @@ class DeleteAllSequenceDataEndpointTest( @Autowired private val submissionControllerClient: SubmissionControllerClient, @Autowired private val dataUseTermsClient: DataUseTermsControllerClient, @Autowired private val useNewerProcessingPipelineVersionTask: UseNewerProcessingPipelineVersionTask, + @Autowired val submissionDatabaseService: SubmissionDatabaseService, @Autowired private val mockMvc: MockMvc, ) { @Test @@ -142,6 +145,27 @@ class DeleteAllSequenceDataEndpointTest( assertThat(extractedDataAfterDeletion, hasSize(NUMBER_OF_SEQUENCES)) } + @Test + fun `GIVEN preprocessing pipeline version 1 WHEN some sequences deleted THEN can update pipeline to version 2`() { + val accessionVersions = submissionConvenienceClient.prepareDataTo(Status.PROCESSED) + + val accessionFirst = accessionVersions.first() + + submissionControllerClient.deleteSequenceEntries( + scope = DeleteSequenceScope.ALL, + accessionVersionsFilter = listOf(accessionFirst), + jwt = jwtForSuperUser, + ) + + val extractedDataVersion2 = submissionConvenienceClient.extractUnprocessedData(pipelineVersion = 2) + val processedDataVersion2 = extractedDataVersion2 + .map { PreparedProcessedData.successfullyProcessed(accession = it.accession, version = it.version) } + submissionConvenienceClient.submitProcessedData(processedDataVersion2, pipelineVersion = 2) + + val canUpdate = submissionDatabaseService.useNewerProcessingPipelineIfPossible() + assertThat("An update to v2 should be possible", canUpdate, `is`(2L)) + } + @Test fun `GIVEN I deleted all sequences WHEN submitting again THEN new sequences are present`() { submissionConvenienceClient.prepareDataTo(Status.APPROVED_FOR_RELEASE)