MODSOURMAN-1204: Using a partitioned journal_records table

mod-source-record-manager-server/pom.xml

@@ -15,7 +15,7 @@
     <sonar.exclusions>**/org/folio/rest/impl/ModTenantAPI.java</sonar.exclusions>
     <sonar.coverage.exclusions>**/services/JournalRecordService.java</sonar.coverage.exclusions>
     <aspectj.version>1.9.19</aspectj.version>
-    <lombok.version>1.18.30</lombok.version>
+    <lombok.version>1.18.34</lombok.version>
     <testcontainers.version>1.16.0</testcontainers.version>
     <kafkaclients.version>3.6.0</kafkaclients.version>
     <kafkajunit.version>3.5.1</kafkajunit.version>

mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_job_log_entries_function.sql

@@ -406,7 +406,7 @@ FROM (
        FROM journal_records
        WHERE journal_records.job_execution_id = ''%1$s'' and entity_type = ''INVOICE'' and title != ''INVOICE''
        GROUP BY journal_records.source_id, journal_records.source_record_order, journal_records.job_execution_id,
-                entity_hrid, title, error, id
+                entity_hrid, title, error, tenant_id, id
        HAVING count(journal_records.source_id) FILTER (WHERE (%3$L IN (''ALL'', ''INVOICE'')) AND (NOT %2$L or journal_records.error <> '''')) > 0
      ) AS records_actions
 

mod-source-record-manager-server/src/main/resources/templates/db_scripts/migration_to_partitioning_logs.sql

@@ -0,0 +1,52 @@
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_entity_type (
+  id                    UUID,
+  job_execution_id      UUID,
+  source_id             UUID,
+  entity_type           text,
+  entity_id             text,
+  entity_hrid           text,
+  action_type           text,
+  action_status         text,
+  action_date           timestamp,
+  source_record_order   integer NULL,
+  error                 text NULL,
+  title                 text NULL,
+  instance_id           text,
+  holdings_id           text,
+  order_id              text,
+  permanent_location_id text,
+  tenant_id             text
+) partition by list (entity_type);
+
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_bibliographic PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_BIBLIOGRAPHIC');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_po_line PARTITION OF journal_records_entity_type FOR VALUES IN ('PO_LINE');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_holdings PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_HOLDINGS');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_marc_authority PARTITION OF journal_records_entity_type FOR VALUES IN ('MARC_AUTHORITY');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_holdings PARTITION OF journal_records_entity_type FOR VALUES IN ('HOLDINGS');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_authority PARTITION OF journal_records_entity_type FOR VALUES IN ('AUTHORITY');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_instance PARTITION OF journal_records_entity_type FOR VALUES IN ('INSTANCE');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_item PARTITION OF journal_records_entity_type FOR VALUES IN ('ITEM');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_order PARTITION OF journal_records_entity_type FOR VALUES IN ('ORDER');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_invoice PARTITION OF journal_records_entity_type FOR VALUES IN ('INVOICE');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_edifact PARTITION OF journal_records_entity_type FOR VALUES IN ('EDIFACT');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_empty_entity_type PARTITION OF journal_records_entity_type FOR VALUES IN ('');
+CREATE TABLE IF NOT EXISTS ${myuniversity}_${mymodule}.journal_records_null_entity_type PARTITION OF journal_records_entity_type FOR VALUES IN (NULL);
+
+INSERT INTO ${myuniversity}_${mymodule}.journal_records_entity_type (id, job_execution_id, source_id, entity_type, entity_id, entity_hrid,
+                                         action_type, action_status, action_date, source_record_order, error, title,
+                                         instance_id, holdings_id, order_id, permanent_location_id, tenant_id)
+SELECT id, job_execution_id, source_id, entity_type, entity_id, entity_hrid, action_type, action_status,
+       action_date, source_record_order, error, title, instance_id, holdings_id, order_id, permanent_location_id,
+       tenant_id
+FROM ${myuniversity}_${mymodule}.journal_records;
+
+CREATE UNIQUE INDEX journal_records_et_pkey ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (id, entity_type);
+CREATE INDEX journal_records_et_job_execution_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (job_execution_id);
+CREATE INDEX journal_records_et_source_id_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (source_id);
+CREATE INDEX journal_records_et_action_type_idx ON ${myuniversity}_${mymodule}.journal_records_entity_type USING btree (action_type);
+
+ALTER TABLE ${myuniversity}_${mymodule}.journal_records RENAME TO journal_records_backup;
+ALTER TABLE ${myuniversity}_${mymodule}.journal_records_entity_type RENAME TO journal_records;
+--ANALIZE VERBOSE ${myuniversity}_${mymodule}.journal_records;
+
+

mod-source-record-manager-server/src/main/resources/templates/db_scripts/schema.json

@@ -306,6 +306,16 @@
       "run": "after",
       "snippetPath": "create_get_job_log_entries_function.sql",
       "fromModuleVersion": "mod-source-record-manager-3.9.0"
+    },
+    {
+      "run": "after",
+      "snippetPath": "create_get_record_processing_log_function.sql",
+      "fromModuleVersion": "mod-source-record-manager-3.10.0"
+    },
+    {
+      "run": "after",
+      "snippetPath": "migration_to_partitioning_logs.sql",
+      "fromModuleVersion": "mod-source-record-manager-3.10.0"
     }
   ]
 }

mod-source-record-manager-server/src/test/java/org/folio/rest/impl/metadataProvider/MetaDataProviderJobLogEntriesAPITest.java

@@ -18,6 +18,7 @@
 import static org.folio.rest.jaxrs.model.JournalRecord.EntityType.MARC_BIBLIOGRAPHIC;
 import static org.folio.rest.jaxrs.model.JournalRecord.EntityType.MARC_HOLDINGS;
 import static org.folio.rest.jaxrs.model.JournalRecord.EntityType.PO_LINE;
+import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.emptyOrNullString;
 import static org.hamcrest.Matchers.in;
@@ -36,6 +37,7 @@
 import org.folio.rest.jaxrs.model.ActionStatus;
 import org.folio.rest.jaxrs.model.JobExecution;
 import org.folio.rest.jaxrs.model.JournalRecord;
+import org.folio.rest.jaxrs.model.RecordProcessingLogDto;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -94,6 +96,8 @@ public void shouldReturnInstanceIdWhenHoldingsCreated(TestContext context) {
         .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + instanceCreationJobExecution.getId() + "/records/" + instanceCreationSourceRecordId)
         .then()
         .statusCode(HttpStatus.SC_OK)
+        .log()
+        .all()
         .body("jobExecutionId", is(instanceCreationJobExecution.getId()))
         .body("incomingRecordId", is(instanceCreationSourceRecordId))
         .body("sourceRecordOrder", is("0"))
@@ -128,6 +132,7 @@ public void shouldReturnPoLineWithOrderIdWhenMarcCreate(TestContext context) {
         .when()
         .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId()+ "/records/" + sourceRecordId)
         .then().log().all()
+        .log().all()
         .statusCode(HttpStatus.SC_OK)
         .body("jobExecutionId", is(createdJobExecution.getId()))
         .body("incomingRecordId", is(sourceRecordId))
@@ -386,6 +391,7 @@ public void shouldReturnMarcBibAndAllEntitiesWithoutErrors(TestContext context)
         .when()
         .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + sourceRecordId)
         .then()
+        .log().all()
         .statusCode(HttpStatus.SC_OK)
         .body("jobExecutionId", is(createdJobExecution.getId()))
         .body("incomingRecordId", is(sourceRecordId))
@@ -482,6 +488,7 @@ public void shouldReturnDiscardedForItemsIfNoItemsCreated(TestContext context) {
         .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + sourceRecordId)
         .then()
         .statusCode(HttpStatus.SC_OK)
+        .log().all()
         .body("jobExecutionId", is(createdJobExecution.getId()))
         .body("incomingRecordId", is(sourceRecordId))
         .body("sourceRecordTitle", is(recordTitle))
@@ -1049,12 +1056,13 @@ public void shouldReturnHoldingsDataIfHoldingsPoweredByMarcHoldingsWasCreated(Te
       .onFailure(context::fail);
 
     future.onComplete(ar -> context.verify(v -> {
-      RestAssured.given()
+      RecordProcessingLogDto recordProcessingLogSummary = RestAssured.given()
         .spec(spec)
         .when()
         .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + incomingRecordId)
         .then()
         .statusCode(HttpStatus.SC_OK)
+        .log().all()
         .body("jobExecutionId", is(createdJobExecution.getId()))
         .body("incomingRecordId", is(incomingRecordId))
         .body("sourceRecordId", is(marcHoldingsId))
@@ -1065,7 +1073,12 @@ public void shouldReturnHoldingsDataIfHoldingsPoweredByMarcHoldingsWasCreated(Te
         .body("relatedHoldingsInfo[0].hrid", is(holdingHrid))
         .body("relatedHoldingsInfo[0].permanentLocationId", is(holdingPermanentLocationId))
         .body("relatedHoldingsInfo[0].error", emptyOrNullString())
-        .body("error", emptyOrNullString());
+        .body("error", emptyOrNullString())
+        .extract().response().body().as(RecordProcessingLogDto.class);
+
+      assertThat(recordProcessingLogSummary.getJobExecutionId(), is(createdJobExecution.getId()));
+      assertThat(recordProcessingLogSummary.getIncomingRecordId(), is(incomingRecordId));
+      assertThat(recordProcessingLogSummary.getSourceRecordId(), is(marcHoldingsId));
       async.complete();
     }));
   }
@@ -1218,6 +1231,7 @@ public void shouldReturnDiscardedForHoldingsIfHoldingsMatchedAndNoOtherAction(Te
         .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + incomingRecordId)
         .then()
         .statusCode(HttpStatus.SC_OK)
+        .log().all()
         .body("jobExecutionId", is(createdJobExecution.getId()))
         .body("incomingRecordId", is(incomingRecordId))
         .body("sourceRecordTitle", is(recordTitle))
@@ -1312,6 +1326,7 @@ public void shouldReturnInstanceCreatedIfInstanceNonMatchAndCreated(TestContext
         .when()
         .get(GET_JOB_EXECUTION_JOURNAL_RECORDS_PATH + "/" + createdJobExecution.getId() + "/records/" + incomingRecordId)
         .then()
+        .log().all()
         .statusCode(HttpStatus.SC_OK)
         .body("jobExecutionId", is(createdJobExecution.getId()))
         .body("incomingRecordId", is(incomingRecordId))