From a6e6f64f83ff0dd1baaf7828890b98010f29cb8c Mon Sep 17 00:00:00 2001 From: TomJKing Date: Mon, 19 Aug 2024 10:02:31 +0100 Subject: [PATCH] Use 'source system' to fetch correct schema --- .../service/api/controllers/LoadController.scala | 2 +- .../services/dataload/DataLoadInitiation.scala | 9 +++++---- .../dataload/MetadataLoadConfiguration.scala | 12 ++++++++++-- .../services/dataload/DataLoadInitiationSpec.scala | 7 ++++--- .../dataload/MetadataLoadConfigurationSpec.scala | 14 +++++++++++++- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/api/controllers/LoadController.scala b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/api/controllers/LoadController.scala index dc439b7..177c4ac 100644 --- a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/api/controllers/LoadController.scala +++ b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/api/controllers/LoadController.scala @@ -40,7 +40,7 @@ class LoadController(dataLoadInitiation: DataLoadInitiation, dataLoadProcessor: .out(jsonBody[String]) val initiateLoadRoute: HttpRoutes[IO] = - Http4sServerInterpreter[IO]().toRoutes(initiateLoadEndpoint.serverLogicSuccess(ac => _ => dataLoadInitiation.initiateConsignmentLoad(ac.token))) + Http4sServerInterpreter[IO]().toRoutes(initiateLoadEndpoint.serverLogicSuccess(ac => input => dataLoadInitiation.initiateConsignmentLoad(ac.token, input))) val completeLoadRoute: HttpRoutes[IO] = Http4sServerInterpreter[IO]().toRoutes(completeLoadEndpoint.serverLogicSuccess(ac => ci => dataLoadProcessor.trigger(ci._2, ac.token))) diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiation.scala b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiation.scala index 1510e14..abdd68e 100644 --- a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiation.scala +++ b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiation.scala @@ -4,25 +4,26 @@ import cats.effect.IO import uk.gov.nationalarchives.tdr.keycloak.Token import uk.gov.nationalarchives.tdr.transfer.service.ApplicationConfig import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.{AWSS3LoadDestination, LoadDetails, TransferConfiguration} +import uk.gov.nationalarchives.tdr.transfer.service.api.model.SourceSystem.SourceSystemEnum.SourceSystem import uk.gov.nationalarchives.tdr.transfer.service.services.GraphQlApiService import uk.gov.nationalarchives.tdr.transfer.service.services.dataload.DataLoadInitiation.s3Config import java.util.UUID class DataLoadInitiation(graphQlApiService: GraphQlApiService) { - def initiateConsignmentLoad(token: Token): IO[LoadDetails] = { + def initiateConsignmentLoad(token: Token, sourceSystem: SourceSystem): IO[LoadDetails] = { for { addConsignmentResult <- graphQlApiService.addConsignment(token) consignmentId = addConsignmentResult.consignmentid.get _ <- graphQlApiService.startUpload(token, consignmentId) - result <- loadDetails(consignmentId, token.userId) + result <- loadDetails(consignmentId, token.userId, sourceSystem) } yield result } - private def loadDetails(transferId: UUID, userId: UUID): IO[LoadDetails] = { + private def loadDetails(transferId: UUID, userId: UUID, sourceSystem: SourceSystem): IO[LoadDetails] = { val recordsS3Bucket = AWSS3LoadDestination(s"${s3Config.recordsUploadBucket}", s"$userId/$transferId") val metadataS3Bucket = AWSS3LoadDestination(s"${s3Config.metadataUploadBucket}", s"$transferId/dataload") - val metadataProperties = MetadataLoadConfiguration.metadataLoadConfiguration() + val metadataProperties = MetadataLoadConfiguration.metadataLoadConfiguration(sourceSystem) val transferConfiguration = TransferConfiguration(metadataProperties) IO(LoadDetails(transferId, recordsLoadDestination = recordsS3Bucket, metadataLoadDestination = metadataS3Bucket, transferConfiguration)) } diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/MetadataLoadConfiguration.scala b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/MetadataLoadConfiguration.scala index d05a6d3..b5dcd0d 100644 --- a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/MetadataLoadConfiguration.scala +++ b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/MetadataLoadConfiguration.scala @@ -2,6 +2,8 @@ package uk.gov.nationalarchives.tdr.transfer.service.services.dataload import uk.gov.nationalarchives.tdr.transfer.service.ApplicationConfig import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.MetadataPropertyDetails +import uk.gov.nationalarchives.tdr.transfer.service.api.model.SourceSystem.SourceSystemEnum +import uk.gov.nationalarchives.tdr.transfer.service.api.model.SourceSystem.SourceSystemEnum.SourceSystem import uk.gov.nationalarchives.tdr.transfer.service.services.schema.SchemaHandler import scala.jdk.CollectionConverters._ @@ -9,8 +11,14 @@ import scala.jdk.CollectionConverters._ object MetadataLoadConfiguration { private val schemaConfig: ApplicationConfig.Schema = ApplicationConfig.appConfig.schema - def metadataLoadConfiguration(): Set[MetadataPropertyDetails] = { - val schema = SchemaHandler.schema(schemaConfig.dataLoadSharePointLocation) + private def sourceSystemSchemaMapping(sourceSystem: SourceSystem): String = sourceSystem match { + case SourceSystemEnum.SharePoint => schemaConfig.dataLoadSharePointLocation + case _ => throw new RuntimeException(s"Source System '$sourceSystem' not mapped to schema") + } + + def metadataLoadConfiguration(sourceSystem: SourceSystem): Set[MetadataPropertyDetails] = { + val schemaLocation = sourceSystemSchemaMapping(sourceSystem) + val schema = SchemaHandler.schema(schemaLocation) val properties = schema.get("properties").properties().asScala val requiredProperties = schema.get("required").asScala.map(_.asText()).toSet properties diff --git a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiationSpec.scala b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiationSpec.scala index 63129b0..3081aca 100644 --- a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiationSpec.scala +++ b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiationSpec.scala @@ -7,6 +7,7 @@ import graphql.codegen.AddConsignment.addConsignment.AddConsignment import uk.gov.nationalarchives.tdr.keycloak.Token import uk.gov.nationalarchives.tdr.transfer.service.BaseSpec import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.{AWSS3LoadDestination, LoadDetails, TransferConfiguration} +import uk.gov.nationalarchives.tdr.transfer.service.api.model.SourceSystem.SourceSystemEnum import uk.gov.nationalarchives.tdr.transfer.service.services.GraphQlApiService import java.util.UUID @@ -35,7 +36,7 @@ class DataLoadInitiationSpec extends BaseSpec { ) val service = new DataLoadInitiation(mockGraphQlApiService) - val result = service.initiateConsignmentLoad(mockToken).unsafeRunSync() + val result = service.initiateConsignmentLoad(mockToken, SourceSystemEnum.SharePoint).unsafeRunSync() result shouldBe expectedResult verify(mockGraphQlApiService, times(1)).addConsignment(mockToken) verify(mockGraphQlApiService, times(1)).startUpload(mockToken, consignmentId, None) @@ -48,7 +49,7 @@ class DataLoadInitiationSpec extends BaseSpec { val service = new DataLoadInitiation(mockGraphQlApiService) val exception = intercept[RuntimeException] { - service.initiateConsignmentLoad(mockToken).attempt.unsafeRunSync() + service.initiateConsignmentLoad(mockToken, SourceSystemEnum.SharePoint).attempt.unsafeRunSync() } exception.getMessage shouldBe "Error adding consignment" verify(mockGraphQlApiService, times(1)).addConsignment(mockToken) @@ -63,7 +64,7 @@ class DataLoadInitiationSpec extends BaseSpec { when(mockGraphQlApiService.startUpload(mockToken, consignmentId)).thenThrow(new RuntimeException("Error starting upload")) val service = new DataLoadInitiation(mockGraphQlApiService) - val response = service.initiateConsignmentLoad(mockToken).attempt.unsafeRunSync() + val response = service.initiateConsignmentLoad(mockToken, SourceSystemEnum.SharePoint).attempt.unsafeRunSync() response.isLeft should equal(true) response.left.value.getMessage should equal("Error starting upload") diff --git a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/MetadataLoadConfigurationSpec.scala b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/MetadataLoadConfigurationSpec.scala index 9c15ade..4d0bc4d 100644 --- a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/MetadataLoadConfigurationSpec.scala +++ b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/MetadataLoadConfigurationSpec.scala @@ -2,14 +2,26 @@ package uk.gov.nationalarchives.tdr.transfer.service.services.dataload import uk.gov.nationalarchives.tdr.transfer.service.BaseSpec import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.MetadataPropertyDetails +import uk.gov.nationalarchives.tdr.transfer.service.api.model.SourceSystem.SourceSystemEnum +import uk.gov.nationalarchives.tdr.transfer.service.services.schema.SchemaHandler class MetadataLoadConfigurationSpec extends BaseSpec { "'metadataLoadConfiguration'" should "return the correct metadata configuration for the given source system" in { - val result = MetadataLoadConfiguration.metadataLoadConfiguration() + val result = MetadataLoadConfiguration.metadataLoadConfiguration(SourceSystemEnum.SharePoint) result.size shouldBe 4 result.contains(MetadataPropertyDetails("FileRef", true)) shouldBe true result.contains(MetadataPropertyDetails("File_x0020_Size", true)) shouldBe true result.contains(MetadataPropertyDetails("SHA256ClientSideChecksum", true)) shouldBe true result.contains(MetadataPropertyDetails("Modified", true)) shouldBe true } + + "'metadataLoadConfiguration'" should "return an error if source system not mapped to a schema" in { + val mockSourceSystem = mock[SourceSystemEnum.SourceSystem] + + val exception = intercept[RuntimeException] { + MetadataLoadConfiguration.metadataLoadConfiguration(mockSourceSystem) + } + + exception.getMessage shouldBe s"Source System '$mockSourceSystem' not mapped to schema" + } }