From a6f68fb956e73d0873cb9e83d4d87e918c17911f Mon Sep 17 00:00:00 2001 From: TomJKing Date: Tue, 13 Aug 2024 10:04:45 +0100 Subject: [PATCH] Pass configuration for metadata back to client Provide calling client with configuration for metadata for the transfer Configuration driven by the relevant schema --- build.sbt | 1 + project/Dependencies.scala | 1 + .../transfer/service/api/model/LoadModel.scala | 7 ++++++- .../services/dataload/DataLoadInitiation.scala | 7 +++++-- .../schema/MetadataConfiguration.scala | 18 ++++++++++++++++++ .../services/schema/SchemaHandler.scala | 16 ++++++++++++++++ .../tdr/transfer/service/BaseSpec.scala | 11 +++++++++++ .../api/TransferServiceServerSpec.scala | 6 ++---- .../dataload/DataLoadInitiationSpec.scala | 4 ++-- 9 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/schema/MetadataConfiguration.scala create mode 100644 src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/schema/SchemaHandler.scala diff --git a/build.sbt b/build.sbt index 34dd76f..cb273f0 100644 --- a/build.sbt +++ b/build.sbt @@ -18,6 +18,7 @@ lazy val root = (project in file(".")) keycloakMock % Test, logbackClassic, logBackEncoder, + metadataSchema, mockito % Test, pekkoTestKitHttp % Test, pureConfig, diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 7e8c383..56e20c2 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -22,6 +22,7 @@ object Dependencies { lazy val logBackEncoder = "net.logstash.logback" % "logstash-logback-encoder" % "7.4" lazy val logbackClassic = "ch.qos.logback" % "logback-classic" % "1.5.6" + lazy val metadataSchema = "uk.gov.nationalarchives" % "da-metadata-schema_3" % "0.0.26" lazy val mockito = "org.mockito" %% "mockito-scala" % mockitoVersion lazy val mockitoScalaTest = "org.mockito" %% "mockito-scala-scalatest" % mockitoVersion diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/api/model/LoadModel.scala b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/api/model/LoadModel.scala index 15eeb3e..70055b9 100644 --- a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/api/model/LoadModel.scala +++ b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/api/model/LoadModel.scala @@ -7,12 +7,17 @@ sealed trait LoadDestinationModel sealed trait MetadataPropertyModel object LoadModel { + case class CustomMetadataConfiguration(required: Boolean = false) extends MetadataPropertyModel case class MetadataPropertyDetails(propertyName: String, required: Boolean) extends MetadataPropertyModel + case class TransferConfiguration( + metadataPropertyDetails: Set[MetadataPropertyDetails] = Set(), + customMetadataConfiguration: CustomMetadataConfiguration = CustomMetadataConfiguration() + ) case class AWSS3LoadDestination(bucketName: String, bucketKeyPrefix: String) extends LoadDestinationModel case class LoadDetails( transferId: UUID, recordsLoadDestination: AWSS3LoadDestination, metadataLoadDestination: AWSS3LoadDestination, - metadataProperties: List[MetadataPropertyDetails] = List() + transferConfiguration: TransferConfiguration = TransferConfiguration() ) extends LoadModel } diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiation.scala b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiation.scala index 8541269..2013dea 100644 --- a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiation.scala +++ b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiation.scala @@ -3,9 +3,10 @@ package uk.gov.nationalarchives.tdr.transfer.service.services.dataload import cats.effect.IO import uk.gov.nationalarchives.tdr.keycloak.Token import uk.gov.nationalarchives.tdr.transfer.service.ApplicationConfig -import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.{AWSS3LoadDestination, LoadDetails} +import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.{AWSS3LoadDestination, LoadDetails, TransferConfiguration} import uk.gov.nationalarchives.tdr.transfer.service.services.GraphQlApiService import uk.gov.nationalarchives.tdr.transfer.service.services.dataload.DataLoadInitiation.s3Config +import uk.gov.nationalarchives.tdr.transfer.service.services.schema.MetadataConfiguration import java.util.UUID @@ -22,7 +23,9 @@ class DataLoadInitiation(graphQlApiService: GraphQlApiService) { private def loadDetails(transferId: UUID, userId: UUID): IO[LoadDetails] = { val recordsS3Bucket = AWSS3LoadDestination(s"${s3Config.recordsUploadBucket}", s"$userId/$transferId") val metadataS3Bucket = AWSS3LoadDestination(s"${s3Config.metadataUploadBucket}", s"$transferId/dataload") - IO(LoadDetails(transferId, recordsLoadDestination = recordsS3Bucket, metadataLoadDestination = metadataS3Bucket)) + val metadataProperties = MetadataConfiguration.metadataConfiguration() + val transferConfiguration = TransferConfiguration(metadataProperties) + IO(LoadDetails(transferId, recordsLoadDestination = recordsS3Bucket, metadataLoadDestination = metadataS3Bucket, transferConfiguration)) } } diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/schema/MetadataConfiguration.scala b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/schema/MetadataConfiguration.scala new file mode 100644 index 0000000..b24b2fc --- /dev/null +++ b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/schema/MetadataConfiguration.scala @@ -0,0 +1,18 @@ +package uk.gov.nationalarchives.tdr.transfer.service.services.schema + +import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.MetadataPropertyDetails + +import scala.jdk.CollectionConverters._ + +object MetadataConfiguration { + def metadataConfiguration(): Set[MetadataPropertyDetails] = { + val schema = SchemaHandler.schema().get("allOf") + val properties = schema.get(0).get("properties").properties().asScala + val requiredProperties = schema.get(0).get("required").asScala.map(_.asText()).toSet + properties + .map(p => { + MetadataPropertyDetails(p.getKey, requiredProperties.contains(p.getKey)) + }) + .toSet + } +} diff --git a/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/schema/SchemaHandler.scala b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/schema/SchemaHandler.scala new file mode 100644 index 0000000..5ff242a --- /dev/null +++ b/src/main/scala/uk/gov/nationalarchives/tdr/transfer/service/services/schema/SchemaHandler.scala @@ -0,0 +1,16 @@ +package uk.gov.nationalarchives.tdr.transfer.service.services.schema + +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} + +import java.io.InputStream + +object SchemaHandler { + private def getJsonNodeFromStreamContent(content: InputStream): JsonNode = { + val mapper = new ObjectMapper() + mapper.readTree(content) + } + + def schema(schemaLocation: String = "/metadata-schema/dataLoadSharePointSchema.schema.json"): JsonNode = { + getJsonNodeFromStreamContent(getClass.getResourceAsStream(schemaLocation)) + } +} diff --git a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/BaseSpec.scala b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/BaseSpec.scala index bb10103..7c51d7f 100644 --- a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/BaseSpec.scala +++ b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/BaseSpec.scala @@ -6,6 +6,7 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import sttp.client3.{HttpURLConnectionBackend, Identity, SttpBackend} import uk.gov.nationalarchives.tdr.keycloak.TdrKeycloakDeployment +import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.{MetadataPropertyDetails, TransferConfiguration} import scala.concurrent.ExecutionContextExecutor @@ -13,4 +14,14 @@ trait BaseSpec extends AnyFlatSpec with MockitoSugar with Matchers with EitherVa implicit val executionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.global implicit val backend: SttpBackend[Identity, Any] = HttpURLConnectionBackend() implicit val tdrKeycloakDeployment: TdrKeycloakDeployment = TdrKeycloakDeployment("authUrl", "realm", 60) + + val expectedMetadataPropertyDetails: Set[MetadataPropertyDetails] = Set( + MetadataPropertyDetails("SHA256ClientSideChecksum", true), + MetadataPropertyDetails("ClientSideFileSize", true), + MetadataPropertyDetails("ClientSideFileLastModifiedDate", true), + MetadataPropertyDetails("ClientSideOriginalFilepath", true), + MetadataPropertyDetails("UUID", true) + ) + + val expectedTransferConfiguration: TransferConfiguration = TransferConfiguration(expectedMetadataPropertyDetails) } diff --git a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/api/TransferServiceServerSpec.scala b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/api/TransferServiceServerSpec.scala index 971d696..f1b0469 100644 --- a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/api/TransferServiceServerSpec.scala +++ b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/api/TransferServiceServerSpec.scala @@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers import org.typelevel.ci.CIString import uk.gov.nationalarchives.tdr.transfer.service.TestUtils.{invalidToken, userId, validUserToken} import uk.gov.nationalarchives.tdr.transfer.service.api.controllers.LoadController -import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.MetadataPropertyDetails +import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.{MetadataPropertyDetails, TransferConfiguration} import uk.gov.nationalarchives.tdr.transfer.service.services.ExternalServicesSpec class TransferServiceServerSpec extends ExternalServicesSpec with Matchers { @@ -55,13 +55,11 @@ class TransferServiceServerSpec extends ExternalServicesSpec with Matchers { "bucketKeyPrefix" := s"$transferId/dataload" ) - val metadataProperties: List[MetadataPropertyDetails] = List() - val expectedResponse = Json.obj( "transferId" := transferId, "recordsLoadDestination" := recordsDestination, "metadataLoadDestination" := metadataLoadDestination, - "metadataProperties" := metadataProperties + "transferConfiguration" := expectedTransferConfiguration ) response.status shouldBe Status.Ok diff --git a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiationSpec.scala b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiationSpec.scala index a82bfc2..63129b0 100644 --- a/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiationSpec.scala +++ b/src/test/scala/uk/gov/nationalarchives/tdr/transfer/service/services/dataload/DataLoadInitiationSpec.scala @@ -6,7 +6,7 @@ import com.nimbusds.oauth2.sdk.token.BearerAccessToken import graphql.codegen.AddConsignment.addConsignment.AddConsignment import uk.gov.nationalarchives.tdr.keycloak.Token import uk.gov.nationalarchives.tdr.transfer.service.BaseSpec -import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.{AWSS3LoadDestination, LoadDetails} +import uk.gov.nationalarchives.tdr.transfer.service.api.model.LoadModel.{AWSS3LoadDestination, LoadDetails, TransferConfiguration} import uk.gov.nationalarchives.tdr.transfer.service.services.GraphQlApiService import java.util.UUID @@ -31,7 +31,7 @@ class DataLoadInitiationSpec extends BaseSpec { consignmentId, AWSS3LoadDestination("s3BucketNameRecords", s"$userId/$consignmentId"), AWSS3LoadDestination("s3BucketNameMetadata", s"$consignmentId/dataload"), - List() + expectedTransferConfiguration ) val service = new DataLoadInitiation(mockGraphQlApiService)