Merge pull request #3805 from nationalarchives/fix-metadata-upload

Use different library for upload
nationalarchives · Mar 22, 2024 · 613e3d9 · 613e3d9
2 parents e414b73 + da4dd2b
commit 613e3d9
Show file tree

Hide file tree

Showing 9 changed files with 88 additions and 85 deletions.
diff --git a/app/configuration/ApplicationConfig.scala b/app/configuration/ApplicationConfig.scala
@@ -21,6 +21,8 @@ class ApplicationConfig @Inject() (configuration: Configuration) {
 
   val metadataValidationBaseUrl: String = configuration.get[String]("metadatavalidation.baseUrl")
 
+  val s3Endpoint: String = configuration.get[String]("s3.endpoint")
+
   val draft_metadata_s3_bucket_name: String = configuration.get[String]("draft_metadata_s3_bucket_name")
 
 }
diff --git a/app/controllers/DraftMetadataUploadController.scala b/app/controllers/DraftMetadataUploadController.scala
@@ -56,7 +56,7 @@ class DraftMetadataUploadController @Inject() (
         firstFilePart <- fromOption(request.body.files.headOption)(new RuntimeException(noDraftMetadataFileUploaded))
         file <- fromOption(request.body.file(firstFilePart.key))(new RuntimeException(noDraftMetadataFileUploaded))
         draftMetadata <- fromOption(Using(scala.io.Source.fromFile(file.ref.getAbsoluteFile))(_.mkString).toOption)(new RuntimeException(noDraftMetadataFileUploaded))
-        _ <- uploadService.uploadDraftMetadata(uploadBucket, uploadKey, draftMetadata)
+        _ <- IO.fromFuture(IO(uploadService.uploadDraftMetadata(uploadBucket, uploadKey, draftMetadata)))
         _ <- IO.fromFuture(IO { draftMetadataService.triggerDraftMetadataValidator(consignmentId, token.bearerAccessToken.getValue) })
         successPage <- IO(play.api.mvc.Results.Redirect(successPage))
       } yield successPage

diff --git a/app/services/UploadService.scala b/app/services/UploadService.scala
@@ -1,22 +1,24 @@
 package services
 
-import cats.effect.IO
 import com.nimbusds.oauth2.sdk.token.BearerAccessToken
-import configuration.GraphQLConfiguration
+import configuration.{ApplicationConfig, GraphQLConfiguration}
 import graphql.codegen.AddFilesAndMetadata.{addFilesAndMetadata => afam}
 import graphql.codegen.StartUpload.{startUpload => su}
 import graphql.codegen.types.{AddFileAndMetadataInput, StartUploadInput}
 import services.ApiErrorHandling.sendApiRequest
 import software.amazon.awssdk.core.internal.async.ByteBuffersAsyncRequestBody
-import software.amazon.awssdk.transfer.s3.model.CompletedUpload
-import uk.gov.nationalarchives.DAS3Client
+import software.amazon.awssdk.services.s3.S3AsyncClient
+import software.amazon.awssdk.services.s3.model.{PutObjectRequest, PutObjectResponse}
+import uk.gov.nationalarchives.aws.utils.s3.S3Clients._
 
 import javax.inject.Inject
 import scala.concurrent.{ExecutionContext, Future}
+import scala.jdk.FutureConverters.CompletionStageOps
 
-class UploadService @Inject() (val graphqlConfiguration: GraphQLConfiguration)(implicit val ec: ExecutionContext) {
+class UploadService @Inject() (val graphqlConfiguration: GraphQLConfiguration, val applicationConfig: ApplicationConfig)(implicit val ec: ExecutionContext) {
   private val startUploadClient = graphqlConfiguration.getClient[su.Data, su.Variables]()
   private val addFilesAndMetadataClient = graphqlConfiguration.getClient[afam.Data, afam.Variables]()
+  private val s3Endpoint = applicationConfig.s3Endpoint
 
   def startUpload(startUploadInput: StartUploadInput, token: BearerAccessToken): Future[String] = {
     val variables = su.Variables(startUploadInput)
@@ -28,13 +30,13 @@ class UploadService @Inject() (val graphqlConfiguration: GraphQLConfiguration)(i
     sendApiRequest(addFilesAndMetadataClient, afam.document, token, variables).map(data => data.addFilesAndMetadata)
   }
 
-  def uploadDraftMetadata(bucket: String, key: String, draftMetadata: String): IO[CompletedUpload] = {
-    uploadDraftMetadata(bucket, key, draftMetadata, DAS3Client[IO]())
+  def uploadDraftMetadata(bucket: String, key: String, draftMetadata: String): Future[PutObjectResponse] = {
+    uploadDraftMetadata(bucket, key, draftMetadata, s3Async(s3Endpoint))
   }
 
-  def uploadDraftMetadata(bucket: String, key: String, draftMetadata: String, s3client: DAS3Client[IO]): IO[CompletedUpload] = {
+  def uploadDraftMetadata(bucket: String, key: String, draftMetadata: String, s3AsyncClient: S3AsyncClient): Future[PutObjectResponse] = {
     val bytes = draftMetadata.getBytes
-    val publisher = ByteBuffersAsyncRequestBody.from("application/octet-stream", bytes)
-    s3client.upload(bucket, key, bytes.size, publisher)
+    val publisher: ByteBuffersAsyncRequestBody = ByteBuffersAsyncRequestBody.from("application/octet-stream", bytes)
+    s3AsyncClient.putObject(PutObjectRequest.builder.bucket(bucket).key(key).build, publisher).asScala
   }
 }
diff --git a/build.sbt b/build.sbt
@@ -34,7 +34,7 @@ libraryDependencies ++= Seq(
   "uk.gov.nationalarchives" %% "tdr-auth-utils" % "0.0.190",
   "uk.gov.nationalarchives" %% "tdr-generated-graphql" % "0.0.361",
   "uk.gov.nationalarchives" %% "tdr-metadata-validation" % "0.0.14",
-  "uk.gov.nationalarchives" %% "da-s3-client" % "0.1.41",
+  "uk.gov.nationalarchives" %% "s3-utils" % "0.1.151",
   "com.github.tototoshi" %% "scala-csv" % "1.3.10",
   "ch.qos.logback" % "logback-classic" % "1.5.0",
   ws,

diff --git a/conf/application.base.conf b/conf/application.base.conf
@@ -11,6 +11,8 @@ consignmentapi.url = ${consignmentapi.domain}"/graphql"
 
 logout.url="http://localhost:9000/signed-out"
 
+s3.endpoint = "https://s3.eu-west-2.amazonaws.com/"
+
 play.filters.enabled += play.filters.https.RedirectHttpsFilter
 
 play {

diff --git a/test/controllers/DraftMetadataUploadControllerSpec.scala b/test/controllers/DraftMetadataUploadControllerSpec.scala
@@ -1,6 +1,5 @@
 package controllers
 
-import cats.effect.IO
 import com.github.tomakehurst.wiremock.WireMockServer
 import configuration.{ApplicationConfig, GraphQLConfiguration, KeycloakConfiguration}
 import org.mockito.ArgumentMatchers.{any, anyString}
@@ -10,14 +9,13 @@ import org.scalatest.matchers.should.Matchers._
 import play.api.Configuration
 import play.api.Play.materializer
 import play.api.libs.Files.SingletonTemporaryFileCreator
-import play.api.mvc.{MultipartFormData, Result}
 import play.api.mvc.MultipartFormData.FilePart
+import play.api.mvc.{MultipartFormData, Result}
 import play.api.test.CSRFTokenHelper._
-import play.api.test.{FakeHeaders, FakeRequest}
 import play.api.test.Helpers.{status => playStatus, _}
+import play.api.test.{FakeHeaders, FakeRequest}
 import services.{ConsignmentService, DraftMetadataService, UploadService}
 import software.amazon.awssdk.services.s3.model.PutObjectResponse
-import software.amazon.awssdk.transfer.s3.model.CompletedUpload
 import testUtils.FrontEndTestHelper
 
 import java.io.{BufferedWriter, File, FileWriter}
@@ -73,8 +71,7 @@ class DraftMetadataUploadControllerSpec extends FrontEndTestHelper {
     "redirect to draft metadata checks page when upload successful" in {
       val uploadServiceMock = mock[UploadService]
       val putObjectResponse = PutObjectResponse.builder().eTag("testEtag").build()
-      val completedUpload = IO(CompletedUpload.builder().response(putObjectResponse).build())
-      when(uploadServiceMock.uploadDraftMetadata(anyString, anyString, anyString)).thenReturn(completedUpload)
+      when(uploadServiceMock.uploadDraftMetadata(anyString, anyString, anyString)).thenReturn(Future.successful(putObjectResponse))
 
       val draftMetadataServiceMock = mock[DraftMetadataService]
       when(draftMetadataServiceMock.triggerDraftMetadataValidator(any[UUID], anyString)).thenReturn(Future.successful(true))
@@ -88,7 +85,8 @@ class DraftMetadataUploadControllerSpec extends FrontEndTestHelper {
 
     "render error page when upload unsuccessful" in {
       val uploadServiceMock = mock[UploadService]
-      when(uploadServiceMock.uploadDraftMetadata(anyString, anyString, anyString)).thenReturn(IO.raiseError[CompletedUpload](new RuntimeException("Upload failed")))
+      when(uploadServiceMock.uploadDraftMetadata(anyString, anyString, anyString))
+        .thenReturn(Future.failed(new RuntimeException("Upload failed")))
       val draftMetadataServiceMock = mock[DraftMetadataService]
       when(draftMetadataServiceMock.triggerDraftMetadataValidator(any[UUID], anyString)).thenReturn(Future.successful(true))
       val response = requestFileUpload(uploadServiceMock, draftMetadataServiceMock)
@@ -101,8 +99,7 @@ class DraftMetadataUploadControllerSpec extends FrontEndTestHelper {
     "render error page when upload success but trigger fails" in {
       val uploadServiceMock = mock[UploadService]
       val putObjectResponse = PutObjectResponse.builder().eTag("testEtag").build()
-      val completedUpload = IO(CompletedUpload.builder().response(putObjectResponse).build())
-      when(uploadServiceMock.uploadDraftMetadata(anyString, anyString, anyString)).thenReturn(completedUpload)
+      when(uploadServiceMock.uploadDraftMetadata(anyString, anyString, anyString)).thenReturn(Future.successful(putObjectResponse))
 
       val draftMetadataServiceMock = mock[DraftMetadataService]
       when(draftMetadataServiceMock.triggerDraftMetadataValidator(any[UUID], anyString)).thenReturn(Future.failed(new RuntimeException("Trigger failed")))
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,6 +11,8 @@ consignmentapi.url = ${consignmentapi.domain}"/graphql" @@
     logout.url="http://localhost:9000/signed-out"
+    s3.endpoint = "https://s3.eu-west-2.amazonaws.com/"
     play.filters.enabled += play.filters.https.RedirectHttpsFilter
     play {
@@ Expand Down @@