TDR-3326 Assign references to file/directories

build.sbt

@@ -63,6 +63,7 @@ libraryDependencies ++= Seq(
   "io.circe" %% "circe-optics" % "0.15.0",
   "io.circe" %% "circe-generic" % circeVersion,
   "io.circe" %% "circe-generic-extras" % "0.14.3",
+  "com.softwaremill.sttp.client3" %% "core" % "3.9.0",
   "uk.gov.nationalarchives" %% "consignment-api-db" % "0.1.36",
   "org.postgresql" % "postgresql" % "42.6.0",
   "com.typesafe.slick" %% "slick" % "3.4.0",

src/main/resources/application.base.conf

@@ -33,8 +33,16 @@ fileUpload {
 
 featureAccessBlock {
   http4s = ${BLOCK_HTTP4S}
+  assignFileReferences = ${BLOCK_ASSIGN_FILE_REFERENCES}
 }
 
+referenceGenerator {
+  referenceGeneratorUrl = ${REFERENCE_GENERATOR_URL}
+  referenceLimit = ${REFERENCE_GENERATOR_LIMIT}
+}
+
+environment = ${ENVIRONMENT}
+
 akka.http {
   server {
     idle-timeout = 180 s

src/main/resources/application.conf

@@ -12,8 +12,16 @@ frontend = {
 
 featureAccessBlock {
   http4s = true
+  assignFileReferences = true
 }
 
+referenceGenerator {
+  referenceGeneratorUrl = "https://q59yzinibd.execute-api.eu-west-2.amazonaws.com"
+  referenceLimit = 500
+}
+
+environment = "intg"
+
 consignmentapi = {
   useIamAuth = false
   db {

src/main/scala/uk/gov/nationalarchives/tdr/api/http/GraphQLServerBase.scala

@@ -5,6 +5,7 @@ import com.typesafe.scalalogging.Logger
 import sangria.execution._
 import sangria.marshalling.ResultMarshaller
 import slick.jdbc.JdbcBackend
+import sttp.client3.SimpleHttpClient
 import uk.gov.nationalarchives.tdr.api.auth.AuthorisationException
 import uk.gov.nationalarchives.tdr.api.consignmentstatevalidation.ConsignmentStateException
 import uk.gov.nationalarchives.tdr.api.db.repository._
@@ -83,14 +84,15 @@ trait GraphQLServerBase {
       new FFIDMetadataService(ffidMetadataRepository, ffidMetadataMatchesRepository, timeSource, uuidSource)
     val fileStatusService = new FileStatusService(fileStatusRepository)
     val displayPropertiesService = new DisplayPropertiesService(displayPropertiesRepository)
+    val referenceGeneratorService = new ReferenceGeneratorService(config, SimpleHttpClient())
     val fileService = new FileService(
       fileRepository,
-      consignmentRepository,
       customMetadataPropertiesRepository,
       ffidMetadataService,
       antivirusMetadataService,
       fileStatusService,
       fileMetadataService,
+      referenceGeneratorService,
       new CurrentTimeSource,
       uuidSource,
       config

src/main/scala/uk/gov/nationalarchives/tdr/api/service/FileService.scala

@@ -28,12 +28,12 @@ import scala.math.min
 
 class FileService(
     fileRepository: FileRepository,
-    consignmentRepository: ConsignmentRepository,
     customMetadataPropertiesRepository: CustomMetadataPropertiesRepository,
     ffidMetadataService: FFIDMetadataService,
     avMetadataService: AntivirusMetadataService,
     fileStatusService: FileStatusService,
     fileMetadataService: FileMetadataService,
+    referenceGeneratorService: ReferenceGeneratorService,
     timeSource: TimeSource,
     uuidSource: UUIDSource,
     config: Config
@@ -43,6 +43,7 @@ class FileService(
 
   private val fileUploadBatchSize: Int = config.getInt("fileUpload.batchSize")
   private val filePageMaxLimit: Int = config.getInt("pagination.filesMaxLimit")
+  private val referenceGeneratorFeatureBlock: Boolean = config.getBoolean("featureAccessBlock.assignFileReferences")
 
   def addFile(addFileAndMetadataInput: AddFileAndMetadataInput, userId: UUID): Future[List[FileMatches]] = {
     val now = Timestamp.from(timeSource.now)
@@ -53,10 +54,26 @@ class FileService(
 
     val row: (UUID, String, String) => FilemetadataRow = FilemetadataRow(uuidSource.uuid, _, _, now, userId, _)
     val rows: Future[List[Rows]] = customMetadataPropertiesRepository.getCustomMetadataValuesWithDefault.map(filePropertyValue => {
-      ((allEmptyDirectoryNodes ++ allFileNodes) map { case (path, treeNode) =>
+      val allNodes = allEmptyDirectoryNodes ++ allFileNodes
+      val allNodesWithReference = if (referenceGeneratorFeatureBlock) {
+        allNodes.keys.zip(allNodes.values.zip(List.fill(allNodes.size)(None))).toMap
+      } else {
+        val generatedReferences = referenceGeneratorService.getReferences(allNodes.size)
+        allNodes.keys.zip(allNodes.values.zip(generatedReferences.map(Some(_)))).toMap
+      }
+      (allNodesWithReference map { case (path, (treeNode, reference)) =>
         val parentId = treeNode.parentPath.map(path => allFileNodes.getOrElse(path, allEmptyDirectoryNodes(path)).id)
         val fileId = treeNode.id
-        val fileRow = FileRow(fileId, consignmentId, userId, now, filetype = Some(treeNode.treeNodeType), filename = Some(treeNode.name), parentid = parentId)
+        val fileRow = FileRow(
+          fileId,
+          consignmentId,
+          userId,
+          now,
+          filetype = Some(treeNode.treeNodeType),
+          filename = Some(treeNode.name),
+          parentid = parentId,
+          filereference = reference
+        )
 
         val commonMetadataRows = List(
           row(fileId, path, ClientSideOriginalFilepath),

src/main/scala/uk/gov/nationalarchives/tdr/api/service/ReferenceGeneratorService.scala

@@ -0,0 +1,54 @@
+package uk.gov.nationalarchives.tdr.api.service
+
+import com.typesafe.config.Config
+import com.typesafe.scalalogging.Logger
+import io.circe.parser._
+import org.apache.http.client.HttpResponseException
+import sttp.client3.{Response, SimpleHttpClient, UriContext, basicRequest}
+import uk.gov.nationalarchives.tdr.api.service.ReferenceGeneratorService.Reference
+
+import scala.annotation.tailrec
+
+class ReferenceGeneratorService(config: Config, client: SimpleHttpClient) {
+  val logger: Logger = Logger("ReferenceGeneratorService")
+  private val environment: String = config.getString("environment")
+  private val refGeneratorUrl: String = config.getString("referenceGenerator.referenceGeneratorUrl")
+  private val refGeneratorLimit: Int = config.getInt("referenceGenerator.referenceLimit")
+
+  def getReferences(numberOfRefs: Int): List[Reference] = {
+    @tailrec
+    def fetchReferences(numberOfRefs: Int, acc: List[Reference]): List[Reference] = {
+      if (numberOfRefs <= 0) acc
+      else {
+        val batchSize = Math.min(numberOfRefs, refGeneratorLimit)
+        fetchReferences(numberOfRefs - batchSize, acc ++ sendRequest(batchSize))
+      }
+    }
+
+    fetchReferences(numberOfRefs, Nil)
+  }
+
+  private def sendRequest(numberOfRefs: Int): List[Reference] = {
+    val response: Response[Either[String, Reference]] = client.send(basicRequest.get(uri"$refGeneratorUrl/$environment/counter?numberofrefs=$numberOfRefs"))
+
+    try {
+      response.body match {
+        case Left(body) =>
+          val message = "Failed to get references from reference generator api"
+          logger.error(s"${response.code} $message:\n$body")
+          throw new HttpResponseException(response.code.code, message)
+        case Right(body) =>
+          val references = parse(body).getOrElse(null)
+          val listOfReferences = references.as[List[Reference]].getOrElse(null)
+          logger.info(s"2xx response to GET:\n$listOfReferences")
+          listOfReferences
+      }
+    } finally {
+      client.close()
+    }
+  }
+}
+
+object ReferenceGeneratorService {
+  type Reference = String
+}

src/test/resources/application.conf

@@ -36,4 +36,12 @@ fileUpload {
 
 featureAccessBlock {
   http4s = true
+  assignFileReferences = true
 }
+
+referenceGenerator {
+  referenceGeneratorUrl = "https://dummy-reference-url.com"
+  referenceLimit = 2
+}
+
+environment = "intg"