Add reports for originals and topleft (#57)

Co-authored-by: Marcin Procyk <[email protected]>

src/main/scala/swiss/dasch/api/ApiProblem.scala

@@ -12,11 +12,7 @@ import zio.schema.{ DeriveSchema, Schema }
 
 sealed trait ApiProblem
 
-case class ProjectNotFound(shortcode: String) extends ApiProblem
-
-object ProjectNotFound {
-  def make(shortcode: ProjectShortcode): ProjectNotFound = ProjectNotFound(shortcode.toString)
-}
+case class ProjectNotFound(shortcode: ProjectShortcode) extends ApiProblem
 
 case class IllegalArguments(errors: List[IllegalArgument]) extends ApiProblem
 
@@ -65,5 +61,5 @@ object ApiProblem {
     invalidHeader("Content-Type", actual.toString, s"expected '$expected'")
 
   // other
-  def projectNotFound(shortcode: ProjectShortcode): ProjectNotFound = ProjectNotFound.make(shortcode)
+  def projectNotFound(shortcode: ProjectShortcode): ProjectNotFound = ProjectNotFound(shortcode)
 }

src/main/scala/swiss/dasch/api/ListProjectsEndpoint.scala

@@ -15,10 +15,8 @@ import zio.http.{ App, Status }
 import zio.json.{ DeriveJsonEncoder, JsonEncoder }
 import zio.schema.{ DeriveSchema, Schema }
 object ListProjectsEndpoint {
-  final case class ProjectResponse(id: String)
+  final case class ProjectResponse(id: ProjectShortcode)
   object ProjectResponse {
-    def make(shortcode: ProjectShortcode): ProjectResponse = ProjectResponse(shortcode.toString)
-
     implicit val schema: Schema[ProjectResponse]           = DeriveSchema.gen[ProjectResponse]
     implicit val jsonEncoder: JsonEncoder[ProjectResponse] = DeriveJsonEncoder.gen[ProjectResponse]
   }
@@ -34,7 +32,7 @@ object ListProjectsEndpoint {
         .listAllProjects()
         .mapBoth(
           ApiProblem.internalError,
-          shortcodes => (EndTotal("items", 0, shortcodes.size, shortcodes.size), shortcodes.map(ProjectResponse.make)),
+          shortcodes => (EndTotal("items", 0, shortcodes.size, shortcodes.size), shortcodes.map(ProjectResponse.apply)),
         )
     )
     .toApp

src/main/scala/swiss/dasch/api/MaintenanceEndpoint.scala

@@ -6,58 +6,45 @@
 package swiss.dasch.api
 
 import swiss.dasch.api.ApiPathCodecSegments.shortcodePathVar
-import swiss.dasch.domain.{ MaintenanceActions, ProjectService }
 import zio.*
 import zio.http.*
 import zio.http.codec.*
 import zio.http.codec.HttpCodec.*
 import zio.http.endpoint.*
 import zio.json.{ DeriveJsonEncoder, JsonEncoder }
-import zio.nio.file
 import zio.schema.{ DeriveSchema, Schema }
 
 object MaintenanceEndpoint {
 
   final case class MappingEntry(internalFilename: String, originalFilename: String)
+
   object MappingEntry {
     implicit val encoder: JsonEncoder[MappingEntry] = DeriveJsonEncoder.gen[MappingEntry]
     implicit val schema: Schema[MappingEntry]       = DeriveSchema.gen[MappingEntry]
   }
 
-  private val applyTopLeftCorrectionEndpoint = Endpoint
-    .post("maintenance" / "apply-top-left-correction" / shortcodePathVar)
+  private val maintenance = "maintenance"
+
+  val applyTopLeftCorrectionEndpoint = Endpoint
+    .post(maintenance / "apply-top-left-correction" / shortcodePathVar)
     .out[String](Status.Accepted)
     .outErrors(
       HttpCodec.error[ProjectNotFound](Status.NotFound),
       HttpCodec.error[IllegalArguments](Status.BadRequest),
       HttpCodec.error[InternalProblem](Status.InternalServerError),
     )
 
-  private val applyTopLeftCorrectionRoute =
-    applyTopLeftCorrectionEndpoint.implement(shortcodeStr =>
-      for {
-        projectPath <- getProjectPath(shortcodeStr)
-        _           <- ZIO.logInfo(s"Creating originals for $projectPath")
-        _           <- MaintenanceActions
-                         .applyTopLeftCorrections(projectPath)
-                         .tap(count => ZIO.logInfo(s"Corrected $count top left images for $projectPath"))
-                         .logError
-                         .forkDaemon
-      } yield "work in progress"
+  val needsTopLeftCorrectionEndpoint = Endpoint
+    .get(maintenance / "needs-top-left-correction")
+    .out[String](Status.Accepted)
+    .outErrors(
+      HttpCodec.error[ProjectNotFound](Status.NotFound),
+      HttpCodec.error[IllegalArguments](Status.BadRequest),
+      HttpCodec.error[InternalProblem](Status.InternalServerError),
     )
 
-  private def getProjectPath(shortcodeStr: String): ZIO[ProjectService, ApiProblem, file.Path] =
-    ApiStringConverters
-      .fromPathVarToProjectShortcode(shortcodeStr)
-      .flatMap(code =>
-        ProjectService.findProject(code).some.mapError {
-          case Some(e) => ApiProblem.internalError(e)
-          case _       => ApiProblem.projectNotFound(code)
-        }
-      )
-
-  private val createOriginalEndpoint = Endpoint
-    .post("maintenance" / "create-originals" / shortcodePathVar)
+  val createOriginalsEndpoint = Endpoint
+    .post(maintenance / "create-originals" / shortcodePathVar)
     .inCodec(ContentCodec.content[Chunk[MappingEntry]](MediaType.application.json))
     .out[String](Status.Accepted)
     .outErrors(
@@ -66,19 +53,13 @@ object MaintenanceEndpoint {
       HttpCodec.error[InternalProblem](Status.InternalServerError),
     )
 
-  private val createOriginalRoute =
-    createOriginalEndpoint.implement {
-      case (shortCodeStr: String, mapping: Chunk[MappingEntry]) =>
-        for {
-          projectPath <- getProjectPath(shortCodeStr)
-          _           <- ZIO.logInfo(s"Creating originals for $projectPath")
-          _           <- MaintenanceActions
-                           .createOriginals(projectPath, mapping.map(e => e.internalFilename -> e.originalFilename).toMap)
-                           .tap(count => ZIO.logInfo(s"Created $count originals for $projectPath"))
-                           .logError
-                           .forkDaemon
-        } yield "work in progress"
-    }
-
-  val app = (createOriginalRoute ++ applyTopLeftCorrectionRoute).toApp
+  val needsOriginalsEndpoint = Endpoint
+    .get(maintenance / "needs-originals")
+    .query(queryBool("imagesOnly").optional)
+    .out[String](Status.Accepted)
+    .outErrors(
+      HttpCodec.error[ProjectNotFound](Status.NotFound),
+      HttpCodec.error[IllegalArguments](Status.BadRequest),
+      HttpCodec.error[InternalProblem](Status.InternalServerError),
+    )
 }

src/main/scala/swiss/dasch/api/MaintenanceEndpointRoutes.scala

@@ -0,0 +1,144 @@
+/*
+ * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package swiss.dasch.api
+
+import swiss.dasch.api.MaintenanceEndpoint.*
+import swiss.dasch.domain.*
+import swiss.dasch.domain.FileFilters.{ isImage, isNonHiddenRegularFile }
+import zio.json.JsonEncoder
+import zio.nio.file
+import zio.nio.file.Files
+import zio.{ Chunk, IO, ZIO }
+
+import java.io.IOException
+
+object MaintenanceEndpointRoutes {
+
+  private def getProjectPath(shortcode: String): ZIO[ProjectService, ApiProblem, file.Path] =
+    ApiStringConverters
+      .fromPathVarToProjectShortcode(shortcode)
+      .flatMap(code =>
+        ProjectService.findProject(code).some.mapError {
+          case Some(e) => ApiProblem.internalError(e)
+          case _       => ApiProblem.projectNotFound(code)
+        }
+      )
+
+  private def saveReport[A](
+      tmpDir: file.Path,
+      name: String,
+      report: A,
+    )(implicit encoder: JsonEncoder[A]
+    ): ZIO[StorageService, Throwable, Unit] =
+    Files.createDirectories(tmpDir / "reports") *>
+      Files.deleteIfExists(tmpDir / "reports" / s"$name.json") *>
+      Files.createFile(tmpDir / "reports" / s"$name.json") *>
+      StorageService.saveJsonFile(tmpDir / "reports" / s"$name.json", report)
+
+  private val needsOriginalsRoute = needsOriginalsEndpoint.implement(imagesOnlyMaybe =>
+    {
+      val imagesOnly = imagesOnlyMaybe.getOrElse(true)
+      val reportName = if (imagesOnly) "needsOriginals_images_only" else "needsOriginals"
+      for {
+        _                 <- ZIO.logInfo(s"Checking for originals")
+        assetDir          <- StorageService.getAssetDirectory()
+        tmpDir            <- StorageService.getTempDirectory()
+        projectShortcodes <- ProjectService.listAllProjects()
+        _                 <- ZIO
+                               .foreach(projectShortcodes)(shortcode =>
+                                 Files
+                                   .walk(assetDir / shortcode.toString)
+                                   .mapZIOPar(8)(originalNotPresent(imagesOnly))
+                                   .filter(identity)
+                                   .as(shortcode)
+                                   .runHead
+                               )
+                               .map(_.flatten)
+                               .flatMap(saveReport(tmpDir, reportName, _))
+                               .zipLeft(ZIO.logInfo(s"Created $reportName.json"))
+                               .logError
+                               .forkDaemon
+      } yield "work in progress"
+    }.logError.mapError(ApiProblem.internalError)
+  )
+
+  private def originalNotPresent(imagesOnly: Boolean)(path: file.Path): IO[IOException, Boolean] = {
+    lazy val assetId                          = AssetId.makeFromPath(path).map(_.toString).getOrElse("unknown-asset-id")
+    def checkIsImageIfNeeded(path: file.Path) = {
+      val shouldNotCheckImages = ZIO.succeed(!imagesOnly)
+      shouldNotCheckImages || isImage(path)
+    }
+
+    isNonHiddenRegularFile(path) &&
+    checkIsImageIfNeeded(path) &&
+    Files
+      .list(path.parent.orNull)
+      .map(_.filename.toString)
+      .filter(name => name.endsWith(".orig") && name.startsWith(assetId))
+      .runHead
+      .map(_.isEmpty)
+  }
+
+  private val createOriginalsRoute =
+    createOriginalsEndpoint.implement {
+      case (shortCodeStr: String, mapping: Chunk[MappingEntry]) =>
+        for {
+          projectPath <- getProjectPath(shortCodeStr)
+          _           <- ZIO.logInfo(s"Creating originals for $projectPath")
+          _           <- MaintenanceActions
+                           .createOriginals(projectPath, mapping.map(e => e.internalFilename -> e.originalFilename).toMap)
+                           .tap(count => ZIO.logInfo(s"Created $count originals for $projectPath"))
+                           .logError
+                           .forkDaemon
+        } yield "work in progress"
+    }
+
+  private val needsTopLeftCorrectionRoute =
+    needsTopLeftCorrectionEndpoint.implement(_ =>
+      (
+        for {
+          _                 <- ZIO.logInfo(s"Checking for top left correction")
+          assetDir          <- StorageService.getAssetDirectory()
+          tmpDir            <- StorageService.getTempDirectory()
+          imageService      <- ZIO.service[ImageService]
+          projectShortcodes <- ProjectService.listAllProjects()
+          _                 <-
+            ZIO
+              .foreach(projectShortcodes)(shortcode =>
+                Files
+                  .walk(assetDir / shortcode.toString)
+                  .mapZIOPar(8)(imageService.needsTopLeftCorrection)
+                  .filter(identity)
+                  .runHead
+                  .map(_.map(_ => shortcode))
+              )
+              .map(_.flatten)
+              .flatMap(saveReport(tmpDir, "needsTopLeftCorrection", _))
+              .zipLeft(ZIO.logInfo(s"Created needsTopLeftCorrection.json"))
+              .logError
+              .forkDaemon
+        } yield "work in progress"
+      ).logError.mapError(ApiProblem.internalError)
+    )
+
+  private val applyTopLeftCorrectionRoute =
+    applyTopLeftCorrectionEndpoint.implement(shortcodeStr =>
+      for {
+        projectPath <- getProjectPath(shortcodeStr)
+        _           <- ZIO.logInfo(s"Creating originals for $projectPath")
+        _           <- MaintenanceActions
+                         .applyTopLeftCorrections(projectPath)
+                         .tap(count => ZIO.logInfo(s"Corrected $count top left images for $projectPath"))
+                         .logError
+                         .forkDaemon
+      } yield "work in progress"
+    )
+
+  val app = (needsOriginalsRoute ++
+    createOriginalsRoute ++
+    needsTopLeftCorrectionRoute ++
+    applyTopLeftCorrectionRoute).toApp
+}

src/main/scala/swiss/dasch/domain/Asset.scala

@@ -16,7 +16,10 @@ object AssetId {
 
   def makeFromPath(file: Path): Option[AssetId] = {
     val filename = file.filename.toString
-    AssetId.make(filename.substring(0, filename.lastIndexOf("."))).toOption
+    filename.contains(".") match {
+      case true  => AssetId.make(filename.substring(0, filename.indexOf("."))).toOption
+      case false => None
+    }
   }
 }
 

src/main/scala/swiss/dasch/domain/Exif.scala

@@ -0,0 +1,22 @@
+/*
+ * Copyright © 2021 - 2023 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package swiss.dasch.domain
+
+// see also https://exiftool.org/TagNames/EXIF.html
+object Exif {
+  object Image {
+    val Orientation = "Exif.Image.Orientation"
+
+    sealed trait OrientationValue { def value: Char }
+    object OrientationValue       {
+      // = Horizontal(normal)
+      case object Horizontal extends OrientationValue { val value = '1' }
+
+      // = Rotate 270 CW
+      case object Rotate270CW extends OrientationValue { val value = '8' }
+    }
+  }
+}

src/main/scala/swiss/dasch/domain/FileFilters.scala

@@ -17,7 +17,7 @@ object FileFilters {
 
   val isJpeg2000: FileFilter = hasFileExtension(Jpx.allExtensions)
 
-  val isImage: FileFilter = hasFileExtension(SipiImageFormat.allExtension)
+  val isImage: FileFilter = hasFileExtension(SipiImageFormat.allExtensions)
 
   val isNonHiddenRegularFile: FileFilter = (path: Path) => Files.isRegularFile(path) && Files.isHidden(path).negate
 

src/main/scala/swiss/dasch/domain/ImageService.scala

@@ -24,26 +24,15 @@ trait ImageService {
     *   the path to the corrected image or None if no correction was needed
     */
   def applyTopLeftCorrection(image: Path): Task[Option[Path]]
+
+  def needsTopLeftCorrection(image: Path): IO[IOException, Boolean]
 }
+
 object ImageService {
   def applyTopLeftCorrection(image: Path): ZIO[ImageService, Throwable, Option[Path]] =
     ZIO.serviceWithZIO[ImageService](_.applyTopLeftCorrection(image))
-}
-
-// see also https://exiftool.org/TagNames/EXIF.html
-object Exif {
-  object Image {
-    val Orientation = "Exif.Image.Orientation"
-
-    sealed trait OrientationValue { def value: Char }
-    object OrientationValue       {
-      // = Horizontal(normal)
-      case object Horizontal extends OrientationValue { val value = '1' }
-
-      // = Rotate 270 CW
-      case object Rotate270CW extends OrientationValue { val value = '8' }
-    }
-  }
+  def needsTopLeftCorrection(image: Path): ZIO[ImageService, IOException, Boolean]    =
+    ZIO.serviceWithZIO[ImageService](_.needsTopLeftCorrection(image))
 }
 
 final case class ImageServiceLive(sipiClient: SipiClient, assetInfos: AssetInfoService) extends ImageService {
@@ -56,7 +45,7 @@ final case class ImageServiceLive(sipiClient: SipiClient, assetInfos: AssetInfoS
         assetInfos.updateAssetInfoForDerivative(image).as(image)
     )
 
-  private def needsTopLeftCorrection(image: Path): IO[IOException, Boolean] =
+  override def needsTopLeftCorrection(image: Path): IO[IOException, Boolean] =
     FileFilters.isImage(image) &&
     sipiClient
       .queryImageFile(image)

src/main/scala/swiss/dasch/domain/ProjectService.scala

@@ -10,17 +10,25 @@ import eu.timepit.refined.refineV
 import eu.timepit.refined.string.MatchesRegex
 import org.apache.commons.io.FileUtils
 import zio.*
+import zio.json.JsonCodec
 import zio.nio.file.Files.{ isDirectory, newDirectoryStream }
 import zio.nio.file.{ Files, Path }
+import zio.schema.Schema
 import zio.stream.ZStream
 
 import java.io.IOException
 
 opaque type ProjectShortcode = String Refined MatchesRegex["""^\p{XDigit}{4,4}$"""]
-type IiifPrefix              = ProjectShortcode
 
 object ProjectShortcode {
+
   def make(shortcode: String): Either[String, ProjectShortcode] = refineV(shortcode.toUpperCase)
+
+  extension (c: ProjectShortcode) { def value: String = c.toString }
+
+  given schema: Schema[ProjectShortcode] = Schema[String].transformOrFail(ProjectShortcode.make, id => Right(id.value))
+
+  given codec: JsonCodec[ProjectShortcode] = JsonCodec[String].transformOrFail(ProjectShortcode.make, _.value)
 }
 
 trait ProjectService {