Skip to content

Commit

Permalink
feat: Download original asset (DEV-3832) (#267)
Browse files Browse the repository at this point in the history
Implementation of `projects / shortcodePathVar / "assets" / assetIdPathVar / "original"`.
  • Loading branch information
siers authored Sep 12, 2024
1 parent 332b519 commit 2c0b3bf
Show file tree
Hide file tree
Showing 16 changed files with 314 additions and 58 deletions.
29 changes: 15 additions & 14 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -81,20 +81,21 @@ lazy val root = (project in file("."))
),
),
libraryDependencies ++= db ++ tapir ++ metrics ++ Seq(
"com.github.jwt-scala" %% "jwt-zio-json" % "10.0.1",
"commons-io" % "commons-io" % "2.16.1",
"dev.zio" %% "zio" % zioVersion,
"dev.zio" %% "zio-config" % zioConfigVersion,
"dev.zio" %% "zio-config-magnolia" % zioConfigVersion,
"dev.zio" %% "zio-config-typesafe" % zioConfigVersion,
"dev.zio" %% "zio-json" % zioJsonVersion,
"dev.zio" %% "zio-json-interop-refined" % zioJsonVersion,
"dev.zio" %% "zio-metrics-connectors" % zioMetricsConnectorsVersion,
"dev.zio" %% "zio-metrics-connectors-prometheus" % zioMetricsConnectorsVersion,
"dev.zio" %% "zio-nio" % zioNioVersion,
"dev.zio" %% "zio-prelude" % zioPreludeVersion,
"dev.zio" %% "zio-streams" % zioVersion,
"eu.timepit" %% "refined" % "0.11.2",
"com.github.jwt-scala" %% "jwt-zio-json" % "10.0.1",
"commons-io" % "commons-io" % "2.16.1",
"dev.zio" %% "zio" % zioVersion,
"dev.zio" %% "zio-config" % zioConfigVersion,
"dev.zio" %% "zio-config-magnolia" % zioConfigVersion,
"dev.zio" %% "zio-config-typesafe" % zioConfigVersion,
"dev.zio" %% "zio-json" % zioJsonVersion,
"dev.zio" %% "zio-json-interop-refined" % zioJsonVersion,
"dev.zio" %% "zio-metrics-connectors" % zioMetricsConnectorsVersion,
"dev.zio" %% "zio-metrics-connectors-prometheus" % zioMetricsConnectorsVersion,
"dev.zio" %% "zio-nio" % zioNioVersion,
"dev.zio" %% "zio-prelude" % zioPreludeVersion,
"dev.zio" %% "zio-streams" % zioVersion,
"eu.timepit" %% "refined" % "0.11.2",
"com.softwaremill.sttp.client3" %% "zio" % "3.9.8",

// csv for reports
"com.github.tototoshi" %% "scala-csv" % "2.0.0",
Expand Down
3 changes: 1 addition & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
services:

ingest:
image: daschswiss/dsp-ingest:latest
ports:
Expand All @@ -21,4 +20,4 @@ services:
- INGEST_BULK_MAX_PARALLEL=10
- SIPI_USE_LOCAL_DEV=false
- DB_JDBC_URL=jdbc:sqlite:/opt/db/ingest.sqlite

- DSP_API_URL=http://localhost:3333
5 changes: 5 additions & 0 deletions src/main/resources/application.conf
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ ingest {
bulk-max-parallel = ${?INGEST_BULK_MAX_PARALLEL}
}

dsp-api {
url = "http://localhost:3333"
url = ${?DSP_API_URL}
}

features {
allow-erase-projects = false
allow-erase-projects = ${?ALLOW_ERASE_PROJECTS}
Expand Down
62 changes: 62 additions & 0 deletions src/main/scala/swiss/dasch/FetchAssetPermissions.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Copyright © 2021 - 2024 Swiss National Data and Service Center for the Humanities and/or DaSCH Service Platform contributors.
* SPDX-License-Identifier: Apache-2.0
*/

package swiss.dasch

import cats.implicits._
import sttp.capabilities.zio.ZioStreams
import sttp.client3.SttpBackend
import sttp.client3.*
import sttp.client3.httpclient.zio.HttpClientZioBackend
import swiss.dasch.config.Configuration
import swiss.dasch.domain.AssetInfo
import zio.*
import zio.json.DecoderOps
import zio.json.DeriveJsonDecoder
import zio.json.JsonDecoder

import scala.concurrent.duration._

import FetchAssetPermissions.PermissionResponse

trait FetchAssetPermissions {
def getPermissionCode(
jwt: String,
assetInfo: AssetInfo,
): Task[Int]
}

class FetchAssetPermissionsLive(
sttp: SttpBackend[Task, ZioStreams],
apiConfig: Configuration.DspApiConfig,
) extends FetchAssetPermissions {
def getPermissionCode(
jwt: String,
assetInfo: AssetInfo,
): Task[Int] =
(for {
uri <-
ZIO.succeed(
uri"${apiConfig.url}/admin/files/${assetInfo.assetRef.belongsToProject}/${assetInfo.derivative.filename}",
)
response <- sttp.send(basicRequest.get(uri).header("Authorization", s"Bearer ${jwt}"))
successBody <- ZIO.fromEither(response.body).mapError(httpError(uri.toString, response.code.code, _))
permissionCode <-
ZIO.fromEither(successBody.fromJson[PermissionResponse].bimap(e => new Exception(e), _.permissionCode))
} yield permissionCode).tapError(e => ZIO.logError(s"FetchAssetPermissions failure: ${e.getMessage}"))

def httpError(uri: String, code: Int, body: String): Throwable =
Exception(s"FetchAssetPermissions: GET $uri returned $code and contents: $body")
}

object FetchAssetPermissions {
final case class PermissionResponse(permissionCode: Int)

implicit val decoder: JsonDecoder[PermissionResponse] = DeriveJsonDecoder.gen[PermissionResponse]

val layer =
HttpClientZioBackend.layer(SttpBackendOptions.connectionTimeout(5.seconds)).orDie >+>
ZLayer.derive[FetchAssetPermissionsLive]
}
1 change: 1 addition & 0 deletions src/main/scala/swiss/dasch/Main.scala
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ object Main extends ZIOAppDefault {
DbHealthIndicator.layer,
DbMigrator.layer,
Endpoints.layer,
FetchAssetPermissions.layer,
FileChecksumServiceLive.layer,
FileSystemHealthIndicatorLive.layer,
HealthCheckServiceLive.layer,
Expand Down
13 changes: 9 additions & 4 deletions src/main/scala/swiss/dasch/api/AuthService.scala
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,18 @@ final case class AuthServiceLive(jwtConfig: JwtConfig) extends AuthService {

def authenticate(jwtString: String): IO[NonEmptyChunk[AuthenticationError], Principal] =
if (jwtConfig.disableAuth) {
ZIO.succeed(Principal("developer", AuthScope(Set(AuthScope.ScopeValue.Admin))))
ZIO.succeed(Principal("developer", AuthScope(Set(AuthScope.ScopeValue.Admin)), "fake jwt claim"))
} else {
ZIO
.fromTry(JwtZIOJson.decode(jwtString, secret, alg))
.mapError(e => NonEmptyChunk(AuthenticationError.jwtProblem(e)))
.flatMap(verifyClaim)
.flatMap(verifyClaim(_, jwtString))
}

private def verifyClaim(claim: JwtClaim): IO[NonEmptyChunk[AuthenticationError], Principal] = {
private def verifyClaim(
claim: JwtClaim,
claimLiteral: String,
): IO[NonEmptyChunk[AuthenticationError], Principal] = {
val audVal = if (claim.audience.getOrElse(Set.empty).contains(audience)) { Validation.succeed(()) }
else { Validation.fail(AuthenticationError.invalidAudience(jwtConfig)) }

Expand All @@ -81,7 +84,9 @@ final case class AuthServiceLive(jwtConfig: JwtConfig) extends AuthService {
.mapError(AuthenticationError.invalidContents)

Validation
.validateWith(authScope, issVal, audVal, subVal)((authScope, _, _, subject) => Principal(subject, authScope))
.validateWith(authScope, issVal, audVal, subVal)((authScope, _, _, subject) =>
Principal(subject, authScope, claimLiteral),
)
.toZIOParallelErrors
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/scala/swiss/dasch/api/BaseEndpoints.scala
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ import sttp.tapir.statusCode
import sttp.tapir.ztapir.*
import swiss.dasch.api.ApiProblem.Unauthorized
import swiss.dasch.api.BaseEndpoints.defaultErrorOutputs
import zio.IO
import zio.ZLayer
import zio._

case class BaseEndpoints(authService: AuthService) {
val publicEndpoint: PublicEndpoint[Unit, ApiProblem, Unit, Any] = endpoint
Expand Down
1 change: 0 additions & 1 deletion src/main/scala/swiss/dasch/api/HandlerFunctions.scala
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import swiss.dasch.api.ApiProblem.{InternalServerError, NotFound}
import swiss.dasch.domain.{AssetRef, ProjectShortcode}

trait HandlerFunctions {

def projectNotFoundOrServerError(mayBeError: Option[Throwable], shortcode: ProjectShortcode): ApiProblem =
mayBeError.map(InternalServerError(_)).getOrElse(NotFound(shortcode))

Expand Down
1 change: 1 addition & 0 deletions src/main/scala/swiss/dasch/api/Principal.scala
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,5 @@ import swiss.dasch.domain.AuthScope
final case class Principal(
subject: String,
scope: AuthScope = AuthScope.Empty,
jwtRaw: String = "",
)
50 changes: 35 additions & 15 deletions src/main/scala/swiss/dasch/api/ProjectsEndpoints.scala
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,28 @@
package swiss.dasch.api

import sttp.capabilities.zio.ZioStreams
import sttp.model.{HeaderNames, StatusCode}
import sttp.model.HeaderNames
import sttp.model.StatusCode
import sttp.tapir.Codec
import sttp.tapir.CodecFormat
import sttp.tapir.EndpointInput
import sttp.tapir.codec.refined.*
import sttp.tapir.generic.auto.*
import sttp.tapir.json.zio.jsonBody
import sttp.tapir.ztapir.*
import sttp.tapir.{Codec, CodecFormat, EndpointInput}
import swiss.dasch.api.ProjectsEndpoints.shortcodePathVar
import swiss.dasch.api.ProjectsEndpointsResponses.{
AssetCheckResultResponse,
AssetInfoResponse,
ProjectResponse,
UploadResponse,
}
import swiss.dasch.domain.*
import swiss.dasch.api.ProjectsEndpoints.{shortcodePathVar, assetIdPathVar}
import swiss.dasch.api.ProjectsEndpointsResponses.AssetCheckResultResponse
import swiss.dasch.api.ProjectsEndpointsResponses.AssetInfoResponse
import swiss.dasch.api.ProjectsEndpointsResponses.ProjectResponse
import swiss.dasch.api.ProjectsEndpointsResponses.UploadResponse
import swiss.dasch.domain.AugmentedPath.ProjectFolder
import zio.json.{DeriveJsonCodec, JsonCodec}
import zio.schema.{DeriveSchema, Schema}
import zio.{Chunk, ZLayer}
import swiss.dasch.domain.*
import zio.Chunk
import zio.ZLayer
import zio.json.DeriveJsonCodec
import zio.json.JsonCodec
import zio.schema.DeriveSchema
import zio.schema.Schema

object ProjectsEndpointsResponses {
final case class ProjectResponse(id: String)
Expand Down Expand Up @@ -177,11 +181,22 @@ final case class ProjectsEndpoints(base: BaseEndpoints) {
)

val getProjectsAssetsInfo = base.secureEndpoint.get
.in(projects / shortcodePathVar / "assets" / path[AssetId]("assetId"))
.in(projects / shortcodePathVar / "assets" / assetIdPathVar)
.out(jsonBody[AssetInfoResponse])
.tag("assets")
.description("Authorization: read:project:1234 scope required.")

val getProjectsAssetsOriginal = base.secureEndpoint.get
.in(projects / shortcodePathVar / "assets" / assetIdPathVar / "original")
.out(header[String]("Content-Disposition"))
.out(header[String]("Content-Type"))
.out(streamBinaryBody(ZioStreams)(CodecFormat.OctetStream()))
.tag("assets")
.description(
"""|Offers the original file for upload, provided the API permisisons allow.
|Authorization: JWT bearer token.""".stripMargin,
)

given filenameCodec: Codec[String, AssetFilename, CodecFormat.TextPlain] =
Codec.string.mapEither(AssetFilename.from)(_.value)
val postProjectAsset = base.secureEndpoint.post
Expand Down Expand Up @@ -261,6 +276,7 @@ final case class ProjectsEndpoints(base: BaseEndpoints) {
getProjectsChecksumReport,
deleteProjectsErase,
getProjectsAssetsInfo,
getProjectsAssetsOriginal,
postProjectAsset,
postBulkIngest,
postBulkIngestFinalize,
Expand All @@ -272,11 +288,15 @@ final case class ProjectsEndpoints(base: BaseEndpoints) {
}

object ProjectsEndpoints {

val shortcodePathVar: EndpointInput.PathCapture[ProjectShortcode] = path[ProjectShortcode]
.name("shortcode")
.description("The shortcode of the project must be an exactly 4 characters long hexadecimal string.")
.example(ProjectShortcode.from("0001").getOrElse(throw Exception("Invalid shortcode.")))

val assetIdPathVar: EndpointInput.PathCapture[AssetId] = path[AssetId]
.name("assetId")
.description("The id of the asset")
.example(AssetId.from("5RMOnH7RmAY-qKzgr431bg7").getOrElse(throw Exception("Invalid AssetId.")))

val layer = ZLayer.derive[ProjectsEndpoints]
}
50 changes: 39 additions & 11 deletions src/main/scala/swiss/dasch/api/ProjectsEndpointsHandler.scala
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,28 @@ package swiss.dasch.api
import sttp.capabilities.zio.ZioStreams
import sttp.model.headers.ContentRange
import sttp.tapir.ztapir.ZServerEndpoint
import swiss.dasch.api.*
import swiss.dasch.FetchAssetPermissions
import swiss.dasch.api.ApiProblem.*
import swiss.dasch.api.ProjectsEndpointsResponses.{
AssetCheckResultResponse,
AssetInfoResponse,
ProjectResponse,
UploadResponse,
}
import swiss.dasch.api.ProjectsEndpointsResponses.AssetCheckResultResponse
import swiss.dasch.api.ProjectsEndpointsResponses.AssetInfoResponse
import swiss.dasch.api.ProjectsEndpointsResponses.ProjectResponse
import swiss.dasch.api.ProjectsEndpointsResponses.UploadResponse
import swiss.dasch.api.*
import swiss.dasch.config.Configuration.Features
import swiss.dasch.domain.BulkIngestError.BulkIngestInProgress
import swiss.dasch.domain.BulkIngestError.ImportFolderDoesNotExist
import swiss.dasch.domain.*
import swiss.dasch.domain.BulkIngestError.{BulkIngestInProgress, ImportFolderDoesNotExist}
import zio.stream.{ZSink, ZStream}
import zio.{ZIO, ZLayer, stream}
import zio.ZIO
import zio.ZLayer
import zio.*
import zio.nio.file.Files
import zio.stream
import zio.stream.ZSink
import zio.stream.ZStream

import java.io.IOException
import zio.nio.file.Files
import java.net.URLEncoder
import java.nio.charset.StandardCharsets

final case class ProjectsEndpointsHandler(
bulkIngestService: BulkIngestService,
Expand All @@ -36,6 +42,7 @@ final case class ProjectsEndpointsHandler(
assetInfoService: AssetInfoService,
authorizationHandler: AuthorizationHandler,
features: Features,
fetchAssetPermissions: FetchAssetPermissions,
) extends HandlerFunctions {

val getProjectsEndpoint: ZServerEndpoint[Any, Any] = projectEndpoints.getProjectsEndpoint
Expand Down Expand Up @@ -110,6 +117,26 @@ final case class ProjectsEndpointsHandler(
},
)

private val getProjectsAssetsOriginalEndpoint: ZServerEndpoint[Any, ZioStreams] =
projectEndpoints.getProjectsAssetsOriginal
.serverLogic(userSession =>
(shortcode, assetId) => {
for {
ref <- ZIO.succeed(AssetRef(assetId, shortcode))
assetInfo <- assetInfoService.findByAssetRef(ref).some.mapError(assetRefNotFoundOrServerError(_, ref))
filenameEncoded = URLEncoder.encode(assetInfo.originalFilename.value, StandardCharsets.UTF_8.toString)
permissionCode <- fetchAssetPermissions
.getPermissionCode(userSession.jwtRaw, assetInfo)
.mapError(_ => InternalServerError("error fetching permissions"))
_ <- ZIO.fail(Forbidden("permission denied")).unless(permissionCode >= 2)
} yield (
s"attachment; filename*=\"${filenameEncoded}\"",
assetInfo.metadata.originalMimeType.map(m => m.stringValue).getOrElse("application/octet-stream"),
ZStream.fromFile(assetInfo.original.file.toFile),
)
},
)

private val postProjectAssetEndpoint: ZServerEndpoint[Any, ZioStreams] = projectEndpoints.postProjectAsset
.serverLogic(principal => { case (shortcode, filename, stream) =>
authorizationHandler.ensureProjectWritable(principal, shortcode) *>
Expand Down Expand Up @@ -226,6 +253,7 @@ final case class ProjectsEndpointsHandler(
getProjectChecksumReportEndpoint,
deleteProjectsEraseEndpoint,
getProjectsAssetsInfoEndpoint,
getProjectsAssetsOriginalEndpoint,
postProjectAssetEndpoint,
postBulkIngestEndpoint,
postBulkIngestEndpointFinalize,
Expand Down
9 changes: 8 additions & 1 deletion src/main/scala/swiss/dasch/config/Configuration.scala
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ object Configuration {
ingest: IngestConfig,
features: Features,
db: DbConfig,
dspApi: DspApiConfig,
)

final case class JwtConfig(
Expand Down Expand Up @@ -50,6 +51,10 @@ object Configuration {

final case class DbConfig(jdbcUrl: String)

final case class DspApiConfig(
url: String,
)

private val configDescriptor = deriveConfig[ApplicationConf].mapKey(toKebabCase)

private type AllConfigs = ServiceConfig
Expand All @@ -59,6 +64,7 @@ object Configuration {
with IngestConfig
with Features
with DbConfig
with DspApiConfig

val layer: ZLayer[Any, Config.Error, AllConfigs] = {
val applicationConf = ZLayer.fromZIO(
Expand All @@ -71,6 +77,7 @@ object Configuration {
applicationConf.project(_.jwt) ++
applicationConf.project(_.sipi) ++
applicationConf.project(_.ingest) ++
applicationConf.project(_.features)
applicationConf.project(_.features) ++
applicationConf.project(_.dspApi)
}
}
Loading

0 comments on commit 2c0b3bf

Please sign in to comment.