Skip to content

Commit

Permalink
support fetching versioned s3 files (#297)
Browse files Browse the repository at this point in the history
* support fetching versioned s3 files

* bump travis
  • Loading branch information
barryoneill authored May 5, 2020
1 parent f4ecde4 commit 0dba143
Show file tree
Hide file tree
Showing 5 changed files with 41 additions and 1 deletion.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,3 +142,4 @@ val result = program
result should be(...)
```
**TODO:** Stream send SQS messages

18 changes: 18 additions & 0 deletions fs2-aws/src/main/scala/fs2/aws/s3/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,24 @@ package object s3 {
closeAfterUse = true
)

def readS3VersionedFile[F[_]](
bucket: String,
key: String,
version: String,
blockingEC: ExecutionContext,
amazonS3: AmazonS3 = AmazonS3ClientBuilder.defaultClient()
)(
implicit F: Effect[F],
cs: ContextShift[F],
s3Client: S3Client[F] = S3Client[F](amazonS3)
): Stream[F, Byte] =
readInputStream[F](
s3Client.getObjectContent(new GetObjectRequest(bucket, key, version)),
chunkSize = 8192,
blocker = Blocker.liftExecutionContext(blockingEC),
closeAfterUse = true
)

def uploadS3FileMultipart[F[_]](
bucket: String,
key: String,
Expand Down
4 changes: 4 additions & 0 deletions fs2-aws/src/test/resources/jsontest.json_vABC
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{"this": 1}
{"is": 2}
{"versioned": 3}
{"content": 4}
13 changes: 13 additions & 0 deletions fs2-aws/src/test/scala/fs2/aws/S3Spec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,19 @@ class S3Spec extends AnyFlatSpec with Matchers {
)
}

"Downloading the versioned JSON test file" should "return the same content" in {
readS3VersionedFile[IO]("resources", "jsontest.json", version = "ABC", blockingEC, mockS3)
.through(fs2.text.utf8Decode)
.through(fs2.text.lines)
.compile
.toVector
.unsafeRunSync
.reduce(_ + _)
.concat("") should be(
"""{"this": 1}{"is": 2}{"versioned": 3}{"content": 4}"""
)
}

"big chunk size but small entire text" should "be trimmed to content" in {
readS3FileMultipart[IO]("resources", "jsontest1.json", 25, mockS3)
.through(fs2.text.utf8Decode)
Expand Down
6 changes: 5 additions & 1 deletion fs2-aws/src/test/scala/fs2/aws/utils/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,11 @@ package object utils {
IO[ByteArrayInputStream] {
val fileContent: Array[Byte] =
try {
Source.fromResource(getObjectRequest.getKey).mkString.getBytes
val testS3Resource = Option(getObjectRequest.getVersionId) match {
case Some(version) => s"${getObjectRequest.getKey}_v$version"
case None => getObjectRequest.getKey
}
Source.fromResource(testS3Resource).mkString.getBytes
} catch {
case _: FileNotFoundException => throw new AmazonS3Exception("File not found")
case e: Throwable => throw e
Expand Down

0 comments on commit 0dba143

Please sign in to comment.