Skip to content

Commit

Permalink
Propagate content length from filesystem through geny.Writable and …
Browse files Browse the repository at this point in the history
…`os.Source` (#320)

This means downstream libraries like Requests-Scala or Cask can properly
set the content length when handling `os.read.stream`s and similar
values in their HTTP requests or responses

Noticed the lack of this when uploading to github failed in
com-lihaoyi/mill#3686 due to the lack of content
length header in the upload

Covered by unit tests
  • Loading branch information
lihaoyi authored Oct 8, 2024
1 parent 07166c5 commit 9b08cb0
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 2 deletions.
6 changes: 4 additions & 2 deletions os/src/Path.scala
Original file line number Diff line number Diff line change
Expand Up @@ -534,8 +534,10 @@ trait ReadablePath {
*/
class Path private[os] (val wrapped: java.nio.file.Path)
extends FilePath with ReadablePath with BasePathImpl {
def toSource: SeekableSource =
new SeekableSource.ChannelSource(java.nio.file.Files.newByteChannel(wrapped))
def toSource: SeekableSource = new SeekableSource.ChannelLengthSource(
java.nio.file.Files.newByteChannel(wrapped),
java.nio.file.Files.size(wrapped)
)

require(wrapped.isAbsolute || Path.driveRelative(wrapped), s"$wrapped is not an absolute path")
def root = Option(wrapped.getRoot).map(_.toString).getOrElse("")
Expand Down
1 change: 1 addition & 0 deletions os/src/ReadWriteOps.scala
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ object read extends Function1[ReadablePath, String] {

object stream extends Function1[ReadablePath, geny.Readable] {
def apply(p: ReadablePath): geny.Readable = new geny.Readable {
override def contentLength: Option[Long] = p.toSource.contentLength
def readBytesThrough[T](f: java.io.InputStream => T): T = {
val is = p.getInputStream
try f(is)
Expand Down
7 changes: 7 additions & 0 deletions os/src/Source.scala
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ object Source extends WritableLowPri {

implicit class WritableSource[T](s: T)(implicit f: T => geny.Writable) extends Source {
val writable = f(s)

override def contentLength: Option[Long] = writable.contentLength
def getHandle() = Left(writable)
}
}
Expand Down Expand Up @@ -115,4 +117,9 @@ object SeekableSource {
implicit class ChannelSource(cn: SeekableByteChannel) extends SeekableSource {
def getHandle() = Right(cn)
}
class ChannelLengthSource(cn: SeekableByteChannel, length: Long) extends SeekableSource {
def getHandle() = Right(cn)

override def contentLength: Option[Long] = Some(length)
}
}
21 changes: 21 additions & 0 deletions os/test/src/SourceTests.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package test.os
import utest.{assert => _, _}

object SourceTests extends TestSuite {

val tests = Tests {
test("contentMetadata") - TestUtil.prep { wd =>
// content type for all files is just treated as application/octet-stream,
// we do not do any clever mime-type inference or guessing
(wd / "folder1/one.txt").toSource.httpContentType ==> Some("application/octet-stream")
// length is taken from the filesystem at the moment at which `.toSource` is called
(wd / "folder1/one.txt").toSource.contentLength ==> Some(22)
(wd / "File.txt").toSource.contentLength ==> Some(8)

// Make sure the `Writable` returned by `os.read.stream` propagates the content length
os.read.stream(wd / "folder1/one.txt").contentLength ==> Some(22)
// Even when converted to an `os.Source`
(os.read.stream(wd / "folder1/one.txt"): os.Source).contentLength ==> Some(22)
}
}
}

0 comments on commit 9b08cb0

Please sign in to comment.