From 0b624385e60c7c874970c289aeade4d4bad09cf8 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 20 Aug 2024 07:12:02 +0800 Subject: [PATCH] Add lightweight syntax for `os.proc().call()` and `os.proc().spawn()` (#292) Now these can be spelled `os.call()` and `os.spawn()`, and we provide `Shellable[TupleN]` conversions to make it convenient to call without constructing a `Seq(...)` every time. So this: ```scala os.proc("ls", "doesnt-exist").call(cwd = wd, check = false, stderr = os.Pipe) ``` Becomes ```scala os.call(cmd = ("ls", "doesnt-exist"), cwd = wd, check = false, stderr = os.Pipe) ``` The original purpose of the `os.proc().call()` style was to avoid the verbosity of constructing a `Seq` each time, and by making it flexible enough to take tuples, this mitigates that issue without the annoying method chaining style. The new style still isn't actually shorter in terms of number of characters, but it is a lot cleaner in terms of "function call taking named/optional arguments" rather than "fluent call chain with first call taking varargs and second call taking named/optional parameters". It also aligns with the Python `subprocess.*` functions which OS-Lib in general is inspired by To support Scala 2, the `Shellable[TupleN]` conversions are defined using codegen. Scala 3 allows a nicer generic-tuple implementation, but we'll be supporting Scala 2 for the foreseeable future. The older `os.proc.*` APIs remain, both for backwards compatibility, as well as to support the `pipeTo` API used to construct process pipelines Duplicated some of the existing subprocess tests to exercise the new APIs. Did not duplicate all of them, as the new APIs are pretty dumb forwarders to the existing ones so we don't need to exercise every flag in detail. Updated the docs to point towards the new APIs, but with a mention that the older `os.proc().call()` style is still supported --- Readme.adoc | 57 +++--- build.sc | 25 +++ os/src/Model.scala | 4 +- os/src/ProcessOps.scala | 64 ++++++- .../SpawningSubprocessesNewTests.scala | 175 ++++++++++++++++++ 5 files changed, 298 insertions(+), 27 deletions(-) create mode 100644 os/test/src-jvm/SpawningSubprocessesNewTests.scala diff --git a/Readme.adoc b/Readme.adoc index c777ebfc..ac7ddf17 100644 --- a/Readme.adoc +++ b/Readme.adoc @@ -1424,8 +1424,9 @@ os.owner.set(wd / "File.txt", originalOwner) === Spawning Subprocesses -Subprocess are spawned using `+os.proc(command: os.Shellable*).foo(...)+` calls, -where the `command: Shellable*` sets up the basic command you wish to run and +Subprocess are spawned using `+os.call(cmd: os.Shellable, ...)+` or +`+os.spawn(cmd: os.Shellable, ...)+` calls, +where the `cmd: Shellable` sets up the basic command you wish to run and `+.foo(...)+` specifies how you want to run it. `os.Shellable` represents a value that can make up part of your subprocess command, and the following values can be used as ``os.Shellable``s: @@ -1436,6 +1437,7 @@ be used as ``os.Shellable``s: * `os.RelPath` * `T: Numeric` * ``Iterable[T]``s of any of the above +* ``TupleN[T1, T2, ...Tn]``s of any of the above Most of the subprocess commands also let you redirect the subprocess's `stdin`/`stdout`/`stderr` streams via `os.ProcessInput` or `os.ProcessOutput` @@ -1467,12 +1469,12 @@ Often, if you are only interested in capturing the standard output of the subprocess but want any errors sent to the console, you might set `stderr = os.Inherit` while leaving `stdout = os.Pipe`. -==== `os.proc.call` +==== `os.call` [source,scala] ---- -os.proc(command: os.Shellable*) - .call(cwd: Path = null, +os.call(cmd: os.Shellable, + cwd: Path = null, env: Map[String, String] = null, stdin: ProcessInput = Pipe, stdout: ProcessOutput = Pipe, @@ -1483,6 +1485,8 @@ os.proc(command: os.Shellable*) propagateEnv: Boolean = true): os.CommandResult ---- +_Also callable via `os.proc(cmd).call(...)`_ + Invokes the given subprocess like a function, passing in input and returning a `CommandResult`. You can then call `result.exitCode` to see how it exited, or `result.out.bytes` or `result.err.string` to access the aggregated stdout and @@ -1508,7 +1512,7 @@ Note that redirecting `stdout`/`stderr` elsewhere means that the respective [source,scala] ---- -val res = os.proc('ls, wd/"folder2").call() +val res = os.call(cmd = ('ls, wd/"folder2")) res.exitCode ==> 0 @@ -1531,13 +1535,13 @@ res.out.bytes // Non-zero exit codes throw an exception by default val thrown = intercept[os.SubprocessException]{ - os.proc('ls, "doesnt-exist").call(cwd = wd) + os.call(cmd = ('ls, "doesnt-exist"), cwd = wd) } assert(thrown.result.exitCode != 0) // Though you can avoid throwing by setting `check = false` -val fail = os.proc('ls, "doesnt-exist").call(cwd = wd, check = false) +val fail = os.call(cmd = ('ls, "doesnt-exist"), cwd = wd, check = false) assert(fail.exitCode != 0) @@ -1547,11 +1551,11 @@ fail.out.text() ==> "" assert(fail.err.text().contains("No such file or directory")) // You can pass in data to a subprocess' stdin -val hash = os.proc("shasum", "-a", "256").call(stdin = "Hello World") +val hash = os.call(cmd = ("shasum", "-a", "256"), stdin = "Hello World") hash.out.trim() ==> "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e -" // Taking input from a file and directing output to another file -os.proc("base64").call(stdin = wd / "File.txt", stdout = wd / "File.txt.b64") +os.call(cmd = ("base64"), stdin = wd / "File.txt", stdout = wd / "File.txt.b64") os.read(wd / "File.txt.b64") ==> "SSBhbSBjb3c=" ---- @@ -1570,7 +1574,8 @@ of `os.proc.call` in a streaming fashion, either on groups of bytes: [source,scala] ---- var lineCount = 1 -os.proc('find, ".").call( +os.call( + cmd = ('find, "."), cwd = wd, stdout = os.ProcessOutput( (buf, len) => lineCount += buf.slice(0, len).count(_ == '\n') @@ -1584,7 +1589,8 @@ Or on lines of output: ---- lineCount ==> 22 var lineCount = 1 -os.proc('find, ".").call( +os.call( + cmd = ('find, "."), cwd = wd, stdout = os.ProcessOutput.Readlines( line => lineCount += 1 @@ -1593,12 +1599,12 @@ os.proc('find, ".").call( lineCount ==> 22 ---- -==== `os.proc.spawn` +==== `os.spawn` [source,scala] ---- -os.proc(command: os.Shellable*) - .spawn(cwd: Path = null, +os.spawn(cmd: os.Shellable, + cwd: Path = null, env: Map[String, String] = null, stdin: os.ProcessInput = os.Pipe, stdout: os.ProcessOutput = os.Pipe, @@ -1607,7 +1613,9 @@ os.proc(command: os.Shellable*) propagateEnv: Boolean = true): os.SubProcess ---- -The most flexible of the `os.proc` calls, `os.proc.spawn` simply configures and +_Also callable via `os.proc(cmd).spawn(...)`_ + +The most flexible of the `os.proc` calls, `os.spawn` simply configures and starts a subprocess, and returns it as a `os.SubProcess`. `os.SubProcess` is a simple wrapper around `java.lang.Process`, which provides `stdin`, `stdout`, and `stderr` streams for you to interact with however you like. e.g. You can sending @@ -1619,10 +1627,7 @@ as the stdin of a second spawned process. Note that if you provide `ProcessOutput` callbacks to `stdout`/`stderr`, the calls to those callbacks take place on newly spawned threads that execute in parallel with the main thread. Thus make sure any data processing you do in -those callbacks is thread safe! For simpler cases, it may be easier to use -`os.proc.stream` which triggers it's `onOut`/`onErr` callbacks -all on the calling thread, avoiding needing to think about multithreading and -concurrency issues. +those callbacks is thread safe! `stdin`, `stdout` and `stderr` are ``java.lang.OutputStream``s and ``java.lang.InputStream``s enhanced with the `.writeLine(s: String)`/`.readLine()` @@ -1631,8 +1636,10 @@ methods for easy reading and writing of character and line-based data. [source,scala] ---- // Start a long-lived python process which you can communicate with -val sub = os.proc("python", "-u", "-c", "while True: print(eval(raw_input()))") - .spawn(cwd = wd) +val sub = os.spawn( + cmd = ("python", "-u", "-c", "while True: print(eval(raw_input()))"), + cwd = wd +) // Sending some text to the subprocess sub.stdin.write("1 + 2") @@ -1654,9 +1661,9 @@ sub.stdout.read() ==> '8'.toByte sub.destroy() // You can chain multiple subprocess' stdin/stdout together -val curl = os.proc("curl", "-L" , "https://git.io/fpfTs").spawn(stderr = os.Inherit) -val gzip = os.proc("gzip", "-n").spawn(stdin = curl.stdout) -val sha = os.proc("shasum", "-a", "256").spawn(stdin = gzip.stdout) +val curl = os.spawn(cmd = ("curl", "-L" , "https://git.io/fpfTs"), stderr = os.Inherit) +val gzip = os.spawn(cmd = ("gzip", "-n"), stdin = curl.stdout) +val sha = os.spawn(cmd = ("shasum", "-a", "256"), stdin = gzip.stdout) sha.stdout.trim ==> "acc142175fa520a1cb2be5b97cbbe9bea092e8bba3fe2e95afa645615908229e -" ---- diff --git a/build.sc b/build.sc index b619614e..dd40a544 100644 --- a/build.sc +++ b/build.sc @@ -112,6 +112,31 @@ trait OsModule extends OsLibModule { outer => def scalaDocOptions = super.scalaDocOptions() ++ conditionalScalaDocOptions() + def generatedSources = T{ + val conversions = for(i <- Range.inclusive(2, 22)) yield { + val ts = Range.inclusive(1, i).map(n => s"T$n").mkString(", ") + val fs = Range.inclusive(1, i).map(n => s"f$n: T$n => R").mkString(", ") + val vs = Range.inclusive(1, i).map(n => s"f$n(t._$n)").mkString(", ") + s""" implicit def tuple${i}Conversion[$ts] + | (t: ($ts)) + | (implicit $fs): R = { + | this.flatten($vs) + | } + |""".stripMargin + } + _root_.os.write( + T.dest / "os" / "GeneratedTupleConversions.scala", + s"""package os + |trait GeneratedTupleConversions[R]{ + | protected def flatten(vs: R*): R + | ${conversions.mkString("\n")} + |} + | + |""".stripMargin, + createFolders = true + ) + Seq(PathRef(T.dest)) + } } object os extends Module { diff --git a/os/src/Model.scala b/os/src/Model.scala index 2d9fb45e..f2389407 100644 --- a/os/src/Model.scala +++ b/os/src/Model.scala @@ -215,7 +215,7 @@ case class SubprocessException(result: CommandResult) extends Exception(result.t * be "interpolated" directly into a subprocess call. */ case class Shellable(value: Seq[String]) -object Shellable { +object Shellable extends os.GeneratedTupleConversions[Shellable] { implicit def StringShellable(s: String): Shellable = Shellable(Seq(s)) implicit def CharSequenceShellable(cs: CharSequence): Shellable = Shellable(Seq(cs.toString)) @@ -232,6 +232,8 @@ object Shellable { implicit def ArrayShellable[T](s: Array[T])(implicit f: T => Shellable): Shellable = Shellable(s.toIndexedSeq.flatMap(f(_).value)) + + protected def flatten(vs: Shellable*): Shellable = IterableShellable(vs) } /** diff --git a/os/src/ProcessOps.scala b/os/src/ProcessOps.scala index fa8cc973..389b34f0 100644 --- a/os/src/ProcessOps.scala +++ b/os/src/ProcessOps.scala @@ -10,6 +10,69 @@ import java.util.concurrent.LinkedBlockingQueue import ProcessOps._ import scala.util.Try +object call { + + /** + * @see [[os.proc.call]] + */ + def apply( + cmd: Shellable, + env: Map[String, String] = null, + // Make sure `cwd` only comes after `env`, so `os.call("foo", path)` is a compile error + // since the correct syntax is `os.call(("foo", path))` + cwd: Path = null, + stdin: ProcessInput = Pipe, + stdout: ProcessOutput = Pipe, + stderr: ProcessOutput = os.Inherit, + mergeErrIntoOut: Boolean = false, + timeout: Long = -1, + check: Boolean = true, + propagateEnv: Boolean = true, + timeoutGracePeriod: Long = 100 + ): CommandResult = { + os.proc(cmd).call( + cwd = cwd, + env = env, + stdin = stdin, + stdout = stdout, + stderr = stderr, + mergeErrIntoOut = mergeErrIntoOut, + timeout = timeout, + check = check, + propagateEnv = propagateEnv, + timeoutGracePeriod = timeoutGracePeriod + ) + } +} +object spawn { + + /** + * @see [[os.proc.spawn]] + */ + def apply( + cmd: Shellable, + // Make sure `cwd` only comes after `env`, so `os.spawn("foo", path)` is a compile error + // since the correct syntax is `os.spawn(("foo", path))` + env: Map[String, String] = null, + cwd: Path = null, + stdin: ProcessInput = Pipe, + stdout: ProcessOutput = Pipe, + stderr: ProcessOutput = os.Inherit, + mergeErrIntoOut: Boolean = false, + propagateEnv: Boolean = true + ): SubProcess = { + os.proc(cmd).spawn( + cwd = cwd, + env = env, + stdin = stdin, + stdout = stdout, + stderr = stderr, + mergeErrIntoOut = mergeErrIntoOut, + propagateEnv = propagateEnv + ) + } +} + /** * Convenience APIs around [[java.lang.Process]] and [[java.lang.ProcessBuilder]]: * @@ -27,7 +90,6 @@ import scala.util.Try * the standard stdin/stdout/stderr streams, using whatever protocol you * want */ - case class proc(command: Shellable*) { def commandChunks: Seq[String] = command.flatMap(_.value) diff --git a/os/test/src-jvm/SpawningSubprocessesNewTests.scala b/os/test/src-jvm/SpawningSubprocessesNewTests.scala new file mode 100644 index 00000000..b753d1e6 --- /dev/null +++ b/os/test/src-jvm/SpawningSubprocessesNewTests.scala @@ -0,0 +1,175 @@ +package test.os + +import java.io.{BufferedReader, InputStreamReader} +import os.ProcessOutput + +import scala.collection.mutable + +import test.os.TestUtil.prep +import utest._ + +object SpawningSubprocessesNewTests extends TestSuite { + + def tests = Tests { + test("proc") { + test("call") { + test - prep { wd => + if (Unix()) { + val res = os.call(cmd = ("ls", wd / "folder2")) + + res.exitCode ==> 0 + + res.out.text() ==> + """nestedA + |nestedB + |""".stripMargin + + res.out.trim() ==> + """nestedA + |nestedB""".stripMargin + + res.out.lines() ==> Seq( + "nestedA", + "nestedB" + ) + + res.out.bytes + + val thrown = intercept[os.SubprocessException] { + os.call(cmd = ("ls", "doesnt-exist"), cwd = wd) + } + + assert(thrown.result.exitCode != 0) + + val fail = + os.call(cmd = ("ls", "doesnt-exist"), cwd = wd, check = false, stderr = os.Pipe) + + assert(fail.exitCode != 0) + + fail.out.text() ==> "" + + assert(fail.err.text().contains("No such file or directory")) + + // You can pass in data to a subprocess' stdin + val hash = os.call(cmd = ("shasum", "-a", "256"), stdin = "Hello World") + hash.out.trim() ==> "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e -" + + // Taking input from a file and directing output to another file + os.call(cmd = ("base64"), stdin = wd / "File.txt", stdout = wd / "File.txt.b64") + + os.read(wd / "File.txt.b64") ==> "SSBhbSBjb3c=\n" + + if (false) { + os.call(cmd = ("vim"), stdin = os.Inherit, stdout = os.Inherit, stderr = os.Inherit) + } + } + } + test - prep { wd => + if (Unix()) { + val ex = intercept[os.SubprocessException] { + os.call(cmd = ("bash", "-c", "echo 123; sleep 10; echo 456"), timeout = 2000) + } + + ex.result.out.trim() ==> "123" + } + } + } + test("stream") { + test - prep { wd => + if (Unix()) { + var lineCount = 1 + os.call( + cmd = ("find", "."), + cwd = wd, + stdout = + os.ProcessOutput((buf, len) => lineCount += buf.slice(0, len).count(_ == '\n')) + ) + lineCount ==> 22 + } + } + test - prep { wd => + if (Unix()) { + var lineCount = 1 + os.call( + cmd = ("find", "."), + cwd = wd, + stdout = os.ProcessOutput.Readlines(line => lineCount += 1) + ) + lineCount ==> 22 + } + } + } + + test("spawn python") { + test - prep { wd => + if (TestUtil.isInstalled("python") && Unix()) { + // Start a long-lived python process which you can communicate with + val sub = os.spawn( + cmd = ( + "python", + "-u", + "-c", + if (TestUtil.isPython3()) "while True: print(eval(input()))" + else "while True: print(eval(raw_input()))" + ), + cwd = wd + ) + + // Sending some text to the subprocess + sub.stdin.write("1 + 2") + sub.stdin.writeLine("+ 4") + sub.stdin.flush() + sub.stdout.readLine() ==> "7" + + sub.stdin.write("'1' + '2'") + sub.stdin.writeLine("+ '4'") + sub.stdin.flush() + sub.stdout.readLine() ==> "124" + + // Sending some bytes to the subprocess + sub.stdin.write("1 * 2".getBytes) + sub.stdin.write("* 4\n".getBytes) + sub.stdin.flush() + sub.stdout.read() ==> '8'.toByte + + sub.destroy() + } + } + } + test("spawn curl") { + if ( + Unix() && // shasum seems to not accept stdin on Windows + TestUtil.isInstalled("curl") && + TestUtil.isInstalled("gzip") && + TestUtil.isInstalled("shasum") + ) { + // You can chain multiple subprocess' stdin/stdout together + val curl = + os.spawn(cmd = ("curl", "-L", ExampleResourcess.RemoteReadme.url), stderr = os.Inherit) + val gzip = os.spawn(cmd = ("gzip", "-n", "-6"), stdin = curl.stdout) + val sha = os.spawn(cmd = ("shasum", "-a", "256"), stdin = gzip.stdout) + sha.stdout.trim() ==> s"${ExampleResourcess.RemoteReadme.gzip6ShaSum256} -" + } + } + test("spawn callback") { + test - prep { wd => + if (TestUtil.isInstalled("echo") && Unix()) { + val output: mutable.Buffer[String] = mutable.Buffer() + val sub = os.spawn( + cmd = ("echo", "output"), + stdout = ProcessOutput((bytes, count) => output += new String(bytes, 0, count)) + ) + val finished = sub.join(5000) + sub.wrapped.getOutputStream().flush() + assert(finished) + assert(sub.exitCode() == 0) + val expectedOutput = "output\n" + val actualOutput = output.mkString("") + assert(actualOutput == expectedOutput) + sub.destroy() + } + } + } + } + } +}