Delimiters in outputOf, map, reduce, etc. for #20

Package.swift

@@ -41,11 +41,17 @@ let package = Package(
         "Script"
       ]
     ),
+    .testTarget(
+      name: "ScriptTests",
+      dependencies: ["Script"]
+    ),
     .testTarget(
       name: "ShwiftTests",
       dependencies: ["Shwift"],
       resources: [
-        .copy("Cat.txt")
+        .copy("Cat.txt"),
+        .copy("Chunks.txt"),
+        .copy("Chunks.bin"),
       ]),
   ]
 )

Sources/Script/List Comprehensions.swift

@@ -1,52 +1,91 @@
 import Shwift
 
 /**
- By default, shell output is processed as a list of lines
- */
+ Converts shell input stream to output stream via transform.
 
-public func map(transform: @Sendable @escaping (String) async throws -> String)
-  -> Shell.PipableCommand<Void>
-{
-  compactMap(transform: transform)
+ By default, input is segmented by newline, and a newline is printed at the end of each output segment.
+ - Parameters:
+ - delimiter: Character to split input into segments (defaults to newline)
+ - terminator: String printed at the end of each output  (defaults to newline)
+ - transform: converts String to String
+ - Returns: ``Shell/PipableCommand``
+ */
+public func map(
+  segmentingInputAt delimiter: Character = Builtin.Input.Lines.eol,
+  withOutputTerminator terminator: String = Builtin.Input.Lines.eolStr,
+  transform: @Sendable @escaping (String) async throws -> String
+) -> Shell.PipableCommand<Void> {
+  compactMap(segmentingInputAt: delimiter, withOutputTerminator: terminator, transform: transform)
 }
 
-public func compactMap(transform: @Sendable @escaping (String) async throws -> String?)
-  -> Shell.PipableCommand<Void>
-{
+/**
+ Converts shell input stream to output stream via transform, ignoring nil input
+
+ By default, input is segmented by newline, and a newline is printed at the end of each output segment.
+ - Parameters:
+ - delimiter: Character to split input into segments (defaults to newline)
+ - terminator: String printed at the end of each output  (defaults to newline)
+ - transform: converts String to String
+ - Returns: ``Shell/PipableCommand``
+ */
+public func compactMap(
+  segmentingInputAt delimiter: Character = Builtin.Input.Lines.eol,
+  withOutputTerminator terminator: String = Builtin.Input.Lines.eolStr,
+  transform: @Sendable @escaping (String) async throws -> String?
+) -> Shell.PipableCommand<Void> {
   Shell.PipableCommand {
     try await Shell.invoke { shell, invocation in
       try await invocation.builtin { channel in
-        for try await line in channel.input.lines.compactMap(transform) {
+        for try await line in channel.input.segmented(by: delimiter).compactMap(transform) {
           try await channel.output.withTextOutputStream { stream in
-            print(line, to: &stream)
+            print(line, terminator: terminator, to: &stream)
           }
         }
       }
     }
   }
 }
 
+/**
+ Returns the result of processing input elements using the given closure and mutable initial value to accumulate the result.
+ - Parameters:
+   - initialResult: mutable initial value to accumulate the result
+   - delimiter: Character to split input into segments passed to closure (defaults to newline)
+   - updateAccumulatingResult: A closure over the mutable accumulator and the next String.
+ - Returns: ``Shell/PipableCommand``
+*/
 public func reduce<T>(
   into initialResult: T,
+  segmentingInputAt delimiter: Character = Builtin.Input.Lines.eol,
   _ updateAccumulatingResult: @escaping (inout T, String) async throws -> Void
 ) -> Shell.PipableCommand<T> {
   Shell.PipableCommand {
     try await Shell.invoke { _, invocation in
       try await invocation.builtin { channel in
-        try await channel.input.lines.reduce(into: initialResult, updateAccumulatingResult)
+        try await channel.input.segmented(by: delimiter)
+          .reduce(into: initialResult, updateAccumulatingResult)
       }
     }
   }
 }
 
+/**
+ Returns the result of processing input elements using the given closure and mutable initial value to accumulate the result.
+ - Parameters:
+ - initialResult: initial value passed to initial closure
+ - delimiter: Character to split input into segments passed to closure (defaults to newline)
+ - nextPartialResult: A closure over the initial or previous value and the next String segment that returns the next value
+ - Returns: ``Shell/PipableCommand``
+ */
 public func reduce<T>(
   _ initialResult: T,
+  segmentingInputAt delimiter: Character = Builtin.Input.Lines.eol,
   _ nextPartialResult: @escaping (T, String) async throws -> T
 ) -> Shell.PipableCommand<T> {
   Shell.PipableCommand {
     try await Shell.invoke { _, invocation in
       try await invocation.builtin { channel in
-        try await channel.input.lines.reduce(initialResult, nextPartialResult)
+        try await channel.input.segmented(by: delimiter).reduce(initialResult, nextPartialResult)
       }
     }
   }

Sources/Script/Output Capture.swift

@@ -1,4 +1,25 @@
-public func outputOf(_ operation: @escaping () async throws -> Void) async throws -> String {
-  let lines = try await Shell.PipableCommand(operation) | reduce(into: []) { $0.append($1) }
-  return lines.joined(separator: "\n")
+import Shwift
+
+/**
+ Captures operation output as text.
+
+ By default, input is processed per line (delimited by newline),
+ and the result String contains each output line separated by newlines (but without an terminating newline).
+
+ - Parameters:
+ - delimiter: Character to split input into segments (defaults to newline)
+   - separator: String used to join output items into a single String (defaults to newlin)
+   - operation: closure writing to output stream to capture
+ - Throws: rethrows errors thrown by underlying operations
+ - Returns: String of output items delimited by separator
+ */
+public func outputOf(
+  segmentingInputAt delimiter: Character = Builtin.Input.Lines.eol,
+  withOutputSeparator separator: String = Builtin.Input.Lines.eolStr,
+  _ operation: @escaping () async throws -> Void
+) async throws -> String {
+  let lines =
+    try await Shell.PipableCommand(operation)
+    | reduce(into: [], segmentingInputAt: delimiter) { $0.append($1) }
+  return lines.joined(separator: separator)
 }

Sources/ScriptExample/Main.swift

@@ -38,6 +38,10 @@ import Script
         let numberOfLines =
           try await echo("Foo", "Bar") | reduce(into: 0, { count, _ in count += 1 })
         print(numberOfLines)
+        let totals =
+          try await echo("-n", "1", "2", "3")
+            | reduce(4, segmentingInputAt: " ") {$0 + (Int($1) ?? -10)}
+        print("10 == \(totals) from 4 + 1 + 2 + 3")
       case .writeToFile:
         try await echo("Foo") > "Test.txt"
       case .infiniteInput:

Sources/Shwift/Builtins.swift

@@ -87,6 +87,8 @@ extension Builtin {
   public struct Input {
 
     public struct Lines: AsyncSequence {
+      public static let eol: Character = "\n"
+      public static let eolStr = "\n"
       public typealias Element = String
 
       public struct AsyncIterator: AsyncIteratorProtocol {
@@ -105,13 +107,13 @@ extension Builtin {
                   at: buffer.readerIndex,
                   length: buffer.readableBytes)!
                 var substring = readString[readString.startIndex...]
-                while let lineBreak = substring.firstIndex(of: "\n") {
+                while let lineBreak = substring.firstIndex(of: delimiter) {
                   let line = substring[substring.startIndex..<lineBreak]
                   substring = substring[substring.index(after: lineBreak)...]
                   continuation.yield(remainder + String(line))
                   remainder = ""
                 }
-                remainder = String(substring)
+                remainder += String(substring)
               }
               if !remainder.isEmpty {
                 continuation.yield(String(remainder))
@@ -126,9 +128,20 @@ extension Builtin {
       }
 
       fileprivate let byteBuffers: ByteBuffers
+      fileprivate let delimiter: Character
     }
+
+    /// Make a Lines iterator splitting at newlines
     public var lines: Lines {
-      Lines(byteBuffers: byteBuffers)
+      segmented()
+    }
+
+    /// Make a Lines iterator yielding text segments between delimiters (like split).
+    ///
+    /// - Parameter delimiter: Character separating input text to yield (and not itself yielded)  Defaults to newline.
+    /// - Returns: Lines segmented by delimiter
+    public func segmented(by delimiter: Character = Lines.eol) -> Lines {
+      Lines(byteBuffers: byteBuffers, delimiter: delimiter)
     }
 
     typealias ByteBuffers = AsyncCompactMapSequence<

Tests/ScriptTests/Script Tests.swift

@@ -0,0 +1,173 @@
+import XCTest
+import Script
+
+final class ScriptCoreTests: XCTestCase {
+
+  func testEcho() async throws {
+    try runInScript {
+      var result: String
+      result = try await outputOf {
+        try await echo("1", "2")
+      }
+      // fyi, trailing newline stripped by outputOf
+      XCTAssertEqual("1 2", result, "echo 1 2")
+
+      result = try await outputOf {
+        try await echo("1", "2", separator: ",", terminator: ";")
+      }
+      XCTAssertEqual("1,2;", result, "echo 1,2;")
+    }
+  }
+
+  func testEchoMap() async throws {
+    try runInScript {
+      var result: String
+      result = try await outputOf {
+        // inject newline so map op runs on head then tail
+        try await echo("1", "2", "\n", "3", "4") | map(){"<\($0)>"}
+      }
+      XCTAssertEqual("<1 2 >\n< 3 4>", result, "echo 1 2 \n 3 4 | map{<$0>}")
+    }
+  }
+
+  /// Demo [#22 dropped remainder](https://github.com/GeorgeLyon/Shwift/issues/22)
+  func testEchoMapWithDelimiters() async throws {
+    try runInScript {
+      let result = try await outputOf {
+        try await echo("1", "2", separator: ",", terminator: "")
+        | compactMap(segmentingInputAt: ",", withOutputTerminator: "|") { "<\($0)>"}
+      }
+      let exp = "<1>|<2>|"
+      XCTAssertEqual(exp, result, "echo 1,2 -map-> <1>|<2>|")
+    }
+  }
+
+  /// Verify `reduce` chunks input by splitAt delimiter (and handles non-string output)
+  func testReduceChunked() async throws {
+    typealias SF = ChunkFixtures
+    let result = SF.Result<SF.Ints>() // reference type
+    let exp: [SF.Ints] = [.i(1), .pair(2, 3), .i(4), .err("not")]
+    try runInScript {
+      let sep: Character = ";"
+
+      // inout, here using class
+      try await echo("1", "2\n3", "4", "not",
+                     separator: "\(sep)", terminator: "")
+      | reduce(into: result, segmentingInputAt: sep) { $0.add(.make($1)) }
+      XCTAssertEqual(exp, result.result, "Reduce (inside)")
+
+      // returning value each time
+      let array: [SF.Ints]
+      = try await echo("1", "2\n3", "4", "not",
+                       separator: "\(sep)", terminator: "")
+      | reduce([], segmentingInputAt: sep) { r, s in r + [SF.Ints.make(s)] }
+      XCTAssertEqual(exp, array, "Reduce (arrays)")
+    }
+    XCTAssertEqual(exp, result.result, "Reduce (outside)")
+  }
+
+  /// Verify `map` chunks input by splitAt delimiter
+  func testMapChunked() async throws {
+    typealias SF = ChunkFixtures
+    let mapped: [SF.Ints] = [.i(1), .pair(2, 3), .i(4), .err("not")]
+    let expect = mapped.map{ $0.double() }.joined(separator: "\n")
+    try runInScript {
+      let sep: Character = ";"
+      let result = try await outputOf {
+        try await echo("1", "2\n3", "4", "not",
+                       separator: "\(sep)", terminator: "")
+        | map(segmentingInputAt: sep) { SF.Ints.make($0).double() }
+      }
+      XCTAssertEqual(expect, result, "Map")
+    }
+  }
+
+  /// Verify `compactMap` chunks input by splitAt delimiter
+  func testCompactMapChunked() async throws {
+    typealias SF = ChunkFixtures
+    let all: [SF.Ints] = [.i(1), .pair(2, 3), .i(4), .err("not")]
+    let evens = all.compactMap{ $0.anyEven() }.joined(separator: "\n")
+    try runInScript {
+      let sep: Character = ";"
+      let result = try await outputOf {
+        try await echo("1", "2\n3", "4", "not",
+                       separator: "\(sep)", terminator: "")
+        | compactMap(segmentingInputAt: sep) { SF.Ints.make($0).anyEven() }
+      }
+      XCTAssertEqual(evens, result, "Map")
+    }
+  }  /// Run test in Script.run() context
+  private func runInScript(
+    _ op: @escaping RunInScriptProxy.Op,
+    caller: StaticString = #function,
+    callerLine: UInt = #line
+  ) throws {
+    let e = expectation(description: "\(caller):\(callerLine)")
+    try RunInScriptProxy(op, e).run() // sync call preferred
+    wait(for: [e], timeout: 2)
+  }
+
+  private struct RunInScriptProxy: Script {
+    typealias Op = () async throws -> Void
+    var op: Op?
+    var e: XCTestExpectation?
+    init(_ op: @escaping Op, _ e: XCTestExpectation) {
+      self.op = op
+      self.e = e
+    }
+    func run() async throws {
+      defer { e?.fulfill() }
+      try await op?()
+    }
+    // Codable
+    init() {}
+    var i = 0
+    private enum CodingKeys: String, CodingKey {
+      case i
+    }
+  }
+  private enum ChunkFixtures {
+    class Result<T> {
+      var result = [T]()
+      func add(_ next: T) {
+        result.append(next)
+      }
+    }
+    enum Ints: Codable, Equatable {
+      case i(Int), pair(Int, Int), err(String)
+
+      static func make(_ s: String) -> Ints {
+        let nums = s.split(separator: "\n")
+        switch nums.count {
+        case 1:
+          if let i = Int(nums[0]) {
+            return .i(i)
+          }
+          return err(s)
+        case 2: return .pair(Int(nums[0]) ?? -1, Int(nums[1]) ?? -2)
+        default: return .err(s)
+        }
+      }
+
+      func double() -> String {
+        switch self {
+        case .i(let n): return "\(2*n)"
+        case .pair(let i, let j): return "\(2*i)\n\(2*j)"
+        case .err(let s): return "\(s)\(s)"
+        }
+      }
+
+      func anyEven() -> String? {
+        func anyEvens(_ i: Int...) -> Bool {
+          nil != i.first { 0 == $0 % 2}
+        }
+        switch self {
+        case .i(let n): return anyEvens(n) ? "\(n)" : nil
+        case .pair(let i, let j): return anyEvens(i, j) ? "\(i)\n\(j)" : nil
+        case .err: return nil
+        }
+      }
+    }
+  }
+}
+

Tests/ShwiftTests/Chunks.bin

@@ -0,0 +1,2 @@
+1;2
+3;4

Tests/ShwiftTests/Chunks.txt

@@ -0,0 +1,2 @@
+1;2
+3;4