Changed folder

Test that definitely fails

.github/workflows/build.yml

@@ -23,6 +23,6 @@ jobs:
         uses: AnimMouse/setup-rclone@v1
 
       - run: ./test-go-direct.sh
-        working-directory: packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image
+        working-directory: packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image
 #      - run: pnpm run build
 #      - run: pnpm run package

packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/Dockerfile

@@ -1,8 +1,29 @@
+FROM golang:1.21-alpine as builder
+
+# git is needed for go mod download to work
+RUN apk add --no-cache git
+
+WORKDIR /build
+
+RUN go env -w GOPROXY=direct
+
+ADD go.mod go.sum ./
+
+RUN go mod download
+
+ADD *.go ./
+
+RUN env GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build rclone-batch.go
+
+
 FROM rclone/rclone:1.64.2
 
-RUN apk add --no-cache aws-cli tini
+# if we want signals to propogate through to the app we need a lightweight initd
+RUN apk add --no-cache tini
+
+# this is where rclone binary ends up by default so we will do our work there
+WORKDIR /data
 
-# a shell script that allows us to take more batch like input to rclone
-COPY rclone-batch-copy.sh /app/
+COPY --from=builder /build/rclone-batch ./
 
-ENTRYPOINT ["/sbin/tini", "--", "/bin/sh", "/app/rclone-batch-copy.sh"]
+ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/rclone-batch"]

packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/README.md

@@ -0,0 +1,8 @@
+# rclone-batch
+
+`rclone-batch` is a Go wrapper around the invocation of `rclone`. Why do we need
+a wrapper? Well we want to get the statistics output of `rclone` in a way
+that we can standardise. Also, we want optionally to be able to send this info
+back to the parent AWS ECS task. Furthermore, there are aspects of signal
+handling that we want to support for AWS Spot that is not quite the same as
+`rclone` out of the box.

packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-go-direct.sh → packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-go-direct.sh

@@ -8,20 +8,19 @@ RCLONE_BINARY=$(which rclone)
 set -e
 
 # bring in some helpful bash assertions
-. test-assert.sh
+. test-assert.sh --invariant
 
-# build our rclone batch executable
-go build
+# build our rclone-batch executable
+go build rclone-batch.go
 
 # make a temporary directory for the copy destination
+# NOTE: we do not remove this on a EXIT trap as that interferes with the assert.sh exit codes
 TEMPD=$(mktemp -d)
 if [ ! -e "$TEMPD" ]; then
     >&2 echo "Failed to create temp directory"
     exit 1
 fi
 
-trap 'rm -rf "$TEMPD"' EXIT
-
 #
 # test 1 (main case of just copying files)
 #
@@ -79,6 +78,8 @@ assert " cat $TEMPD/result.json | jq -r '.\"1\" | .lastError' " "Skipped due to
 
 rm "$TEMPD/result.json"
 
+assert "5" "10"
+
 #
 # end overall testing and set return code
 #

packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/Dockerfile

@@ -0,0 +1,8 @@
+FROM rclone/rclone:1.64.2
+
+RUN apk add --no-cache aws-cli tini
+
+# a shell script that allows us to take more batch like input to rclone
+COPY rclone-batch-copy.sh /app/
+
+ENTRYPOINT ["/sbin/tini", "--", "/bin/sh", "/app/rclone-batch-copy.sh"]