From 4299262ef0f99c3a280e4a8765553c9e3b7709da Mon Sep 17 00:00:00 2001 From: andrewpatto Date: Wed, 29 Nov 2023 14:21:50 +1100 Subject: [PATCH] Changed folder Test that definitely fails --- .github/workflows/build.yml | 2 +- .../.dockerignore | 0 .../.gitignore | 0 .../rclone-batch-copy-docker-image/Dockerfile | 29 ++++++++++++++++--- .../rclone-batch-copy-docker-image/README.md | 8 +++++ .../go.mod | 0 .../go.sum | 0 .../rclone-batch.go} | 0 .../test-assert.sh | 0 .../test-docker-direct.sh | 0 .../test-go-direct.sh | 11 +++---- .../testfile1.txt | 0 .../testfile2.txt | 0 .../Dockerfile | 8 +++++ .../rclone-batch-copy.sh | 0 .../Dockerfile | 28 ------------------ 16 files changed, 48 insertions(+), 38 deletions(-) rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image => rclone-batch-copy-docker-image}/.dockerignore (100%) rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image => rclone-batch-copy-docker-image}/.gitignore (100%) create mode 100644 packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/README.md rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image => rclone-batch-copy-docker-image}/go.mod (100%) rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image => rclone-batch-copy-docker-image}/go.sum (100%) rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image/main.go => rclone-batch-copy-docker-image/rclone-batch.go} (100%) rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image => rclone-batch-copy-docker-image}/test-assert.sh (100%) rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image => rclone-batch-copy-docker-image}/test-docker-direct.sh (100%) rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image => rclone-batch-copy-docker-image}/test-go-direct.sh (92%) rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image => rclone-batch-copy-docker-image}/testfile1.txt (100%) rename packages/aws-copy-out-sharer/construct/{rclone-go-batch-copy-docker-image => rclone-batch-copy-docker-image}/testfile2.txt (100%) create mode 100644 packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/Dockerfile rename packages/aws-copy-out-sharer/construct/{rclone-batch-copy-docker-image => rclone-batch-shell-copy-docker-image}/rclone-batch-copy.sh (100%) delete mode 100644 packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/Dockerfile diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7868f28..ee18cda 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,6 @@ jobs: uses: AnimMouse/setup-rclone@v1 - run: ./test-go-direct.sh - working-directory: packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image + working-directory: packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image # - run: pnpm run build # - run: pnpm run package diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.dockerignore b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/.dockerignore similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.dockerignore rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/.dockerignore diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.gitignore b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/.gitignore similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.gitignore rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/.gitignore diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/Dockerfile b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/Dockerfile index 8b29bee..78753aa 100644 --- a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/Dockerfile +++ b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/Dockerfile @@ -1,8 +1,29 @@ +FROM golang:1.21-alpine as builder + +# git is needed for go mod download to work +RUN apk add --no-cache git + +WORKDIR /build + +RUN go env -w GOPROXY=direct + +ADD go.mod go.sum ./ + +RUN go mod download + +ADD *.go ./ + +RUN env GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build rclone-batch.go + + FROM rclone/rclone:1.64.2 -RUN apk add --no-cache aws-cli tini +# if we want signals to propogate through to the app we need a lightweight initd +RUN apk add --no-cache tini + +# this is where rclone binary ends up by default so we will do our work there +WORKDIR /data -# a shell script that allows us to take more batch like input to rclone -COPY rclone-batch-copy.sh /app/ +COPY --from=builder /build/rclone-batch ./ -ENTRYPOINT ["/sbin/tini", "--", "/bin/sh", "/app/rclone-batch-copy.sh"] +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/rclone-batch"] diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/README.md b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/README.md new file mode 100644 index 0000000..38dced5 --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/README.md @@ -0,0 +1,8 @@ +# rclone-batch + +`rclone-batch` is a Go wrapper around the invocation of `rclone`. Why do we need +a wrapper? Well we want to get the statistics output of `rclone` in a way +that we can standardise. Also, we want optionally to be able to send this info +back to the parent AWS ECS task. Furthermore, there are aspects of signal +handling that we want to support for AWS Spot that is not quite the same as +`rclone` out of the box. diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/go.mod b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/go.mod similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/go.mod rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/go.mod diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/go.sum b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/go.sum similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/go.sum rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/go.sum diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/main.go b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/rclone-batch.go similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/main.go rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/rclone-batch.go diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-assert.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-assert.sh similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-assert.sh rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-assert.sh diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-docker-direct.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-docker-direct.sh similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-docker-direct.sh rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-docker-direct.sh diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-go-direct.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-go-direct.sh similarity index 92% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-go-direct.sh rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-go-direct.sh index e540d5c..575314a 100755 --- a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-go-direct.sh +++ b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-go-direct.sh @@ -8,20 +8,19 @@ RCLONE_BINARY=$(which rclone) set -e # bring in some helpful bash assertions -. test-assert.sh +. test-assert.sh --invariant -# build our rclone batch executable -go build +# build our rclone-batch executable +go build rclone-batch.go # make a temporary directory for the copy destination +# NOTE: we do not remove this on a EXIT trap as that interferes with the assert.sh exit codes TEMPD=$(mktemp -d) if [ ! -e "$TEMPD" ]; then >&2 echo "Failed to create temp directory" exit 1 fi -trap 'rm -rf "$TEMPD"' EXIT - # # test 1 (main case of just copying files) # @@ -79,6 +78,8 @@ assert " cat $TEMPD/result.json | jq -r '.\"1\" | .lastError' " "Skipped due to rm "$TEMPD/result.json" +assert "5" "10" + # # end overall testing and set return code # diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/testfile1.txt b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/testfile1.txt similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/testfile1.txt rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/testfile1.txt diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/testfile2.txt b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/testfile2.txt similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/testfile2.txt rename to packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/testfile2.txt diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/Dockerfile b/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/Dockerfile new file mode 100644 index 0000000..8b29bee --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/Dockerfile @@ -0,0 +1,8 @@ +FROM rclone/rclone:1.64.2 + +RUN apk add --no-cache aws-cli tini + +# a shell script that allows us to take more batch like input to rclone +COPY rclone-batch-copy.sh /app/ + +ENTRYPOINT ["/sbin/tini", "--", "/bin/sh", "/app/rclone-batch-copy.sh"] diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/rclone-batch-copy.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/rclone-batch-copy.sh similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/rclone-batch-copy.sh rename to packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/rclone-batch-copy.sh diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/Dockerfile b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/Dockerfile deleted file mode 100644 index 9121c34..0000000 --- a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM golang:1.21-alpine as builder - -RUN apk add --no-cache git - -WORKDIR /build - -RUN go env -w GOPROXY=direct - -ADD go.mod go.sum ./ - -RUN go mod download - -ADD main.go ./ - -RUN env GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build - - -FROM rclone/rclone:1.64.2 - -# if we want signals to propogate through to the app we need a lightweight initd -RUN apk add --no-cache tini - -# this is where rclone binary ends up by default so we will do our work there -WORKDIR /data - -COPY --from=builder /build/rclone-batch ./ - -ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/rclone-batch"]