diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/rclone-batch.go b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/rclone-batch.go deleted file mode 100644 index d41fc16..0000000 --- a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/rclone-batch.go +++ /dev/null @@ -1,157 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/sfn" - "log" - "os" - "os/exec" - "os/signal" - "strings" - "syscall" -) - -// retryError:false serverSideCopies:0 serverSideCopyBytes:0 serverSideMoveBytes:0 serverSideMoves:0 speed:0 totalChecks:0 totalTransfers:1 transferTime:0.000586292 transfers:1 -type RcloneStats struct { - Bytes int `json:"bytes"` - Checks int `json:"checks"` - ElapsedTime float64 `json:"elapsedTime"` - Errors int `json:"errors"` - LastError string `json:"lastError"` - FatalError bool `json:"fatalError"` - TotalBytes int `json:"totalBytes"` -} - -func If[T any](cond bool, vtrue, vfalse T) T { - if cond { - return vtrue - } - return vfalse -} - -func main() { - cfg, err := config.LoadDefaultConfig(context.TODO()) - - if err != nil { - log.Fatalf("unable to load AWS config, %v", err) - } - - // NOTE: if this was a traditional command line tool we would take these in as command - // line parameters. However, we are invoking this as an ECS Task and it turns out easier - // to pass these down via environment variables - saving the command line *only* for the list - // of files we want to copy - destination, destinationOk := os.LookupEnv("DESTINATION") - - if !destinationOk { - log.Fatalf("No environment variable DESTINATION telling us where to copy the objects", err) - } - - taskToken, taskTokenOk := os.LookupEnv("TASK_TOKEN") - - // special environment variables that we can use for some debug/testing - debugBandwidth, debugBandwidthOk := os.LookupEnv("DEBUG_BANDWIDTH") - - results := make(map[int]any) - - signalChannel := make(chan os.Signal) - - // set as soon as we receive a SIGTERM - so that we will then just quickly skip the rest of the files - interrupted := false - - for i := 2; i < len(os.Args); i++ { - if !interrupted { - // setup an rclone copy with capture stats (noting that stats are sent to stderr) - cmd := exec.Command(os.Args[1], - "--use-json-log", - "--stats-log-level", "NOTICE", - "--stats-one-line", - // only display stats at the end (after 10000 hours) - "--stats", "10000h", - // normally no bandwidth limiting ("0") - but can institute bandwidth limit if asked - "--bwlimit", If(debugBandwidthOk, debugBandwidth, "0"), - "copy", os.Args[i], destination) - - // we are only interested in stderr - stderrStringBuilder := new(strings.Builder) - cmd.Stderr = stderrStringBuilder - - // we need to be able handling getting a SIGTERM when AWS wants to reclaim our SPOT instance - signal.Notify(signalChannel, os.Interrupt, syscall.SIGTERM) - go func() { - sig := <-signalChannel - switch sig { - case syscall.SIGTERM: - // terminate the currently running rclone - cmd.Process.Signal(syscall.SIGTERM) - // indicate we don't want future rclones to run - interrupted = true - } - }() - - err := cmd.Run() - - if err != nil { - results[i-2] = map[string]any{"lastError": "Interrupted by SIGTERM", "systemError": err} - } - - // each line of stderr output is stats in JSON format or possibly other random messages - stderrStringLines := strings.Split(strings.TrimSuffix(stderrStringBuilder.String(), "\n"), "\n") - - // attempt to process each line of log output to stderr as JSON (if not then ignore) - for _, line := range stderrStringLines { - var logLineJson map[string]any - - logLineJsonErr := json.Unmarshal([]byte(line), &logLineJson) - - if logLineJsonErr == nil { - - statsValue, statsOk := logLineJson["stats"].(map[string]any) - - if statsOk { - results[i-2] = statsValue - } - } - } - } else { - results[i-2] = map[string]any{"lastError": "Skipped due to SIGTERM received"} - } - - // if asked we sleep for delay after seconds (this gives the outer environment - // time to send us a SIGINT etc) - /*if debugDelayAfterOk { - debugDelayAfterFloat, err := strconv.ParseFloat(debugDelayAfter, 64) - if err == nil { - time.Sleep(time.Duration(debugDelayAfterFloat) * time.Second) - } - } */ - - } - - resultsJson, err := json.MarshalIndent(results, "", " ") - - if err != nil { - log.Fatalf("Could not marshall the rclone outputs to JSON", err) - } - - resultsString := string(resultsJson) - - // the normal mechanism by which we will send back results to our caller is - // Steps SendTask - which sends back JSON - if taskTokenOk { - sfnSvc := sfn.NewFromConfig(cfg) - - sfnSvc.SendTaskSuccess(context.TODO(), &sfn.SendTaskSuccessInput{ - Output: aws.String(resultsString), - TaskToken: aws.String(taskToken), - }) - - fmt.Println(taskToken) - } else { - // if no task token was given then we just print the results - fmt.Println(resultsString) - } -} diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-docker-direct.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-docker-direct.sh deleted file mode 100755 index 25b51f2..0000000 --- a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-docker-direct.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# we want to exit immediately on error (especially for Go/Docker build errors) -set -e - -docker build . -t rclone-batch - -docker run rclone-batch -e DESTINATION=/tmp diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-go-direct.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-go-direct.sh deleted file mode 100755 index 575314a..0000000 --- a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-go-direct.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash - -# this is a setting which is system dependent - feel free to change to the location of your -# dev system rclone binary (this one is brew installed on a Mac M1) -RCLONE_BINARY=$(which rclone) - -# we want to exit immediately on error (especially for Go compile errors) -set -e - -# bring in some helpful bash assertions -. test-assert.sh --invariant - -# build our rclone-batch executable -go build rclone-batch.go - -# make a temporary directory for the copy destination -# NOTE: we do not remove this on a EXIT trap as that interferes with the assert.sh exit codes -TEMPD=$(mktemp -d) -if [ ! -e "$TEMPD" ]; then - >&2 echo "Failed to create temp directory" - exit 1 -fi - -# -# test 1 (main case of just copying files) -# - -DESTINATION="$TEMPD/test1" ./rclone-batch "$RCLONE_BINARY" ./testfile1.txt ./testfile2.txt > "$TEMPD/result.json" - -# Uncomment to debug invalid result -# cat "$TEMPD/result.json" - -assert " find $TEMPD/test1 -type f | awk 'END{print NR}' " "2" -assert " cat $TEMPD/result.json | jq -r '.\"0\" | .bytes' " "20" -assert " cat $TEMPD/result.json | jq -r '.\"1\" | .bytes' " "37" - -rm "$TEMPD/result.json" - -# -# test 2 (error cases with one file to fail) -# - -DESTINATION="$TEMPD/test2" ./rclone-batch "$RCLONE_BINARY" ./afilethatdoesnotexist.txt ./testfile2.txt > "$TEMPD/result.json" - -# Uncomment to debug invalid result -# cat "$TEMPD/result.json" - -assert "find $TEMPD/test2 -type f | awk 'END{print NR}'" "1" -assert " cat $TEMPD/result.json | jq -r '.\"0\" | .lastError' " "directory not found" -assert " cat $TEMPD/result.json | jq -r '.\"0\" | .bytes' " "0" -assert " cat $TEMPD/result.json | jq -r '.\"1\" | .lastError' " "null" -assert " cat $TEMPD/result.json | jq -r '.\"1\" | .bytes' " "37" - -rm "$TEMPD/result.json" - -# -# test 3 (signal intercept) -# -# this is a test that app will intercept a SIGTERM, pass it to any running rclone process, -# and return sensible results -# - -# we set the bandwidth to 1B so that it is slow enough that our TERM signal will come mid-process -# we start this execution in the background -DESTINATION="$TEMPD/test3" DEBUG_BANDWIDTH="1B" ./rclone-batch "$RCLONE_BINARY" ./testfile1.txt ./testfile2.txt > "$TEMPD/result.json" & - -# wait a small amount -sleep 1 - -# now send a SIGTERM to the launched job -kill %1 - -# Uncomment to debug invalid result -cat "$TEMPD/result.json" - -assert " cat $TEMPD/result.json | jq -r '.\"0\" | .lastError' " "Interrupted by SIGTERM" -assert " cat $TEMPD/result.json | jq -r '.\"1\" | .lastError' " "Skipped due to SIGTERM received" - -rm "$TEMPD/result.json" - -assert "5" "10" - -# -# end overall testing and set return code -# - -assert_end examples diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/.dockerignore b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/.dockerignore similarity index 73% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/.dockerignore rename to packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/.dockerignore index adab58a..0c530a3 100644 --- a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/.dockerignore +++ b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/.dockerignore @@ -1,3 +1,4 @@ +Dockerfile test/* test-*.sh rclone-batch diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/.gitignore b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/.gitignore similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/.gitignore rename to packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/.gitignore diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/Dockerfile b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/Dockerfile similarity index 83% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/Dockerfile rename to packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/Dockerfile index 78753aa..38f0731 100644 --- a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/Dockerfile +++ b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/Dockerfile @@ -26,4 +26,6 @@ WORKDIR /data COPY --from=builder /build/rclone-batch ./ -ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/rclone-batch"] +ENV RB_RCLONE_BINARY "/usr/local/bin/rclone" + +ENTRYPOINT ["/sbin/tini", "-vvv", "--", "/data/rclone-batch"] diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/README.md b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/README.md similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/README.md rename to packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/README.md diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/go.mod b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/go.mod similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/go.mod rename to packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/go.mod diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/go.sum b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/go.sum similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/go.sum rename to packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/go.sum diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/rclone-batch.go b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/rclone-batch.go new file mode 100644 index 0000000..9baee7a --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/rclone-batch.go @@ -0,0 +1,197 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sfn" + "log" + "os" + "os/exec" + "os/signal" + "strings" + "syscall" +) + +// NOTE: we use a prefix of RB (rclone-batch) just so we don't accidentally clash with a real +// env variable that has meaning to rclone (for example) +const rcloneBinaryEnvName = "RB_RCLONE_BINARY" +const destinationEnvName = "RB_DESTINATION" + +/** + * A ternaryish operator + */ +func If[T any](cond bool, vtrue, vfalse T) T { + if cond { + return vtrue + } + return vfalse +} + +/** + * A command line wrapper for invoking rclone one by one and return stats/error messages + * to the parent caller. Finishes by sending the stats back to the AWS parent task if asked. + * + * Inputs + * os.Args the source object paths to copy (rclone syntax e.g s3:bucket:key) + * Env + * RB_RCLONE_BINARY the path to an rclone binary to use + * RB_DESTINATION the path to send the objects (rclone syntax e.g s3:bucket:key) + * RB_TASK_TOKEN if present, the task token to use to send the copy results back to the parent + * RB_DEBUG_BANDWIDTH if present, a rclone bandwidth setting (just for debug/testing) + * ...any other rclone settings needed... + * RCLONE_CONFIG_S3_PROVIDER... + */ +func main() { + // NOTE: if this was a traditional command line tool we would take these in as command + // line parameters. However, we are invoking this as an ECS Task and it turns out easier + // to pass these down via environment variables - saving the command line args *only* for the list + // of files we want to copy + + rcloneBinary, rcloneBinaryOk := os.LookupEnv(rcloneBinaryEnvName) + + if !rcloneBinaryOk { + log.Fatalf("No environment variable %s telling us the path to an rclone executable", rcloneBinaryEnvName) + } + + if !strings.Contains(rcloneBinary, "rclone") { + // given we are a program that executes another program - just a little sanity check that what we + // are invoking vaguely makes sense + // (feel free to remove this if you have a use case where the binary is named otherwise) + log.Fatalf("The environment variable %s should have the string rclone in it somewhere", rcloneBinaryEnvName) + } + + destination, destinationOk := os.LookupEnv(destinationEnvName) + + if !destinationOk { + log.Fatalf("No environment variable %s telling us where to copy the objects", destinationEnvName) + } + + // a task token that ECS/Steps can pass us so we can return data + taskToken, taskTokenOk := os.LookupEnv("RB_TASK_TOKEN") + + // now that we know whether we want to use the task token - we will definitely need AWS config to work + // - so no need starting copying if we will fail at the end + cfg, cfgErr := config.LoadDefaultConfig(context.TODO()) + + if taskTokenOk { + if cfgErr != nil { + log.Fatalf("Unable to load AWS config, %v", cfgErr) + } + } + + // special environment variables that we can use for some debug/testing + debugBandwidth, debugBandwidthOk := os.LookupEnv("RB_DEBUG_BANDWIDTH") + + // we end up with a result array entry for each object we have been asked to copy + results := make([]any, len(os.Args)-1) + + signalChannel := make(chan os.Signal) + + // set as soon as we receive a SIGTERM - so that we will then just quickly skip the rest of the files + interrupted := false + + for i := 1; i < len(os.Args); i++ { + + // what we are processing in this iteration + which := i - 1 + source := os.Args[i] + + log.Printf("Asked to copy %s as the %d object to copy", source, which) + + if !interrupted { + // setup an rclone copy with capture stats (noting that stats are sent to stderr) + cmd := exec.Command(rcloneBinary, + "--use-json-log", + "--stats-log-level", "NOTICE", + "--stats-one-line", + // only display stats at the end (after 10000 hours) + "--stats", "10000h", + // normally no bandwidth limiting ("0") - but can institute bandwidth limit if asked + "--bwlimit", If(debugBandwidthOk, debugBandwidth, "0"), + "copy", source, destination) + + // we are only interested in stderr + stderrStringBuilder := new(strings.Builder) + cmd.Stderr = stderrStringBuilder + + // we need to be able handling getting a SIGTERM when AWS wants to reclaim our SPOT instance + signal.Notify(signalChannel, os.Interrupt, syscall.SIGTERM) + go func() { + sig := <-signalChannel + switch sig { + case syscall.SIGTERM: + // terminate the currently running rclone + cmd.Process.Signal(syscall.SIGTERM) + // indicate we don't want future rclones to run + interrupted = true + } + }() + + err := cmd.Run() + + if err != nil { + log.Printf("rclone Run() failed with %v", err) + results[which] = map[string]any{ + "lastError": "Interrupted by SIGTERM", + "systemError": fmt.Sprintf("%v", err), + "source": source} + } else { + log.Printf("rclone Run() succeeded") + } + + // each line of stderr output is stats in JSON format or possibly other random messages + stderrStringLines := strings.Split(strings.TrimSuffix(stderrStringBuilder.String(), "\n"), "\n") + + // attempt to process each line of log output to stderr as JSON (if not then ignore) + for _, line := range stderrStringLines { + var logLineJson map[string]any + + logLineJsonErr := json.Unmarshal([]byte(line), &logLineJson) + + if logLineJsonErr == nil { + + statsValue, statsOk := logLineJson["stats"].(map[string]any) + + if statsOk { + // insert information about the file we were copying + statsValue["source"] = source + results[which] = statsValue + } + } + } + } else { + results[which] = map[string]any{ + "lastError": "Skipped due to SIGTERM received", + "source": source} + } + } + + resultsJson, err := json.MarshalIndent(results, "", " ") + + if err != nil { + log.Fatalf("Could not marshall the rclone outputs to JSON", err) + } + + resultsString := string(resultsJson) + + // the normal mechanism by which we will send back results to our caller is + // Steps SendTask - which sends back JSON + if taskTokenOk { + sfnSvc := sfn.NewFromConfig(cfg) + + // output + // The JSON output of the task. Length constraints apply to the payload size, and are expressed as bytes in UTF-8 encoding. + // Type: String + // Length Constraints: Maximum length of 262144. + sfnSvc.SendTaskSuccess(context.TODO(), &sfn.SendTaskSuccessInput{ + Output: aws.String(resultsString), + TaskToken: aws.String(taskToken), + }) + } else { + // if no task token was given then we just print the results + fmt.Println(resultsString) + } +} diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-assert.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/test-assert.sh similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/test-assert.sh rename to packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/test-assert.sh diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/test-docker-direct.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/test-docker-direct.sh new file mode 100755 index 0000000..d94dabf --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/test-docker-direct.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# we want to exit immediately on error (especially for Go/Docker build errors) +set -e + +docker build . -t rclone-batch-image + +# make a temporary directory for the copy destination +# NOTE: we do not remove this on a EXIT trap as that interferes with the assert.sh exit codes +TEMPD=$(mktemp -d) +if [ ! -e "$TEMPD" ]; then + >&2 echo "Failed to create temp directory" + exit 1 +fi + +# note the /etc files here are not important! We are just using them as source files +# that happen to already exist in the docker image by default +docker run -ti --rm \ + --env RB_DESTINATION=/tmp \ + --mount "type=bind,source=$TEMPD,target=/tmp" \ + rclone-batch-image \ + /etc/alpine-release /etc/os-release /etc/services + +# need to write some assertions here (need to separate out various outputs) + +ls -al $TEMPD diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/test-go-direct.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/test-go-direct.sh new file mode 100755 index 0000000..bc6538b --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/test-go-direct.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# we want to exit immediately on error (especially for Go compile errors) +set -e + +# bring in some helpful bash assertions +. test-assert.sh --invariant + +# build our rclone-batch executable +go build rclone-batch.go + +# make a temporary directory for the copy destination +# NOTE: we do not remove this on a EXIT trap as that interferes with the assert.sh exit codes +TEMPD=$(mktemp -d) +if [ ! -e "$TEMPD" ]; then + >&2 echo "Failed to create temp directory" + exit 1 +fi + +RB_RCLONE_BINARY="$(which rclone)" +if [ ! -e "$RB_RCLONE_BINARY" ]; then + >&2 echo "Failed to locate rclone binary to use" + exit 1 +fi + +export RB_RCLONE_BINARY + +# +# Test 1 +# +echo "Test 1 - copying two files" + +RB_DESTINATION="$TEMPD/test1" ./rclone-batch ./testfile1.txt ./testfile2.txt > "$TEMPD/result.json" + +cat "$TEMPD/result.json" + +assert " find $TEMPD/test1 -type f | awk 'END{print NR}' " "2" +assert " cat $TEMPD/result.json | jq -r '.[0] | .bytes' " "20" +assert " cat $TEMPD/result.json | jq -r '.[1] | .bytes' " "37" + +rm "$TEMPD/result.json" + +# +# Test 2 +# +echo "Test 2 - copying two files but one not present/fails" + +RB_DESTINATION="$TEMPD/test2" ./rclone-batch ./afilethatdoesnotexist.txt ./testfile2.txt > "$TEMPD/result.json" + +cat "$TEMPD/result.json" + +assert "find $TEMPD/test2 -type f | awk 'END{print NR}'" "1" +assert " cat $TEMPD/result.json | jq -r '.[0] | .lastError' " "directory not found" +assert " cat $TEMPD/result.json | jq -r '.[0] | .bytes' " "0" +assert " cat $TEMPD/result.json | jq -r '.[1] | .lastError' " "null" +assert " cat $TEMPD/result.json | jq -r '.[1] | .bytes' " "37" + +rm "$TEMPD/result.json" + +# +# Test 3 +# +# this is a test that app will intercept a SIGTERM, pass it to any running rclone process, +# and return sensible results +# +echo "Test 3 - copying two files but signals tells us to stop" + +# we set the bandwidth to 1B so that it is slow enough that our TERM signal will come mid-process +# we start this execution in the background +RB_DESTINATION="$TEMPD/test3" RB_DEBUG_BANDWIDTH="1B" ./rclone-batch ./testfile1.txt ./testfile2.txt > "$TEMPD/result.json" & + +# wait a small amount +sleep 1 + +# now send a SIGTERM to the launched job +kill %1 + +cat "$TEMPD/result.json" + +assert " cat $TEMPD/result.json | jq -r '.[0] | .lastError' " "Interrupted by SIGTERM" +assert " cat $TEMPD/result.json | jq -r '.[1] | .lastError' " "Skipped due to SIGTERM received" + +rm "$TEMPD/result.json" + +# +# end overall testing and set return code +# + +assert_end examples diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/testfile1.txt b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/testfile1.txt similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/testfile1.txt rename to packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/testfile1.txt diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/testfile2.txt b/packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/testfile2.txt similarity index 100% rename from packages/aws-copy-out-sharer/construct/rclone-batch-copy-docker-image/testfile2.txt rename to packages/aws-copy-out-sharer/construct/rclone-batch-docker-image/testfile2.txt diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/Dockerfile b/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/Dockerfile deleted file mode 100644 index 8b29bee..0000000 --- a/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM rclone/rclone:1.64.2 - -RUN apk add --no-cache aws-cli tini - -# a shell script that allows us to take more batch like input to rclone -COPY rclone-batch-copy.sh /app/ - -ENTRYPOINT ["/sbin/tini", "--", "/bin/sh", "/app/rclone-batch-copy.sh"] diff --git a/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/rclone-batch-copy.sh b/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/rclone-batch-copy.sh deleted file mode 100644 index f499461..0000000 --- a/packages/aws-copy-out-sharer/construct/rclone-batch-shell-copy-docker-image/rclone-batch-copy.sh +++ /dev/null @@ -1,15 +0,0 @@ - -# If you are familiar with rsync, rclone always works -# as if you had written a trailing / - meaning "copy the -# contents of this directory". This applies to all commands -# and whether you are talking about the source or destination. - -echo "Destination = $destination" -echo "Task token = $tasktoken" - -for src in "$@"; do - echo "Source arg = $src" - /usr/local/bin/rclone copy --stats-log-level NOTICE --checksum "$src" "$destination" -done - -aws stepfunctions send-task-success --task-token "$tasktoken" --task-output '{ "0": "OK", "1": "OK" }'