From ef5bef62c47bf806738a439469efc78b8922660a Mon Sep 17 00:00:00 2001
From: Dale Newby <dale_newby@sil.org>
Date: Mon, 24 Jun 2024 09:04:03 -0400
Subject: [PATCH 1/3] Initial commit

---
 .dockerignore    |  5 +++++
 .gitignore       |  2 ++
 Dockerfile       | 10 ++++++++++
 README.md        | 39 +++++++++++++++++++++++++++++++++++++--
 local.env.dist   | 19 +++++++++++++++++++
 sync-s3-to-b2.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 119 insertions(+), 2 deletions(-)
 create mode 100644 .dockerignore
 create mode 100644 Dockerfile
 create mode 100755 local.env.dist
 create mode 100755 sync-s3-to-b2.sh

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..100be07
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,5 @@
+.git
+.gitignore
+LICENSE
+local.env.dist
+README.md
diff --git a/.gitignore b/.gitignore
index f76f79a..a4f1772 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,5 @@
 node_modules/
 vendor/
 .terraform/
+
+local.env
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..39e3675
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,10 @@
+FROM alpine:3
+
+RUN apk update \
+ && apk add --no-cache bash curl unzip \
+ && curl https://rclone.org/install.sh | bash
+
+COPY sync-s3-to-b2.sh /data/
+WORKDIR /data
+
+CMD ["./sync-s3-to-b2.sh"]
diff --git a/README.md b/README.md
index f3dd90e..a97e5aa 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,37 @@
-# template-public
-general template for public repositories
+# sync-s3-to-b2
+Service to synchronize an AWS S3 bucket path to a Backblaze B2 bucket path
+
+This process uses `rclone sync` to copy changes from a specified directory in an AWS S3 bucket to a specified directory in a Backblaze B2 bucket. After configuring `rclone`, the script executes `rclone sync --checksum --create-empty-src-dirs --metadata --transfers 32 ${RCLONE_ARGUMENTS} s3bucket:${S3_BUCKET}/${S3_PATH} b2bucket:${B2_BUCKET}/${B2_PATH}`.
+
+## How to use it
+1. Create an AWS access key that allows read access to the source S3 bucket.
+2. Create a Backblaze Application Key that allows read, write, and delete access to the destination B2 bucket.
+3. Supply all appropriate environment variables.
+4. Run the sync.
+5. Verify the destination bucket is identical to the source bucket.
+
+**Note:** Empty directories in the AWS S3 bucket may not be created on the Backblaze B2 bucket. See the `rclone` documentation for details.
+
+### Environment variables
+`AWS_ACCESS_KEY` - used to access the AWS S3 bucket
+
+`AWS_SECRET_KEY` - used to access the AWS S3 bucket
+
+`AWS_REGION` - AWS region the S3 bucket was created in
+
+`B2_APPLICATION_KEY_ID` - used to access the Backblaze B2 bucket
+
+`B2_APPLICATION_KEY` - used to access the Backblaze B2 bucket
+
+`S3_BUCKET` - AWS S3 bucket name, e.g., _my-s3-storage-bucket_
+
+`S3_PATH` - path within the AWS S3 bucket
+
+`B2_BUCKET` - Backblaze B2 bucket name, e.g., _my-b2-copy-bucket_
+
+`B2_PATH` - path within the Backblaze B2 bucket
+
+`RCLONE_ARGUMENTS` - (optional) Extra rclone arguments. For example, adding `--combined -` to `RCLONE_ARGUMENTS` will "list all file paths with a symbol and then a space and then the path to tell you what happened to it".
+
+## Docker Hub
+This image is built automatically on Docker Hub as [silintl/sync-s3-to-b2](https://hub.docker.com/r/silintl/sync-s3-to-b2/).
diff --git a/local.env.dist b/local.env.dist
new file mode 100755
index 0000000..afadd80
--- /dev/null
+++ b/local.env.dist
@@ -0,0 +1,19 @@
+# AWS access variables
+AWS_ACCESS_KEY=
+AWS_SECRET_KEY=
+AWS_REGION=
+
+# Backblaze access variables
+B2_APPLICATION_KEY_ID=
+B2_APPLICATION_KEY=
+
+# Extra rclone arguments
+RCLONE_ARGUMENTS=
+
+# AWS S3 bucket and path
+S3_BUCKET=
+S3_PATH=
+
+# Backblaze B2 bucket and path
+B2_BUCKET=
+B2_PATH=
diff --git a/sync-s3-to-b2.sh b/sync-s3-to-b2.sh
new file mode 100755
index 0000000..f41214e
--- /dev/null
+++ b/sync-s3-to-b2.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env sh
+
+MYNAME="sync-s3-to-b2"
+STATUS=0
+
+echo "${MYNAME}: Started"
+
+echo "${MYNAME}: Configuring rclone"
+
+mkdir -p ~/.config/rclone
+cat > ~/.config/rclone/rclone.conf << EOF
+# Created by sync-s3-to-b2.sh
+
+[b2bucket]
+type = b2
+hard_delete = true
+account = ${B2_APPLICATION_KEY_ID}
+key = ${B2_APPLICATION_KEY}
+
+[s3bucket]
+type = s3
+provider = AWS
+access_key_id = ${AWS_ACCESS_KEY}
+secret_access_key = ${AWS_SECRET_KEY}
+region = ${AWS_REGION}
+location_constraint = ${AWS_REGION}
+
+EOF
+
+# Adding "--combined - " to RCLONE_ARGUMENTS will list all file paths with a
+# symbol and then a space and then the path to tell you what happened to it.
+
+echo "${MYNAME}: Starting rclone sync"
+
+start=$(date +%s)
+rclone sync --checksum --create-empty-src-dirs --metadata --transfers 32 ${RCLONE_ARGUMENTS} s3bucket:${S3_BUCKET}/${S3_PATH} b2bucket:${B2_BUCKET}/${B2_PATH} || STATUS=$?
+end=$(date +%s)
+
+if [ $STATUS -ne 0 ]; then
+    echo "${MYNAME}: FATAL: Sync of ${S3_BUCKET}/${S3_PATH} to ${B2_BUCKET}/${B2_PATH} returned non-zero status ($STATUS) in $(expr ${end} - ${start}) seconds."
+    exit $STATUS
+else
+    echo "${MYNAME}: Sync of ${S3_BUCKET}/${S3_PATH} to ${B2_BUCKET}/${B2_PATH}  completed in $(expr ${end} - ${start}) seconds."
+fi
+
+echo "${MYNAME}: Completed"

From 4aa356e507abf9c5834752f304f41d90a127e803 Mon Sep 17 00:00:00 2001
From: Dale Newby <dale_newby@sil.org>
Date: Mon, 24 Jun 2024 10:31:39 -0400
Subject: [PATCH 2/3] Add GitHub Actions to build Docker image

---
 .github/workflows/build-and-publish.yml | 37 +++++++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 .github/workflows/build-and-publish.yml

diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml
new file mode 100644
index 0000000..4237f6f
--- /dev/null
+++ b/.github/workflows/build-and-publish.yml
@@ -0,0 +1,37 @@
+name: Build and Publish
+
+on:
+  push:
+
+jobs:
+  build-and-publish:
+    name: Build and Publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ vars.DOCKER_ORG }}/${{ github.event.repository.name }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=tag
+            # set latest tag for master branch
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

From f610a39a72aba8123a0a797255870d1bee0ddf10 Mon Sep 17 00:00:00 2001
From: Dale Newby <dale_newby@sil.org>
Date: Thu, 18 Jul 2024 14:37:57 -0400
Subject: [PATCH 3/3] Verify required environment variables are set

---
 sync-s3-to-b2.sh | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/sync-s3-to-b2.sh b/sync-s3-to-b2.sh
index f41214e..273ba97 100755
--- a/sync-s3-to-b2.sh
+++ b/sync-s3-to-b2.sh
@@ -5,6 +5,45 @@ STATUS=0
 
 echo "${MYNAME}: Started"
 
+if [ "${B2_APPLICATION_KEY_ID}" = "" ]; then
+	echo "${MYNAME}: FATAL: environment variable B2_APPLICATION_KEY_ID is required."
+	STATUS=1
+fi
+
+if [ "${B2_APPLICATION_KEY}" = "" ]; then
+	echo "${MYNAME}: FATAL: environment variable B2_APPLICATION_KEY is required."
+	STATUS=1
+fi
+
+if [ "${AWS_ACCESS_KEY}" = "" ]; then
+	echo "${MYNAME}: FATAL: environment variable AWS_ACCESS_KEY is required."
+	STATUS=1
+fi
+
+if [ "${AWS_SECRET_KEY}" = "" ]; then
+	echo "${MYNAME}: FATAL: environment variable AWS_SECRET_KEY is required."
+	STATUS=1
+fi
+
+if [ "${AWS_REGION}" = "" ]; then
+	echo "${MYNAME}: FATAL: environment variable AWS_REGION is required."
+	STATUS=1
+fi
+
+if [ "${B2_BUCKET}" = "" ]; then
+	echo "${MYNAME}: FATAL: environment variable B2_BUCKET is required."
+	STATUS=1
+fi
+
+if [ "${S3_BUCKET}" = "" ]; then
+	echo "${MYNAME}: FATAL: environment variable S3_BUCKET is required."
+	STATUS=1
+fi
+
+if [ $STATUS -ne 0 ]; then
+	exit $STATUS
+fi
+
 echo "${MYNAME}: Configuring rclone"
 
 mkdir -p ~/.config/rclone