added local minio s3 and backup:mongodb script

.env.template

@@ -9,3 +9,9 @@ REDIS_URI=redis://redis:6379
 GOOGLE_CLIENT_ID=_
 GOOGLE_CLIENT_SECRET=_
 SESSION_SECRET=_
+
+S3_ENDPOINT=minio
+S3_PORT=9000
+S3_ACCESS_KEY_ID=root
+S3_SECRET_ACCESS_KEY=password
+S3_MONGO_BACKUP_BUCKET=mongo-backup

backend/Dockerfile

@@ -1,6 +1,12 @@
 FROM node:16.14.2-slim AS builder
 RUN mkdir /backend
 WORKDIR /backend
+RUN  apt-get update \
+  && apt-get install -y wget \
+  && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu1804-arm64-100.9.0.deb && \
+    apt-get -y install ./mongodb-database-tools-*100.9.0.deb && \
+    rm -f mongodb-database-tools-*.deb
 COPY package.json .
 RUN npm install --prefer-offline --no-audit
 COPY . .

backend/package.json

@@ -10,7 +10,9 @@
     "test": "jest --watch",
     "coverage": "jest --coverage",
     "generate": "graphql-codegen --config codegen.ts",
-    "lint": "tsc --noEmit && eslint --ext .ts,.tsx ."
+    "lint": "tsc --noEmit && eslint --ext .ts,.tsx .",
+    "backup:mongodb": "node ./build/scripts/backup-mongodb.js",
+    "download:mongodb": "node ./build/scripts/download-mongodb.js"
   },
   "repository": {
     "type": "git",
@@ -35,7 +37,7 @@
     "@types/helmet": "0.0.46",
     "@types/jest": "^25.2.1",
     "@types/lodash": "^4.14.186",
-    "@types/node": "^13.13.5",
+    "@types/node": "^13.13.52",
     "@types/passport-google-oauth20": "^2.0.11",
     "@typescript-eslint/eslint-plugin": "^5.40.1",
     "@typescript-eslint/parser": "^5.40.1",
@@ -61,6 +63,7 @@
     "graphql-type-json": "^0.3.2",
     "helmet": "^3.22.0",
     "lodash": "^4.17.21",
+    "minio": "^7.1.3",
     "mongodb": "^4.10.0",
     "mongoose": "^6.8.0",
     "passport": "^0.6.0",

backend/src/config.ts

@@ -24,6 +24,13 @@ export interface Config {
   SESSION_SECRET: string;
   GOOGLE_CLIENT_ID: string;
   GOOGLE_CLIENT_SECRET: string;
+  s3: {
+    endpoint: string,
+    port: number,
+    access_key_id: string,
+    secret_access_key: string,
+    mongo_backup_bucket: string
+  }
 }
 
 // All your secrets, keys go here
@@ -39,4 +46,11 @@ export const config: Config = {
   SESSION_SECRET: env("SESSION_SECRET"),
   GOOGLE_CLIENT_ID: env("GOOGLE_CLIENT_ID"),
   GOOGLE_CLIENT_SECRET: env("GOOGLE_CLIENT_SECRET"),
+  s3: {
+    endpoint: env("S3_ENDPOINT"),
+    port: parseInt(env("S3_PORT")),
+    access_key_id: env("S3_ACCESS_KEY_ID"),
+    secret_access_key: env("S3_SECRET_ACCESS_KEY"),
+    mongo_backup_bucket: env("S3_MONGO_BACKUP_BUCKET")
+  }
 };

backend/src/scripts/backup-mongodb.ts

@@ -0,0 +1,114 @@
+import * as Minio from 'minio';
+import * as child from 'child_process';
+import * as Fs from "fs";
+import { config } from "../config";
+
+const ensureBucketExists = async (client : Minio.Client) => {
+    const exists = await client.bucketExists(config.s3.mongo_backup_bucket)
+    if (!exists) {
+        const res = await client.makeBucket(config.s3.mongo_backup_bucket)
+        console.log(`Bucket ${config.s3.mongo_backup_bucket} created`)
+    } 
+    const bucketLifeCycleExists = await client.getBucketLifecycle(config.s3.mongo_backup_bucket);
+    if (bucketLifeCycleExists) return;
+
+    const lifecycleConfig = {
+        Rule: [
+          {
+            ID: 'Transition and Expiration Rule',
+            Status: 'Enabled',
+            Filter: {
+              Prefix: '',
+            },
+            Expiration: {
+              Days: '7',
+            },
+          },
+        ],
+      }
+
+    const err = await client.setBucketLifecycle(config.s3.mongo_backup_bucket, lifecycleConfig)
+    if (err != null) {
+        console.error("Could not set bucket lifecycle")
+    }
+}
+
+const error = async (client : Minio.Client, toDelete : string[]) => {
+    client.removeObjects(config.s3.mongo_backup_bucket, toDelete, (e) => {
+        if (e) {
+            console.error("Unable to delete incorrect objects. Please do so manually.")
+        }
+    });
+}
+
+(async () => {
+
+    const client = new Minio.Client({
+        endPoint: config.s3.endpoint,
+        port: config.s3.port,
+        useSSL: config.s3.endpoint != "minio",
+        accessKey: config.s3.access_key_id,
+        secretKey: config.s3.secret_access_key,
+    })
+
+    await ensureBucketExists(client)
+
+    const dump = child.spawn("mongodump", ["--uri", config.mongoDB.uri])
+
+    console.log("RUNNING MONGODUMP\n---------------------\n")
+
+    const dumpedFiles:string[] = [];
+
+    const fileFinder = new RegExp(/(?:writing )(.*)(?: to )(.*)\n/)
+    dump.stderr.on('data', (data) => {
+        // mongodump output goes by default to stderr
+        const matches = data.toString().match(fileFinder)
+        if (matches != null) {
+            dumpedFiles.push(matches[2])
+            dumpedFiles.push(matches[2].substring(0, matches[2].length - 5) + ".metadata.json")
+        }
+        console.log(data.toString())
+    });
+
+    dump.on('close', (code) => {
+        if (code != 0) {
+            console.error('mongodump failed')
+            process.exit(1);
+        }
+        const complete:string[] = []
+        let puts = 0
+        const folderName = (new Date()).getTime().toString()
+        console.log(`--------------------------\nSAVING BACKUP TO: ${folderName} \n\n`)
+        let errored = false
+        dumpedFiles.forEach((file:string) => {
+            const fileStream = Fs.createReadStream(file)
+            Fs.stat(file, (err, stats) => {
+                if (err) {
+                    errored = true
+                    return
+                }
+                const name = `${folderName}/${file}`
+                client.putObject(config.s3.mongo_backup_bucket, name, fileStream, stats.size, (err:any, objInfo:any) => {
+                    puts++
+                    if (err) {
+                        console.error(err) // err should be null
+                        errored = true
+                    } else {
+                        console.log('Object Created:', objInfo)
+                        complete.push(name)
+                    }
+                    if (puts == dumpedFiles.length) {
+                        if (errored) {
+                            error(client, complete)
+                            console.log("\n--------------------\nBACKUP FAILED")
+                        } else {
+                            console.log("\n--------------------\nBACKUP SUCCESSFUL")
+                        }
+                    }
+                })
+            })
+        })
+
+    }); 
+
+})();

backend/src/scripts/download-mongodb.ts

@@ -0,0 +1,58 @@
+import * as Minio from 'minio';
+import { config } from "../config";
+import * as Fs from "fs";
+
+const getBucketExists = async (client : Minio.Client) => {
+    return await client.bucketExists(config.s3.mongo_backup_bucket)
+}
+
+(async () => {
+
+    const client = new Minio.Client({
+        endPoint: config.s3.endpoint,
+        port: config.s3.port,
+        useSSL: config.s3.endpoint != "minio",
+        accessKey: config.s3.access_key_id,
+        secretKey: config.s3.secret_access_key,
+    })
+
+    const exists = await getBucketExists(client)
+
+    if (!exists) {
+        console.error("Bucket does not exist")
+        return
+    }
+
+    if (process.argv.length < 3) {
+        console.error("Please give timestamp argument")
+        return
+    }
+
+    const timeStamp = parseInt(process.argv[2])
+
+    const objectStream = client.listObjectsV2(config.s3.mongo_backup_bucket, timeStamp + "/dump/bt/")
+
+    const objects:string[] = []
+
+    objectStream.on("data", (chunk : any) => {
+        objects.push(chunk.name)
+    })
+
+    objectStream.on("end", () => {
+        Fs.mkdir(`/backend/dump/bt`, { recursive: true }, (err) => { console.error(err) })
+        objects.forEach(async (objectName) => {
+            const dataStream = await client.getObject(config.s3.mongo_backup_bucket, objectName)
+            const fileStream = Fs.createWriteStream("/backend/" + objectName.substring(timeStamp.toString().length + 1))
+            fileStream.on("open", () => {
+                dataStream.on("data", (chunk : any) => {
+                    fileStream.write(chunk)
+                })
+                dataStream.on("end", () => {
+                    fileStream.close()
+                    console.log(`${"/backend/" + objectName.substring(timeStamp.toString().length + 1)} done writing`)
+                })
+            })
+        })
+    })
+
+})();

docker-compose.yml

@@ -55,5 +55,23 @@ services:
     ports:
       - 6379:6379
     restart: always
+    networks:
+      - bt   
+  # LOCAL S3 STORAGE - FOR TESTING MONGODB BACKUP SCRIPT
+  minio:
+    image: 'minio/minio:latest'
+    ports:
+      - '9000:9000'
+      - '9090:9090'
+    environment:
+      MINIO_ROOT_USER: 'root'
+      MINIO_ROOT_PASSWORD: 'password'
     networks:
       - bt
+    volumes:
+      - 'minio:/data/minio'
+    command: minio server /data/minio --console-address ":9090"
+volumes:
+  minio:
+    driver: local
+