Add minio backend for uploads

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:12.16.3-buster-slim AS build
+FROM node:16.20.2-bookworm-slim AS build
 WORKDIR /app
 
 COPY package.json yarn.lock ./
@@ -7,7 +7,7 @@ RUN yarn install --frozen-lockfile && yarn cache clean
 COPY . .
 RUN yarn build
 
-FROM node:12.16.3-buster-slim AS run
+FROM node:16.20.2-bookworm-slim AS run
 WORKDIR /app
 
 COPY --from=build /app/yarn.lock /app/package.json /app/

package.json

@@ -35,21 +35,23 @@
     }
   },
   "dependencies": {
+    "@fastify/cors": "8.4.0",
+    "@fastify/helmet": "11.1.1",
+    "@fastify/static": "6.11.2",
     "@google-cloud/storage": "5.3.0",
-    "aws-sdk": "2.726.0",
+    "@types/minio": "7.1.0",
+    "aws-sdk": "2.1390.0",
     "content-disposition": "0.5.3",
     "data-uri-to-buffer": "3.0.1",
     "deepmerge": "4.2.2",
     "dotenv": "8.2.0",
     "email-validator": "2.0.4",
-    "fastify": "3.2.0",
-    "fastify-cors": "4.1.0",
-    "fastify-helmet": "5.0.0",
-    "fastify-static": "3.2.0",
+    "fastify": "4.23.2",
     "got": "11.5.2",
     "hyperid": "2.0.5",
     "jss-plugin-global": "10.4.0",
     "mailgun-js": "0.22.0",
+    "minio": "7.1.0",
     "mustache": "4.0.1",
     "node-pg-migrate": "5.5.0",
     "nodemailer": "6.4.16",
@@ -65,8 +67,8 @@
     "@types/nodemailer": "6.4.0",
     "@types/pg": "7.14.5",
     "@types/uuid": "8.3.0",
-    "@typescript-eslint/eslint-plugin": "4.0.0",
-    "@typescript-eslint/parser": "3.10.1",
+    "@typescript-eslint/eslint-plugin": "6.7.5",
+    "@typescript-eslint/parser": "6.7.5",
     "ava": "3.12.1",
     "babel-eslint": "10.1.0",
     "babel-plugin-transform-export-extensions": "6.22.0",
@@ -108,7 +110,7 @@
     "supertest": "4.0.2",
     "svg-sprite-loader": "5.0.0",
     "type-fest": "0.16.0",
-    "typescript": "3.9.7"
+    "typescript": "5.2.2"
   },
   "description": "rctf is RedpwnCTF's CTF platform. It is developed and maintained by the [redpwn](https://redpwn.net) CTF team.",
   "repository": {

server/app.js

@@ -1,10 +1,10 @@
 import path from 'path'
 import fastify from 'fastify'
-import fastifyStatic from 'fastify-static'
-import helmet from 'fastify-helmet'
+import fastifyStatic from '@fastify/static'
+import helmet from '@fastify/helmet'
 import hyperid from 'hyperid'
 import config from './config/server'
-import { serveIndex, getRealIp } from './util'
+import { serveIndex, serveMinioFiles, getRealIp } from './util'
 import { init as uploadProviderInit } from './uploads'
 import api, { logSerializers as apiLogSerializers } from './api'
 
@@ -20,7 +20,7 @@ const app = fastify({
         version: req.headers['accept-version'],
         hostname: req.hostname,
         remoteAddress: getRealIp(req),
-        remotePort: req.connection.remotePort,
+        remotePort: req.socket.remotePort,
         userAgent: req.headers['user-agent']
       })
     }
@@ -49,13 +49,18 @@ app.register(helmet, {
   }
 })
 
-uploadProviderInit(app)
+const uploadProvider = uploadProviderInit(app)
 
 app.register(api, {
   prefix: '/api/v1/',
   logSerializers: apiLogSerializers
 })
 
+if (config.uploadProvider.name === 'uploads/minio') {
+  // if minio (private) as uploads provider, we need a route to proxy the files download
+  app.register(serveMinioFiles(uploadProvider), {})
+}
+
 const staticPath = path.join(__dirname, '../build')
 
 app.register(serveIndex, {

server/index.js

@@ -16,7 +16,7 @@ const runMain = async () => {
     const port = process.env.PORT || 3000
 
     const { default: app } = await import('./app')
-    app.listen(port, '::', err => {
+    app.listen({ port: port, host: '::' }, err => {
       if (err) {
         app.log.error(err)
       }

server/providers/challenges/database/index.ts

@@ -40,12 +40,11 @@ class DatabaseProvider extends EventEmitter implements Provider {
   challengeToRow (chall: Challenge): DatabaseChallenge {
     chall = deepCopy(chall)
 
-    const id = chall.id
-    delete chall.id
+    const { id, ...challWithoutId } = chall
 
     return {
       id,
-      data: chall
+      data: { ...challWithoutId }
     }
   }
 

server/providers/emails/ses/index.ts

@@ -63,7 +63,11 @@ export default class SesProvider implements Provider {
         Source: mail.from
       })
     } catch (e) {
-      throw new SesError(e)
+      if (e instanceof Error) {
+        throw new SesError(e)
+      } else {
+        // handle
+      }
     }
   }
 }

server/providers/uploads/local/index.ts

@@ -5,7 +5,7 @@ import fs from 'fs'
 import crypto from 'crypto'
 import config from '../../../config/server'
 import { FastifyInstance } from 'fastify'
-import fastifyStatic from 'fastify-static'
+import fastifyStatic from '@fastify/static'
 import contentDisposition from 'content-disposition'
 
 interface LocalProviderOptions {

server/providers/uploads/minio/index.ts

@@ -0,0 +1,118 @@
+import crypto from 'crypto'
+import { Provider } from '../../../uploads/provider'
+import { Client as MinioClient } from 'minio'
+import { Stream } from 'stream'
+
+interface MinioProviderOptions {
+  accessKey: string;
+  secretKey: string;
+  endPoint: string;
+  pathStyle: boolean;
+  port: number;
+  bucketName: string;
+  useSSL: boolean;
+}
+
+class MinioFile {
+  key: string
+  name: string
+  sha256: string
+  bucket: string
+  url: string
+
+  constructor (sha256: string, name: string, bucket: string) {
+    this.key = `uploads/${sha256}/${name}`
+    this.name = name
+    this.sha256 = sha256
+    this.bucket = bucket
+    this.url = `/proxy/file/${this.key}`
+  }
+
+  async exists (minioClient: MinioClient): Promise<boolean> {
+    const stat = await new Promise((resolve, reject) => {
+      return minioClient.statObject(
+        this.bucket,
+        this.key,
+        (err, stat) => {
+          if (err) {
+            console.log(err)
+            return resolve(null)
+          }
+          return resolve(stat)
+        }
+      )
+    })
+    return !!stat
+  }
+}
+
+export default class MinioProvider implements Provider {
+  private bucketName: string
+  private minioClient: MinioClient
+
+  constructor (_options: Partial<MinioProviderOptions>) {
+    const options: Required<MinioProviderOptions> = {
+      accessKey: _options.accessKey || process.env.RCTF_MINIO_ACCESS_KEY as string,
+      secretKey: _options.secretKey || process.env.RCTF_MINIO_SECRET_KEY as string,
+      endPoint: _options.endPoint || process.env.RCTF_MINIO_ENDPOINT as string || 'minio',
+      port: _options.port || (process.env.RCTF_MINIO_PORT as unknown) as number || 9000,
+      bucketName: _options.bucketName || process.env.RCTF_MINIO_BUCKET_NAME as string || 'rctf',
+      pathStyle: _options.pathStyle || true,
+      useSSL: _options.useSSL || false
+    }
+
+    // TODO: validate that all options are indeed provided
+    this.minioClient = new MinioClient({
+      accessKey: options.accessKey,
+      secretKey: options.secretKey,
+      endPoint: options.endPoint,
+      useSSL: options.useSSL,
+      pathStyle: options.pathStyle,
+      port: options.port
+    })
+
+    this.bucketName = options.bucketName
+  }
+
+  private getMinioFile = (sha256: string, name: string): MinioFile => {
+    return new MinioFile(sha256, name, this.bucketName)
+  }
+
+  upload = async (data: Buffer, name: string): Promise<string> => {
+    const hash = crypto.createHash('sha256').update(data).digest('hex')
+    const file = this.getMinioFile(hash, name)
+    const exists = await file.exists(this.minioClient)
+
+    if (!exists) {
+      await this.minioClient
+        .putObject(this.bucketName, file.key, data)
+        .catch((e) => {
+          console.log('Error while creating object: ', e)
+          throw e
+        })
+    }
+    return file.url
+  }
+
+  async getUrl (sha256: string, name: string): Promise<string|null> {
+    const file = new MinioFile(sha256, name, this.bucketName)
+    const exists = await file.exists(this.minioClient)
+
+    if (!exists) return null
+    return file.url
+  }
+
+  private async stream2buffer (stream: Stream): Promise<Buffer> {
+    return new Promise <Buffer>((resolve, reject) => {
+      const _buf: any[] = []
+      stream.on('data', chunk => _buf.push(chunk))
+      stream.on('end', () => resolve(Buffer.concat(_buf)))
+      stream.on('error', err => reject(err))
+    })
+  }
+
+  async streamFile (name: string): Promise<Buffer> {
+    const fileStream = await this.minioClient.getObject(this.bucketName, name)
+    return this.stream2buffer(fileStream)
+  }
+}

server/uploads/index.ts

@@ -5,14 +5,15 @@ import { FastifyInstance } from 'fastify'
 
 let provider: Provider | null = null
 
-export const init = (app: FastifyInstance | null): void => {
+export const init = (app: FastifyInstance | null): Provider => {
   const name = app === null ? 'uploads/dummy' : config.uploadProvider.name
 
   // FIXME: use async loading
   // eslint-disable-next-line @typescript-eslint/no-var-requires
   const { default: ProviderClass } = require(path.join('../providers', name)) as { default: ProviderConstructor }
 
   provider = new ProviderClass(config.uploadProvider.options ?? {}, app)
+  return provider
 }
 
 export const upload = (data: Buffer, name: string): Promise<string> => {

server/util/index.ts

@@ -3,6 +3,8 @@ import clientConfig from '../config/client'
 import { promises as fs } from 'fs'
 import mustache from 'mustache'
 import { FastifyPluginAsync, FastifyRequest, RouteHandlerMethod } from 'fastify'
+import { Readable } from 'stream'
+import MinioProvider from '../providers/uploads/minio'
 
 export * as normalize from './normalize'
 export * as validate from './validate'
@@ -17,6 +19,26 @@ export const deepCopy = <T>(data: T): T => {
   return JSON.parse(JSON.stringify(data)) as T
 }
 
+export const serveMinioFiles = (uploadProvider: MinioProvider) => {
+  const serve: FastifyPluginAsync = async (fastify, opts) => {
+    fastify.get('/proxy/file/*', async (req, reply): Promise<void> => {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment
+      const filePath = (req.params as any)['*']
+      const buffer = await uploadProvider.streamFile(filePath)
+
+      const myStream = new Readable({
+        read () {
+          this.push(buffer)
+          this.push(null)
+        }
+      })
+      // eslint-disable-next-line @typescript-eslint/no-floating-promises
+      reply.send(myStream)
+    })
+  }
+  return serve
+}
+
 export const serveIndex: FastifyPluginAsync<{ indexPath: string; }> = async (fastify, opts) => {
   const indexTemplate = (await fs.readFile(opts.indexPath)).toString()