add zip output

package.json

@@ -118,6 +118,7 @@
 	"optionalDependencies": {
 		"erlpack": "^0.1.4",
 		"jimp": "^0.22.12",
+		"jszip": "^3.10.1",
 		"mysql": "^2.18.1",
 		"nodemailer-mailgun-transport": "^2.1.5",
 		"nodemailer-mailjet-transport": "github:n0script22/nodemailer-mailjet-transport",

src/api/routes/users/#id/harvest.ts

@@ -28,12 +28,21 @@ import {
 	Application,
 	ChannelTypeString,
 	getRights,
+	JSZipType,
 } from "@spacebar/util";
 import { storage } from "../../../../cdn/util/Storage";
 
 import { Request, Response, Router } from "express";
 import { promises as fs } from "node:fs";
 import path from "node:path";
+import { HTTPError } from "lambert-server";
+
+let JSZip: JSZipType | undefined = undefined;
+try {
+	JSZip = require("jszip") as JSZipType;
+} catch {
+	// empty
+}
 
 const router = Router();
 
@@ -53,6 +62,19 @@ router.get(
 		},
 	}),
 	async (req: Request, res: Response) => {
+		let zip: JSZipType;
+		try {
+			JSZip = require("jszip");
+			// @ts-expect-error Missing types for constructor
+			zip = new JSZip();
+		} catch (e) {
+			console.error("[Data export/Harvest] Failed to import JSZip", e);
+			throw new HTTPError(
+				'The NPM package "jszip" has to be installed for data exports/harvests to work.',
+				500,
+			);
+		}
+
 		if (req.params.id != "@me") {
 			const rights = await getRights(req.user_id);
 			rights.hasThrow("MANAGE_USERS");
@@ -115,49 +137,45 @@ router.get(
 		if (applications.length > 0) {
 			await fs.mkdir(path.join(harvestPath, "account", "applications"));
 			for await (const app of applications) {
-				await fs.mkdir(
-					path.join(harvestPath, "account", "applications", app.id),
+				const appPath = path.join(
+					harvestPath,
+					"account",
+					"applications",
+					app.id,
 				);
+				await fs.mkdir(appPath);
 				await fs.writeFile(
-					path.join(
-						harvestPath,
-						"account",
-						"applications",
-						app.id,
-						"application.json",
-					),
+					path.join(appPath, "application.json"),
 					JSON.stringify(app, null, "\t"),
 				);
 
 				if (app.icon) {
 					const iconFile = await storage.get(
-						"applications/" + app.id + "/" + app.icon,
+						"app-icons/" + app.id + "/" + app.icon,
 					);
 					if (iconFile)
 						await fs.writeFile(
-							path.join(
-								harvestPath,
-								"account",
-								"applications",
-								app.id,
-								"icon.png",
-							),
+							path.join(appPath, "icon.png"),
 							iconFile,
 						);
 				}
+				if (app.cover_image) {
+					const coverImageFile = await storage.get(
+						"applications/" + app.id + "/" + app.cover_image,
+					);
+					if (coverImageFile)
+						await fs.writeFile(
+							path.join(appPath, "cover_image.png"),
+							coverImageFile,
+						);
+				}
 				if (app.bot && app.bot.avatar) {
 					const botAvatarFile = await storage.get(
 						"avatars/" + app.bot.id + "/" + app.bot.avatar,
 					);
 					if (botAvatarFile)
 						await fs.writeFile(
-							path.join(
-								harvestPath,
-								"account",
-								"applications",
-								app.id,
-								"bot_avatar.png",
-							),
+							path.join(appPath, "bot_avatar.png"),
 							botAvatarFile,
 						);
 				}
@@ -166,7 +184,7 @@ router.get(
 
 		const messages = await Message.find({
 			where: { author_id: req.user_id },
-			relations: ["channel"],
+			relations: ["channel", "guild", "attachments"],
 		});
 		await fs.mkdir(path.join(harvestPath, "messages"));
 
@@ -183,13 +201,14 @@ router.get(
 					id: msg.channel_id,
 					type: ChannelTypeString[msg.channel.type],
 					name: msg.channel.name,
-					guild: msg.guild
-						? {
-								id: msg.channel.guild_id,
-								name: msg.guild.name,
-						  }
-						: null,
 				};
+				if (msg.guild)
+					// @ts-ignore yes a new property is fine
+					transformedChannel.guild = {
+						id: msg.guild.id,
+						name: msg.guild.name,
+					};
+
 				await fs.writeFile(
 					path.join(
 						harvestPath,
@@ -204,8 +223,11 @@ router.get(
 			const transformedMsg = {
 				ID: msg.id,
 				Timestamp: msg.timestamp.toISOString(),
-				Contents: msg.content,
-				Attachments: msg.attachments,
+				Contents: msg.content || "",
+				Attachments:
+					msg.attachments && msg.attachments.length > 0
+						? JSON.stringify(msg.attachments)
+						: "",
 			};
 			if (channelMessages.has(msg.channel_id))
 				channelMessages.get(msg.channel_id).push(transformedMsg);
@@ -305,6 +327,32 @@ router.get(
 				);
 			}
 		}
+
+		const files = await fs.readdir(harvestPath, {
+			withFileTypes: true,
+			recursive: true,
+		});
+		const promises = [];
+		for await (const file of files) {
+			if (!file.isFile()) continue;
+			promises.push(
+				fs
+					.readFile(path.join(file.parentPath, file.name))
+					.then((data) =>
+						zip.file(
+							path
+								.join(file.parentPath, file.name)
+								.replace(harvestPath, "")
+								.slice(path.sep.length),
+							data,
+						),
+					),
+			);
+		}
+		await Promise.all(promises);
+
+		const buffer = await zip.generateAsync({ type: "nodebuffer" });
+		await fs.writeFile(path.join(harvestPath, "harvest.zip"), buffer);
 	},
 );
 

src/util/entities/Application.ts

@@ -62,7 +62,7 @@ export class Application extends BaseClass {
 	verify_key: string;
 
 	@Column({ nullable: true })
-	// TODO: @RelationId((application: Application) => application.owner)
+	@RelationId((application: Application) => application.owner)
 	owner_id: string;
 
 	@JoinColumn({ name: "owner_id" })

src/util/imports/JSZip.ts

@@ -0,0 +1,24 @@
+/*
+	Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
+	Copyright (C) 2023 Spacebar and Spacebar Contributors
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published
+	by the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+export type JSZipType = {
+	generateAsync: (options: object) => any;
+	file: (name: string, content: any) => void;
+};

src/util/imports/index.ts

@@ -1,21 +1,22 @@
 /*
 	Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
 	Copyright (C) 2023 Spacebar and Spacebar Contributors
-	
+
 	This program is free software: you can redistribute it and/or modify
 	it under the terms of the GNU Affero General Public License as published
 	by the Free Software Foundation, either version 3 of the License, or
 	(at your option) any later version.
-	
+
 	This program is distributed in the hope that it will be useful,
 	but WITHOUT ANY WARRANTY; without even the implied warranty of
 	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 	GNU Affero General Public License for more details.
-	
+
 	You should have received a copy of the GNU Affero General Public License
 	along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
 export * from "./OrmUtils";
 export * from "./Erlpack";
 export * from "./Jimp";
+export * from "./JSZip";