From ed2b4bb01d15bd2404e5119c65ef97c91849d56f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Thu, 11 Mar 2021 22:35:58 +0000 Subject: [PATCH 001/120] Client now properly emits ready event on hot reload Client will emit the standard ready event once all of its shards are ready and connected. --- client.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client.js b/client.js index 2b4eec7..95ebadf 100644 --- a/client.js +++ b/client.js @@ -1,6 +1,7 @@ "use strict"; require("./init.js"); +const { Constants } = require("discord.js"); const Discord = require("./classes.js"); const actions = require("./actions.js"); const pkg = require("./package.json"); From 09526952293e9611313462ccfd1b7e15221c6ae6 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 12 Mar 2021 20:58:12 +0000 Subject: [PATCH 002/120] Added directory property and added all session cached to folder which is gitignored --- .gitignore | 4 ++++ client.js | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 627a9f1..802d68a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ package-lock.json .vscode .eslintrc.json .gitattributes +.pnp.js +.yarnrc.yml +yarn.lock +.sessions diff --git a/client.js b/client.js index 95ebadf..20b709b 100644 --- a/client.js +++ b/client.js @@ -23,6 +23,7 @@ Discord.Client = class Client extends Discord.Client { super(options); actions(this); if(options.hotreload) { + this.cacheFilePath = `${process.cwd()}/.sessions`; this.ws._hotreload = {}; if (options.sessionID && options.sequence) { if (!Array.isArray(options.sessionID) && !Array.isArray(options.sequence)) { @@ -38,7 +39,7 @@ Discord.Client = class Client extends Discord.Client { } else { try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${process.cwd()}/.sessions.json`, "utf8")); + this.ws._hotreload = JSON.parse(fs.readFileSync(`${process.cwd()}/.sessions/sessions.json`, "utf8")); } catch(e) { this.ws._hotreload = {}; } @@ -49,7 +50,7 @@ Discord.Client = class Client extends Discord.Client { for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, () => { try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${process.cwd()}/.sessions.json`, "utf8")); + this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); } catch(e) { this.ws._hotreload = {}; } @@ -62,7 +63,7 @@ Discord.Client = class Client extends Discord.Client { } }; })); - fs.writeFileSync(`${process.cwd()}/.sessions.json`, JSON.stringify(this.ws._hotreload)); + fs.writeFileSync(`${this.cacheFilePath}/sessions.json`, JSON.stringify(this.ws._hotreload)); if(eventType !== "exit") { process.exit(); } From 3e1bde3658f02c48ccc486cb79bc3626954f9ef6 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 12 Mar 2021 20:58:51 +0000 Subject: [PATCH 003/120] Creates folder if it does not exist --- client.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client.js b/client.js index 20b709b..3774759 100644 --- a/client.js +++ b/client.js @@ -49,6 +49,7 @@ Discord.Client = class Client extends Discord.Client { }); for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, () => { + if (!fs.existsSync(this.cacheFilePath)) { fs.mkdirSync(this.cacheFilePath); } try { this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); } catch(e) { From fe0b745e86af4ae6b2d583cb4d5dcc86819cc220 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 12 Mar 2021 23:17:04 +0000 Subject: [PATCH 004/120] Added hot reloading explanation to README --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 07d6f95..a1f1858 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,17 @@ client.login("TOKEN").catch(console.error); Generally, usage should be identical to discord.js and you can safely refer to its documentation as long as you respect the caching differences explained below. +### Hot reloading + +**THIS FEATURE IS CURRENTLY EXPERIMENTAL USE AT YOUR OWN RISK!** + +When developing bots you will often want to prototype through trial and error. This often requires turning your bot on and off lots of times potentially using [nodemon](https://nodemon.io/) +by doing this you are connecting to the Discord websocket gateway each time which can often take a few seconds as well as cuts into your 1000 daily identifies. + +To solve this problem you can use hot reloading which is a client option allowing you to simply resume the previous session rather than create a new one. + +You can also use Hot reloading in your production bot by supplying a sequence and session ID or by just letting us take care of it with cache files found in the `.sessions` folder + ## Client Options The following client options are available to control caching behavior: From f4c66ea63f6cc8b1f44a9108d5ef4bb3147e3cb6 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 12 Mar 2021 23:18:05 +0000 Subject: [PATCH 005/120] Added temporary file loading and parsing. Fails with roles --- client.js | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 3774759..65d18ac 100644 --- a/client.js +++ b/client.js @@ -1,7 +1,6 @@ "use strict"; require("./init.js"); -const { Constants } = require("discord.js"); const Discord = require("./classes.js"); const actions = require("./actions.js"); const pkg = require("./package.json"); @@ -18,6 +17,7 @@ Discord.Client = class Client extends Discord.Client { cacheEmojis: false, cacheMembers: false, disabledEvents: [], + restoreCache: ["guilds"], ..._options }; super(options); @@ -39,11 +39,29 @@ Discord.Client = class Client extends Discord.Client { } else { try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${process.cwd()}/.sessions/sessions.json`, "utf8")); + this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); } catch(e) { this.ws._hotreload = {}; } } + try { + for (const toCache of options.restoreCache) { + const data = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/${toCache}.json`, "utf8")); + switch (toCache) { + case "guilds": { + console.log("Created"); + data.cache.forEach(i => { + console.log(i); + this.guilds.cache.set(i.id, new Discord.Guild(this, i)); + console.log(i.id); + }); + break; + } + } + } + } catch(e) { + // Do nothing + } this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); @@ -65,6 +83,11 @@ Discord.Client = class Client extends Discord.Client { }; })); fs.writeFileSync(`${this.cacheFilePath}/sessions.json`, JSON.stringify(this.ws._hotreload)); + if (options.restoreCache) { + for (const toCache of options.restoreCache) { + fs.writeFileSync(`${this.cacheFilePath}/${toCache}.json`, JSON.stringify(this[toCache])); + } + } if(eventType !== "exit") { process.exit(); } From 1ee78f07e63140756102d26c3f91d882f3a4e8c0 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:40:38 +0000 Subject: [PATCH 006/120] Added _unpatch method to guild class Not finished roles and emojis need to be completed and the rest of the properties need to be looked at and confirmed to be correct --- classes.js | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/classes.js b/classes.js index 12d4789..32e6f87 100644 --- a/classes.js +++ b/classes.js @@ -216,6 +216,115 @@ Discord.Structures.extend("Guild", G => { } } } + _unpatch() { + /** + * Discord raw guild data as documented: https://github.com/discordjs/discord-api-types/blob/main/v8/payloads/guild.ts + */ + return { + id: this.id, + unavailable: this.available, + name: this.name, + icon: this.icon, + splash: this.splash, + banner: this.banner, + description: this.description, + features: this.features, + verification_level: this.verificationLevel, + discovery_splash: this.discoverySplash, + owner_id: this.ownerID, + region: this.region, + afk_channel_id: this.afkChannelID, + afk_timeout: this.afkTimeout, + widget_enabled: this.widgetEnabled, + widget_channel_id: this.widgetChannelID, + default_message_notifications: this.defaultMessageNotifications, + explicit_content_filter: this.explicitContentFilter, + /** + * Roles in the guild + * + * See https://discord.com/developers/docs/topics/permissions#role-object + */ + roles: this.roles.cache.map(r => ({ + name: r.name, + color: r.color + })), + /** + * Custom guild emojis + * + * See https://discord.com/developers/docs/resources/emoji#emoji-object + */ + emojis: this.emojis.cache.map(e => ({ + id: e.id, + name: e.name + })), + mfa_level: this.mfaLevel, + application_id: this.applicationID, + system_channel_id: this.systemChannelID, + system_channel_flags: this.systemChannelFlags, + rules_channel_id: this.rulesChannelID, + joined_at: this.joinedAt, + large: this.large, + member_count: this.memberCount, + voice_states: this.voiceStates.cache.map(v => ({ + guild_id: v.guild.id, + channel_id: v.channelID, + user_id: v.userID, + session_id: v.sessionID, + deaf: v.deaf, + mute: v.mute, + self_deaf: v.selfDeaf, + self_mute: v.selfMute, + suppress: v.suppress + })), + members: this.members.cache.map(m => ({ + user: m.user, + nick: m.nickname, + roles: m.roles, + joined_at: m.joinedAt, + premium_since: m.premiumSinceTimestamp, + deaf: m.deaf, + mute: m.mute, + pending: m.pending, + permissions: m.permissions + })), + channels: this.channels.cache.map(c => ({ + id: c.id, + type: c.type, + guild_id: c.guild.id, + position: c.position, + permission_overwrites: c.permissionOverwrites, + name: c.name, + topic: c.topic, + nsfw: c.nsfw, + last_message_id: c.lastMessageID, + bitrate: c.bitrate, + user_limit: c.userLimit, + rate_limit_per_user: c.rateLimitPerUser, + recipients: c.recipients, + icon: c.icon, + owner_id: c.ownerID, + application_id: c.applicationID, + parent_id: c.parentID, + last_pin_timestamp: c.lastPinTimestamp + })), + presences: this.presences.cache.map(p => ({ + user: p.user, + guild_id: p.guild.id, + status: p.status, + activities: p.activities, + client_status: p.clientStatus + })), + max_presences: this.maximumPresences, + max_members: this.maximumMembers, + vanity_url_code: this.vanityURLCode, + premium_tier: this.premiumTier, + premium_subscription_count: this.premiumSubscriptionCount, + preferred_locale: this.preferredLocale || "en-US", + public_updates_channel_id: this.publicUpdatesChannelID, + approximate_member_count: this.approximateMemberCount, + approximate_presence_count: this.approximatePresenceCount + }; + } get nameAcronym() { return this.name ? super.nameAcronym : void 0; } From c1a6254bdae214aae4b363aa6d5a50ae17c7c3dd Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:47:10 +0000 Subject: [PATCH 007/120] Added new client options --- client.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 65d18ac..e74f634 100644 --- a/client.js +++ b/client.js @@ -17,8 +17,11 @@ Discord.Client = class Client extends Discord.Client { cacheEmojis: false, cacheMembers: false, disabledEvents: [], - restoreCache: ["guilds"], - ..._options + sessions: {}, + ..._options, + restoreCache: { + guilds: true, + } }; super(options); actions(this); From 37a23c7331ab5129e33f85aac4b3c27a76a3f6d4 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:48:41 +0000 Subject: [PATCH 008/120] Greatly simplified session and sequence client option handling It was very cumbersome using an array so I removed the sessionID and sequence options and just replaced it with a session object which should be structured identically to the _hotreload object for simplicity --- client.js | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/client.js b/client.js index e74f634..697fa8d 100644 --- a/client.js +++ b/client.js @@ -27,18 +27,8 @@ Discord.Client = class Client extends Discord.Client { actions(this); if(options.hotreload) { this.cacheFilePath = `${process.cwd()}/.sessions`; - this.ws._hotreload = {}; - if (options.sessionID && options.sequence) { - if (!Array.isArray(options.sessionID) && !Array.isArray(options.sequence)) { - options.sessionID = [options.sessionID]; - options.sequence = [options.sequence]; - } - for (let shard = 0; shard < options.sessionID.length; shard++) { - this.ws._hotreload[shard] = { - id: options.sessionID[shard], - seq: options.sequence[shard] - }; - } + if (options.sessions && Object.keys(options.sessions).length) { + this.ws._hotreload = options.sessions; } else { try { From 9285dc337834dbf086f617d10c99aec16d6b8fee Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:56:46 +0000 Subject: [PATCH 009/120] Users can specify the exit events they desire or use default ones --- client.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 697fa8d..ff7161f 100644 --- a/client.js +++ b/client.js @@ -22,6 +22,7 @@ Discord.Client = class Client extends Discord.Client { restoreCache: { guilds: true, } + exitEvents: ["exit", "uncaughtException", "SIGINT", "SIGTERM", ..._options.exitEvents] }; super(options); actions(this); @@ -58,7 +59,6 @@ Discord.Client = class Client extends Discord.Client { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); - for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, () => { if (!fs.existsSync(this.cacheFilePath)) { fs.mkdirSync(this.cacheFilePath); } try { @@ -79,6 +79,7 @@ Discord.Client = class Client extends Discord.Client { if (options.restoreCache) { for (const toCache of options.restoreCache) { fs.writeFileSync(`${this.cacheFilePath}/${toCache}.json`, JSON.stringify(this[toCache])); + for (const eventType of options.exitEvents) { } } if(eventType !== "exit") { From 79644c42c2a5184c54e8e33f0038bf6d5a0c0e4e Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:59:24 +0000 Subject: [PATCH 010/120] Added uncaughtExceptionOnExit bool to stop loop of uncaughExceptions If there is an uncaught exception inside of the event function it will trigger itself causing an infinite loop. I added a uncaughtExceptionOnExit bool variable so that the uncaughtException event can only trigger once and after that it will exit the process without running the buggy code --- client.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index ff7161f..852804e 100644 --- a/client.js +++ b/client.js @@ -79,12 +79,19 @@ Discord.Client = class Client extends Discord.Client { if (options.restoreCache) { for (const toCache of options.restoreCache) { fs.writeFileSync(`${this.cacheFilePath}/${toCache}.json`, JSON.stringify(this[toCache])); + this._uncaughtExceptionOnExit = false; for (const eventType of options.exitEvents) { + process.on(eventType, async () => { + if (eventType === "uncaughtException") { + this._uncaughtExceptionOnExit = true; + } } } - if(eventType !== "exit") { - process.exit(); + else { + console.error("There was an uncaughtException inside your exit loop causing an infinite loop. Your exit function was not run"); + process.exit(1); } + }); } } From 84afdb3e26f71eefcf64b2683518996f255db293 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:01:40 +0000 Subject: [PATCH 011/120] Added dumpCache method with sessions and client args --- client.js | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/client.js b/client.js index 852804e..2e9632e 100644 --- a/client.js +++ b/client.js @@ -59,26 +59,9 @@ Discord.Client = class Client extends Discord.Client { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); - process.on(eventType, () => { - if (!fs.existsSync(this.cacheFilePath)) { fs.mkdirSync(this.cacheFilePath); } - try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); - } catch(e) { - this.ws._hotreload = {}; + this.dumpCache = (sessions, client) => { } - Object.assign(this.ws._hotreload, ...this.ws.shards.map(s => { - s.connection.close(); - return { - [s.id]: { - id: s.sessionID, - seq: s.sequence - } - }; - })); - fs.writeFileSync(`${this.cacheFilePath}/sessions.json`, JSON.stringify(this.ws._hotreload)); - if (options.restoreCache) { - for (const toCache of options.restoreCache) { - fs.writeFileSync(`${this.cacheFilePath}/${toCache}.json`, JSON.stringify(this[toCache])); + }; this._uncaughtExceptionOnExit = false; for (const eventType of options.exitEvents) { process.on(eventType, async () => { From 382516e42a9800e7614394923a76d79ca4731e1b Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:06:14 +0000 Subject: [PATCH 012/120] Assigns new shards to hotreload and calls dumpCache asynchronously if not exit --- client.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/client.js b/client.js index 2e9632e..855a4f4 100644 --- a/client.js +++ b/client.js @@ -68,6 +68,19 @@ Discord.Client = class Client extends Discord.Client { if (eventType === "uncaughtException") { this._uncaughtExceptionOnExit = true; } + if (!this._uncaughtExceptionOnExit) { + Object.assign(this.ws._hotreload, ...this.ws.shards.map(s => { + s.connection.close(); + return { + [s.id]: { + id: s.sessionID, + seq: s.sequence + } + }; + })); + if (eventType !== "exit") { + await this.dumpCache(this.ws._hotreload, this); + process.exit(); } } else { From dc31079225c8e740cbd907abffb117f84b2974fd Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:11:18 +0000 Subject: [PATCH 013/120] dumpCache sync file storing function Currently just stores the bot user and guilds to disk. Needs improvement --- client.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 855a4f4..8c9dd98 100644 --- a/client.js +++ b/client.js @@ -60,7 +60,23 @@ Discord.Client = class Client extends Discord.Client { if(!this.readyAt) { this.ws.checkShardsReady(); } }); this.dumpCache = (sessions, client) => { - } + if (!fs.existsSync(client.cacheFilePath)) { fs.mkdirSync(client.cacheFilePath); } + try { + client.ws._hotreload = JSON.parse(fs.readFileSync(`${client.cacheFilePath}/sessions.json`, "utf8")); + } catch(e) { + client.ws._hotreload = {}; + } + client.ws._hotreload = { + ...client.ws._hotreload, + ...sessions + }; + fs.writeFileSync(`${client.cacheFilePath}/sessions.json`, JSON.stringify(client.ws._hotreload)); + if (options.restoreCache.guilds) { + const discordGuilds = client.guilds.cache.map(g => g._unpatch()); + fs.writeFileSync(`${client.cacheFilePath}/guilds.json`, JSON.stringify(discordGuilds)); + const discordMe = client.user._unpatch(); + fs.writeFileSync(`${client.cacheFilePath}/me.json`, JSON.stringify(discordMe)); + } }; this._uncaughtExceptionOnExit = false; for (const eventType of options.exitEvents) { From 329bff9d63e5da8858f8a9712b6b7660a4eefcfb Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:14:16 +0000 Subject: [PATCH 014/120] Restores guilds and users --- client.js | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/client.js b/client.js index 8c9dd98..80d9272 100644 --- a/client.js +++ b/client.js @@ -20,13 +20,20 @@ Discord.Client = class Client extends Discord.Client { sessions: {}, ..._options, restoreCache: { + channels: false, guilds: true, - } + presences: false, + roles: false, + overwrites: false, + emojis: false, + members: false, + ..._options.restoreCache + }, exitEvents: ["exit", "uncaughtException", "SIGINT", "SIGTERM", ..._options.exitEvents] }; super(options); actions(this); - if(options.hotreload) { + if (options.hotreload) { this.cacheFilePath = `${process.cwd()}/.sessions`; if (options.sessions && Object.keys(options.sessions).length) { this.ws._hotreload = options.sessions; @@ -38,23 +45,13 @@ Discord.Client = class Client extends Discord.Client { this.ws._hotreload = {}; } } - try { - for (const toCache of options.restoreCache) { - const data = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/${toCache}.json`, "utf8")); - switch (toCache) { - case "guilds": { - console.log("Created"); - data.cache.forEach(i => { - console.log(i); - this.guilds.cache.set(i.id, new Discord.Guild(this, i)); - console.log(i.id); - }); - break; - } - } + + if (options.restoreCache.guilds) { + const discordGuildData = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8")); + for (const guild of discordGuildData) { + this.guilds.cache.set(guild.id, new Discord.Guild(this, guild)); } - } catch(e) { - // Do nothing + this.user = new Discord.User(this, JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8"))); } this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } From f1f9161f1bd20a8b27e177296e2d0dec0c31f55b Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:15:22 +0000 Subject: [PATCH 015/120] Added restore cache client options --- client.d.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/client.d.ts b/client.d.ts index 9162719..28636b2 100644 --- a/client.d.ts +++ b/client.d.ts @@ -58,6 +58,15 @@ declare module "discord.js-light" { cacheEmojis?:boolean cacheMembers?:boolean disabledEvents?: Array + restoreCache: { + channels: boolean + guilds: boolean + presences: boolean + roles: boolean + overwrites: boolean + emojis: boolean + members: boolean + } } interface ClientEvents { rest:[{path:string,method:string,response?:Promise}] From e9770e7bf5981ddde699c7df4dd0452140654230 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:18:15 +0000 Subject: [PATCH 016/120] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a1f1858..cfc1824 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,10 @@ by doing this you are connecting to the Discord websocket gateway each time whic To solve this problem you can use hot reloading which is a client option allowing you to simply resume the previous session rather than create a new one. -You can also use Hot reloading in your production bot by supplying a sequence and session ID or by just letting us take care of it with cache files found in the `.sessions` folder +You can also use Hot reloading in your production bot by supplying a session object along with preferences for caching restoration or by just letting us take care of it with cache files found in the `.sessions` folder + +By setting the client.dumpCache method you can run a custom async function to store your caches and session IDs in your database of choice. The dumpCache method is +called with the (session, client) params where session is your up to date sessoins ## Client Options From 1eec95b54b95c189cd8c0c8dd1b33153bec5f1bc Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 19:46:35 +0000 Subject: [PATCH 017/120] Updated client options and added typings --- client.d.ts | 23 ++++++++++++++--------- client.js | 21 +++++---------------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/client.d.ts b/client.d.ts index 28636b2..ecb02c0 100644 --- a/client.d.ts +++ b/client.d.ts @@ -48,6 +48,19 @@ type ReactionUserFetchOptions = { after?: Discord.Snowflake } +type SessionData = { + [shardID: string]: { + id: string + sequence: number + } +} + +type HotReloadOptions = { + sessionData: SessionData + unpatchOnExit: boolean + patchSource: string | Object +} + declare module "discord.js-light" { interface ClientOptions { cacheChannels?:boolean @@ -58,15 +71,7 @@ declare module "discord.js-light" { cacheEmojis?:boolean cacheMembers?:boolean disabledEvents?: Array - restoreCache: { - channels: boolean - guilds: boolean - presences: boolean - roles: boolean - overwrites: boolean - emojis: boolean - members: boolean - } + hotReload?: boolean | HotReloadOptions } interface ClientEvents { rest:[{path:string,method:string,response?:Promise}] diff --git a/client.js b/client.js index 80d9272..934981d 100644 --- a/client.js +++ b/client.js @@ -17,23 +17,12 @@ Discord.Client = class Client extends Discord.Client { cacheEmojis: false, cacheMembers: false, disabledEvents: [], - sessions: {}, - ..._options, - restoreCache: { - channels: false, - guilds: true, - presences: false, - roles: false, - overwrites: false, - emojis: false, - members: false, - ..._options.restoreCache - }, - exitEvents: ["exit", "uncaughtException", "SIGINT", "SIGTERM", ..._options.exitEvents] + hotReload: false, + ..._options }; super(options); actions(this); - if (options.hotreload) { + if (options.hotReload) { this.cacheFilePath = `${process.cwd()}/.sessions`; if (options.sessions && Object.keys(options.sessions).length) { this.ws._hotreload = options.sessions; @@ -46,7 +35,7 @@ Discord.Client = class Client extends Discord.Client { } } - if (options.restoreCache.guilds) { + if (options.cacheGuilds) { const discordGuildData = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8")); for (const guild of discordGuildData) { this.guilds.cache.set(guild.id, new Discord.Guild(this, guild)); @@ -68,7 +57,7 @@ Discord.Client = class Client extends Discord.Client { ...sessions }; fs.writeFileSync(`${client.cacheFilePath}/sessions.json`, JSON.stringify(client.ws._hotreload)); - if (options.restoreCache.guilds) { + if (options.cacheGuilds) { const discordGuilds = client.guilds.cache.map(g => g._unpatch()); fs.writeFileSync(`${client.cacheFilePath}/guilds.json`, JSON.stringify(discordGuilds)); const discordMe = client.user._unpatch(); From ea895b624d07c69f6b73ebb3fc4a60ce39baba76 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 21:18:06 +0000 Subject: [PATCH 018/120] Added new typedefs --- client.d.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/client.d.ts b/client.d.ts index ecb02c0..808f46d 100644 --- a/client.d.ts +++ b/client.d.ts @@ -55,10 +55,16 @@ type SessionData = { } } +type CacheData = { + guilds?: Array + channels?: Array + users?: Array +} + type HotReloadOptions = { - sessionData: SessionData - unpatchOnExit: boolean - patchSource: string | Object + sessionData?: SessionData + cacheData?: CacheData + onUnload?: Function } declare module "discord.js-light" { From a8be027dd3340e9b90f1958099114738efe2b4fb Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 21:42:32 +0000 Subject: [PATCH 019/120] Added djs-light validate options --- client.js | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/client.js b/client.js index 934981d..36259b2 100644 --- a/client.js +++ b/client.js @@ -22,6 +22,7 @@ Discord.Client = class Client extends Discord.Client { }; super(options); actions(this); + this._validateOptionsLight(); if (options.hotReload) { this.cacheFilePath = `${process.cwd()}/.sessions`; if (options.sessions && Object.keys(options.sessions).length) { @@ -111,6 +112,48 @@ Discord.Client = class Client extends Discord.Client { guild.channels.cache.sweep(t => !this.channels.cache.has(t.id)); } } + /** + * Validates the client options. + * @param {object} options Options to validate + * @private + */ + _validateOptionsLight(options) { + if (typeof options.cacheChannels !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheChannels", "a boolean"); + } + if (typeof options.cacheGuilds !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheGuilds", "a boolean"); + } + if (typeof options.cachePresences !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cachePresences", "a boolean"); + } + if (typeof options.cacheRoles !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheRoles", "a boolean"); + } + if (typeof options.cacheOverwrites !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheOverwrites", "a boolean"); + } + if (typeof options.cacheEmojis !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheEmojis", "a boolean"); + } + if (typeof options.cacheMembers !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheMembers", "a boolean"); + } + if (!Array.isArray(options.disabledEvents)) { + throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); + } + if (options.hotReload) { + if (typeof options.hotReload.sessionData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); + } + if (typeof options.hotReload.cacheData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); + } + if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { + throw new TypeError("CLIENT_INVALID_OPTION", "onUnload", "a function"); + } + } + } }; Discord.version = `${pkg.version} (${Discord.version})`; From 24fc69bcde814ac286450a3d63623fd2eacb0f2f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 21:48:13 +0000 Subject: [PATCH 020/120] Added change for sessionData --- client.js | 5 ++--- init.js | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index 36259b2..f44ef31 100644 --- a/client.js +++ b/client.js @@ -25,8 +25,8 @@ Discord.Client = class Client extends Discord.Client { this._validateOptionsLight(); if (options.hotReload) { this.cacheFilePath = `${process.cwd()}/.sessions`; - if (options.sessions && Object.keys(options.sessions).length) { - this.ws._hotreload = options.sessions; + if (options.hotReload.sessionData && Object.keys(options.hotReload.sessionData).length) { + this.ws._hotreload = options.hotReload.sessionData; } else { try { @@ -66,7 +66,6 @@ Discord.Client = class Client extends Discord.Client { } }; this._uncaughtExceptionOnExit = false; - for (const eventType of options.exitEvents) { process.on(eventType, async () => { if (eventType === "uncaughtException") { this._uncaughtExceptionOnExit = true; diff --git a/init.js b/init.js index 3deeea1..dab02df 100644 --- a/init.js +++ b/init.js @@ -52,7 +52,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { const data = this.manager._hotreload[this.id]; if(data && !this.sessionID) { this.sessionID = data.id; - this.closeSequence = this.sequence = data.seq; + this.closeSequence = this.sequence = data.sequence; delete this.manager._hotreload[this.id]; } } From b9a16811e706e9a473e0d27222c120cc1ed16ac3 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:35:32 +0000 Subject: [PATCH 021/120] Reads session files from .sessions/sessions/{shardID}.json --- client.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index f44ef31..59f40a6 100644 --- a/client.js +++ b/client.js @@ -24,14 +24,23 @@ Discord.Client = class Client extends Discord.Client { actions(this); this._validateOptionsLight(); if (options.hotReload) { + this.on(Discord.Constants.Events.SHARD_RESUME, () => { + if (!this.readyAt) { this.ws.checkShardsReady(); } + }); this.cacheFilePath = `${process.cwd()}/.sessions`; + this.ws._hotreload = {}; if (options.hotReload.sessionData && Object.keys(options.hotReload.sessionData).length) { this.ws._hotreload = options.hotReload.sessionData; } else { try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); - } catch(e) { + const shards = fs.readdirSync(`${this.cacheFilePath}/sessions`) + .filter(file => file.endsWith(".json")) + .map(shardSession => shardSession.substr(0, shardSession.lastIndexOf("."))); + for (const shardID of shards) { + this.ws._hotreload[shardID] = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, "utf8")); + } + } catch (e) { this.ws._hotreload = {}; } } From bb43047a540aef10951a3abbe20af348d87595e7 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:37:56 +0000 Subject: [PATCH 022/120] Extra checks for validation of options and added back events --- client.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index 59f40a6..62b6356 100644 --- a/client.js +++ b/client.js @@ -22,7 +22,7 @@ Discord.Client = class Client extends Discord.Client { }; super(options); actions(this); - this._validateOptionsLight(); + this._validateOptionsLight(options); if (options.hotReload) { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if (!this.readyAt) { this.ws.checkShardsReady(); } @@ -59,7 +59,7 @@ Discord.Client = class Client extends Discord.Client { if (!fs.existsSync(client.cacheFilePath)) { fs.mkdirSync(client.cacheFilePath); } try { client.ws._hotreload = JSON.parse(fs.readFileSync(`${client.cacheFilePath}/sessions.json`, "utf8")); - } catch(e) { + } catch (e) { client.ws._hotreload = {}; } client.ws._hotreload = { @@ -75,6 +75,7 @@ Discord.Client = class Client extends Discord.Client { } }; this._uncaughtExceptionOnExit = false; + for (const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async () => { if (eventType === "uncaughtException") { this._uncaughtExceptionOnExit = true; @@ -151,10 +152,10 @@ Discord.Client = class Client extends Discord.Client { throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); } if (options.hotReload) { - if (typeof options.hotReload.sessionData !== "object") { + if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); } - if (typeof options.hotReload.cacheData !== "object") { + if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); } if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { From 3bc84f43dbd368251ff2a1133764f31c16c66269 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:52:42 +0000 Subject: [PATCH 023/120] Created _loadSesssions to load sessions from disk --- client.js | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/client.js b/client.js index 62b6356..2f0d596 100644 --- a/client.js +++ b/client.js @@ -33,16 +33,7 @@ Discord.Client = class Client extends Discord.Client { this.ws._hotreload = options.hotReload.sessionData; } else { - try { - const shards = fs.readdirSync(`${this.cacheFilePath}/sessions`) - .filter(file => file.endsWith(".json")) - .map(shardSession => shardSession.substr(0, shardSession.lastIndexOf("."))); - for (const shardID of shards) { - this.ws._hotreload[shardID] = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, "utf8")); - } - } catch (e) { - this.ws._hotreload = {}; - } + this._loadSessions(); } if (options.cacheGuilds) { @@ -104,6 +95,22 @@ Discord.Client = class Client extends Discord.Client { } } } + /** + * Loads all of the stored sessions on disk into memory + * @private + */ + _loadSessions() { + try { + const shards = fs.readdirSync(`${this.cacheFilePath}/sessions`) + .filter(file => file.endsWith(".json")) + .map(shardSession => shardSession.substr(0, shardSession.lastIndexOf("."))); + for (const shardID of shards) { + this.ws._hotreload[shardID] = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, "utf8")); + } + } catch (e) { + this.ws._hotreload = {}; + } + } sweepUsers(_lifetime = 86400) { const lifetime = _lifetime * 1000; this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); From 63e6929677051e1a5a9c91d9ec84f21b3d19ced7 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 23:23:16 +0000 Subject: [PATCH 024/120] Added helper methods Added unLoadSessions to unload sessions from memory to disk Added makeDir to create a directory if it doesn't already exist Added loadCache to load all caches from disk Added onUnload method which can be overwritten by user to store caches --- client.js | 78 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/client.js b/client.js index 2f0d596..44f86b3 100644 --- a/client.js +++ b/client.js @@ -35,34 +35,23 @@ Discord.Client = class Client extends Discord.Client { else { this._loadSessions(); } - - if (options.cacheGuilds) { - const discordGuildData = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8")); - for (const guild of discordGuildData) { - this.guilds.cache.set(guild.id, new Discord.Guild(this, guild)); - } - this.user = new Discord.User(this, JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8"))); - } - this.on(Discord.Constants.Events.SHARD_RESUME, () => { - if(!this.readyAt) { this.ws.checkShardsReady(); } - }); - this.dumpCache = (sessions, client) => { - if (!fs.existsSync(client.cacheFilePath)) { fs.mkdirSync(client.cacheFilePath); } - try { - client.ws._hotreload = JSON.parse(fs.readFileSync(`${client.cacheFilePath}/sessions.json`, "utf8")); - } catch (e) { - client.ws._hotreload = {}; - } - client.ws._hotreload = { - ...client.ws._hotreload, + this.onUnload = (sessions, cache) => { + this._makeDir(this.cacheFilePath); + this._makeDir(`${this.cacheFilePath}/sessions`); + this._loadSessions(); + this.ws._hotreload = { + ...this.ws._hotreload, ...sessions }; - fs.writeFileSync(`${client.cacheFilePath}/sessions.json`, JSON.stringify(client.ws._hotreload)); + this._unLoadSessions(); if (options.cacheGuilds) { - const discordGuilds = client.guilds.cache.map(g => g._unpatch()); - fs.writeFileSync(`${client.cacheFilePath}/guilds.json`, JSON.stringify(discordGuilds)); - const discordMe = client.user._unpatch(); - fs.writeFileSync(`${client.cacheFilePath}/me.json`, JSON.stringify(discordMe)); + this._makeDir(`${this.cacheFilePath}/guilds`); + } + if (options.cacheChannels) { + this._makeDir(`${this.cacheFilePath}/channels`); + } + if (options.cacheMembers) { + this._makeDir(`${this.cacheFilePath}/users`); } }; this._uncaughtExceptionOnExit = false; @@ -95,6 +84,29 @@ Discord.Client = class Client extends Discord.Client { } } } + /** + * Loads all of the stored caches on disk into memory + * @returns {object} All of the stored cache + * @private + */ + _loadCache() { + const allCache = {}; + for (const cache of ["guilds", "channels", "users"]) { + try { + const cachedFiles = fs.readdirSync(`${this.cacheFilePath}/${cache}`) + .filter(file => file.endsWith(".json")) + .map(c => c.substr(0, c.lastIndexOf("."))); + if (cachedFiles.length) { continue; } + allCache[cache] = []; + for (const id of cachedFiles) { + allCache[cache].push(JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${id}.json`, "utf8"))); + } + } catch (d) { + // Do nothing + } + } + return allCache; + } /** * Loads all of the stored sessions on disk into memory * @private @@ -111,6 +123,22 @@ Discord.Client = class Client extends Discord.Client { this.ws._hotreload = {}; } } + /** + * Unloads all of the stored sessions in memory onto disk + * @private + */ + _unLoadSessions() { + for (const [shardID, session] of this.ws._hotreload) { + fs.writeFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, JSON.stringify(session)); + } + } + /** + * Creates a directory if it does not already exist + * @private + */ + _makeDir(dir) { + if (!fs.existsSync(dir)) { fs.mkdirSync(dir); } + } sweepUsers(_lifetime = 86400) { const lifetime = _lifetime * 1000; this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); From b3afb405b9563b210aec351828b3a0db166d2855 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 23:24:02 +0000 Subject: [PATCH 025/120] Added dumpCache and patchCache methods --- client.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 44f86b3..bb2eb83 100644 --- a/client.js +++ b/client.js @@ -35,6 +35,7 @@ Discord.Client = class Client extends Discord.Client { else { this._loadSessions(); } + this._patchCache(options.hotReload.cacheData || this._loadCache()); this.onUnload = (sessions, cache) => { this._makeDir(this.cacheFilePath); this._makeDir(`${this.cacheFilePath}/sessions`); @@ -71,7 +72,7 @@ Discord.Client = class Client extends Discord.Client { }; })); if (eventType !== "exit") { - await this.dumpCache(this.ws._hotreload, this); + await this.onUnload(this.ws._hotreload, this.dumpCache()); process.exit(); } } @@ -84,6 +85,13 @@ Discord.Client = class Client extends Discord.Client { } } } + /** + * Generates a complete dump of the current stored cache + * @param {object} options Options to validate + * @returns {object} All of the cache + */ + dumpCache() { + } /** * Loads all of the stored caches on disk into memory * @returns {object} All of the stored cache @@ -107,6 +115,12 @@ Discord.Client = class Client extends Discord.Client { } return allCache; } + /** + * Patches raw discord api objects into the discord.js cache + * @private + */ + _patchCache({ guilds, channels, users }) { + } /** * Loads all of the stored sessions on disk into memory * @private From ca7ebd1e8644b9864a6e99305d87e83f2d068e34 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Mon, 22 Mar 2021 21:45:37 +0000 Subject: [PATCH 026/120] Added a _write method to write an array of cache data to disk --- client.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/client.js b/client.js index bb2eb83..54bfed8 100644 --- a/client.js +++ b/client.js @@ -47,12 +47,15 @@ Discord.Client = class Client extends Discord.Client { this._unLoadSessions(); if (options.cacheGuilds) { this._makeDir(`${this.cacheFilePath}/guilds`); + this._write("guilds", guilds); } if (options.cacheChannels) { this._makeDir(`${this.cacheFilePath}/channels`); + this._write("channels", channels); } if (options.cacheMembers) { this._makeDir(`${this.cacheFilePath}/users`); + this._write("users", users); } }; this._uncaughtExceptionOnExit = false; @@ -153,6 +156,17 @@ Discord.Client = class Client extends Discord.Client { _makeDir(dir) { if (!fs.existsSync(dir)) { fs.mkdirSync(dir); } } + /** + * Writes a cache array to multiple files indexed by ID to disk using the cached file path and JSON format + * @param {string} path The path to write the data to + * @param {Array} data An array of all of the data items to write + * @private + */ + _write(path, data) { + for (const item of data) { + fs.writeFileSync(`${this.cacheFilePath}/${path}/${item.id}.json`, JSON.stringify(item)); + } + } sweepUsers(_lifetime = 86400) { const lifetime = _lifetime * 1000; this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); From 6a915d21266559e92a19df32a16de2667ef2abb1 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Mon, 22 Mar 2021 21:46:04 +0000 Subject: [PATCH 027/120] Implemented dumpCache --- client.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 54bfed8..a22f873 100644 --- a/client.js +++ b/client.js @@ -36,7 +36,7 @@ Discord.Client = class Client extends Discord.Client { this._loadSessions(); } this._patchCache(options.hotReload.cacheData || this._loadCache()); - this.onUnload = (sessions, cache) => { + this.onUnload = (sessions, { guilds, channels, users }) => { this._makeDir(this.cacheFilePath); this._makeDir(`${this.cacheFilePath}/sessions`); this._loadSessions(); @@ -94,6 +94,11 @@ Discord.Client = class Client extends Discord.Client { * @returns {object} All of the cache */ dumpCache() { + return { + guilds: this.guilds.cache.map(g => g._unpatch()), + channels: this.channels.cache.map(c => c._unpatch()), + users: this.users.cache.map(u => u._unpatch()) + }; } /** * Loads all of the stored caches on disk into memory From 7fffe3b8b13036ab653a7c06a240400afc904c58 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Mon, 22 Mar 2021 22:11:34 +0000 Subject: [PATCH 028/120] Added better error handling for infinite uncaught exception loop --- client.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index a22f873..d563c2e 100644 --- a/client.js +++ b/client.js @@ -60,7 +60,7 @@ Discord.Client = class Client extends Discord.Client { }; this._uncaughtExceptionOnExit = false; for (const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { - process.on(eventType, async () => { + process.on(eventType, async (...args) => { if (eventType === "uncaughtException") { this._uncaughtExceptionOnExit = true; } @@ -79,11 +79,11 @@ Discord.Client = class Client extends Discord.Client { process.exit(); } } - else { - console.error("There was an uncaughtException inside your exit loop causing an infinite loop. Your exit function was not run"); + else if (eventType !== "exit") { + console.error(args[0]); + console.error("UNCAUGHT_EXCEPTION_LOOP", "There was an uncaughtException inside your exit loop causing an infinite loop. Your exit function was not run or failed"); process.exit(1); } - }); } } From 176ae7b1ec0c020291dfb72a1917c12a7d4b1a14 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Mon, 22 Mar 2021 22:11:42 +0000 Subject: [PATCH 029/120] Client options --- client.js | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/client.js b/client.js index d563c2e..35a98a9 100644 --- a/client.js +++ b/client.js @@ -219,15 +219,20 @@ Discord.Client = class Client extends Discord.Client { if (!Array.isArray(options.disabledEvents)) { throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); } - if (options.hotReload) { - if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { - throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); - } - if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); + if (typeof options.hotReload !== "boolean") { + if (options.hotReload && typeof options.hotReload === "object") { + if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); + } + if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); + } + if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { + throw new TypeError("CLIENT_INVALID_OPTION", "onUnload", "a function"); + } } - if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { - throw new TypeError("CLIENT_INVALID_OPTION", "onUnload", "a function"); + else { + throw new TypeError("CLIENT_INVALID_OPTION", "hotReload", "a boolean or an object"); } } } From 7dced34897cfd685f21097c41a9f5381a4f53b3e Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Wed, 31 Mar 2021 18:23:14 +0100 Subject: [PATCH 030/120] implement _unpatch() Channel unpatching needs to be done in another file --- classes.js | 231 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 138 insertions(+), 93 deletions(-) diff --git a/classes.js b/classes.js index 32e6f87..c5decca 100644 --- a/classes.js +++ b/classes.js @@ -120,6 +120,22 @@ Discord.Structures.extend("Message", M => { }; }); +Discord.Strictures.extend("User", U => { + return class User extends U { + _unpatch() { + return { + id: this.id, + username: this.username, + bot: this.bot, + discriminator: this.discriminator, + avatar: this.avatar, + system: this.system, + public_flags: this.flags.valueOf() + } + } + } +}); + Discord.Structures.extend("GuildMember", G => { return class GuildMember extends G { _patch(data) { @@ -138,6 +154,16 @@ Discord.Structures.extend("GuildMember", G => { } } } + _unpatch() { + return { + user: this.user._unpatch(), + nick: this.nickname, + joined_at: this.joinedTimestamp, + premium_since: this.premiumSinceTimestamp, + roles: this._roles, + pending: this.pending + } + } equals(member) { return member && this.deleted === member.deleted && this.nickname === member.nickname && this._roles.length === member._roles.length; } @@ -217,112 +243,48 @@ Discord.Structures.extend("Guild", G => { } } _unpatch() { - /** - * Discord raw guild data as documented: https://github.com/discordjs/discord-api-types/blob/main/v8/payloads/guild.ts - */ return { - id: this.id, - unavailable: this.available, + unavailable: !this.available, + shardID: this.shardID, name: this.name, icon: this.icon, splash: this.splash, - banner: this.banner, - description: this.description, - features: this.features, - verification_level: this.verificationLevel, discovery_splash: this.discoverySplash, - owner_id: this.ownerID, region: this.region, - afk_channel_id: this.afkChannelID, + member_count: this.memberCount, + large: this.large, + features: this.features, + application_id: this.applicationID, afk_timeout: this.afkTimeout, + afk_channel_id: this.afkChannelID, + system_channel_id: this.systemChannelID, + premium_tier: this.premiumTier, + premium_subscription_count: this.premiumSubscriptionCount, widget_enabled: this.widgetEnabled, widget_channel_id: this.widgetChannelID, - default_message_notifications: this.defaultMessageNotifications, - explicit_content_filter: this.explicitContentFilter, - /** - * Roles in the guild - * - * See https://discord.com/developers/docs/topics/permissions#role-object - */ - roles: this.roles.cache.map(r => ({ - name: r.name, - color: r.color - })), - /** - * Custom guild emojis - * - * See https://discord.com/developers/docs/resources/emoji#emoji-object - */ - emojis: this.emojis.cache.map(e => ({ - id: e.id, - name: e.name - })), + verification_level: Discord.Util.Constants.VerificationLevels.indexOf(this.verificationLevel), + explicit_content_filter: Discord.Util.Constants.ExplicitContentFilterLevels.indexOf(this.explicitContentFilter), mfa_level: this.mfaLevel, - application_id: this.applicationID, - system_channel_id: this.systemChannelID, - system_channel_flags: this.systemChannelFlags, - rules_channel_id: this.rulesChannelID, - joined_at: this.joinedAt, - large: this.large, - member_count: this.memberCount, - voice_states: this.voiceStates.cache.map(v => ({ - guild_id: v.guild.id, - channel_id: v.channelID, - user_id: v.userID, - session_id: v.sessionID, - deaf: v.deaf, - mute: v.mute, - self_deaf: v.selfDeaf, - self_mute: v.selfMute, - suppress: v.suppress - })), - members: this.members.cache.map(m => ({ - user: m.user, - nick: m.nickname, - roles: m.roles, - joined_at: m.joinedAt, - premium_since: m.premiumSinceTimestamp, - deaf: m.deaf, - mute: m.mute, - pending: m.pending, - permissions: m.permissions - })), - channels: this.channels.cache.map(c => ({ - id: c.id, - type: c.type, - guild_id: c.guild.id, - position: c.position, - permission_overwrites: c.permissionOverwrites, - name: c.name, - topic: c.topic, - nsfw: c.nsfw, - last_message_id: c.lastMessageID, - bitrate: c.bitrate, - user_limit: c.userLimit, - rate_limit_per_user: c.rateLimitPerUser, - recipients: c.recipients, - icon: c.icon, - owner_id: c.ownerID, - application_id: c.applicationID, - parent_id: c.parentID, - last_pin_timestamp: c.lastPinTimestamp - })), - presences: this.presences.cache.map(p => ({ - user: p.user, - guild_id: p.guild.id, - status: p.status, - activities: p.activities, - client_status: p.clientStatus - })), - max_presences: this.maximumPresences, + joinedTimestamp: this.joinedTimestamp, + default_message_notifications: this.defaultMessageNotifications, + system_channel_flags: this.systemChannelFlags.valueOf(), max_members: this.maximumMembers, + max_presences: this.maximumPresences, + approximate_member_count: this.approximateMemberCount, + approximate_presence_count: this.approximatePresenceCount, vanity_url_code: this.vanityURLCode, - premium_tier: this.premiumTier, - premium_subscription_count: this.premiumSubscriptionCount, - preferred_locale: this.preferredLocale || "en-US", + description: this.description, + banner: this.banner, + id: this.id, + rules_channel_id: this.rulesChannelID, public_updates_channel_id: this.publicUpdatesChannelID, - approximate_member_count: this.approximateMemberCount, - approximate_presence_count: this.approximatePresenceCount + preferred_locale: this.preferredLocale, + roles: this.roles.cache.map(x => x._unpatch()), + members: this.members.cache.map(x => x._unpatch()), + owner_id: this.ownerID, + presences: this.presences.cache.map(x => x._unpatch()), + voice_states: this.voiceStates.cache.map(x => x._unpatch()), + emojis: this.emojis.cache.map(x => x._unpatch()) }; } get nameAcronym() { @@ -384,6 +346,18 @@ Discord.Structures.extend("GuildEmoji", E => { this._author = data.user.id; } } + _unpatch() { + return { + animated: this.animated, + name: this.name, + id: this.id, + require_colons: this.requiresColons, + managed: this.managed, + available: this.available, + roles: this._roles, + user: this.author ? this.author._unpatch() : void 0; + } + } async fetchAuthor(cache = true) { if(this.managed) { throw new Error("EMOJI_MANAGED"); @@ -400,6 +374,28 @@ Discord.Structures.extend("GuildEmoji", E => { }; }); +Discord.Structures.extend("Role", R => { + return class Role extends R { + _unpatch() { + return { + id: this.id, + name: this.name, + color: this.color, + hoist: this.hoist, + position: this.rawPosition, + permissions: this.permissions.valueOf().toString(), + managed: this.managed, + mentionable: this.mentionable, + tags: { + bot_id: this.tags.botID, + integration_id: this.tags.integrationID, + premium_subscriber: this.tags.premiumSubscriberRole + } + } + } + } +}); + Discord.Structures.extend("VoiceState", V => { return class VoiceState extends V { _patch(data) { @@ -409,6 +405,19 @@ Discord.Structures.extend("VoiceState", V => { } return this; } + _unpatch() { + return { + user_id: this.id, + deaf: this.serverDeaf, + mute: this.serverMute, + self_deaf: this.selfDeaf, + self_mute: this.selfMute, + self_video: this.selfVideo, + session_id: this.sessionID, + self_stream: this.streaming, + channel_id: this.channelID + } + } get channel() { return this.channelID ? this.client.channels.cache.get(this.channelID) || this.client.channels.add({ id: this.channelID, @@ -463,6 +472,42 @@ Discord.Structures.extend("Presence", P => { } return this; } + _unpatch() { + return { + user: { + id: this.userID + }, + status: this.status, + activities: this.activities.map(a => ({ + name: a.name, + type: a.type, + url: a.url, + details: a.details, + state: a.state, + application_id: a.applicationID, + timestamps: { + start: a.timestamps.start ? a.timestamps.start.getTime() : null, + end: a.timestamps.end ? a.timestamps.end.getTime() : null + }, + party: a.party, + assets: { + large_text: a.assets.largeText, + small_text: a.assets.smallText, + large_image: a.assets.largeImage, + small_image: a.assets.smallImage + }, + sync_id: a.syncID, + flags: a.flags.valueOf(), + emoji: { + animated: a.emoji.animated, + name: a.emoji.name, + id: a.emoji.id + }, + created_at: a.createdTimestamp + })), + client_status: this.clientStatus + } + } get user() { return this.client.users.cache.get(this.userID) || this.client.users.add((this._member || {}).user || { id: this.userID }, false); } From 75f15da0290c831f5d2dfa812482c9a90ceedd15 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Wed, 31 Mar 2021 19:30:14 +0100 Subject: [PATCH 031/120] Channel unpatching --- init.js | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/init.js b/init.js index dab02df..915d2da 100644 --- a/init.js +++ b/init.js @@ -258,6 +258,19 @@ require.cache[GCPath].exports = class GuildChannel extends GC { }); } } + _unpatch() { + let obj = super._unpatch(); + obj.name = this.name; + obj.position = this.rawPosition; + obj.parent_id = this.parentID; + obj.permission_overwrites = this.permissionOverwrites.map(x => ({ + id: x.id, + type: Constants.OverwriteTypes[x.type], + deny: x.deny.valueOf().toString(), + allow: x.allow.valueOf().toString() + })); + return obj; + } get deletable() { if(this.deleted) { return false; } if(!this.client.options.cacheRoles && !this.guild.roles.cache.size) { return false; } @@ -265,6 +278,44 @@ require.cache[GCPath].exports = class GuildChannel extends GC { } }; +const CPath = resolve(require.resolve("discord.js").replace("index.js", "/structures/Channel.js")); +const C = require(CPath); +require.cache[CPath].exports = class Channel extends C { + _unpatch() { + let obj = { + type: Constants.ChannelTypes[this.type.toUpperCase()], + id: this.id + }; + if(this.messages) { + obj.last_message_id: this.lastMessageID; + obj.last_pin_timestamp: this.lastPinTimestamp; + } + switch(this.type) { + case "dm": { + obj.recipients: [this.recipient._unpatch()]; + break; + } + case "text": case "news": { + obj.nsfw: this.nsfw; + obj.topic: this.topic; + obj.rate_limit_per_user: this.rateLimitPerUser; + obj.messages = this.messages.cache.map(x => x._unpatch()); + break; + } + case "voice": { + obj.bitrate: this.bitrate; + obj.user_limit: this.userLimit + break; + } + case "store": { + obj.nsfw: this.nsfw; + break; + } + } + return obj; + } +}; + const Action = require(resolve(require.resolve("discord.js").replace("index.js", "/client/actions/Action.js"))); Action.prototype.getPayload = function(data, manager, id, partialType, cache) { return manager.cache.get(id) || manager.add(data, cache); From c12961cb0bc4cb21c7c123d3af1e96e56bb7e1ec Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Wed, 31 Mar 2021 20:42:44 +0100 Subject: [PATCH 032/120] message unpatch plus fixes --- classes.js | 91 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/classes.js b/classes.js index c5decca..10f8c74 100644 --- a/classes.js +++ b/classes.js @@ -93,6 +93,81 @@ Discord.Structures.extend("Message", M => { } } } + _unpatch() { + return { + id: this.id, + type: Discord.Constants.MessageTypes.indexOf(this.type), + content: this.content, + author: this.author._unpatch(), + pinned: this.pinned, + tts: this.tts, + nonce: this.nonce, + embeds: this.embeds.map(x => x.toJSON()), + attachments: this.attachments.map(x => ({ + filename: x.name, + id: x.id, + size: x.size, + url: x.url, + proxy_url: x.proxyURL, + height: x.height, + width: x.width + })), + edited_timestamp: this.editedTimestamp, + reactions: this.reactions.cache.map(x => ({ + me: x.me, + emoji: { + animated: x.emoji.animated, + name: x.emoji.name, + id: x.emoji.id + }, + count: x.count + })), + mentions: this.mentions.users.map(x => x._unpatch()), + mention_roles: this.mentions.roles.map(x => x._unpatch()), + mention_everyone: this.mentions.everyone, + mention_channels: this.mentions.crosspostedChannels.map(x => ({ + id: x.channelID, + guild_id: x.guildID, + type: Discord.Constants.ChannelTypes[x.type.toUpperCase()], + name: x.name + })), + webhook_id: this.webhookID, + application: this.application ? { + id: this.application.id, + name: this.application.name, + description: this.application.description, + icon: this.application.icon, + cover_image: this.application.cover, + rpc_origins: this.application.rpcOrigins, + bot_require_code_grant: this.botRequireCodeGrant, + bot_public: this.application.botPublic, + team: this.application.owner instanceof Discord.Team ? { + id: this.application.owner.id, + name: this.application.owner.name, + icon: this.application.owner.icon, + owner_user_id: this.application.owner.ownerID, + members: this.application.owner.members.map(x => ({ + permissions: x.permissions, + membership_state: x.membershipState, + user: x.user._unpatch() + })) + } : void 0, + owner: this.application.owner instanceof Discord.User ? this.application.owner._unpatch() : void 0 + } : void 0, + activity: this.activity ? { + party_id: this.activity.partyID, + type: this.activity.type + } : void 0, + member: this.member._unpatch(), + flags: this.flags.valueOf(), + message_reference: this.reference ? { + channel_id: this.reference.channelID, + guild_id: this.reference.guildID, + message_id: this.reference.messageID + } : void 0, + referenced_message: this.client.guilds.cache.get(this.reference?.guildID)?.channels.cache.get(this.reference?.channelID)?.messages.cache.get(this.reference?.messageID)?._unpatch() + } + } get member() { if(!this.guild) { return null; } const id = (this.author || {}).id || (this._member || {}).id; @@ -262,8 +337,8 @@ Discord.Structures.extend("Guild", G => { premium_subscription_count: this.premiumSubscriptionCount, widget_enabled: this.widgetEnabled, widget_channel_id: this.widgetChannelID, - verification_level: Discord.Util.Constants.VerificationLevels.indexOf(this.verificationLevel), - explicit_content_filter: Discord.Util.Constants.ExplicitContentFilterLevels.indexOf(this.explicitContentFilter), + verification_level: Discord.Constants.VerificationLevels.indexOf(this.verificationLevel), + explicit_content_filter: Discord.Constants.ExplicitContentFilterLevels.indexOf(this.explicitContentFilter), mfa_level: this.mfaLevel, joinedTimestamp: this.joinedTimestamp, default_message_notifications: this.defaultMessageNotifications, @@ -485,24 +560,24 @@ Discord.Structures.extend("Presence", P => { details: a.details, state: a.state, application_id: a.applicationID, - timestamps: { + timestamps: a.timestamps ? { start: a.timestamps.start ? a.timestamps.start.getTime() : null, end: a.timestamps.end ? a.timestamps.end.getTime() : null - }, + } : void 0, party: a.party, - assets: { + assets: a.assets ? { large_text: a.assets.largeText, small_text: a.assets.smallText, large_image: a.assets.largeImage, small_image: a.assets.smallImage - }, + } : void 0, sync_id: a.syncID, flags: a.flags.valueOf(), - emoji: { + emoji: a.emoji ? { animated: a.emoji.animated, name: a.emoji.name, id: a.emoji.id - }, + } : void 0, created_at: a.createdTimestamp })), client_status: this.clientStatus From 422857d0f244d9c9572bdbfc9647bf188c20a5aa Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Thu, 1 Apr 2021 01:02:04 +0100 Subject: [PATCH 033/120] wip: needs rethinking we dont want to load ALL caches from ALL shards we need to separate files by shard ids and only load the right caches in the right processes --- client.js | 296 +++++++++++++++++++++++------------------------------- 1 file changed, 126 insertions(+), 170 deletions(-) diff --git a/client.js b/client.js index 35a98a9..049dd37 100644 --- a/client.js +++ b/client.js @@ -6,6 +6,52 @@ const actions = require("./actions.js"); const pkg = require("./package.json"); const fs = require("fs"); +/** + * Validate client options. + * @param {object} options Options to validate + * @private + */ +validateOptions(options) { + if(typeof options.cacheChannels !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheChannels", "a boolean"); + } + if(typeof options.cacheGuilds !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheGuilds", "a boolean"); + } + if(typeof options.cachePresences !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cachePresences", "a boolean"); + } + if(typeof options.cacheRoles !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheRoles", "a boolean"); + } + if(typeof options.cacheOverwrites !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheOverwrites", "a boolean"); + } + if(typeof options.cacheEmojis !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheEmojis", "a boolean"); + } + if(typeof options.cacheMembers !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheMembers", "a boolean"); + } + if(!Array.isArray(options.disabledEvents)) { + throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); + } + if(options.hotReload && typeof options.hotReload === "object") { + if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); + } + if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "an object"); + } + if (options.hotReload.onExit && typeof options.hotReload.onExit !== "function") { + throw new TypeError("CLIENT_INVALID_OPTION", "onExit", "a function"); + } + } + else if(typeof options.hotReload !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "hotReload", "a boolean or an object"); + } +} + Discord.Client = class Client extends Discord.Client { constructor(_options = {}) { const options = { @@ -16,82 +62,60 @@ Discord.Client = class Client extends Discord.Client { cacheOverwrites: false, cacheEmojis: false, cacheMembers: false, - disabledEvents: [], hotReload: false, + disabledEvents: [], ..._options }; + validateOptions(options); super(options); actions(this); - this._validateOptionsLight(options); - if (options.hotReload) { + if(options.hotReload) { this.on(Discord.Constants.Events.SHARD_RESUME, () => { - if (!this.readyAt) { this.ws.checkShardsReady(); } + if(!this.readyAt) { this.ws.checkShardsReady(); } }); - this.cacheFilePath = `${process.cwd()}/.sessions`; - this.ws._hotreload = {}; - if (options.hotReload.sessionData && Object.keys(options.hotReload.sessionData).length) { - this.ws._hotreload = options.hotReload.sessionData; - } - else { - this._loadSessions(); - } - this._patchCache(options.hotReload.cacheData || this._loadCache()); - this.onUnload = (sessions, { guilds, channels, users }) => { - this._makeDir(this.cacheFilePath); - this._makeDir(`${this.cacheFilePath}/sessions`); - this._loadSessions(); - this.ws._hotreload = { - ...this.ws._hotreload, - ...sessions - }; - this._unLoadSessions(); - if (options.cacheGuilds) { - this._makeDir(`${this.cacheFilePath}/guilds`); - this._write("guilds", guilds); - } - if (options.cacheChannels) { - this._makeDir(`${this.cacheFilePath}/channels`); - this._write("channels", channels); - } - if (options.cacheMembers) { - this._makeDir(`${this.cacheFilePath}/users`); - this._write("users", users); - } - }; - this._uncaughtExceptionOnExit = false; - for (const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { + this._patchCache(options.hotReload.cacheData || this._loadCache()); + for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { - if (eventType === "uncaughtException") { - this._uncaughtExceptionOnExit = true; - } - if (!this._uncaughtExceptionOnExit) { - Object.assign(this.ws._hotreload, ...this.ws.shards.map(s => { - s.connection.close(); - return { - [s.id]: { - id: s.sessionID, - seq: s.sequence - } - }; - })); - if (eventType !== "exit") { - await this.onUnload(this.ws._hotreload, this.dumpCache()); - process.exit(); + let cache = this.dumpCache(); + let sessions = this.dumpSessions(); + if(options.hotReload.onExit) { + await options.hotReload.onExit(cache, sessions); // async will not work on exit and exception but might work on SIGINT and SIGTERM + } else { + for(const folder of ["websocket", "users", "guilds", "channels"]) { + if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } } + for(const shard of sessions) + } + if(eventType === "uncaughtException") { + console.error(...args); } - else if (eventType !== "exit") { - console.error(args[0]); - console.error("UNCAUGHT_EXCEPTION_LOOP", "There was an uncaughtException inside your exit loop causing an infinite loop. Your exit function was not run or failed"); - process.exit(1); + if(eventType !== "exit") { + process.exit(process.exitCode); } }); } } } + sweepUsers(_lifetime = 86400) { + const lifetime = _lifetime * 1000; + this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); + for(const guild of this.guilds.cache.values()) { + guild.members.cache.sweep(t => !this.users.cache.has(t.id)); + guild.presences.cache.sweep(t => !this.users.cache.has(t.id) && !this.options.cachePresences); + } + } + sweepChannels(_lifetime = 86400) { + const lifetime = _lifetime * 1000; + if(this.options.cacheChannels) { return; } + const connections = this.voice ? this.voice.connections.map(t => t.channel.id) : []; + this.channels.cache.sweep(t => !connections.includes(t.id) && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); + for(const guild of this.guilds.cache.values()) { + guild.channels.cache.sweep(t => !this.channels.cache.has(t.id)); + } + } /** * Generates a complete dump of the current stored cache - * @param {object} options Options to validate - * @returns {object} All of the cache + * @returns {object} Cache data */ dumpCache() { return { @@ -100,139 +124,71 @@ Discord.Client = class Client extends Discord.Client { users: this.users.cache.map(u => u._unpatch()) }; } + /** + * Generates a complete dump of the current stored cache + * @returns {object} Session data + */ + dumpSessions() { + return this.ws.shards.map(s => ({ + [s.id]: { + id: s.sessionID, + sequence: s.sequence + } + })); + } /** * Loads all of the stored caches on disk into memory * @returns {object} All of the stored cache * @private */ _loadCache() { - const allCache = {}; - for (const cache of ["guilds", "channels", "users"]) { + const allCache = { + guilds: [], + channels: [], + users: [] + }; + for(const cache of ["guilds", "channels", "users"]) { + const files = []; try { - const cachedFiles = fs.readdirSync(`${this.cacheFilePath}/${cache}`) - .filter(file => file.endsWith(".json")) - .map(c => c.substr(0, c.lastIndexOf("."))); - if (cachedFiles.length) { continue; } - allCache[cache] = []; - for (const id of cachedFiles) { - allCache[cache].push(JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${id}.json`, "utf8"))); - } - } catch (d) { - // Do nothing + files = fs.readdirSync(`${process.cwd()}/.sessions/${cache}`).filter(file => file.endsWith(".json")); + } catch(e) { /* no-op */ } + for(const file of files) { + try { + const json = fs.readFileSync(`${process.cwd()}/.sessions/${cache}/${file}`, "utf8"); + const obj = JSON.parse(json); + allCache[cache].push(obj); + } catch(e) { /* no-op */ } } } return allCache; } - /** - * Patches raw discord api objects into the discord.js cache - * @private - */ - _patchCache({ guilds, channels, users }) { - } /** * Loads all of the stored sessions on disk into memory * @private */ _loadSessions() { + let data = {} + let files = []; try { - const shards = fs.readdirSync(`${this.cacheFilePath}/sessions`) - .filter(file => file.endsWith(".json")) - .map(shardSession => shardSession.substr(0, shardSession.lastIndexOf("."))); - for (const shardID of shards) { - this.ws._hotreload[shardID] = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, "utf8")); - } - } catch (e) { - this.ws._hotreload = {}; - } - } - /** - * Unloads all of the stored sessions in memory onto disk - * @private - */ - _unLoadSessions() { - for (const [shardID, session] of this.ws._hotreload) { - fs.writeFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, JSON.stringify(session)); - } - } - /** - * Creates a directory if it does not already exist - * @private - */ - _makeDir(dir) { - if (!fs.existsSync(dir)) { fs.mkdirSync(dir); } - } - /** - * Writes a cache array to multiple files indexed by ID to disk using the cached file path and JSON format - * @param {string} path The path to write the data to - * @param {Array} data An array of all of the data items to write - * @private - */ - _write(path, data) { - for (const item of data) { - fs.writeFileSync(`${this.cacheFilePath}/${path}/${item.id}.json`, JSON.stringify(item)); - } - } - sweepUsers(_lifetime = 86400) { - const lifetime = _lifetime * 1000; - this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); - for(const guild of this.guilds.cache.values()) { - guild.members.cache.sweep(t => !this.users.cache.has(t.id)); - guild.presences.cache.sweep(t => !this.users.cache.has(t.id) && !this.options.cachePresences); - } - } - sweepChannels(_lifetime = 86400) { - const lifetime = _lifetime * 1000; - if(this.options.cacheChannels) { return; } - const connections = this.voice ? this.voice.connections.map(t => t.channel.id) : []; - this.channels.cache.sweep(t => !connections.includes(t.id) && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); - for(const guild of this.guilds.cache.values()) { - guild.channels.cache.sweep(t => !this.channels.cache.has(t.id)); + files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); + } catch (e) { /* no-op */ } + for(const file of files) { + try { + const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); + const obj = JSON.parse(json); + data[file.slice(0, -5)] = obj; + } catch(e) { /* no-op */ } } + return data; } /** - * Validates the client options. - * @param {object} options Options to validate + * Patches raw discord api objects into the discord.js cache * @private */ - _validateOptionsLight(options) { - if (typeof options.cacheChannels !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheChannels", "a boolean"); - } - if (typeof options.cacheGuilds !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheGuilds", "a boolean"); - } - if (typeof options.cachePresences !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cachePresences", "a boolean"); - } - if (typeof options.cacheRoles !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheRoles", "a boolean"); - } - if (typeof options.cacheOverwrites !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheOverwrites", "a boolean"); - } - if (typeof options.cacheEmojis !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheEmojis", "a boolean"); - } - if (typeof options.cacheMembers !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheMembers", "a boolean"); - } - if (!Array.isArray(options.disabledEvents)) { - throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); - } - if (typeof options.hotReload !== "boolean") { - if (options.hotReload && typeof options.hotReload === "object") { - if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { - throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); - } - if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); - } - if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { - throw new TypeError("CLIENT_INVALID_OPTION", "onUnload", "a function"); - } - } - else { - throw new TypeError("CLIENT_INVALID_OPTION", "hotReload", "a boolean or an object"); + _patchCache(data) { + for(const [cache, items] of Object.entries(data)) { + for(const item of items) { + this[cache].add(item); } } } From fa3f48cd0deadb0ace36af7d5ba9ffde70b64194 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Thu, 1 Apr 2021 15:46:44 +0100 Subject: [PATCH 034/120] Linting --- classes.js | 24 +++++++++++------------- client.js | 14 +++++++------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/classes.js b/classes.js index 10f8c74..e7bcde6 100644 --- a/classes.js +++ b/classes.js @@ -166,7 +166,7 @@ Discord.Structures.extend("Message", M => { message_id: this.reference.messageID } : void 0, referenced_message: this.client.guilds.cache.get(this.reference?.guildID)?.channels.cache.get(this.reference?.channelID)?.messages.cache.get(this.reference?.messageID)?._unpatch() - } + }; } get member() { if(!this.guild) { return null; } @@ -206,9 +206,9 @@ Discord.Strictures.extend("User", U => { avatar: this.avatar, system: this.system, public_flags: this.flags.valueOf() - } + }; } - } + }; }); Discord.Structures.extend("GuildMember", G => { @@ -237,7 +237,7 @@ Discord.Structures.extend("GuildMember", G => { premium_since: this.premiumSinceTimestamp, roles: this._roles, pending: this.pending - } + }; } equals(member) { return member && this.deleted === member.deleted && this.nickname === member.nickname && this._roles.length === member._roles.length; @@ -430,8 +430,8 @@ Discord.Structures.extend("GuildEmoji", E => { managed: this.managed, available: this.available, roles: this._roles, - user: this.author ? this.author._unpatch() : void 0; - } + user: this.author ? this.author._unpatch() : void 0 + }; } async fetchAuthor(cache = true) { if(this.managed) { @@ -466,9 +466,9 @@ Discord.Structures.extend("Role", R => { integration_id: this.tags.integrationID, premium_subscriber: this.tags.premiumSubscriberRole } - } + }; } - } + }; }); Discord.Structures.extend("VoiceState", V => { @@ -491,7 +491,7 @@ Discord.Structures.extend("VoiceState", V => { session_id: this.sessionID, self_stream: this.streaming, channel_id: this.channelID - } + }; } get channel() { return this.channelID ? this.client.channels.cache.get(this.channelID) || this.client.channels.add({ @@ -549,9 +549,7 @@ Discord.Structures.extend("Presence", P => { } _unpatch() { return { - user: { - id: this.userID - }, + user: { id: this.userID }, status: this.status, activities: this.activities.map(a => ({ name: a.name, @@ -581,7 +579,7 @@ Discord.Structures.extend("Presence", P => { created_at: a.createdTimestamp })), client_status: this.clientStatus - } + }; } get user() { return this.client.users.cache.get(this.userID) || this.client.users.add((this._member || {}).user || { id: this.userID }, false); diff --git a/client.js b/client.js index 049dd37..5d05687 100644 --- a/client.js +++ b/client.js @@ -11,7 +11,7 @@ const fs = require("fs"); * @param {object} options Options to validate * @private */ -validateOptions(options) { +function validateOptions(options) { if(typeof options.cacheChannels !== "boolean") { throw new TypeError("CLIENT_INVALID_OPTION", "cacheChannels", "a boolean"); } @@ -73,18 +73,18 @@ Discord.Client = class Client extends Discord.Client { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); - this._patchCache(options.hotReload.cacheData || this._loadCache()); + this._patchCache(options.hotReload.cacheData || this._loadCache()); for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { - let cache = this.dumpCache(); - let sessions = this.dumpSessions(); + const cache = this.dumpCache(); + const sessions = this.dumpSessions(); if(options.hotReload.onExit) { await options.hotReload.onExit(cache, sessions); // async will not work on exit and exception but might work on SIGINT and SIGTERM } else { for(const folder of ["websocket", "users", "guilds", "channels"]) { if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } } - for(const shard of sessions) + for(const shard of sessions) { /* no-op */ } } if(eventType === "uncaughtException") { console.error(...args); @@ -148,7 +148,7 @@ Discord.Client = class Client extends Discord.Client { users: [] }; for(const cache of ["guilds", "channels", "users"]) { - const files = []; + let files = []; try { files = fs.readdirSync(`${process.cwd()}/.sessions/${cache}`).filter(file => file.endsWith(".json")); } catch(e) { /* no-op */ } @@ -167,7 +167,7 @@ Discord.Client = class Client extends Discord.Client { * @private */ _loadSessions() { - let data = {} + const data = {}; let files = []; try { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); From ea0b8ee0d6a1167c14958afa3ccf144997e24f0f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Thu, 1 Apr 2021 15:50:23 +0100 Subject: [PATCH 035/120] Try catch around user supplied function Promises are supported inside of uncaughtException, SIGINT and SIGTERM but not exit. Most restarts call SIGINT or SIGTERM before they exit. I don't know why you removed the check for an infinite loop of uncaught exceptions? If there is an uncaught exception inside of the user defined function or our function it will loop forever. I added a try catch block around the user function so it won't happen for user functions but if you don't want a check for it we need to be 100% sure our code won't cause a loop --- client.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 5d05687..6f7da8b 100644 --- a/client.js +++ b/client.js @@ -79,7 +79,9 @@ Discord.Client = class Client extends Discord.Client { const cache = this.dumpCache(); const sessions = this.dumpSessions(); if(options.hotReload.onExit) { - await options.hotReload.onExit(cache, sessions); // async will not work on exit and exception but might work on SIGINT and SIGTERM + try { + await options.hotReload.onExit(cache, sessions); // async will not work on exit + } catch (e) { /* no-op */ } } else { for(const folder of ["websocket", "users", "guilds", "channels"]) { if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } From 7838707c3b66462edead72b54da8bdc0dcf82d85 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Thu, 1 Apr 2021 16:04:32 +0100 Subject: [PATCH 036/120] prevent exit events from bleeding into each other --- client.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/client.js b/client.js index 6f7da8b..585d058 100644 --- a/client.js +++ b/client.js @@ -74,19 +74,21 @@ Discord.Client = class Client extends Discord.Client { if(!this.readyAt) { this.ws.checkShardsReady(); } }); this._patchCache(options.hotReload.cacheData || this._loadCache()); + let dumped = false; for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { - const cache = this.dumpCache(); - const sessions = this.dumpSessions(); - if(options.hotReload.onExit) { - try { - await options.hotReload.onExit(cache, sessions); // async will not work on exit - } catch (e) { /* no-op */ } - } else { - for(const folder of ["websocket", "users", "guilds", "channels"]) { - if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } + if(!dumped) { + dumped = true; + const cache = this.dumpCache(); + const sessions = this.dumpSessions(); + if(options.hotReload.onExit) { + await options.hotReload.onExit(cache, sessions).catch(() => {}); // async will not work on exit + } else { + for(const folder of ["websocket", "users", "guilds", "channels"]) { + if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } + } + for(const shard of sessions) { /* no-op */ } } - for(const shard of sessions) { /* no-op */ } } if(eventType === "uncaughtException") { console.error(...args); From 2ce132051398607d87868ff10add3171a6def805 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Thu, 1 Apr 2021 22:11:02 +0100 Subject: [PATCH 037/120] store caches --- client.js | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/client.js b/client.js index 585d058..313c02e 100644 --- a/client.js +++ b/client.js @@ -73,7 +73,6 @@ Discord.Client = class Client extends Discord.Client { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); - this._patchCache(options.hotReload.cacheData || this._loadCache()); let dumped = false; for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { @@ -82,12 +81,9 @@ Discord.Client = class Client extends Discord.Client { const cache = this.dumpCache(); const sessions = this.dumpSessions(); if(options.hotReload.onExit) { - await options.hotReload.onExit(cache, sessions).catch(() => {}); // async will not work on exit + await options.hotReload.onExit(sessions, cache).catch(() => {}); // async will not work on exit } else { - for(const folder of ["websocket", "users", "guilds", "channels"]) { - if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } - } - for(const shard of sessions) { /* no-op */ } + this._storeData(sessions, cache); } } if(eventType === "uncaughtException") { @@ -123,22 +119,24 @@ Discord.Client = class Client extends Discord.Client { */ dumpCache() { return { - guilds: this.guilds.cache.map(g => g._unpatch()), - channels: this.channels.cache.map(c => c._unpatch()), - users: this.users.cache.map(u => u._unpatch()) - }; + guilds: this.guilds.cache.reduce((a, g) => { + a[g.id] = g._unpatch(); + return a; + }, {}) + } } /** * Generates a complete dump of the current stored cache * @returns {object} Session data */ dumpSessions() { - return this.ws.shards.map(s => ({ - [s.id]: { + return this.ws.shards.reduce((a, s) => { + a[s.id] = { id: s.sessionID, sequence: s.sequence - } - })); + }; + return a; + }, {}); } /** * Loads all of the stored caches on disk into memory @@ -196,6 +194,23 @@ Discord.Client = class Client extends Discord.Client { } } } + /** + * Built-in cache storing + * @private + */ + _storeData(sessions, cache) { + for(const [id, data] of Object.entries(sessions)) { + if(!fs.existsSync(`${process.cwd()}/.sessions/websocket`)) { fs.mkdirSync(`${process.cwd()}/.sessions/websocket`, { recursive: true }); } + let obj = JSON.stringify(data); + fs.writeFileSync(`${process.cwd()}/.sessions/websocket/${id}.json`, obj, "utf8"); + } + for(const folder of Object.keys(cache)) { + for(const [id, data] of Object.entries(cache[folder])) { + let obj = JSON.stringify(data); + fs.writeFileSync(`${process.cwd()}/.sessions/${folder}/${id}.json`, obj, "utf8"); + } + } + } }; Discord.version = `${pkg.version} (${Discord.version})`; From 3ff1f9a68abd2aedc7225896cf9f644014febed8 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Thu, 1 Apr 2021 22:13:15 +0100 Subject: [PATCH 038/120] unpatch channels in the guild object --- classes.js | 1 + 1 file changed, 1 insertion(+) diff --git a/classes.js b/classes.js index e7bcde6..1ebdfef 100644 --- a/classes.js +++ b/classes.js @@ -356,6 +356,7 @@ Discord.Structures.extend("Guild", G => { preferred_locale: this.preferredLocale, roles: this.roles.cache.map(x => x._unpatch()), members: this.members.cache.map(x => x._unpatch()), + channels: this.channels.cache.map(x => x._unpatch()), owner_id: this.ownerID, presences: this.presences.cache.map(x => x._unpatch()), voice_states: this.voiceStates.cache.map(x => x._unpatch()), From ff30a01a5a80c2a6def4e8d1008b706cbc6e9dc4 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 00:25:25 +0100 Subject: [PATCH 039/120] Small syntax fixes Fixed some copy paste fails assigning an object to a value with : as well as a spelling error and creating folders for each of the caches --- .eslintrc.json | 3 ++- classes.js | 10 +++++----- client.js | 1 + init.js | 18 +++++++++--------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 9d84e28..0b431c1 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -111,6 +111,7 @@ "prefer-numeric-literals":"error", "prefer-spread":"error", "prefer-template":"error", - "symbol-description":"error" + "symbol-description":"error", + "no-func-loop": false } } diff --git a/classes.js b/classes.js index 1ebdfef..ce84735 100644 --- a/classes.js +++ b/classes.js @@ -195,7 +195,7 @@ Discord.Structures.extend("Message", M => { }; }); -Discord.Strictures.extend("User", U => { +Discord.Structures.extend("User", U => { return class User extends U { _unpatch() { return { @@ -205,7 +205,7 @@ Discord.Strictures.extend("User", U => { discriminator: this.discriminator, avatar: this.avatar, system: this.system, - public_flags: this.flags.valueOf() + public_flags: this.flags?.valueOf() }; } }; @@ -463,9 +463,9 @@ Discord.Structures.extend("Role", R => { managed: this.managed, mentionable: this.mentionable, tags: { - bot_id: this.tags.botID, - integration_id: this.tags.integrationID, - premium_subscriber: this.tags.premiumSubscriberRole + bot_id: this.tags?.botID, + integration_id: this.tags?.integrationID, + premium_subscriber: this.tags?.premiumSubscriberRole } }; } diff --git a/client.js b/client.js index 313c02e..b43fd60 100644 --- a/client.js +++ b/client.js @@ -205,6 +205,7 @@ Discord.Client = class Client extends Discord.Client { fs.writeFileSync(`${process.cwd()}/.sessions/websocket/${id}.json`, obj, "utf8"); } for(const folder of Object.keys(cache)) { + if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } for(const [id, data] of Object.entries(cache[folder])) { let obj = JSON.stringify(data); fs.writeFileSync(`${process.cwd()}/.sessions/${folder}/${id}.json`, obj, "utf8"); diff --git a/init.js b/init.js index 915d2da..8e0348d 100644 --- a/init.js +++ b/init.js @@ -287,28 +287,28 @@ require.cache[CPath].exports = class Channel extends C { id: this.id }; if(this.messages) { - obj.last_message_id: this.lastMessageID; - obj.last_pin_timestamp: this.lastPinTimestamp; + obj.last_message_id = this.lastMessageID; + obj.last_pin_timestamp = this.lastPinTimestamp; } switch(this.type) { case "dm": { - obj.recipients: [this.recipient._unpatch()]; + obj.recipients = [this.recipient._unpatch()]; break; } case "text": case "news": { - obj.nsfw: this.nsfw; - obj.topic: this.topic; - obj.rate_limit_per_user: this.rateLimitPerUser; + obj.nsfw = this.nsfw; + obj.topic = this.topic; + obj.rate_limit_per_user = this.rateLimitPerUser; obj.messages = this.messages.cache.map(x => x._unpatch()); break; } case "voice": { - obj.bitrate: this.bitrate; - obj.user_limit: this.userLimit + obj.bitrate = this.bitrate; + obj.user_limit = this.userLimit break; } case "store": { - obj.nsfw: this.nsfw; + obj.nsfw = this.nsfw; break; } } From 3713d0c0aa647ed9bf53d058831a496d21c6033f Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 2 Apr 2021 15:54:28 +0100 Subject: [PATCH 040/120] rework cache loading and storing --- client.js | 64 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/client.js b/client.js index b43fd60..4941374 100644 --- a/client.js +++ b/client.js @@ -115,6 +115,7 @@ Discord.Client = class Client extends Discord.Client { } /** * Generates a complete dump of the current stored cache + * Only guild cache is dumped for now * @returns {object} Cache data */ dumpCache() { @@ -139,47 +140,54 @@ Discord.Client = class Client extends Discord.Client { }, {}); } /** - * Loads all of the stored caches on disk into memory - * @returns {object} All of the stored cache + * Loads the selected stored cache on disk into memory + * @returns {object} The stored cache * @private */ - _loadCache() { - const allCache = { - guilds: [], - channels: [], - users: [] - }; - for(const cache of ["guilds", "channels", "users"]) { - let files = []; + _loadCache(cacheType, filter) { + const cache = {}; + if(typeof cacheType !== "string" || !["guilds"].includes(cacheType.toLowerCase())) { return cache; } // to allow expanding in the future + let files = []; + try { + files = fs.readdirSync(`${process.cwd()}/.sessions/${cacheType}`).filter(file => file.endsWith(".json")); + } catch(e) { /* no-op */ } + for(const file of files) { + let name = file.slice(0, -5); + if(typeof filter === "function" && !filter(name)) { continue; } try { - files = fs.readdirSync(`${process.cwd()}/.sessions/${cache}`).filter(file => file.endsWith(".json")); + const json = fs.readFileSync(`${process.cwd()}/.sessions/${cacheType}/${file}`, "utf8"); + const obj = JSON.parse(json); + cache[name] = obj; } catch(e) { /* no-op */ } - for(const file of files) { - try { - const json = fs.readFileSync(`${process.cwd()}/.sessions/${cache}/${file}`, "utf8"); - const obj = JSON.parse(json); - allCache[cache].push(obj); - } catch(e) { /* no-op */ } - } } - return allCache; + return cache; } /** - * Loads all of the stored sessions on disk into memory + * Loads the selected stored sessions on disk into memory * @private */ - _loadSessions() { - const data = {}; + _loadSessions(id) { + let data = {}; let files = []; try { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); } catch (e) { /* no-op */ } - for(const file of files) { - try { - const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); - const obj = JSON.parse(json); - data[file.slice(0, -5)] = obj; - } catch(e) { /* no-op */ } + if(id) { + const file = files.find(file => Number(file.slice(0, -5) === id); + if(file) { + try { + const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); + data = JSON.parse(json); + } catch(e) { /* no-op */ } + } + } else { + for(const file of files) { + try { + const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); + const shard = Number(file.slice(0, -5); + data[shard] = JSON.parse(json); + } catch(e) { /* no-op */ } + } } return data; } From 1655dd6dc5debcfd9dd5cf4100307b7c888cf1af Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 2 Apr 2021 16:05:00 +0100 Subject: [PATCH 041/120] load cache on identify --- init.js | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/init.js b/init.js index 8e0348d..39450fb 100644 --- a/init.js +++ b/init.js @@ -6,6 +6,7 @@ const Constants = require(resolve(require.resolve("discord.js").replace("index.j const APIMessage = require(resolve(require.resolve("discord.js").replace("index.js", "/structures/APIMessage.js"))); const Util = require(resolve(require.resolve("discord.js").replace("index.js", "/util/Util.js"))); const { Error: DJSError } = require(resolve(require.resolve("discord.js").replace("index.js", "/errors"))); +const ShardClientUtil = require(resolve(require.resolve("discord.js").replace("index.js", "/sharding/ShardClientUtil.js"))); const RHPath = resolve(require.resolve("discord.js").replace("index.js", "/rest/APIRequest.js")); const RH = require(RHPath); @@ -48,12 +49,26 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { }, 15000); } identify() { - if(this.manager.client.options.hotreload && this.manager._hotreload) { - const data = this.manager._hotreload[this.id]; - if(data && !this.sessionID) { + let hotReload = this.manager.client.options.hotReload; + if(hotReload) { + const data = hotReload.sessionData?[this.id] || this.manager.client._loadSessions(this.id); + if(data?.id && !this.sessionID) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; - delete this.manager._hotreload[this.id]; + } + const cache = this.manager.client.options.hotReload.cacheData; + if(cache?.guilds && typeof cache.guilds === "object") { + const keys = Object.keys(cache.guilds); + for(const id of keys) { + if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { + this.manager.client.guilds.add(cache.guilds[id]); + } + } + } else { + const guilds = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards)); + for(const guild of Object.values(guilds)) { + this.manager.guilds.add(guild); + } } } return super.identify(); From 50b1d95f67c42b672737854498b7cce0e87cf30c Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 2 Apr 2021 16:10:03 +0100 Subject: [PATCH 042/120] fix conflict --- init.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init.js b/init.js index 39450fb..029c6b2 100644 --- a/init.js +++ b/init.js @@ -7,6 +7,8 @@ const APIMessage = require(resolve(require.resolve("discord.js").replace("index. const Util = require(resolve(require.resolve("discord.js").replace("index.js", "/util/Util.js"))); const { Error: DJSError } = require(resolve(require.resolve("discord.js").replace("index.js", "/errors"))); const ShardClientUtil = require(resolve(require.resolve("discord.js").replace("index.js", "/sharding/ShardClientUtil.js"))); +const Util = require(resolve(require.resolve("discord.js").replace("index.js", "/util/Util.js"))); +const { Error: DJSError } = require(resolve(require.resolve("discord.js").replace("index.js", "/errors"))); const RHPath = resolve(require.resolve("discord.js").replace("index.js", "/rest/APIRequest.js")); const RH = require(RHPath); From 652635b7e7256fd06885104e163c466da7f27fc4 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 17:46:14 +0100 Subject: [PATCH 043/120] Small fixes Removed obj variable as it JSON.stringify() can be called directly inside of write file --- client.js | 12 +++++------- init.js | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/client.js b/client.js index 4941374..848fb02 100644 --- a/client.js +++ b/client.js @@ -80,7 +80,7 @@ Discord.Client = class Client extends Discord.Client { dumped = true; const cache = this.dumpCache(); const sessions = this.dumpSessions(); - if(options.hotReload.onExit) { + if(options.hotReload?.onExit) { await options.hotReload.onExit(sessions, cache).catch(() => {}); // async will not work on exit } else { this._storeData(sessions, cache); @@ -173,7 +173,7 @@ Discord.Client = class Client extends Discord.Client { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); } catch (e) { /* no-op */ } if(id) { - const file = files.find(file => Number(file.slice(0, -5) === id); + const file = files.find(file => Number(file.slice(0, -5) === id)); if(file) { try { const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); @@ -184,7 +184,7 @@ Discord.Client = class Client extends Discord.Client { for(const file of files) { try { const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); - const shard = Number(file.slice(0, -5); + const shard = Number(file.slice(0, -5)); data[shard] = JSON.parse(json); } catch(e) { /* no-op */ } } @@ -209,14 +209,12 @@ Discord.Client = class Client extends Discord.Client { _storeData(sessions, cache) { for(const [id, data] of Object.entries(sessions)) { if(!fs.existsSync(`${process.cwd()}/.sessions/websocket`)) { fs.mkdirSync(`${process.cwd()}/.sessions/websocket`, { recursive: true }); } - let obj = JSON.stringify(data); - fs.writeFileSync(`${process.cwd()}/.sessions/websocket/${id}.json`, obj, "utf8"); + fs.writeFileSync(`${process.cwd()}/.sessions/websocket/${id}.json`, JSON.stringify(data), "utf8"); } for(const folder of Object.keys(cache)) { if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } for(const [id, data] of Object.entries(cache[folder])) { - let obj = JSON.stringify(data); - fs.writeFileSync(`${process.cwd()}/.sessions/${folder}/${id}.json`, obj, "utf8"); + fs.writeFileSync(`${process.cwd()}/.sessions/${folder}/${id}.json`, JSON.stringify(data), "utf8"); } } } diff --git a/init.js b/init.js index 029c6b2..ae087da 100644 --- a/init.js +++ b/init.js @@ -53,7 +53,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const data = hotReload.sessionData?[this.id] || this.manager.client._loadSessions(this.id); + const data = hotReload.sessionData?.[this.id] || this.manager.client._loadSessions(this.id) if(data?.id && !this.sessionID) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; From f53ae0cf186136515ceb3092f5d906c9d025b5f8 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 2 Apr 2021 16:40:29 +0100 Subject: [PATCH 044/120] readme --- README.md | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index cfc1824..6c7749c 100644 --- a/README.md +++ b/README.md @@ -91,20 +91,6 @@ client.login("TOKEN").catch(console.error); Generally, usage should be identical to discord.js and you can safely refer to its documentation as long as you respect the caching differences explained below. -### Hot reloading - -**THIS FEATURE IS CURRENTLY EXPERIMENTAL USE AT YOUR OWN RISK!** - -When developing bots you will often want to prototype through trial and error. This often requires turning your bot on and off lots of times potentially using [nodemon](https://nodemon.io/) -by doing this you are connecting to the Discord websocket gateway each time which can often take a few seconds as well as cuts into your 1000 daily identifies. - -To solve this problem you can use hot reloading which is a client option allowing you to simply resume the previous session rather than create a new one. - -You can also use Hot reloading in your production bot by supplying a session object along with preferences for caching restoration or by just letting us take care of it with cache files found in the `.sessions` folder - -By setting the client.dumpCache method you can run a custom async function to store your caches and session IDs in your database of choice. The dumpCache method is -called with the (session, client) params where session is your up to date sessoins - ## Client Options The following client options are available to control caching behavior: @@ -118,6 +104,7 @@ The following client options are available to control caching behavior: | cacheEmojis | boolean | false | Enables caching of all Emojis at login | | cachePresences | boolean | false | Enables caching of all Presences. If not enabled, Presences will be cached only for cached Users | | cacheMembers | boolean | false | Enables caching of Users and Members when possible | +| hotReload | boolean or object | false | Enables hot reloading, an experimental feature that enables instantly restarting the bot | | disabledEvents | array | [] | An array of events to ignore ([Discord events](https://github.com/discordjs/discord.js/blob/master/src/util/Constants.js#L339), not Discord.JS events). Use this in combination with intents for fine tuning which events your bot should process | This library implements its own partials system, therefore the `partials` client option is not available. All other discord.js client options continue to be available and should work normally. @@ -162,6 +149,30 @@ Voice States will be cached if the `GUILD_VOICE_STATES` intent is enabled (requi Messages are cached only if the Channel they belong to is cached. Message caching can further be controlled via discord.js's `messageCacheMaxSize`, `messageCacheLifetime` and `messageSweepInterval` client options as usual. Additionally, the `messageEditHistoryMaxSize` client option is set to `1` by default (instead of infinity). +## Hot reloading + +**THIS FEATURE IS CURRENTLY EXPERIMENTAL USE AT YOUR OWN RISK!** + +When developing bots you will likely do lots of trial and error which often requires restartig your bot. +Each restart requires reconnecting to the Discord gateway on every shard which can often take a long time and eat up your daily identifies. + +Hot reloading provides a way to simply resume the previous gateway sessions on startup rather than creating new ones. +This is done by storing the process data outside the process right before it exits and reloading the data into it when it starts. + +This option can be overriden with a few custom methods: + +```js +new Discord.Client({ + hotReload: { + cacheData: cache // user-supplied cache data. if not present, cache will be loaded from disk + sessionData: sessions // user-supplied session data. if not present, sessions will be loaded from disk + onExit: (sessions, cache) => {} // user-supplied data storing function. if not present, data will be stored to disk. This function can be async if the process is exited by using SIGINT (Ctrl+C), any other ways of exit are sync-only. + } +}) +``` + +Session and cache data can also be obtained at runtime using `client.dumpCache()` and `client.dumpSessions()` for storage before a manually induced exit. In this case the user should pass an empty function to onExit to override the built-in disk storage. + ## Events Most events should be identical to the originals aside from the caching behavior plus they always emit regardless of caching state. When required data is missing, a partial structure where only an id is guaranteed will be given (the `.partial` property is not guaranteed to exist on all partials). From 8d5f0c60e6e54840bcc712632938845af443dbda Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 17:58:25 +0100 Subject: [PATCH 045/120] Added a last connected property to session data to avoid unnecessary identifies Sessions timeout after about 1-2 minutes so we should check for when the session was last connected and if it is more than a minute old ( therefore probably stale ) we shouldn't try and reconnect as the identify will probably fail. --- client.js | 3 ++- init.js | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/client.js b/client.js index 848fb02..309e022 100644 --- a/client.js +++ b/client.js @@ -134,7 +134,8 @@ Discord.Client = class Client extends Discord.Client { return this.ws.shards.reduce((a, s) => { a[s.id] = { id: s.sessionID, - sequence: s.sequence + sequence: s.sequence, + lastConnected: Date.now() }; return a; }, {}); diff --git a/init.js b/init.js index ae087da..3577a70 100644 --- a/init.js +++ b/init.js @@ -53,8 +53,8 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const data = hotReload.sessionData?.[this.id] || this.manager.client._loadSessions(this.id) - if(data?.id && !this.sessionID) { + const data = (hotReload.sessionData || this.manager.client._loadSessions(this.id))?.[this.id] + if(data?.id && data?.sequence && !this.sessionID && data.lastConnected + 60000 > Date.now()) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } From 658c0ac639876eef69722537f3420772abd42a9e Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 18:03:26 +0100 Subject: [PATCH 046/120] Updated cache and session types. Maybe remove from readme --- README.md | 24 +++++++++++++++++++----- client.d.ts | 7 ++++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6c7749c..4057c63 100644 --- a/README.md +++ b/README.md @@ -153,19 +153,33 @@ Messages are cached only if the Channel they belong to is cached. Message cachin **THIS FEATURE IS CURRENTLY EXPERIMENTAL USE AT YOUR OWN RISK!** -When developing bots you will likely do lots of trial and error which often requires restartig your bot. +When developing bots you will likely do lots of trial and error which often requires restarting your bot. Each restart requires reconnecting to the Discord gateway on every shard which can often take a long time and eat up your daily identifies. Hot reloading provides a way to simply resume the previous gateway sessions on startup rather than creating new ones. This is done by storing the process data outside the process right before it exits and reloading the data into it when it starts. - -This option can be overriden with a few custom methods: + +This option can be overridden with a few custom methods: ```js +const cache = { + guilds: { + "581072557512458241": {} // Discord api type + } +} + +const sessions = { + "0": { + id: "3e961ec59a7c24b91198d07dd3189fa0", // Session ID + sequence: 19, // The close sequence of the session + lastConnected: 1617382560867 // Time the session was closed at + } +} + new Discord.Client({ hotReload: { - cacheData: cache // user-supplied cache data. if not present, cache will be loaded from disk - sessionData: sessions // user-supplied session data. if not present, sessions will be loaded from disk + cacheData: cache, // user-supplied cache data. if not present, cache will be loaded from disk + sessionData: sessions, // user-supplied session data. if not present, sessions will be loaded from disk onExit: (sessions, cache) => {} // user-supplied data storing function. if not present, data will be stored to disk. This function can be async if the process is exited by using SIGINT (Ctrl+C), any other ways of exit are sync-only. } }) diff --git a/client.d.ts b/client.d.ts index 808f46d..3ebd5a6 100644 --- a/client.d.ts +++ b/client.d.ts @@ -52,6 +52,7 @@ type SessionData = { [shardID: string]: { id: string sequence: number + lastConnected: number } } @@ -62,9 +63,9 @@ type CacheData = { } type HotReloadOptions = { - sessionData?: SessionData cacheData?: CacheData - onUnload?: Function + sessionData?: SessionData + onExit?: Function } declare module "discord.js-light" { @@ -76,8 +77,8 @@ declare module "discord.js-light" { cacheOverwrites?:boolean cacheEmojis?:boolean cacheMembers?:boolean - disabledEvents?: Array hotReload?: boolean | HotReloadOptions + disabledEvents?: Array } interface ClientEvents { rest:[{path:string,method:string,response?:Promise}] From 848658edac05f620176856d47e069f7c2831fb2e Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 18:33:11 +0100 Subject: [PATCH 047/120] Small fixes and checks --- client.js | 4 ++-- init.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index 309e022..27bf0ae 100644 --- a/client.js +++ b/client.js @@ -173,7 +173,7 @@ Discord.Client = class Client extends Discord.Client { try { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); } catch (e) { /* no-op */ } - if(id) { + if(id && files.length) { const file = files.find(file => Number(file.slice(0, -5) === id)); if(file) { try { @@ -181,7 +181,7 @@ Discord.Client = class Client extends Discord.Client { data = JSON.parse(json); } catch(e) { /* no-op */ } } - } else { + } else if (files.length) { for(const file of files) { try { const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); diff --git a/init.js b/init.js index 3577a70..5dad715 100644 --- a/init.js +++ b/init.js @@ -54,7 +54,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { let hotReload = this.manager.client.options.hotReload; if(hotReload) { const data = (hotReload.sessionData || this.manager.client._loadSessions(this.id))?.[this.id] - if(data?.id && data?.sequence && !this.sessionID && data.lastConnected + 60000 > Date.now()) { + if(data?.id && data.sequence > 0 && !this.sessionID && data.lastConnected + 60000 > Date.now()) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } @@ -69,7 +69,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { } else { const guilds = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards)); for(const guild of Object.values(guilds)) { - this.manager.guilds.add(guild); + this.manager.client.guilds.add(guild); } } } From 86fb045576c7c8e48369fe6af3b483f3623e4507 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 19:03:45 +0100 Subject: [PATCH 048/120] Fixed _loadSession + now returns data in line with user supplied data Spend ages trying to figure out why shard 0 wasn't being treated as truthy... Of course JS treats the number 0 falsy xD --- client.js | 8 ++++---- init.js | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/client.js b/client.js index 27bf0ae..de5dc3e 100644 --- a/client.js +++ b/client.js @@ -167,18 +167,18 @@ Discord.Client = class Client extends Discord.Client { * Loads the selected stored sessions on disk into memory * @private */ - _loadSessions(id) { + _loadSession(id) { let data = {}; let files = []; try { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); } catch (e) { /* no-op */ } - if(id && files.length) { - const file = files.find(file => Number(file.slice(0, -5) === id)); + if(Number(id) >= 0 && files.length) { + const file = files.find(file => Number(file.slice(0, -5)) === id); if(file) { try { const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); - data = JSON.parse(json); + data[id] = JSON.parse(json); } catch(e) { /* no-op */ } } } else if (files.length) { diff --git a/init.js b/init.js index 5dad715..e7e82da 100644 --- a/init.js +++ b/init.js @@ -53,7 +53,9 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const data = (hotReload.sessionData || this.manager.client._loadSessions(this.id))?.[this.id] + const t = hotReload.sessionData || this.manager.client._loadSession(this.id) + console.log(t); + const data = (t)?.[this.id] if(data?.id && data.sequence > 0 && !this.sessionID && data.lastConnected + 60000 > Date.now()) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; From accb61802e57fb8e01989a927ce29ad2c3070e28 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:20:01 +0100 Subject: [PATCH 049/120] Made data more consistent Different data types were being assigned to the same variables change the loadcache method to reflect the same data that the user will supply as it makes it easier to work with. Did a hard compare for filter to stop any falsy value returns ( like shard id 0 being wrongly filtered because the number 0 is falsy ) --- client.js | 7 +++---- init.js | 15 ++++++--------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/client.js b/client.js index de5dc3e..e911db4 100644 --- a/client.js +++ b/client.js @@ -154,14 +154,13 @@ Discord.Client = class Client extends Discord.Client { } catch(e) { /* no-op */ } for(const file of files) { let name = file.slice(0, -5); - if(typeof filter === "function" && !filter(name)) { continue; } + if(typeof filter === "function" && filter(name) === false) { continue; } try { const json = fs.readFileSync(`${process.cwd()}/.sessions/${cacheType}/${file}`, "utf8"); - const obj = JSON.parse(json); - cache[name] = obj; + cache[name] = JSON.parse(json); } catch(e) { /* no-op */ } } - return cache; + return { [cacheType]: cache}; } /** * Loads the selected stored sessions on disk into memory diff --git a/init.js b/init.js index e7e82da..8242fd9 100644 --- a/init.js +++ b/init.js @@ -53,23 +53,20 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const t = hotReload.sessionData || this.manager.client._loadSession(this.id) - console.log(t); - const data = (t)?.[this.id] + const data = (hotReload.sessionData || this.manager.client._loadSession(this.id))?.[this.id] if(data?.id && data.sequence > 0 && !this.sessionID && data.lastConnected + 60000 > Date.now()) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } - const cache = this.manager.client.options.hotReload.cacheData; - if(cache?.guilds && typeof cache.guilds === "object") { - const keys = Object.keys(cache.guilds); - for(const id of keys) { + const { guilds } = this.manager.client.options.hotReload.cacheData; + if(guilds) { + for(const [id, guild] of Object.entries(guilds)) { if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { - this.manager.client.guilds.add(cache.guilds[id]); + this.manager.client.guilds.add(guild); } } } else { - const guilds = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards)); + const { guilds } = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id); for(const guild of Object.values(guilds)) { this.manager.client.guilds.add(guild); } From 68634f4da289946b0cbd0d0197230daedaf2235f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:30:11 +0100 Subject: [PATCH 050/120] Identify without loading cache if creating a new session If session resume checks fail the client will try to identify and not resume. If they are identifying they will receive new data so we don't need to restore old data --- init.js | 1 + 1 file changed, 1 insertion(+) diff --git a/init.js b/init.js index 8242fd9..9b34683 100644 --- a/init.js +++ b/init.js @@ -58,6 +58,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } + else { return super.identify(); } const { guilds } = this.manager.client.options.hotReload.cacheData; if(guilds) { for(const [id, guild] of Object.entries(guilds)) { From dc630ac0d7860aba1a2ea202e32de9f8279a0193 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:32:38 +0100 Subject: [PATCH 051/120] Update init.js --- init.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/init.js b/init.js index 9b34683..5b8537d 100644 --- a/init.js +++ b/init.js @@ -59,9 +59,9 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { this.closeSequence = this.sequence = data.sequence; } else { return super.identify(); } - const { guilds } = this.manager.client.options.hotReload.cacheData; - if(guilds) { - for(const [id, guild] of Object.entries(guilds)) { + const cache = this.manager.client.options.hotReload.cacheData; + if(cache?.guilds) { + for(const [id, guild] of Object.entries(cache.guilds)) { if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { this.manager.client.guilds.add(guild); } From fa814e319f097cb02f9e20ca0b0ef8f0477941f6 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:52:13 +0100 Subject: [PATCH 052/120] Moved loading of cache into resume event I kept getting 4003 Not authenticated errors from the gateway meaning we were sending packets before identifying. There must be some WS connection in our load cache method so now we wait until after the session has resumed before loading cache --- init.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/init.js b/init.js index 5b8537d..0f113f8 100644 --- a/init.js +++ b/init.js @@ -58,20 +58,21 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } - else { return super.identify(); } - const cache = this.manager.client.options.hotReload.cacheData; - if(cache?.guilds) { - for(const [id, guild] of Object.entries(cache.guilds)) { - if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { + this.once(Constants.ShardEvents.RESUMED, () => { + const cache = this.manager.client.options.hotReload.cacheData; + if(cache?.guilds) { + for(const [id, guild] of Object.entries(cache.guilds)) { + if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { + this.manager.client.guilds.add(guild); + } + } + } else { + const { guilds } = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id); + for(const guild of Object.values(guilds)) { this.manager.client.guilds.add(guild); } } - } else { - const { guilds } = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id); - for(const guild of Object.values(guilds)) { - this.manager.client.guilds.add(guild); - } - } + }) } return super.identify(); } From 27f302366e56f8ccb72e3541cd7bddc12695f7d5 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:53:39 +0100 Subject: [PATCH 053/120] Temporary fix until we load client user --- classes.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classes.js b/classes.js index ce84735..34d6cb7 100644 --- a/classes.js +++ b/classes.js @@ -287,8 +287,8 @@ Discord.Structures.extend("Guild", G => { this.members.add(member); } } - if(!this.members.cache.has(this.client.user.id)) { - this.members.fetch(this.client.user.id).catch(() => {}); + if(!this.members.cache.has(this.client.user?.id)) { + this.members.fetch(this.client.user?.id).catch(() => {}); } } if(Array.isArray(data.presences)) { From 49f27b3b37d597a2ce3e1dfa96f83766157e20ba Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 01:38:19 +0100 Subject: [PATCH 054/120] Checks the close sequence of the last shard and doesn't wait if it exists Fixes https://github.com/timotejroiko/discord.js-light/pull/41#issuecomment-812631299 --- init.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init.js b/init.js index 0f113f8..bcdd185 100644 --- a/init.js +++ b/init.js @@ -161,7 +161,7 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { } } // If we have multiple shards add a 5s delay if identifying or no delay if resuming - if (this.shardQueue.size && Object.keys(this._hotreload).length) { + if (this.shardQueue.size && this.shards.last().closeSequence) { this.debug(`Shard Queue Size: ${this.shardQueue.size} with sessions; continuing immediately`); return this.createShards(); } else if (this.shardQueue.size) { From d0db3c6561d2d37e2b2638c9f7df057a110add3a Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 12:15:32 +0100 Subject: [PATCH 055/120] Moved session data to createShards First loops through all shards and requeues them if they need to identify --- init.js | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/init.js b/init.js index bcdd185..ddeb6f5 100644 --- a/init.js +++ b/init.js @@ -53,11 +53,6 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const data = (hotReload.sessionData || this.manager.client._loadSession(this.id))?.[this.id] - if(data?.id && data.sequence > 0 && !this.sessionID && data.lastConnected + 60000 > Date.now()) { - this.sessionID = data.id; - this.closeSequence = this.sequence = data.sequence; - } this.once(Constants.ShardEvents.RESUMED, () => { const cache = this.manager.client.options.hotReload.cacheData; if(cache?.guilds) { @@ -90,6 +85,23 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { const [shard] = this.shardQueue; + // Pushes shards that require reidentifying to the back of the queue + const hotReload = this.client.options.hotReload; + if (hotReload) { + const data = (hotReload.sessionData || this.client._loadSession(shard.id))?.[shard.id] + if(data?.id && data.sequence > 0 && !shard.sessionID && data.lastConnected + 60000 > Date.now()) { + shard.sessionID = data.id; + shard.closeSequence = shard.sequence = data.sequence; + } + else if (this.shardQueue.size > 1 && !shard.requeued) { + shard.requeued = true; + this.shardQueue.delete(shard); + this.shardQueue.add(shard); + this.debug("Shard required to identify, pushed to the back of the queue", shard); + return this.createShards(); + } + } + this.shardQueue.delete(shard); if (!shard.eventsAttached) { From b9d28abfc042d0e08510fd52fda8605f0042768c Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 12:34:28 +0100 Subject: [PATCH 056/120] Stores shardCount in session object Checks if the session is required to reidentify. Added a few extra debug logs --- client.d.ts | 1 + client.js | 4 +++- init.js | 8 +++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/client.d.ts b/client.d.ts index 3ebd5a6..897caaa 100644 --- a/client.d.ts +++ b/client.d.ts @@ -53,6 +53,7 @@ type SessionData = { id: string sequence: number lastConnected: number + shardCount: number } } diff --git a/client.js b/client.js index e911db4..5c2c32d 100644 --- a/client.js +++ b/client.js @@ -77,6 +77,7 @@ Discord.Client = class Client extends Discord.Client { for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { if(!dumped) { + this.ws.debug(`${eventType} Exit event. Storing shard sessions and cache.`); dumped = true; const cache = this.dumpCache(); const sessions = this.dumpSessions(); @@ -135,7 +136,8 @@ Discord.Client = class Client extends Discord.Client { a[s.id] = { id: s.sessionID, sequence: s.sequence, - lastConnected: Date.now() + lastConnected: Date.now(), + shardCount: this.ws.shards.size }; return a; }, {}); diff --git a/init.js b/init.js index ddeb6f5..9d21b2a 100644 --- a/init.js +++ b/init.js @@ -54,6 +54,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { let hotReload = this.manager.client.options.hotReload; if(hotReload) { this.once(Constants.ShardEvents.RESUMED, () => { + this.debug("Shard session resumed. Restoring cache"); const cache = this.manager.client.options.hotReload.cacheData; if(cache?.guilds) { for(const [id, guild] of Object.entries(cache.guilds)) { @@ -89,15 +90,16 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { const hotReload = this.client.options.hotReload; if (hotReload) { const data = (hotReload.sessionData || this.client._loadSession(shard.id))?.[shard.id] - if(data?.id && data.sequence > 0 && !shard.sessionID && data.lastConnected + 60000 > Date.now()) { + if(data?.id && data.sequence > 0 && !shard.sessionID && data.shardCount === this.totalShards && data.lastConnected + 60000 > Date.now()) { shard.sessionID = data.id; shard.closeSequence = shard.sequence = data.sequence; + this.debug("Loaded sessions from cache, resuming previous session.", shard); } else if (this.shardQueue.size > 1 && !shard.requeued) { shard.requeued = true; this.shardQueue.delete(shard); this.shardQueue.add(shard); - this.debug("Shard required to identify, pushed to the back of the queue", shard); + this.debug("Shard required to identify, pushed to the back of the queue.", shard); return this.createShards(); } } @@ -174,7 +176,7 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { } // If we have multiple shards add a 5s delay if identifying or no delay if resuming if (this.shardQueue.size && this.shards.last().closeSequence) { - this.debug(`Shard Queue Size: ${this.shardQueue.size} with sessions; continuing immediately`); + this.debug(`Shard Queue Size: ${this.shardQueue.size} with sessions; continuing immediately.`); return this.createShards(); } else if (this.shardQueue.size) { this.debug(`Shard Queue Size: ${this.shardQueue.size}; continuing in 5s seconds...`); From d806d78ee4ec26796ecd5a6f26e216804ef105e9 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 13:35:29 +0100 Subject: [PATCH 057/120] shards.size -> totalShards --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index 5c2c32d..ed494ce 100644 --- a/client.js +++ b/client.js @@ -137,7 +137,7 @@ Discord.Client = class Client extends Discord.Client { id: s.sessionID, sequence: s.sequence, lastConnected: Date.now(), - shardCount: this.ws.shards.size + shardCount: this.ws.totalShards }; return a; }, {}); From 57c86018741da852dee678008f3650eed9b04e9e Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 13:57:29 +0100 Subject: [PATCH 058/120] Moved cache patching and added 15s timeout for resuming Moved cache patching to WebSocketManager and added timeout of 15s to remove the loadCache event. Now only attaches the event if a session ID is present to stop the event being attached on an identify connection attempt I don't know the best way to remove the event listener in the timeout so let me know how you would do it. --- init.js | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/init.js b/init.js index 9d21b2a..e477445 100644 --- a/init.js +++ b/init.js @@ -50,28 +50,6 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { this.emitReady(); }, 15000); } - identify() { - let hotReload = this.manager.client.options.hotReload; - if(hotReload) { - this.once(Constants.ShardEvents.RESUMED, () => { - this.debug("Shard session resumed. Restoring cache"); - const cache = this.manager.client.options.hotReload.cacheData; - if(cache?.guilds) { - for(const [id, guild] of Object.entries(cache.guilds)) { - if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { - this.manager.client.guilds.add(guild); - } - } - } else { - const { guilds } = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id); - for(const guild of Object.values(guilds)) { - this.manager.client.guilds.add(guild); - } - } - }) - } - return super.identify(); - } }; const SHMPath = resolve(require.resolve("discord.js").replace("index.js", "/client/websocket/WebSocketManager.js")); @@ -156,6 +134,34 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { this.reconnect(); }); + const hotReload = this.client.options.hotReload; + if(hotReload && shard.sessionID) { + shard.once(Constants.ShardEvents.RESUMED, () => { + this.debug("Shard session resumed. Restoring cache", shard); + shard.loadCacheTimeout = null; + this.client.clearTimeout(this.loadCacheTimeout); + const cache = hotReload.cacheData; + if(cache?.guilds) { + for(const [id, guild] of Object.entries(cache.guilds)) { + if(ShardClientUtil.shardIDForGuildID(id, this.totalShards) === shard.id) { + this.client.guilds.add(guild); + } + } + } else { + const { guilds } = this.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.totalShards) === shard.id); + for(const guild of Object.values(guilds)) { + this.client.guilds.add(guild); + } + } + }) + + shard.loadCacheTimeout = this.client.setTimeout(() => { + this.debug("Shard cache was never loaded as the session didn't resume in 15s", shard); + shard.loadCacheTimeout = null; + shard.removeEventListener(Constants.ShardEvents.RESUMED) // Remove the event in a better way? + }, 15000); + } + shard.eventsAttached = true; } From 48ab68f553ba02fa3d5eef4fe7ca386a798970f1 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 14:09:08 +0100 Subject: [PATCH 059/120] Event listener removed --- init.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init.js b/init.js index e477445..035bfcf 100644 --- a/init.js +++ b/init.js @@ -138,8 +138,8 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { if(hotReload && shard.sessionID) { shard.once(Constants.ShardEvents.RESUMED, () => { this.debug("Shard session resumed. Restoring cache", shard); + this.client.clearTimeout(shard.loadCacheTimeout); shard.loadCacheTimeout = null; - this.client.clearTimeout(this.loadCacheTimeout); const cache = hotReload.cacheData; if(cache?.guilds) { for(const [id, guild] of Object.entries(cache.guilds)) { @@ -158,7 +158,7 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { shard.loadCacheTimeout = this.client.setTimeout(() => { this.debug("Shard cache was never loaded as the session didn't resume in 15s", shard); shard.loadCacheTimeout = null; - shard.removeEventListener(Constants.ShardEvents.RESUMED) // Remove the event in a better way? + shard.removeListener(Constants.ShardEvents.RESUMED, shard.listeners(Constants.ShardEvents.RESUMED)[0]) // Remove the event in a better way? }, 15000); } From c9542dbb8adc148d92bcec548e9749dee2c2bb38 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Thu, 11 Mar 2021 22:35:58 +0000 Subject: [PATCH 060/120] Client now properly emits ready event on hot reload Client will emit the standard ready event once all of its shards are ready and connected. --- client.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client.js b/client.js index 2b4eec7..95ebadf 100644 --- a/client.js +++ b/client.js @@ -1,6 +1,7 @@ "use strict"; require("./init.js"); +const { Constants } = require("discord.js"); const Discord = require("./classes.js"); const actions = require("./actions.js"); const pkg = require("./package.json"); From a5b25460629a06465b7175b4ad1dbaa18087ef59 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 12 Mar 2021 20:58:12 +0000 Subject: [PATCH 061/120] Added directory property and added all session cached to folder which is gitignored --- .gitignore | 4 ++++ client.js | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 627a9f1..802d68a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ package-lock.json .vscode .eslintrc.json .gitattributes +.pnp.js +.yarnrc.yml +yarn.lock +.sessions diff --git a/client.js b/client.js index 95ebadf..20b709b 100644 --- a/client.js +++ b/client.js @@ -23,6 +23,7 @@ Discord.Client = class Client extends Discord.Client { super(options); actions(this); if(options.hotreload) { + this.cacheFilePath = `${process.cwd()}/.sessions`; this.ws._hotreload = {}; if (options.sessionID && options.sequence) { if (!Array.isArray(options.sessionID) && !Array.isArray(options.sequence)) { @@ -38,7 +39,7 @@ Discord.Client = class Client extends Discord.Client { } else { try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${process.cwd()}/.sessions.json`, "utf8")); + this.ws._hotreload = JSON.parse(fs.readFileSync(`${process.cwd()}/.sessions/sessions.json`, "utf8")); } catch(e) { this.ws._hotreload = {}; } @@ -49,7 +50,7 @@ Discord.Client = class Client extends Discord.Client { for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, () => { try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${process.cwd()}/.sessions.json`, "utf8")); + this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); } catch(e) { this.ws._hotreload = {}; } @@ -62,7 +63,7 @@ Discord.Client = class Client extends Discord.Client { } }; })); - fs.writeFileSync(`${process.cwd()}/.sessions.json`, JSON.stringify(this.ws._hotreload)); + fs.writeFileSync(`${this.cacheFilePath}/sessions.json`, JSON.stringify(this.ws._hotreload)); if(eventType !== "exit") { process.exit(); } From 84d1aefc1ae98ac545b1764ef8501514717af5a3 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 12 Mar 2021 20:58:51 +0000 Subject: [PATCH 062/120] Creates folder if it does not exist --- client.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client.js b/client.js index 20b709b..3774759 100644 --- a/client.js +++ b/client.js @@ -49,6 +49,7 @@ Discord.Client = class Client extends Discord.Client { }); for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, () => { + if (!fs.existsSync(this.cacheFilePath)) { fs.mkdirSync(this.cacheFilePath); } try { this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); } catch(e) { From 4966abf7dcdf3815a5c27ada897c1cd8be846e2f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 12 Mar 2021 23:17:04 +0000 Subject: [PATCH 063/120] Added hot reloading explanation to README --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 07d6f95..a1f1858 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,17 @@ client.login("TOKEN").catch(console.error); Generally, usage should be identical to discord.js and you can safely refer to its documentation as long as you respect the caching differences explained below. +### Hot reloading + +**THIS FEATURE IS CURRENTLY EXPERIMENTAL USE AT YOUR OWN RISK!** + +When developing bots you will often want to prototype through trial and error. This often requires turning your bot on and off lots of times potentially using [nodemon](https://nodemon.io/) +by doing this you are connecting to the Discord websocket gateway each time which can often take a few seconds as well as cuts into your 1000 daily identifies. + +To solve this problem you can use hot reloading which is a client option allowing you to simply resume the previous session rather than create a new one. + +You can also use Hot reloading in your production bot by supplying a sequence and session ID or by just letting us take care of it with cache files found in the `.sessions` folder + ## Client Options The following client options are available to control caching behavior: From f6f17d7ba7df57bf72520c056c2fe4ce4a6c88c3 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 12 Mar 2021 23:18:05 +0000 Subject: [PATCH 064/120] Added temporary file loading and parsing. Fails with roles --- client.js | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 3774759..65d18ac 100644 --- a/client.js +++ b/client.js @@ -1,7 +1,6 @@ "use strict"; require("./init.js"); -const { Constants } = require("discord.js"); const Discord = require("./classes.js"); const actions = require("./actions.js"); const pkg = require("./package.json"); @@ -18,6 +17,7 @@ Discord.Client = class Client extends Discord.Client { cacheEmojis: false, cacheMembers: false, disabledEvents: [], + restoreCache: ["guilds"], ..._options }; super(options); @@ -39,11 +39,29 @@ Discord.Client = class Client extends Discord.Client { } else { try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${process.cwd()}/.sessions/sessions.json`, "utf8")); + this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); } catch(e) { this.ws._hotreload = {}; } } + try { + for (const toCache of options.restoreCache) { + const data = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/${toCache}.json`, "utf8")); + switch (toCache) { + case "guilds": { + console.log("Created"); + data.cache.forEach(i => { + console.log(i); + this.guilds.cache.set(i.id, new Discord.Guild(this, i)); + console.log(i.id); + }); + break; + } + } + } + } catch(e) { + // Do nothing + } this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); @@ -65,6 +83,11 @@ Discord.Client = class Client extends Discord.Client { }; })); fs.writeFileSync(`${this.cacheFilePath}/sessions.json`, JSON.stringify(this.ws._hotreload)); + if (options.restoreCache) { + for (const toCache of options.restoreCache) { + fs.writeFileSync(`${this.cacheFilePath}/${toCache}.json`, JSON.stringify(this[toCache])); + } + } if(eventType !== "exit") { process.exit(); } From 51523a6afd33bd4bdac6d5b74a76129a3abc63b0 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:40:38 +0000 Subject: [PATCH 065/120] Added _unpatch method to guild class Not finished roles and emojis need to be completed and the rest of the properties need to be looked at and confirmed to be correct --- classes.js | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/classes.js b/classes.js index 8ee49ca..1857e24 100644 --- a/classes.js +++ b/classes.js @@ -217,6 +217,115 @@ Discord.Structures.extend("Guild", G => { } } } + _unpatch() { + /** + * Discord raw guild data as documented: https://github.com/discordjs/discord-api-types/blob/main/v8/payloads/guild.ts + */ + return { + id: this.id, + unavailable: this.available, + name: this.name, + icon: this.icon, + splash: this.splash, + banner: this.banner, + description: this.description, + features: this.features, + verification_level: this.verificationLevel, + discovery_splash: this.discoverySplash, + owner_id: this.ownerID, + region: this.region, + afk_channel_id: this.afkChannelID, + afk_timeout: this.afkTimeout, + widget_enabled: this.widgetEnabled, + widget_channel_id: this.widgetChannelID, + default_message_notifications: this.defaultMessageNotifications, + explicit_content_filter: this.explicitContentFilter, + /** + * Roles in the guild + * + * See https://discord.com/developers/docs/topics/permissions#role-object + */ + roles: this.roles.cache.map(r => ({ + name: r.name, + color: r.color + })), + /** + * Custom guild emojis + * + * See https://discord.com/developers/docs/resources/emoji#emoji-object + */ + emojis: this.emojis.cache.map(e => ({ + id: e.id, + name: e.name + })), + mfa_level: this.mfaLevel, + application_id: this.applicationID, + system_channel_id: this.systemChannelID, + system_channel_flags: this.systemChannelFlags, + rules_channel_id: this.rulesChannelID, + joined_at: this.joinedAt, + large: this.large, + member_count: this.memberCount, + voice_states: this.voiceStates.cache.map(v => ({ + guild_id: v.guild.id, + channel_id: v.channelID, + user_id: v.userID, + session_id: v.sessionID, + deaf: v.deaf, + mute: v.mute, + self_deaf: v.selfDeaf, + self_mute: v.selfMute, + suppress: v.suppress + })), + members: this.members.cache.map(m => ({ + user: m.user, + nick: m.nickname, + roles: m.roles, + joined_at: m.joinedAt, + premium_since: m.premiumSinceTimestamp, + deaf: m.deaf, + mute: m.mute, + pending: m.pending, + permissions: m.permissions + })), + channels: this.channels.cache.map(c => ({ + id: c.id, + type: c.type, + guild_id: c.guild.id, + position: c.position, + permission_overwrites: c.permissionOverwrites, + name: c.name, + topic: c.topic, + nsfw: c.nsfw, + last_message_id: c.lastMessageID, + bitrate: c.bitrate, + user_limit: c.userLimit, + rate_limit_per_user: c.rateLimitPerUser, + recipients: c.recipients, + icon: c.icon, + owner_id: c.ownerID, + application_id: c.applicationID, + parent_id: c.parentID, + last_pin_timestamp: c.lastPinTimestamp + })), + presences: this.presences.cache.map(p => ({ + user: p.user, + guild_id: p.guild.id, + status: p.status, + activities: p.activities, + client_status: p.clientStatus + })), + max_presences: this.maximumPresences, + max_members: this.maximumMembers, + vanity_url_code: this.vanityURLCode, + premium_tier: this.premiumTier, + premium_subscription_count: this.premiumSubscriptionCount, + preferred_locale: this.preferredLocale || "en-US", + public_updates_channel_id: this.publicUpdatesChannelID, + approximate_member_count: this.approximateMemberCount, + approximate_presence_count: this.approximatePresenceCount + }; + } get nameAcronym() { return this.name ? super.nameAcronym : void 0; } From bac00086e0e5a488489fe97c345dd50876d33047 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:47:10 +0000 Subject: [PATCH 066/120] Added new client options --- client.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index 65d18ac..e74f634 100644 --- a/client.js +++ b/client.js @@ -17,8 +17,11 @@ Discord.Client = class Client extends Discord.Client { cacheEmojis: false, cacheMembers: false, disabledEvents: [], - restoreCache: ["guilds"], - ..._options + sessions: {}, + ..._options, + restoreCache: { + guilds: true, + } }; super(options); actions(this); From 6129fbd110856947cdda21a7e5882d2c1f7c3311 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:48:41 +0000 Subject: [PATCH 067/120] Greatly simplified session and sequence client option handling It was very cumbersome using an array so I removed the sessionID and sequence options and just replaced it with a session object which should be structured identically to the _hotreload object for simplicity --- client.js | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/client.js b/client.js index e74f634..697fa8d 100644 --- a/client.js +++ b/client.js @@ -27,18 +27,8 @@ Discord.Client = class Client extends Discord.Client { actions(this); if(options.hotreload) { this.cacheFilePath = `${process.cwd()}/.sessions`; - this.ws._hotreload = {}; - if (options.sessionID && options.sequence) { - if (!Array.isArray(options.sessionID) && !Array.isArray(options.sequence)) { - options.sessionID = [options.sessionID]; - options.sequence = [options.sequence]; - } - for (let shard = 0; shard < options.sessionID.length; shard++) { - this.ws._hotreload[shard] = { - id: options.sessionID[shard], - seq: options.sequence[shard] - }; - } + if (options.sessions && Object.keys(options.sessions).length) { + this.ws._hotreload = options.sessions; } else { try { From 10e9d7ff9cf30e3d3689e66f2f473fc4c9403ed2 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:56:46 +0000 Subject: [PATCH 068/120] Users can specify the exit events they desire or use default ones --- client.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 697fa8d..ff7161f 100644 --- a/client.js +++ b/client.js @@ -22,6 +22,7 @@ Discord.Client = class Client extends Discord.Client { restoreCache: { guilds: true, } + exitEvents: ["exit", "uncaughtException", "SIGINT", "SIGTERM", ..._options.exitEvents] }; super(options); actions(this); @@ -58,7 +59,6 @@ Discord.Client = class Client extends Discord.Client { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); - for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, () => { if (!fs.existsSync(this.cacheFilePath)) { fs.mkdirSync(this.cacheFilePath); } try { @@ -79,6 +79,7 @@ Discord.Client = class Client extends Discord.Client { if (options.restoreCache) { for (const toCache of options.restoreCache) { fs.writeFileSync(`${this.cacheFilePath}/${toCache}.json`, JSON.stringify(this[toCache])); + for (const eventType of options.exitEvents) { } } if(eventType !== "exit") { From e5d0a1edda95c631b85fde98039350cbdab9af60 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 16:59:24 +0000 Subject: [PATCH 069/120] Added uncaughtExceptionOnExit bool to stop loop of uncaughExceptions If there is an uncaught exception inside of the event function it will trigger itself causing an infinite loop. I added a uncaughtExceptionOnExit bool variable so that the uncaughtException event can only trigger once and after that it will exit the process without running the buggy code --- client.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index ff7161f..852804e 100644 --- a/client.js +++ b/client.js @@ -79,12 +79,19 @@ Discord.Client = class Client extends Discord.Client { if (options.restoreCache) { for (const toCache of options.restoreCache) { fs.writeFileSync(`${this.cacheFilePath}/${toCache}.json`, JSON.stringify(this[toCache])); + this._uncaughtExceptionOnExit = false; for (const eventType of options.exitEvents) { + process.on(eventType, async () => { + if (eventType === "uncaughtException") { + this._uncaughtExceptionOnExit = true; + } } } - if(eventType !== "exit") { - process.exit(); + else { + console.error("There was an uncaughtException inside your exit loop causing an infinite loop. Your exit function was not run"); + process.exit(1); } + }); } } From efc65ee199e82a2bb3edd8dadeccdfcdf2bf730c Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:01:40 +0000 Subject: [PATCH 070/120] Added dumpCache method with sessions and client args --- client.js | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/client.js b/client.js index 852804e..2e9632e 100644 --- a/client.js +++ b/client.js @@ -59,26 +59,9 @@ Discord.Client = class Client extends Discord.Client { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); - process.on(eventType, () => { - if (!fs.existsSync(this.cacheFilePath)) { fs.mkdirSync(this.cacheFilePath); } - try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); - } catch(e) { - this.ws._hotreload = {}; + this.dumpCache = (sessions, client) => { } - Object.assign(this.ws._hotreload, ...this.ws.shards.map(s => { - s.connection.close(); - return { - [s.id]: { - id: s.sessionID, - seq: s.sequence - } - }; - })); - fs.writeFileSync(`${this.cacheFilePath}/sessions.json`, JSON.stringify(this.ws._hotreload)); - if (options.restoreCache) { - for (const toCache of options.restoreCache) { - fs.writeFileSync(`${this.cacheFilePath}/${toCache}.json`, JSON.stringify(this[toCache])); + }; this._uncaughtExceptionOnExit = false; for (const eventType of options.exitEvents) { process.on(eventType, async () => { From 3fb04d1acd52c2d3f8ec4547917d17a838964917 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:06:14 +0000 Subject: [PATCH 071/120] Assigns new shards to hotreload and calls dumpCache asynchronously if not exit --- client.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/client.js b/client.js index 2e9632e..855a4f4 100644 --- a/client.js +++ b/client.js @@ -68,6 +68,19 @@ Discord.Client = class Client extends Discord.Client { if (eventType === "uncaughtException") { this._uncaughtExceptionOnExit = true; } + if (!this._uncaughtExceptionOnExit) { + Object.assign(this.ws._hotreload, ...this.ws.shards.map(s => { + s.connection.close(); + return { + [s.id]: { + id: s.sessionID, + seq: s.sequence + } + }; + })); + if (eventType !== "exit") { + await this.dumpCache(this.ws._hotreload, this); + process.exit(); } } else { From bc7900a05627fff6daa60aec4ef3d8726656cf9f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:11:18 +0000 Subject: [PATCH 072/120] dumpCache sync file storing function Currently just stores the bot user and guilds to disk. Needs improvement --- client.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 855a4f4..8c9dd98 100644 --- a/client.js +++ b/client.js @@ -60,7 +60,23 @@ Discord.Client = class Client extends Discord.Client { if(!this.readyAt) { this.ws.checkShardsReady(); } }); this.dumpCache = (sessions, client) => { - } + if (!fs.existsSync(client.cacheFilePath)) { fs.mkdirSync(client.cacheFilePath); } + try { + client.ws._hotreload = JSON.parse(fs.readFileSync(`${client.cacheFilePath}/sessions.json`, "utf8")); + } catch(e) { + client.ws._hotreload = {}; + } + client.ws._hotreload = { + ...client.ws._hotreload, + ...sessions + }; + fs.writeFileSync(`${client.cacheFilePath}/sessions.json`, JSON.stringify(client.ws._hotreload)); + if (options.restoreCache.guilds) { + const discordGuilds = client.guilds.cache.map(g => g._unpatch()); + fs.writeFileSync(`${client.cacheFilePath}/guilds.json`, JSON.stringify(discordGuilds)); + const discordMe = client.user._unpatch(); + fs.writeFileSync(`${client.cacheFilePath}/me.json`, JSON.stringify(discordMe)); + } }; this._uncaughtExceptionOnExit = false; for (const eventType of options.exitEvents) { From 4fd675b83a25e6d0226bccbc3e4907401c5e8d4f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:14:16 +0000 Subject: [PATCH 073/120] Restores guilds and users --- client.js | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/client.js b/client.js index 8c9dd98..80d9272 100644 --- a/client.js +++ b/client.js @@ -20,13 +20,20 @@ Discord.Client = class Client extends Discord.Client { sessions: {}, ..._options, restoreCache: { + channels: false, guilds: true, - } + presences: false, + roles: false, + overwrites: false, + emojis: false, + members: false, + ..._options.restoreCache + }, exitEvents: ["exit", "uncaughtException", "SIGINT", "SIGTERM", ..._options.exitEvents] }; super(options); actions(this); - if(options.hotreload) { + if (options.hotreload) { this.cacheFilePath = `${process.cwd()}/.sessions`; if (options.sessions && Object.keys(options.sessions).length) { this.ws._hotreload = options.sessions; @@ -38,23 +45,13 @@ Discord.Client = class Client extends Discord.Client { this.ws._hotreload = {}; } } - try { - for (const toCache of options.restoreCache) { - const data = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/${toCache}.json`, "utf8")); - switch (toCache) { - case "guilds": { - console.log("Created"); - data.cache.forEach(i => { - console.log(i); - this.guilds.cache.set(i.id, new Discord.Guild(this, i)); - console.log(i.id); - }); - break; - } - } + + if (options.restoreCache.guilds) { + const discordGuildData = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8")); + for (const guild of discordGuildData) { + this.guilds.cache.set(guild.id, new Discord.Guild(this, guild)); } - } catch(e) { - // Do nothing + this.user = new Discord.User(this, JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8"))); } this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } From 81109cacb6f42c560a8f20c7e055d19e1ca64f9c Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:15:22 +0000 Subject: [PATCH 074/120] Added restore cache client options --- client.d.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/client.d.ts b/client.d.ts index 9162719..28636b2 100644 --- a/client.d.ts +++ b/client.d.ts @@ -58,6 +58,15 @@ declare module "discord.js-light" { cacheEmojis?:boolean cacheMembers?:boolean disabledEvents?: Array + restoreCache: { + channels: boolean + guilds: boolean + presences: boolean + roles: boolean + overwrites: boolean + emojis: boolean + members: boolean + } } interface ClientEvents { rest:[{path:string,method:string,response?:Promise}] From f66811bd56dd3ed1a596ce00af2cb7806a23889f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 17:18:15 +0000 Subject: [PATCH 075/120] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a1f1858..cfc1824 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,10 @@ by doing this you are connecting to the Discord websocket gateway each time whic To solve this problem you can use hot reloading which is a client option allowing you to simply resume the previous session rather than create a new one. -You can also use Hot reloading in your production bot by supplying a sequence and session ID or by just letting us take care of it with cache files found in the `.sessions` folder +You can also use Hot reloading in your production bot by supplying a session object along with preferences for caching restoration or by just letting us take care of it with cache files found in the `.sessions` folder + +By setting the client.dumpCache method you can run a custom async function to store your caches and session IDs in your database of choice. The dumpCache method is +called with the (session, client) params where session is your up to date sessoins ## Client Options From b226bb816c73216fe86407358630a20e5cc8c83d Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 19:46:35 +0000 Subject: [PATCH 076/120] Updated client options and added typings --- client.d.ts | 23 ++++++++++++++--------- client.js | 21 +++++---------------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/client.d.ts b/client.d.ts index 28636b2..ecb02c0 100644 --- a/client.d.ts +++ b/client.d.ts @@ -48,6 +48,19 @@ type ReactionUserFetchOptions = { after?: Discord.Snowflake } +type SessionData = { + [shardID: string]: { + id: string + sequence: number + } +} + +type HotReloadOptions = { + sessionData: SessionData + unpatchOnExit: boolean + patchSource: string | Object +} + declare module "discord.js-light" { interface ClientOptions { cacheChannels?:boolean @@ -58,15 +71,7 @@ declare module "discord.js-light" { cacheEmojis?:boolean cacheMembers?:boolean disabledEvents?: Array - restoreCache: { - channels: boolean - guilds: boolean - presences: boolean - roles: boolean - overwrites: boolean - emojis: boolean - members: boolean - } + hotReload?: boolean | HotReloadOptions } interface ClientEvents { rest:[{path:string,method:string,response?:Promise}] diff --git a/client.js b/client.js index 80d9272..934981d 100644 --- a/client.js +++ b/client.js @@ -17,23 +17,12 @@ Discord.Client = class Client extends Discord.Client { cacheEmojis: false, cacheMembers: false, disabledEvents: [], - sessions: {}, - ..._options, - restoreCache: { - channels: false, - guilds: true, - presences: false, - roles: false, - overwrites: false, - emojis: false, - members: false, - ..._options.restoreCache - }, - exitEvents: ["exit", "uncaughtException", "SIGINT", "SIGTERM", ..._options.exitEvents] + hotReload: false, + ..._options }; super(options); actions(this); - if (options.hotreload) { + if (options.hotReload) { this.cacheFilePath = `${process.cwd()}/.sessions`; if (options.sessions && Object.keys(options.sessions).length) { this.ws._hotreload = options.sessions; @@ -46,7 +35,7 @@ Discord.Client = class Client extends Discord.Client { } } - if (options.restoreCache.guilds) { + if (options.cacheGuilds) { const discordGuildData = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8")); for (const guild of discordGuildData) { this.guilds.cache.set(guild.id, new Discord.Guild(this, guild)); @@ -68,7 +57,7 @@ Discord.Client = class Client extends Discord.Client { ...sessions }; fs.writeFileSync(`${client.cacheFilePath}/sessions.json`, JSON.stringify(client.ws._hotreload)); - if (options.restoreCache.guilds) { + if (options.cacheGuilds) { const discordGuilds = client.guilds.cache.map(g => g._unpatch()); fs.writeFileSync(`${client.cacheFilePath}/guilds.json`, JSON.stringify(discordGuilds)); const discordMe = client.user._unpatch(); From 10e41d8d9398a99c3d70e03795411c351a53b3e3 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 21:18:06 +0000 Subject: [PATCH 077/120] Added new typedefs --- client.d.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/client.d.ts b/client.d.ts index ecb02c0..808f46d 100644 --- a/client.d.ts +++ b/client.d.ts @@ -55,10 +55,16 @@ type SessionData = { } } +type CacheData = { + guilds?: Array + channels?: Array + users?: Array +} + type HotReloadOptions = { - sessionData: SessionData - unpatchOnExit: boolean - patchSource: string | Object + sessionData?: SessionData + cacheData?: CacheData + onUnload?: Function } declare module "discord.js-light" { From d3a29aa857824b065895c057ff16edf341f789d7 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 21:42:32 +0000 Subject: [PATCH 078/120] Added djs-light validate options --- client.js | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/client.js b/client.js index 934981d..36259b2 100644 --- a/client.js +++ b/client.js @@ -22,6 +22,7 @@ Discord.Client = class Client extends Discord.Client { }; super(options); actions(this); + this._validateOptionsLight(); if (options.hotReload) { this.cacheFilePath = `${process.cwd()}/.sessions`; if (options.sessions && Object.keys(options.sessions).length) { @@ -111,6 +112,48 @@ Discord.Client = class Client extends Discord.Client { guild.channels.cache.sweep(t => !this.channels.cache.has(t.id)); } } + /** + * Validates the client options. + * @param {object} options Options to validate + * @private + */ + _validateOptionsLight(options) { + if (typeof options.cacheChannels !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheChannels", "a boolean"); + } + if (typeof options.cacheGuilds !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheGuilds", "a boolean"); + } + if (typeof options.cachePresences !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cachePresences", "a boolean"); + } + if (typeof options.cacheRoles !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheRoles", "a boolean"); + } + if (typeof options.cacheOverwrites !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheOverwrites", "a boolean"); + } + if (typeof options.cacheEmojis !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheEmojis", "a boolean"); + } + if (typeof options.cacheMembers !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheMembers", "a boolean"); + } + if (!Array.isArray(options.disabledEvents)) { + throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); + } + if (options.hotReload) { + if (typeof options.hotReload.sessionData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); + } + if (typeof options.hotReload.cacheData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); + } + if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { + throw new TypeError("CLIENT_INVALID_OPTION", "onUnload", "a function"); + } + } + } }; Discord.version = `${pkg.version} (${Discord.version})`; From b0ef9520658e3485ed61ea0a290bef22010e9b8b Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 21:48:13 +0000 Subject: [PATCH 079/120] Added change for sessionData --- client.js | 5 ++--- init.js | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index 36259b2..f44ef31 100644 --- a/client.js +++ b/client.js @@ -25,8 +25,8 @@ Discord.Client = class Client extends Discord.Client { this._validateOptionsLight(); if (options.hotReload) { this.cacheFilePath = `${process.cwd()}/.sessions`; - if (options.sessions && Object.keys(options.sessions).length) { - this.ws._hotreload = options.sessions; + if (options.hotReload.sessionData && Object.keys(options.hotReload.sessionData).length) { + this.ws._hotreload = options.hotReload.sessionData; } else { try { @@ -66,7 +66,6 @@ Discord.Client = class Client extends Discord.Client { } }; this._uncaughtExceptionOnExit = false; - for (const eventType of options.exitEvents) { process.on(eventType, async () => { if (eventType === "uncaughtException") { this._uncaughtExceptionOnExit = true; diff --git a/init.js b/init.js index 3deeea1..dab02df 100644 --- a/init.js +++ b/init.js @@ -52,7 +52,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { const data = this.manager._hotreload[this.id]; if(data && !this.sessionID) { this.sessionID = data.id; - this.closeSequence = this.sequence = data.seq; + this.closeSequence = this.sequence = data.sequence; delete this.manager._hotreload[this.id]; } } From 86ba96550972081a8b7e46a7a8f7642436ad5ede Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:35:32 +0000 Subject: [PATCH 080/120] Reads session files from .sessions/sessions/{shardID}.json --- client.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/client.js b/client.js index f44ef31..59f40a6 100644 --- a/client.js +++ b/client.js @@ -24,14 +24,23 @@ Discord.Client = class Client extends Discord.Client { actions(this); this._validateOptionsLight(); if (options.hotReload) { + this.on(Discord.Constants.Events.SHARD_RESUME, () => { + if (!this.readyAt) { this.ws.checkShardsReady(); } + }); this.cacheFilePath = `${process.cwd()}/.sessions`; + this.ws._hotreload = {}; if (options.hotReload.sessionData && Object.keys(options.hotReload.sessionData).length) { this.ws._hotreload = options.hotReload.sessionData; } else { try { - this.ws._hotreload = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions.json`, "utf8")); - } catch(e) { + const shards = fs.readdirSync(`${this.cacheFilePath}/sessions`) + .filter(file => file.endsWith(".json")) + .map(shardSession => shardSession.substr(0, shardSession.lastIndexOf("."))); + for (const shardID of shards) { + this.ws._hotreload[shardID] = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, "utf8")); + } + } catch (e) { this.ws._hotreload = {}; } } From 019a28c6601e5a8b14f84f17905952f4f8df63cf Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:37:56 +0000 Subject: [PATCH 081/120] Extra checks for validation of options and added back events --- client.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index 59f40a6..62b6356 100644 --- a/client.js +++ b/client.js @@ -22,7 +22,7 @@ Discord.Client = class Client extends Discord.Client { }; super(options); actions(this); - this._validateOptionsLight(); + this._validateOptionsLight(options); if (options.hotReload) { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if (!this.readyAt) { this.ws.checkShardsReady(); } @@ -59,7 +59,7 @@ Discord.Client = class Client extends Discord.Client { if (!fs.existsSync(client.cacheFilePath)) { fs.mkdirSync(client.cacheFilePath); } try { client.ws._hotreload = JSON.parse(fs.readFileSync(`${client.cacheFilePath}/sessions.json`, "utf8")); - } catch(e) { + } catch (e) { client.ws._hotreload = {}; } client.ws._hotreload = { @@ -75,6 +75,7 @@ Discord.Client = class Client extends Discord.Client { } }; this._uncaughtExceptionOnExit = false; + for (const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async () => { if (eventType === "uncaughtException") { this._uncaughtExceptionOnExit = true; @@ -151,10 +152,10 @@ Discord.Client = class Client extends Discord.Client { throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); } if (options.hotReload) { - if (typeof options.hotReload.sessionData !== "object") { + if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); } - if (typeof options.hotReload.cacheData !== "object") { + if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); } if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { From 49777bd0f463dc2c6631a0c744f9631be7adc06b Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:52:42 +0000 Subject: [PATCH 082/120] Created _loadSesssions to load sessions from disk --- client.js | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/client.js b/client.js index 62b6356..2f0d596 100644 --- a/client.js +++ b/client.js @@ -33,16 +33,7 @@ Discord.Client = class Client extends Discord.Client { this.ws._hotreload = options.hotReload.sessionData; } else { - try { - const shards = fs.readdirSync(`${this.cacheFilePath}/sessions`) - .filter(file => file.endsWith(".json")) - .map(shardSession => shardSession.substr(0, shardSession.lastIndexOf("."))); - for (const shardID of shards) { - this.ws._hotreload[shardID] = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, "utf8")); - } - } catch (e) { - this.ws._hotreload = {}; - } + this._loadSessions(); } if (options.cacheGuilds) { @@ -104,6 +95,22 @@ Discord.Client = class Client extends Discord.Client { } } } + /** + * Loads all of the stored sessions on disk into memory + * @private + */ + _loadSessions() { + try { + const shards = fs.readdirSync(`${this.cacheFilePath}/sessions`) + .filter(file => file.endsWith(".json")) + .map(shardSession => shardSession.substr(0, shardSession.lastIndexOf("."))); + for (const shardID of shards) { + this.ws._hotreload[shardID] = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, "utf8")); + } + } catch (e) { + this.ws._hotreload = {}; + } + } sweepUsers(_lifetime = 86400) { const lifetime = _lifetime * 1000; this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); From 09940cceefd8af1377d98de97571c0762456f3ee Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 23:23:16 +0000 Subject: [PATCH 083/120] Added helper methods Added unLoadSessions to unload sessions from memory to disk Added makeDir to create a directory if it doesn't already exist Added loadCache to load all caches from disk Added onUnload method which can be overwritten by user to store caches --- client.js | 78 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/client.js b/client.js index 2f0d596..44f86b3 100644 --- a/client.js +++ b/client.js @@ -35,34 +35,23 @@ Discord.Client = class Client extends Discord.Client { else { this._loadSessions(); } - - if (options.cacheGuilds) { - const discordGuildData = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8")); - for (const guild of discordGuildData) { - this.guilds.cache.set(guild.id, new Discord.Guild(this, guild)); - } - this.user = new Discord.User(this, JSON.parse(fs.readFileSync(`${this.cacheFilePath}/guilds.json`, "utf8"))); - } - this.on(Discord.Constants.Events.SHARD_RESUME, () => { - if(!this.readyAt) { this.ws.checkShardsReady(); } - }); - this.dumpCache = (sessions, client) => { - if (!fs.existsSync(client.cacheFilePath)) { fs.mkdirSync(client.cacheFilePath); } - try { - client.ws._hotreload = JSON.parse(fs.readFileSync(`${client.cacheFilePath}/sessions.json`, "utf8")); - } catch (e) { - client.ws._hotreload = {}; - } - client.ws._hotreload = { - ...client.ws._hotreload, + this.onUnload = (sessions, cache) => { + this._makeDir(this.cacheFilePath); + this._makeDir(`${this.cacheFilePath}/sessions`); + this._loadSessions(); + this.ws._hotreload = { + ...this.ws._hotreload, ...sessions }; - fs.writeFileSync(`${client.cacheFilePath}/sessions.json`, JSON.stringify(client.ws._hotreload)); + this._unLoadSessions(); if (options.cacheGuilds) { - const discordGuilds = client.guilds.cache.map(g => g._unpatch()); - fs.writeFileSync(`${client.cacheFilePath}/guilds.json`, JSON.stringify(discordGuilds)); - const discordMe = client.user._unpatch(); - fs.writeFileSync(`${client.cacheFilePath}/me.json`, JSON.stringify(discordMe)); + this._makeDir(`${this.cacheFilePath}/guilds`); + } + if (options.cacheChannels) { + this._makeDir(`${this.cacheFilePath}/channels`); + } + if (options.cacheMembers) { + this._makeDir(`${this.cacheFilePath}/users`); } }; this._uncaughtExceptionOnExit = false; @@ -95,6 +84,29 @@ Discord.Client = class Client extends Discord.Client { } } } + /** + * Loads all of the stored caches on disk into memory + * @returns {object} All of the stored cache + * @private + */ + _loadCache() { + const allCache = {}; + for (const cache of ["guilds", "channels", "users"]) { + try { + const cachedFiles = fs.readdirSync(`${this.cacheFilePath}/${cache}`) + .filter(file => file.endsWith(".json")) + .map(c => c.substr(0, c.lastIndexOf("."))); + if (cachedFiles.length) { continue; } + allCache[cache] = []; + for (const id of cachedFiles) { + allCache[cache].push(JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${id}.json`, "utf8"))); + } + } catch (d) { + // Do nothing + } + } + return allCache; + } /** * Loads all of the stored sessions on disk into memory * @private @@ -111,6 +123,22 @@ Discord.Client = class Client extends Discord.Client { this.ws._hotreload = {}; } } + /** + * Unloads all of the stored sessions in memory onto disk + * @private + */ + _unLoadSessions() { + for (const [shardID, session] of this.ws._hotreload) { + fs.writeFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, JSON.stringify(session)); + } + } + /** + * Creates a directory if it does not already exist + * @private + */ + _makeDir(dir) { + if (!fs.existsSync(dir)) { fs.mkdirSync(dir); } + } sweepUsers(_lifetime = 86400) { const lifetime = _lifetime * 1000; this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); From bac06e4632f0843e91e12e5908cfbd3de2a32743 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sun, 21 Mar 2021 23:24:02 +0000 Subject: [PATCH 084/120] Added dumpCache and patchCache methods --- client.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 44f86b3..bb2eb83 100644 --- a/client.js +++ b/client.js @@ -35,6 +35,7 @@ Discord.Client = class Client extends Discord.Client { else { this._loadSessions(); } + this._patchCache(options.hotReload.cacheData || this._loadCache()); this.onUnload = (sessions, cache) => { this._makeDir(this.cacheFilePath); this._makeDir(`${this.cacheFilePath}/sessions`); @@ -71,7 +72,7 @@ Discord.Client = class Client extends Discord.Client { }; })); if (eventType !== "exit") { - await this.dumpCache(this.ws._hotreload, this); + await this.onUnload(this.ws._hotreload, this.dumpCache()); process.exit(); } } @@ -84,6 +85,13 @@ Discord.Client = class Client extends Discord.Client { } } } + /** + * Generates a complete dump of the current stored cache + * @param {object} options Options to validate + * @returns {object} All of the cache + */ + dumpCache() { + } /** * Loads all of the stored caches on disk into memory * @returns {object} All of the stored cache @@ -107,6 +115,12 @@ Discord.Client = class Client extends Discord.Client { } return allCache; } + /** + * Patches raw discord api objects into the discord.js cache + * @private + */ + _patchCache({ guilds, channels, users }) { + } /** * Loads all of the stored sessions on disk into memory * @private From 5a7ec3c43419e07446b628bf9c3b5843926c366c Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Mon, 22 Mar 2021 21:45:37 +0000 Subject: [PATCH 085/120] Added a _write method to write an array of cache data to disk --- client.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/client.js b/client.js index bb2eb83..54bfed8 100644 --- a/client.js +++ b/client.js @@ -47,12 +47,15 @@ Discord.Client = class Client extends Discord.Client { this._unLoadSessions(); if (options.cacheGuilds) { this._makeDir(`${this.cacheFilePath}/guilds`); + this._write("guilds", guilds); } if (options.cacheChannels) { this._makeDir(`${this.cacheFilePath}/channels`); + this._write("channels", channels); } if (options.cacheMembers) { this._makeDir(`${this.cacheFilePath}/users`); + this._write("users", users); } }; this._uncaughtExceptionOnExit = false; @@ -153,6 +156,17 @@ Discord.Client = class Client extends Discord.Client { _makeDir(dir) { if (!fs.existsSync(dir)) { fs.mkdirSync(dir); } } + /** + * Writes a cache array to multiple files indexed by ID to disk using the cached file path and JSON format + * @param {string} path The path to write the data to + * @param {Array} data An array of all of the data items to write + * @private + */ + _write(path, data) { + for (const item of data) { + fs.writeFileSync(`${this.cacheFilePath}/${path}/${item.id}.json`, JSON.stringify(item)); + } + } sweepUsers(_lifetime = 86400) { const lifetime = _lifetime * 1000; this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); From 4cc9bade2fed0d22071fd189825a4a47061f3ce0 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Mon, 22 Mar 2021 21:46:04 +0000 Subject: [PATCH 086/120] Implemented dumpCache --- client.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 54bfed8..a22f873 100644 --- a/client.js +++ b/client.js @@ -36,7 +36,7 @@ Discord.Client = class Client extends Discord.Client { this._loadSessions(); } this._patchCache(options.hotReload.cacheData || this._loadCache()); - this.onUnload = (sessions, cache) => { + this.onUnload = (sessions, { guilds, channels, users }) => { this._makeDir(this.cacheFilePath); this._makeDir(`${this.cacheFilePath}/sessions`); this._loadSessions(); @@ -94,6 +94,11 @@ Discord.Client = class Client extends Discord.Client { * @returns {object} All of the cache */ dumpCache() { + return { + guilds: this.guilds.cache.map(g => g._unpatch()), + channels: this.channels.cache.map(c => c._unpatch()), + users: this.users.cache.map(u => u._unpatch()) + }; } /** * Loads all of the stored caches on disk into memory From a526fe7a072316fc3f952279d7ac242378ea0c64 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Mon, 22 Mar 2021 22:11:34 +0000 Subject: [PATCH 087/120] Added better error handling for infinite uncaught exception loop --- client.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index a22f873..d563c2e 100644 --- a/client.js +++ b/client.js @@ -60,7 +60,7 @@ Discord.Client = class Client extends Discord.Client { }; this._uncaughtExceptionOnExit = false; for (const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { - process.on(eventType, async () => { + process.on(eventType, async (...args) => { if (eventType === "uncaughtException") { this._uncaughtExceptionOnExit = true; } @@ -79,11 +79,11 @@ Discord.Client = class Client extends Discord.Client { process.exit(); } } - else { - console.error("There was an uncaughtException inside your exit loop causing an infinite loop. Your exit function was not run"); + else if (eventType !== "exit") { + console.error(args[0]); + console.error("UNCAUGHT_EXCEPTION_LOOP", "There was an uncaughtException inside your exit loop causing an infinite loop. Your exit function was not run or failed"); process.exit(1); } - }); } } From 5c3234e63889689ae32b3ee17c4d3c045ce98830 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Mon, 22 Mar 2021 22:11:42 +0000 Subject: [PATCH 088/120] Client options --- client.js | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/client.js b/client.js index d563c2e..35a98a9 100644 --- a/client.js +++ b/client.js @@ -219,15 +219,20 @@ Discord.Client = class Client extends Discord.Client { if (!Array.isArray(options.disabledEvents)) { throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); } - if (options.hotReload) { - if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { - throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); - } - if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); + if (typeof options.hotReload !== "boolean") { + if (options.hotReload && typeof options.hotReload === "object") { + if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); + } + if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); + } + if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { + throw new TypeError("CLIENT_INVALID_OPTION", "onUnload", "a function"); + } } - if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { - throw new TypeError("CLIENT_INVALID_OPTION", "onUnload", "a function"); + else { + throw new TypeError("CLIENT_INVALID_OPTION", "hotReload", "a boolean or an object"); } } } From 39f33227ae8410910ceefde60d39c06e834aa4c2 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Wed, 31 Mar 2021 18:23:14 +0100 Subject: [PATCH 089/120] implement _unpatch() Channel unpatching needs to be done in another file --- classes.js | 231 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 138 insertions(+), 93 deletions(-) diff --git a/classes.js b/classes.js index 1857e24..e992735 100644 --- a/classes.js +++ b/classes.js @@ -121,6 +121,22 @@ Discord.Structures.extend("Message", M => { }; }); +Discord.Strictures.extend("User", U => { + return class User extends U { + _unpatch() { + return { + id: this.id, + username: this.username, + bot: this.bot, + discriminator: this.discriminator, + avatar: this.avatar, + system: this.system, + public_flags: this.flags.valueOf() + } + } + } +}); + Discord.Structures.extend("GuildMember", G => { return class GuildMember extends G { _patch(data) { @@ -139,6 +155,16 @@ Discord.Structures.extend("GuildMember", G => { } } } + _unpatch() { + return { + user: this.user._unpatch(), + nick: this.nickname, + joined_at: this.joinedTimestamp, + premium_since: this.premiumSinceTimestamp, + roles: this._roles, + pending: this.pending + } + } equals(member) { return member && this.deleted === member.deleted && this.nickname === member.nickname && this._roles.length === member._roles.length; } @@ -218,112 +244,48 @@ Discord.Structures.extend("Guild", G => { } } _unpatch() { - /** - * Discord raw guild data as documented: https://github.com/discordjs/discord-api-types/blob/main/v8/payloads/guild.ts - */ return { - id: this.id, - unavailable: this.available, + unavailable: !this.available, + shardID: this.shardID, name: this.name, icon: this.icon, splash: this.splash, - banner: this.banner, - description: this.description, - features: this.features, - verification_level: this.verificationLevel, discovery_splash: this.discoverySplash, - owner_id: this.ownerID, region: this.region, - afk_channel_id: this.afkChannelID, + member_count: this.memberCount, + large: this.large, + features: this.features, + application_id: this.applicationID, afk_timeout: this.afkTimeout, + afk_channel_id: this.afkChannelID, + system_channel_id: this.systemChannelID, + premium_tier: this.premiumTier, + premium_subscription_count: this.premiumSubscriptionCount, widget_enabled: this.widgetEnabled, widget_channel_id: this.widgetChannelID, - default_message_notifications: this.defaultMessageNotifications, - explicit_content_filter: this.explicitContentFilter, - /** - * Roles in the guild - * - * See https://discord.com/developers/docs/topics/permissions#role-object - */ - roles: this.roles.cache.map(r => ({ - name: r.name, - color: r.color - })), - /** - * Custom guild emojis - * - * See https://discord.com/developers/docs/resources/emoji#emoji-object - */ - emojis: this.emojis.cache.map(e => ({ - id: e.id, - name: e.name - })), + verification_level: Discord.Util.Constants.VerificationLevels.indexOf(this.verificationLevel), + explicit_content_filter: Discord.Util.Constants.ExplicitContentFilterLevels.indexOf(this.explicitContentFilter), mfa_level: this.mfaLevel, - application_id: this.applicationID, - system_channel_id: this.systemChannelID, - system_channel_flags: this.systemChannelFlags, - rules_channel_id: this.rulesChannelID, - joined_at: this.joinedAt, - large: this.large, - member_count: this.memberCount, - voice_states: this.voiceStates.cache.map(v => ({ - guild_id: v.guild.id, - channel_id: v.channelID, - user_id: v.userID, - session_id: v.sessionID, - deaf: v.deaf, - mute: v.mute, - self_deaf: v.selfDeaf, - self_mute: v.selfMute, - suppress: v.suppress - })), - members: this.members.cache.map(m => ({ - user: m.user, - nick: m.nickname, - roles: m.roles, - joined_at: m.joinedAt, - premium_since: m.premiumSinceTimestamp, - deaf: m.deaf, - mute: m.mute, - pending: m.pending, - permissions: m.permissions - })), - channels: this.channels.cache.map(c => ({ - id: c.id, - type: c.type, - guild_id: c.guild.id, - position: c.position, - permission_overwrites: c.permissionOverwrites, - name: c.name, - topic: c.topic, - nsfw: c.nsfw, - last_message_id: c.lastMessageID, - bitrate: c.bitrate, - user_limit: c.userLimit, - rate_limit_per_user: c.rateLimitPerUser, - recipients: c.recipients, - icon: c.icon, - owner_id: c.ownerID, - application_id: c.applicationID, - parent_id: c.parentID, - last_pin_timestamp: c.lastPinTimestamp - })), - presences: this.presences.cache.map(p => ({ - user: p.user, - guild_id: p.guild.id, - status: p.status, - activities: p.activities, - client_status: p.clientStatus - })), - max_presences: this.maximumPresences, + joinedTimestamp: this.joinedTimestamp, + default_message_notifications: this.defaultMessageNotifications, + system_channel_flags: this.systemChannelFlags.valueOf(), max_members: this.maximumMembers, + max_presences: this.maximumPresences, + approximate_member_count: this.approximateMemberCount, + approximate_presence_count: this.approximatePresenceCount, vanity_url_code: this.vanityURLCode, - premium_tier: this.premiumTier, - premium_subscription_count: this.premiumSubscriptionCount, - preferred_locale: this.preferredLocale || "en-US", + description: this.description, + banner: this.banner, + id: this.id, + rules_channel_id: this.rulesChannelID, public_updates_channel_id: this.publicUpdatesChannelID, - approximate_member_count: this.approximateMemberCount, - approximate_presence_count: this.approximatePresenceCount + preferred_locale: this.preferredLocale, + roles: this.roles.cache.map(x => x._unpatch()), + members: this.members.cache.map(x => x._unpatch()), + owner_id: this.ownerID, + presences: this.presences.cache.map(x => x._unpatch()), + voice_states: this.voiceStates.cache.map(x => x._unpatch()), + emojis: this.emojis.cache.map(x => x._unpatch()) }; } get nameAcronym() { @@ -385,6 +347,18 @@ Discord.Structures.extend("GuildEmoji", E => { this._author = data.user.id; } } + _unpatch() { + return { + animated: this.animated, + name: this.name, + id: this.id, + require_colons: this.requiresColons, + managed: this.managed, + available: this.available, + roles: this._roles, + user: this.author ? this.author._unpatch() : void 0; + } + } async fetchAuthor(cache = true) { if(this.managed) { throw new Error("EMOJI_MANAGED"); @@ -401,6 +375,28 @@ Discord.Structures.extend("GuildEmoji", E => { }; }); +Discord.Structures.extend("Role", R => { + return class Role extends R { + _unpatch() { + return { + id: this.id, + name: this.name, + color: this.color, + hoist: this.hoist, + position: this.rawPosition, + permissions: this.permissions.valueOf().toString(), + managed: this.managed, + mentionable: this.mentionable, + tags: { + bot_id: this.tags.botID, + integration_id: this.tags.integrationID, + premium_subscriber: this.tags.premiumSubscriberRole + } + } + } + } +}); + Discord.Structures.extend("VoiceState", V => { return class VoiceState extends V { _patch(data) { @@ -410,6 +406,19 @@ Discord.Structures.extend("VoiceState", V => { } return this; } + _unpatch() { + return { + user_id: this.id, + deaf: this.serverDeaf, + mute: this.serverMute, + self_deaf: this.selfDeaf, + self_mute: this.selfMute, + self_video: this.selfVideo, + session_id: this.sessionID, + self_stream: this.streaming, + channel_id: this.channelID + } + } get channel() { return this.channelID ? this.client.channels.cache.get(this.channelID) || this.client.channels.add({ id: this.channelID, @@ -464,6 +473,42 @@ Discord.Structures.extend("Presence", P => { } return this; } + _unpatch() { + return { + user: { + id: this.userID + }, + status: this.status, + activities: this.activities.map(a => ({ + name: a.name, + type: a.type, + url: a.url, + details: a.details, + state: a.state, + application_id: a.applicationID, + timestamps: { + start: a.timestamps.start ? a.timestamps.start.getTime() : null, + end: a.timestamps.end ? a.timestamps.end.getTime() : null + }, + party: a.party, + assets: { + large_text: a.assets.largeText, + small_text: a.assets.smallText, + large_image: a.assets.largeImage, + small_image: a.assets.smallImage + }, + sync_id: a.syncID, + flags: a.flags.valueOf(), + emoji: { + animated: a.emoji.animated, + name: a.emoji.name, + id: a.emoji.id + }, + created_at: a.createdTimestamp + })), + client_status: this.clientStatus + } + } get user() { return this.client.users.cache.get(this.userID) || this.client.users.add((this._member || {}).user || { id: this.userID }, false); } From 35bd529a09000063a87cabe78f744ec751e91540 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Wed, 31 Mar 2021 19:30:14 +0100 Subject: [PATCH 090/120] Channel unpatching --- init.js | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/init.js b/init.js index dab02df..915d2da 100644 --- a/init.js +++ b/init.js @@ -258,6 +258,19 @@ require.cache[GCPath].exports = class GuildChannel extends GC { }); } } + _unpatch() { + let obj = super._unpatch(); + obj.name = this.name; + obj.position = this.rawPosition; + obj.parent_id = this.parentID; + obj.permission_overwrites = this.permissionOverwrites.map(x => ({ + id: x.id, + type: Constants.OverwriteTypes[x.type], + deny: x.deny.valueOf().toString(), + allow: x.allow.valueOf().toString() + })); + return obj; + } get deletable() { if(this.deleted) { return false; } if(!this.client.options.cacheRoles && !this.guild.roles.cache.size) { return false; } @@ -265,6 +278,44 @@ require.cache[GCPath].exports = class GuildChannel extends GC { } }; +const CPath = resolve(require.resolve("discord.js").replace("index.js", "/structures/Channel.js")); +const C = require(CPath); +require.cache[CPath].exports = class Channel extends C { + _unpatch() { + let obj = { + type: Constants.ChannelTypes[this.type.toUpperCase()], + id: this.id + }; + if(this.messages) { + obj.last_message_id: this.lastMessageID; + obj.last_pin_timestamp: this.lastPinTimestamp; + } + switch(this.type) { + case "dm": { + obj.recipients: [this.recipient._unpatch()]; + break; + } + case "text": case "news": { + obj.nsfw: this.nsfw; + obj.topic: this.topic; + obj.rate_limit_per_user: this.rateLimitPerUser; + obj.messages = this.messages.cache.map(x => x._unpatch()); + break; + } + case "voice": { + obj.bitrate: this.bitrate; + obj.user_limit: this.userLimit + break; + } + case "store": { + obj.nsfw: this.nsfw; + break; + } + } + return obj; + } +}; + const Action = require(resolve(require.resolve("discord.js").replace("index.js", "/client/actions/Action.js"))); Action.prototype.getPayload = function(data, manager, id, partialType, cache) { return manager.cache.get(id) || manager.add(data, cache); From 6fa4af274edd3709c2aa3b3ad3352fa6f24cdba7 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Wed, 31 Mar 2021 20:42:44 +0100 Subject: [PATCH 091/120] message unpatch plus fixes --- classes.js | 91 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/classes.js b/classes.js index e992735..9ac8272 100644 --- a/classes.js +++ b/classes.js @@ -94,6 +94,81 @@ Discord.Structures.extend("Message", M => { } } } + _unpatch() { + return { + id: this.id, + type: Discord.Constants.MessageTypes.indexOf(this.type), + content: this.content, + author: this.author._unpatch(), + pinned: this.pinned, + tts: this.tts, + nonce: this.nonce, + embeds: this.embeds.map(x => x.toJSON()), + attachments: this.attachments.map(x => ({ + filename: x.name, + id: x.id, + size: x.size, + url: x.url, + proxy_url: x.proxyURL, + height: x.height, + width: x.width + })), + edited_timestamp: this.editedTimestamp, + reactions: this.reactions.cache.map(x => ({ + me: x.me, + emoji: { + animated: x.emoji.animated, + name: x.emoji.name, + id: x.emoji.id + }, + count: x.count + })), + mentions: this.mentions.users.map(x => x._unpatch()), + mention_roles: this.mentions.roles.map(x => x._unpatch()), + mention_everyone: this.mentions.everyone, + mention_channels: this.mentions.crosspostedChannels.map(x => ({ + id: x.channelID, + guild_id: x.guildID, + type: Discord.Constants.ChannelTypes[x.type.toUpperCase()], + name: x.name + })), + webhook_id: this.webhookID, + application: this.application ? { + id: this.application.id, + name: this.application.name, + description: this.application.description, + icon: this.application.icon, + cover_image: this.application.cover, + rpc_origins: this.application.rpcOrigins, + bot_require_code_grant: this.botRequireCodeGrant, + bot_public: this.application.botPublic, + team: this.application.owner instanceof Discord.Team ? { + id: this.application.owner.id, + name: this.application.owner.name, + icon: this.application.owner.icon, + owner_user_id: this.application.owner.ownerID, + members: this.application.owner.members.map(x => ({ + permissions: x.permissions, + membership_state: x.membershipState, + user: x.user._unpatch() + })) + } : void 0, + owner: this.application.owner instanceof Discord.User ? this.application.owner._unpatch() : void 0 + } : void 0, + activity: this.activity ? { + party_id: this.activity.partyID, + type: this.activity.type + } : void 0, + member: this.member._unpatch(), + flags: this.flags.valueOf(), + message_reference: this.reference ? { + channel_id: this.reference.channelID, + guild_id: this.reference.guildID, + message_id: this.reference.messageID + } : void 0, + referenced_message: this.client.guilds.cache.get(this.reference?.guildID)?.channels.cache.get(this.reference?.channelID)?.messages.cache.get(this.reference?.messageID)?._unpatch() + } + } get member() { if(!this.guild) { return null; } const id = (this.author || {}).id || (this._member || {}).id; @@ -263,8 +338,8 @@ Discord.Structures.extend("Guild", G => { premium_subscription_count: this.premiumSubscriptionCount, widget_enabled: this.widgetEnabled, widget_channel_id: this.widgetChannelID, - verification_level: Discord.Util.Constants.VerificationLevels.indexOf(this.verificationLevel), - explicit_content_filter: Discord.Util.Constants.ExplicitContentFilterLevels.indexOf(this.explicitContentFilter), + verification_level: Discord.Constants.VerificationLevels.indexOf(this.verificationLevel), + explicit_content_filter: Discord.Constants.ExplicitContentFilterLevels.indexOf(this.explicitContentFilter), mfa_level: this.mfaLevel, joinedTimestamp: this.joinedTimestamp, default_message_notifications: this.defaultMessageNotifications, @@ -486,24 +561,24 @@ Discord.Structures.extend("Presence", P => { details: a.details, state: a.state, application_id: a.applicationID, - timestamps: { + timestamps: a.timestamps ? { start: a.timestamps.start ? a.timestamps.start.getTime() : null, end: a.timestamps.end ? a.timestamps.end.getTime() : null - }, + } : void 0, party: a.party, - assets: { + assets: a.assets ? { large_text: a.assets.largeText, small_text: a.assets.smallText, large_image: a.assets.largeImage, small_image: a.assets.smallImage - }, + } : void 0, sync_id: a.syncID, flags: a.flags.valueOf(), - emoji: { + emoji: a.emoji ? { animated: a.emoji.animated, name: a.emoji.name, id: a.emoji.id - }, + } : void 0, created_at: a.createdTimestamp })), client_status: this.clientStatus From 4de14423664bede349e986ae04ecdb9312577da4 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Thu, 1 Apr 2021 01:02:04 +0100 Subject: [PATCH 092/120] wip: needs rethinking we dont want to load ALL caches from ALL shards we need to separate files by shard ids and only load the right caches in the right processes --- client.js | 296 +++++++++++++++++++++++------------------------------- 1 file changed, 126 insertions(+), 170 deletions(-) diff --git a/client.js b/client.js index 35a98a9..049dd37 100644 --- a/client.js +++ b/client.js @@ -6,6 +6,52 @@ const actions = require("./actions.js"); const pkg = require("./package.json"); const fs = require("fs"); +/** + * Validate client options. + * @param {object} options Options to validate + * @private + */ +validateOptions(options) { + if(typeof options.cacheChannels !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheChannels", "a boolean"); + } + if(typeof options.cacheGuilds !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheGuilds", "a boolean"); + } + if(typeof options.cachePresences !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cachePresences", "a boolean"); + } + if(typeof options.cacheRoles !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheRoles", "a boolean"); + } + if(typeof options.cacheOverwrites !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheOverwrites", "a boolean"); + } + if(typeof options.cacheEmojis !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheEmojis", "a boolean"); + } + if(typeof options.cacheMembers !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheMembers", "a boolean"); + } + if(!Array.isArray(options.disabledEvents)) { + throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); + } + if(options.hotReload && typeof options.hotReload === "object") { + if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); + } + if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { + throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "an object"); + } + if (options.hotReload.onExit && typeof options.hotReload.onExit !== "function") { + throw new TypeError("CLIENT_INVALID_OPTION", "onExit", "a function"); + } + } + else if(typeof options.hotReload !== "boolean") { + throw new TypeError("CLIENT_INVALID_OPTION", "hotReload", "a boolean or an object"); + } +} + Discord.Client = class Client extends Discord.Client { constructor(_options = {}) { const options = { @@ -16,82 +62,60 @@ Discord.Client = class Client extends Discord.Client { cacheOverwrites: false, cacheEmojis: false, cacheMembers: false, - disabledEvents: [], hotReload: false, + disabledEvents: [], ..._options }; + validateOptions(options); super(options); actions(this); - this._validateOptionsLight(options); - if (options.hotReload) { + if(options.hotReload) { this.on(Discord.Constants.Events.SHARD_RESUME, () => { - if (!this.readyAt) { this.ws.checkShardsReady(); } + if(!this.readyAt) { this.ws.checkShardsReady(); } }); - this.cacheFilePath = `${process.cwd()}/.sessions`; - this.ws._hotreload = {}; - if (options.hotReload.sessionData && Object.keys(options.hotReload.sessionData).length) { - this.ws._hotreload = options.hotReload.sessionData; - } - else { - this._loadSessions(); - } - this._patchCache(options.hotReload.cacheData || this._loadCache()); - this.onUnload = (sessions, { guilds, channels, users }) => { - this._makeDir(this.cacheFilePath); - this._makeDir(`${this.cacheFilePath}/sessions`); - this._loadSessions(); - this.ws._hotreload = { - ...this.ws._hotreload, - ...sessions - }; - this._unLoadSessions(); - if (options.cacheGuilds) { - this._makeDir(`${this.cacheFilePath}/guilds`); - this._write("guilds", guilds); - } - if (options.cacheChannels) { - this._makeDir(`${this.cacheFilePath}/channels`); - this._write("channels", channels); - } - if (options.cacheMembers) { - this._makeDir(`${this.cacheFilePath}/users`); - this._write("users", users); - } - }; - this._uncaughtExceptionOnExit = false; - for (const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { + this._patchCache(options.hotReload.cacheData || this._loadCache()); + for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { - if (eventType === "uncaughtException") { - this._uncaughtExceptionOnExit = true; - } - if (!this._uncaughtExceptionOnExit) { - Object.assign(this.ws._hotreload, ...this.ws.shards.map(s => { - s.connection.close(); - return { - [s.id]: { - id: s.sessionID, - seq: s.sequence - } - }; - })); - if (eventType !== "exit") { - await this.onUnload(this.ws._hotreload, this.dumpCache()); - process.exit(); + let cache = this.dumpCache(); + let sessions = this.dumpSessions(); + if(options.hotReload.onExit) { + await options.hotReload.onExit(cache, sessions); // async will not work on exit and exception but might work on SIGINT and SIGTERM + } else { + for(const folder of ["websocket", "users", "guilds", "channels"]) { + if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } } + for(const shard of sessions) + } + if(eventType === "uncaughtException") { + console.error(...args); } - else if (eventType !== "exit") { - console.error(args[0]); - console.error("UNCAUGHT_EXCEPTION_LOOP", "There was an uncaughtException inside your exit loop causing an infinite loop. Your exit function was not run or failed"); - process.exit(1); + if(eventType !== "exit") { + process.exit(process.exitCode); } }); } } } + sweepUsers(_lifetime = 86400) { + const lifetime = _lifetime * 1000; + this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); + for(const guild of this.guilds.cache.values()) { + guild.members.cache.sweep(t => !this.users.cache.has(t.id)); + guild.presences.cache.sweep(t => !this.users.cache.has(t.id) && !this.options.cachePresences); + } + } + sweepChannels(_lifetime = 86400) { + const lifetime = _lifetime * 1000; + if(this.options.cacheChannels) { return; } + const connections = this.voice ? this.voice.connections.map(t => t.channel.id) : []; + this.channels.cache.sweep(t => !connections.includes(t.id) && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); + for(const guild of this.guilds.cache.values()) { + guild.channels.cache.sweep(t => !this.channels.cache.has(t.id)); + } + } /** * Generates a complete dump of the current stored cache - * @param {object} options Options to validate - * @returns {object} All of the cache + * @returns {object} Cache data */ dumpCache() { return { @@ -100,139 +124,71 @@ Discord.Client = class Client extends Discord.Client { users: this.users.cache.map(u => u._unpatch()) }; } + /** + * Generates a complete dump of the current stored cache + * @returns {object} Session data + */ + dumpSessions() { + return this.ws.shards.map(s => ({ + [s.id]: { + id: s.sessionID, + sequence: s.sequence + } + })); + } /** * Loads all of the stored caches on disk into memory * @returns {object} All of the stored cache * @private */ _loadCache() { - const allCache = {}; - for (const cache of ["guilds", "channels", "users"]) { + const allCache = { + guilds: [], + channels: [], + users: [] + }; + for(const cache of ["guilds", "channels", "users"]) { + const files = []; try { - const cachedFiles = fs.readdirSync(`${this.cacheFilePath}/${cache}`) - .filter(file => file.endsWith(".json")) - .map(c => c.substr(0, c.lastIndexOf("."))); - if (cachedFiles.length) { continue; } - allCache[cache] = []; - for (const id of cachedFiles) { - allCache[cache].push(JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${id}.json`, "utf8"))); - } - } catch (d) { - // Do nothing + files = fs.readdirSync(`${process.cwd()}/.sessions/${cache}`).filter(file => file.endsWith(".json")); + } catch(e) { /* no-op */ } + for(const file of files) { + try { + const json = fs.readFileSync(`${process.cwd()}/.sessions/${cache}/${file}`, "utf8"); + const obj = JSON.parse(json); + allCache[cache].push(obj); + } catch(e) { /* no-op */ } } } return allCache; } - /** - * Patches raw discord api objects into the discord.js cache - * @private - */ - _patchCache({ guilds, channels, users }) { - } /** * Loads all of the stored sessions on disk into memory * @private */ _loadSessions() { + let data = {} + let files = []; try { - const shards = fs.readdirSync(`${this.cacheFilePath}/sessions`) - .filter(file => file.endsWith(".json")) - .map(shardSession => shardSession.substr(0, shardSession.lastIndexOf("."))); - for (const shardID of shards) { - this.ws._hotreload[shardID] = JSON.parse(fs.readFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, "utf8")); - } - } catch (e) { - this.ws._hotreload = {}; - } - } - /** - * Unloads all of the stored sessions in memory onto disk - * @private - */ - _unLoadSessions() { - for (const [shardID, session] of this.ws._hotreload) { - fs.writeFileSync(`${this.cacheFilePath}/sessions/${shardID}.json`, JSON.stringify(session)); - } - } - /** - * Creates a directory if it does not already exist - * @private - */ - _makeDir(dir) { - if (!fs.existsSync(dir)) { fs.mkdirSync(dir); } - } - /** - * Writes a cache array to multiple files indexed by ID to disk using the cached file path and JSON format - * @param {string} path The path to write the data to - * @param {Array} data An array of all of the data items to write - * @private - */ - _write(path, data) { - for (const item of data) { - fs.writeFileSync(`${this.cacheFilePath}/${path}/${item.id}.json`, JSON.stringify(item)); - } - } - sweepUsers(_lifetime = 86400) { - const lifetime = _lifetime * 1000; - this.users.cache.sweep(t => t.id !== this.user.id && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); - for(const guild of this.guilds.cache.values()) { - guild.members.cache.sweep(t => !this.users.cache.has(t.id)); - guild.presences.cache.sweep(t => !this.users.cache.has(t.id) && !this.options.cachePresences); - } - } - sweepChannels(_lifetime = 86400) { - const lifetime = _lifetime * 1000; - if(this.options.cacheChannels) { return; } - const connections = this.voice ? this.voice.connections.map(t => t.channel.id) : []; - this.channels.cache.sweep(t => !connections.includes(t.id) && (!t.lastMessageID || Date.now() - Discord.SnowflakeUtil.deconstruct(t.lastMessageID).timestamp > lifetime)); - for(const guild of this.guilds.cache.values()) { - guild.channels.cache.sweep(t => !this.channels.cache.has(t.id)); + files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); + } catch (e) { /* no-op */ } + for(const file of files) { + try { + const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); + const obj = JSON.parse(json); + data[file.slice(0, -5)] = obj; + } catch(e) { /* no-op */ } } + return data; } /** - * Validates the client options. - * @param {object} options Options to validate + * Patches raw discord api objects into the discord.js cache * @private */ - _validateOptionsLight(options) { - if (typeof options.cacheChannels !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheChannels", "a boolean"); - } - if (typeof options.cacheGuilds !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheGuilds", "a boolean"); - } - if (typeof options.cachePresences !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cachePresences", "a boolean"); - } - if (typeof options.cacheRoles !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheRoles", "a boolean"); - } - if (typeof options.cacheOverwrites !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheOverwrites", "a boolean"); - } - if (typeof options.cacheEmojis !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheEmojis", "a boolean"); - } - if (typeof options.cacheMembers !== "boolean") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheMembers", "a boolean"); - } - if (!Array.isArray(options.disabledEvents)) { - throw new TypeError("CLIENT_INVALID_OPTION", "disabledEvents", "an array"); - } - if (typeof options.hotReload !== "boolean") { - if (options.hotReload && typeof options.hotReload === "object") { - if (options.hotReload.sessionData && typeof options.hotReload.sessionData !== "object") { - throw new TypeError("CLIENT_INVALID_OPTION", "sessionData", "an object"); - } - if (options.hotReload.cacheData && typeof options.hotReload.cacheData !== "object") { - throw new TypeError("CLIENT_INVALID_OPTION", "cacheData", "a object"); - } - if (options.hotReload.onUnload && typeof options.hotReload.onUnload !== "function") { - throw new TypeError("CLIENT_INVALID_OPTION", "onUnload", "a function"); - } - } - else { - throw new TypeError("CLIENT_INVALID_OPTION", "hotReload", "a boolean or an object"); + _patchCache(data) { + for(const [cache, items] of Object.entries(data)) { + for(const item of items) { + this[cache].add(item); } } } From 7be37e3f03f6eae9ff68529a281977b0e2e733fb Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Thu, 1 Apr 2021 15:46:44 +0100 Subject: [PATCH 093/120] Linting --- classes.js | 24 +++++++++++------------- client.js | 14 +++++++------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/classes.js b/classes.js index 9ac8272..d32cd14 100644 --- a/classes.js +++ b/classes.js @@ -167,7 +167,7 @@ Discord.Structures.extend("Message", M => { message_id: this.reference.messageID } : void 0, referenced_message: this.client.guilds.cache.get(this.reference?.guildID)?.channels.cache.get(this.reference?.channelID)?.messages.cache.get(this.reference?.messageID)?._unpatch() - } + }; } get member() { if(!this.guild) { return null; } @@ -207,9 +207,9 @@ Discord.Strictures.extend("User", U => { avatar: this.avatar, system: this.system, public_flags: this.flags.valueOf() - } + }; } - } + }; }); Discord.Structures.extend("GuildMember", G => { @@ -238,7 +238,7 @@ Discord.Structures.extend("GuildMember", G => { premium_since: this.premiumSinceTimestamp, roles: this._roles, pending: this.pending - } + }; } equals(member) { return member && this.deleted === member.deleted && this.nickname === member.nickname && this._roles.length === member._roles.length; @@ -431,8 +431,8 @@ Discord.Structures.extend("GuildEmoji", E => { managed: this.managed, available: this.available, roles: this._roles, - user: this.author ? this.author._unpatch() : void 0; - } + user: this.author ? this.author._unpatch() : void 0 + }; } async fetchAuthor(cache = true) { if(this.managed) { @@ -467,9 +467,9 @@ Discord.Structures.extend("Role", R => { integration_id: this.tags.integrationID, premium_subscriber: this.tags.premiumSubscriberRole } - } + }; } - } + }; }); Discord.Structures.extend("VoiceState", V => { @@ -492,7 +492,7 @@ Discord.Structures.extend("VoiceState", V => { session_id: this.sessionID, self_stream: this.streaming, channel_id: this.channelID - } + }; } get channel() { return this.channelID ? this.client.channels.cache.get(this.channelID) || this.client.channels.add({ @@ -550,9 +550,7 @@ Discord.Structures.extend("Presence", P => { } _unpatch() { return { - user: { - id: this.userID - }, + user: { id: this.userID }, status: this.status, activities: this.activities.map(a => ({ name: a.name, @@ -582,7 +580,7 @@ Discord.Structures.extend("Presence", P => { created_at: a.createdTimestamp })), client_status: this.clientStatus - } + }; } get user() { return this.client.users.cache.get(this.userID) || this.client.users.add((this._member || {}).user || { id: this.userID }, false); diff --git a/client.js b/client.js index 049dd37..5d05687 100644 --- a/client.js +++ b/client.js @@ -11,7 +11,7 @@ const fs = require("fs"); * @param {object} options Options to validate * @private */ -validateOptions(options) { +function validateOptions(options) { if(typeof options.cacheChannels !== "boolean") { throw new TypeError("CLIENT_INVALID_OPTION", "cacheChannels", "a boolean"); } @@ -73,18 +73,18 @@ Discord.Client = class Client extends Discord.Client { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); - this._patchCache(options.hotReload.cacheData || this._loadCache()); + this._patchCache(options.hotReload.cacheData || this._loadCache()); for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { - let cache = this.dumpCache(); - let sessions = this.dumpSessions(); + const cache = this.dumpCache(); + const sessions = this.dumpSessions(); if(options.hotReload.onExit) { await options.hotReload.onExit(cache, sessions); // async will not work on exit and exception but might work on SIGINT and SIGTERM } else { for(const folder of ["websocket", "users", "guilds", "channels"]) { if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } } - for(const shard of sessions) + for(const shard of sessions) { /* no-op */ } } if(eventType === "uncaughtException") { console.error(...args); @@ -148,7 +148,7 @@ Discord.Client = class Client extends Discord.Client { users: [] }; for(const cache of ["guilds", "channels", "users"]) { - const files = []; + let files = []; try { files = fs.readdirSync(`${process.cwd()}/.sessions/${cache}`).filter(file => file.endsWith(".json")); } catch(e) { /* no-op */ } @@ -167,7 +167,7 @@ Discord.Client = class Client extends Discord.Client { * @private */ _loadSessions() { - let data = {} + const data = {}; let files = []; try { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); From 00d6e3c2ab02301f037935f8290d693a4eea9160 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Thu, 1 Apr 2021 15:50:23 +0100 Subject: [PATCH 094/120] Try catch around user supplied function Promises are supported inside of uncaughtException, SIGINT and SIGTERM but not exit. Most restarts call SIGINT or SIGTERM before they exit. I don't know why you removed the check for an infinite loop of uncaught exceptions? If there is an uncaught exception inside of the user defined function or our function it will loop forever. I added a try catch block around the user function so it won't happen for user functions but if you don't want a check for it we need to be 100% sure our code won't cause a loop --- client.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client.js b/client.js index 5d05687..6f7da8b 100644 --- a/client.js +++ b/client.js @@ -79,7 +79,9 @@ Discord.Client = class Client extends Discord.Client { const cache = this.dumpCache(); const sessions = this.dumpSessions(); if(options.hotReload.onExit) { - await options.hotReload.onExit(cache, sessions); // async will not work on exit and exception but might work on SIGINT and SIGTERM + try { + await options.hotReload.onExit(cache, sessions); // async will not work on exit + } catch (e) { /* no-op */ } } else { for(const folder of ["websocket", "users", "guilds", "channels"]) { if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } From 0d6ef2d4717fc1cf8e2482abe8e34716b8d2235d Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Thu, 1 Apr 2021 16:04:32 +0100 Subject: [PATCH 095/120] prevent exit events from bleeding into each other --- client.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/client.js b/client.js index 6f7da8b..585d058 100644 --- a/client.js +++ b/client.js @@ -74,19 +74,21 @@ Discord.Client = class Client extends Discord.Client { if(!this.readyAt) { this.ws.checkShardsReady(); } }); this._patchCache(options.hotReload.cacheData || this._loadCache()); + let dumped = false; for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { - const cache = this.dumpCache(); - const sessions = this.dumpSessions(); - if(options.hotReload.onExit) { - try { - await options.hotReload.onExit(cache, sessions); // async will not work on exit - } catch (e) { /* no-op */ } - } else { - for(const folder of ["websocket", "users", "guilds", "channels"]) { - if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } + if(!dumped) { + dumped = true; + const cache = this.dumpCache(); + const sessions = this.dumpSessions(); + if(options.hotReload.onExit) { + await options.hotReload.onExit(cache, sessions).catch(() => {}); // async will not work on exit + } else { + for(const folder of ["websocket", "users", "guilds", "channels"]) { + if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } + } + for(const shard of sessions) { /* no-op */ } } - for(const shard of sessions) { /* no-op */ } } if(eventType === "uncaughtException") { console.error(...args); From 996c5d6dec67cab21e04bd28a1a7853ca1983fff Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Thu, 1 Apr 2021 22:11:02 +0100 Subject: [PATCH 096/120] store caches --- client.js | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/client.js b/client.js index 585d058..313c02e 100644 --- a/client.js +++ b/client.js @@ -73,7 +73,6 @@ Discord.Client = class Client extends Discord.Client { this.on(Discord.Constants.Events.SHARD_RESUME, () => { if(!this.readyAt) { this.ws.checkShardsReady(); } }); - this._patchCache(options.hotReload.cacheData || this._loadCache()); let dumped = false; for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { @@ -82,12 +81,9 @@ Discord.Client = class Client extends Discord.Client { const cache = this.dumpCache(); const sessions = this.dumpSessions(); if(options.hotReload.onExit) { - await options.hotReload.onExit(cache, sessions).catch(() => {}); // async will not work on exit + await options.hotReload.onExit(sessions, cache).catch(() => {}); // async will not work on exit } else { - for(const folder of ["websocket", "users", "guilds", "channels"]) { - if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } - } - for(const shard of sessions) { /* no-op */ } + this._storeData(sessions, cache); } } if(eventType === "uncaughtException") { @@ -123,22 +119,24 @@ Discord.Client = class Client extends Discord.Client { */ dumpCache() { return { - guilds: this.guilds.cache.map(g => g._unpatch()), - channels: this.channels.cache.map(c => c._unpatch()), - users: this.users.cache.map(u => u._unpatch()) - }; + guilds: this.guilds.cache.reduce((a, g) => { + a[g.id] = g._unpatch(); + return a; + }, {}) + } } /** * Generates a complete dump of the current stored cache * @returns {object} Session data */ dumpSessions() { - return this.ws.shards.map(s => ({ - [s.id]: { + return this.ws.shards.reduce((a, s) => { + a[s.id] = { id: s.sessionID, sequence: s.sequence - } - })); + }; + return a; + }, {}); } /** * Loads all of the stored caches on disk into memory @@ -196,6 +194,23 @@ Discord.Client = class Client extends Discord.Client { } } } + /** + * Built-in cache storing + * @private + */ + _storeData(sessions, cache) { + for(const [id, data] of Object.entries(sessions)) { + if(!fs.existsSync(`${process.cwd()}/.sessions/websocket`)) { fs.mkdirSync(`${process.cwd()}/.sessions/websocket`, { recursive: true }); } + let obj = JSON.stringify(data); + fs.writeFileSync(`${process.cwd()}/.sessions/websocket/${id}.json`, obj, "utf8"); + } + for(const folder of Object.keys(cache)) { + for(const [id, data] of Object.entries(cache[folder])) { + let obj = JSON.stringify(data); + fs.writeFileSync(`${process.cwd()}/.sessions/${folder}/${id}.json`, obj, "utf8"); + } + } + } }; Discord.version = `${pkg.version} (${Discord.version})`; From 46fb575ff1fd5a92ab7112b6fb4c985bc68577d6 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Thu, 1 Apr 2021 22:13:15 +0100 Subject: [PATCH 097/120] unpatch channels in the guild object --- classes.js | 1 + 1 file changed, 1 insertion(+) diff --git a/classes.js b/classes.js index d32cd14..3a1867f 100644 --- a/classes.js +++ b/classes.js @@ -357,6 +357,7 @@ Discord.Structures.extend("Guild", G => { preferred_locale: this.preferredLocale, roles: this.roles.cache.map(x => x._unpatch()), members: this.members.cache.map(x => x._unpatch()), + channels: this.channels.cache.map(x => x._unpatch()), owner_id: this.ownerID, presences: this.presences.cache.map(x => x._unpatch()), voice_states: this.voiceStates.cache.map(x => x._unpatch()), From f233e0349975c5d65471348e84a8e1d26a621b11 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 00:25:25 +0100 Subject: [PATCH 098/120] Small syntax fixes Fixed some copy paste fails assigning an object to a value with : as well as a spelling error and creating folders for each of the caches --- .eslintrc.json | 3 ++- classes.js | 10 +++++----- client.js | 1 + init.js | 18 +++++++++--------- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 9d84e28..0b431c1 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -111,6 +111,7 @@ "prefer-numeric-literals":"error", "prefer-spread":"error", "prefer-template":"error", - "symbol-description":"error" + "symbol-description":"error", + "no-func-loop": false } } diff --git a/classes.js b/classes.js index 3a1867f..070a988 100644 --- a/classes.js +++ b/classes.js @@ -196,7 +196,7 @@ Discord.Structures.extend("Message", M => { }; }); -Discord.Strictures.extend("User", U => { +Discord.Structures.extend("User", U => { return class User extends U { _unpatch() { return { @@ -206,7 +206,7 @@ Discord.Strictures.extend("User", U => { discriminator: this.discriminator, avatar: this.avatar, system: this.system, - public_flags: this.flags.valueOf() + public_flags: this.flags?.valueOf() }; } }; @@ -464,9 +464,9 @@ Discord.Structures.extend("Role", R => { managed: this.managed, mentionable: this.mentionable, tags: { - bot_id: this.tags.botID, - integration_id: this.tags.integrationID, - premium_subscriber: this.tags.premiumSubscriberRole + bot_id: this.tags?.botID, + integration_id: this.tags?.integrationID, + premium_subscriber: this.tags?.premiumSubscriberRole } }; } diff --git a/client.js b/client.js index 313c02e..b43fd60 100644 --- a/client.js +++ b/client.js @@ -205,6 +205,7 @@ Discord.Client = class Client extends Discord.Client { fs.writeFileSync(`${process.cwd()}/.sessions/websocket/${id}.json`, obj, "utf8"); } for(const folder of Object.keys(cache)) { + if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } for(const [id, data] of Object.entries(cache[folder])) { let obj = JSON.stringify(data); fs.writeFileSync(`${process.cwd()}/.sessions/${folder}/${id}.json`, obj, "utf8"); diff --git a/init.js b/init.js index 915d2da..8e0348d 100644 --- a/init.js +++ b/init.js @@ -287,28 +287,28 @@ require.cache[CPath].exports = class Channel extends C { id: this.id }; if(this.messages) { - obj.last_message_id: this.lastMessageID; - obj.last_pin_timestamp: this.lastPinTimestamp; + obj.last_message_id = this.lastMessageID; + obj.last_pin_timestamp = this.lastPinTimestamp; } switch(this.type) { case "dm": { - obj.recipients: [this.recipient._unpatch()]; + obj.recipients = [this.recipient._unpatch()]; break; } case "text": case "news": { - obj.nsfw: this.nsfw; - obj.topic: this.topic; - obj.rate_limit_per_user: this.rateLimitPerUser; + obj.nsfw = this.nsfw; + obj.topic = this.topic; + obj.rate_limit_per_user = this.rateLimitPerUser; obj.messages = this.messages.cache.map(x => x._unpatch()); break; } case "voice": { - obj.bitrate: this.bitrate; - obj.user_limit: this.userLimit + obj.bitrate = this.bitrate; + obj.user_limit = this.userLimit break; } case "store": { - obj.nsfw: this.nsfw; + obj.nsfw = this.nsfw; break; } } From c5c3dbc7e881ec187e1db24e405084b15328208d Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 2 Apr 2021 15:54:28 +0100 Subject: [PATCH 099/120] rework cache loading and storing --- client.js | 64 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/client.js b/client.js index b43fd60..4941374 100644 --- a/client.js +++ b/client.js @@ -115,6 +115,7 @@ Discord.Client = class Client extends Discord.Client { } /** * Generates a complete dump of the current stored cache + * Only guild cache is dumped for now * @returns {object} Cache data */ dumpCache() { @@ -139,47 +140,54 @@ Discord.Client = class Client extends Discord.Client { }, {}); } /** - * Loads all of the stored caches on disk into memory - * @returns {object} All of the stored cache + * Loads the selected stored cache on disk into memory + * @returns {object} The stored cache * @private */ - _loadCache() { - const allCache = { - guilds: [], - channels: [], - users: [] - }; - for(const cache of ["guilds", "channels", "users"]) { - let files = []; + _loadCache(cacheType, filter) { + const cache = {}; + if(typeof cacheType !== "string" || !["guilds"].includes(cacheType.toLowerCase())) { return cache; } // to allow expanding in the future + let files = []; + try { + files = fs.readdirSync(`${process.cwd()}/.sessions/${cacheType}`).filter(file => file.endsWith(".json")); + } catch(e) { /* no-op */ } + for(const file of files) { + let name = file.slice(0, -5); + if(typeof filter === "function" && !filter(name)) { continue; } try { - files = fs.readdirSync(`${process.cwd()}/.sessions/${cache}`).filter(file => file.endsWith(".json")); + const json = fs.readFileSync(`${process.cwd()}/.sessions/${cacheType}/${file}`, "utf8"); + const obj = JSON.parse(json); + cache[name] = obj; } catch(e) { /* no-op */ } - for(const file of files) { - try { - const json = fs.readFileSync(`${process.cwd()}/.sessions/${cache}/${file}`, "utf8"); - const obj = JSON.parse(json); - allCache[cache].push(obj); - } catch(e) { /* no-op */ } - } } - return allCache; + return cache; } /** - * Loads all of the stored sessions on disk into memory + * Loads the selected stored sessions on disk into memory * @private */ - _loadSessions() { - const data = {}; + _loadSessions(id) { + let data = {}; let files = []; try { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); } catch (e) { /* no-op */ } - for(const file of files) { - try { - const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); - const obj = JSON.parse(json); - data[file.slice(0, -5)] = obj; - } catch(e) { /* no-op */ } + if(id) { + const file = files.find(file => Number(file.slice(0, -5) === id); + if(file) { + try { + const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); + data = JSON.parse(json); + } catch(e) { /* no-op */ } + } + } else { + for(const file of files) { + try { + const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); + const shard = Number(file.slice(0, -5); + data[shard] = JSON.parse(json); + } catch(e) { /* no-op */ } + } } return data; } From 52efd4c07c0bef8b9f4d3e26f4815bddf8bfe6b0 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 2 Apr 2021 16:05:00 +0100 Subject: [PATCH 100/120] load cache on identify --- init.js | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/init.js b/init.js index 8e0348d..39450fb 100644 --- a/init.js +++ b/init.js @@ -6,6 +6,7 @@ const Constants = require(resolve(require.resolve("discord.js").replace("index.j const APIMessage = require(resolve(require.resolve("discord.js").replace("index.js", "/structures/APIMessage.js"))); const Util = require(resolve(require.resolve("discord.js").replace("index.js", "/util/Util.js"))); const { Error: DJSError } = require(resolve(require.resolve("discord.js").replace("index.js", "/errors"))); +const ShardClientUtil = require(resolve(require.resolve("discord.js").replace("index.js", "/sharding/ShardClientUtil.js"))); const RHPath = resolve(require.resolve("discord.js").replace("index.js", "/rest/APIRequest.js")); const RH = require(RHPath); @@ -48,12 +49,26 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { }, 15000); } identify() { - if(this.manager.client.options.hotreload && this.manager._hotreload) { - const data = this.manager._hotreload[this.id]; - if(data && !this.sessionID) { + let hotReload = this.manager.client.options.hotReload; + if(hotReload) { + const data = hotReload.sessionData?[this.id] || this.manager.client._loadSessions(this.id); + if(data?.id && !this.sessionID) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; - delete this.manager._hotreload[this.id]; + } + const cache = this.manager.client.options.hotReload.cacheData; + if(cache?.guilds && typeof cache.guilds === "object") { + const keys = Object.keys(cache.guilds); + for(const id of keys) { + if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { + this.manager.client.guilds.add(cache.guilds[id]); + } + } + } else { + const guilds = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards)); + for(const guild of Object.values(guilds)) { + this.manager.guilds.add(guild); + } } } return super.identify(); From 984a73bde78aac51407ceeb1223d5d40fa24d04e Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 2 Apr 2021 16:10:03 +0100 Subject: [PATCH 101/120] fix conflict --- init.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init.js b/init.js index 39450fb..029c6b2 100644 --- a/init.js +++ b/init.js @@ -7,6 +7,8 @@ const APIMessage = require(resolve(require.resolve("discord.js").replace("index. const Util = require(resolve(require.resolve("discord.js").replace("index.js", "/util/Util.js"))); const { Error: DJSError } = require(resolve(require.resolve("discord.js").replace("index.js", "/errors"))); const ShardClientUtil = require(resolve(require.resolve("discord.js").replace("index.js", "/sharding/ShardClientUtil.js"))); +const Util = require(resolve(require.resolve("discord.js").replace("index.js", "/util/Util.js"))); +const { Error: DJSError } = require(resolve(require.resolve("discord.js").replace("index.js", "/errors"))); const RHPath = resolve(require.resolve("discord.js").replace("index.js", "/rest/APIRequest.js")); const RH = require(RHPath); From 7ea9e94b6aa303848afefe9a1345d47f140499e9 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 17:46:14 +0100 Subject: [PATCH 102/120] Small fixes Removed obj variable as it JSON.stringify() can be called directly inside of write file --- client.js | 12 +++++------- init.js | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/client.js b/client.js index 4941374..848fb02 100644 --- a/client.js +++ b/client.js @@ -80,7 +80,7 @@ Discord.Client = class Client extends Discord.Client { dumped = true; const cache = this.dumpCache(); const sessions = this.dumpSessions(); - if(options.hotReload.onExit) { + if(options.hotReload?.onExit) { await options.hotReload.onExit(sessions, cache).catch(() => {}); // async will not work on exit } else { this._storeData(sessions, cache); @@ -173,7 +173,7 @@ Discord.Client = class Client extends Discord.Client { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); } catch (e) { /* no-op */ } if(id) { - const file = files.find(file => Number(file.slice(0, -5) === id); + const file = files.find(file => Number(file.slice(0, -5) === id)); if(file) { try { const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); @@ -184,7 +184,7 @@ Discord.Client = class Client extends Discord.Client { for(const file of files) { try { const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); - const shard = Number(file.slice(0, -5); + const shard = Number(file.slice(0, -5)); data[shard] = JSON.parse(json); } catch(e) { /* no-op */ } } @@ -209,14 +209,12 @@ Discord.Client = class Client extends Discord.Client { _storeData(sessions, cache) { for(const [id, data] of Object.entries(sessions)) { if(!fs.existsSync(`${process.cwd()}/.sessions/websocket`)) { fs.mkdirSync(`${process.cwd()}/.sessions/websocket`, { recursive: true }); } - let obj = JSON.stringify(data); - fs.writeFileSync(`${process.cwd()}/.sessions/websocket/${id}.json`, obj, "utf8"); + fs.writeFileSync(`${process.cwd()}/.sessions/websocket/${id}.json`, JSON.stringify(data), "utf8"); } for(const folder of Object.keys(cache)) { if(!fs.existsSync(`${process.cwd()}/.sessions/${folder}`)) { fs.mkdirSync(`${process.cwd()}/.sessions/${folder}`, { recursive: true }); } for(const [id, data] of Object.entries(cache[folder])) { - let obj = JSON.stringify(data); - fs.writeFileSync(`${process.cwd()}/.sessions/${folder}/${id}.json`, obj, "utf8"); + fs.writeFileSync(`${process.cwd()}/.sessions/${folder}/${id}.json`, JSON.stringify(data), "utf8"); } } } diff --git a/init.js b/init.js index 029c6b2..ae087da 100644 --- a/init.js +++ b/init.js @@ -53,7 +53,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const data = hotReload.sessionData?[this.id] || this.manager.client._loadSessions(this.id); + const data = hotReload.sessionData?.[this.id] || this.manager.client._loadSessions(this.id) if(data?.id && !this.sessionID) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; From 36236e7fce9c4f9f2b130019fad7f2c8284c99e2 Mon Sep 17 00:00:00 2001 From: Timotej Rojko <33236065+timotejroiko@users.noreply.github.com> Date: Fri, 2 Apr 2021 16:40:29 +0100 Subject: [PATCH 103/120] readme --- README.md | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index cfc1824..6c7749c 100644 --- a/README.md +++ b/README.md @@ -91,20 +91,6 @@ client.login("TOKEN").catch(console.error); Generally, usage should be identical to discord.js and you can safely refer to its documentation as long as you respect the caching differences explained below. -### Hot reloading - -**THIS FEATURE IS CURRENTLY EXPERIMENTAL USE AT YOUR OWN RISK!** - -When developing bots you will often want to prototype through trial and error. This often requires turning your bot on and off lots of times potentially using [nodemon](https://nodemon.io/) -by doing this you are connecting to the Discord websocket gateway each time which can often take a few seconds as well as cuts into your 1000 daily identifies. - -To solve this problem you can use hot reloading which is a client option allowing you to simply resume the previous session rather than create a new one. - -You can also use Hot reloading in your production bot by supplying a session object along with preferences for caching restoration or by just letting us take care of it with cache files found in the `.sessions` folder - -By setting the client.dumpCache method you can run a custom async function to store your caches and session IDs in your database of choice. The dumpCache method is -called with the (session, client) params where session is your up to date sessoins - ## Client Options The following client options are available to control caching behavior: @@ -118,6 +104,7 @@ The following client options are available to control caching behavior: | cacheEmojis | boolean | false | Enables caching of all Emojis at login | | cachePresences | boolean | false | Enables caching of all Presences. If not enabled, Presences will be cached only for cached Users | | cacheMembers | boolean | false | Enables caching of Users and Members when possible | +| hotReload | boolean or object | false | Enables hot reloading, an experimental feature that enables instantly restarting the bot | | disabledEvents | array | [] | An array of events to ignore ([Discord events](https://github.com/discordjs/discord.js/blob/master/src/util/Constants.js#L339), not Discord.JS events). Use this in combination with intents for fine tuning which events your bot should process | This library implements its own partials system, therefore the `partials` client option is not available. All other discord.js client options continue to be available and should work normally. @@ -162,6 +149,30 @@ Voice States will be cached if the `GUILD_VOICE_STATES` intent is enabled (requi Messages are cached only if the Channel they belong to is cached. Message caching can further be controlled via discord.js's `messageCacheMaxSize`, `messageCacheLifetime` and `messageSweepInterval` client options as usual. Additionally, the `messageEditHistoryMaxSize` client option is set to `1` by default (instead of infinity). +## Hot reloading + +**THIS FEATURE IS CURRENTLY EXPERIMENTAL USE AT YOUR OWN RISK!** + +When developing bots you will likely do lots of trial and error which often requires restartig your bot. +Each restart requires reconnecting to the Discord gateway on every shard which can often take a long time and eat up your daily identifies. + +Hot reloading provides a way to simply resume the previous gateway sessions on startup rather than creating new ones. +This is done by storing the process data outside the process right before it exits and reloading the data into it when it starts. + +This option can be overriden with a few custom methods: + +```js +new Discord.Client({ + hotReload: { + cacheData: cache // user-supplied cache data. if not present, cache will be loaded from disk + sessionData: sessions // user-supplied session data. if not present, sessions will be loaded from disk + onExit: (sessions, cache) => {} // user-supplied data storing function. if not present, data will be stored to disk. This function can be async if the process is exited by using SIGINT (Ctrl+C), any other ways of exit are sync-only. + } +}) +``` + +Session and cache data can also be obtained at runtime using `client.dumpCache()` and `client.dumpSessions()` for storage before a manually induced exit. In this case the user should pass an empty function to onExit to override the built-in disk storage. + ## Events Most events should be identical to the originals aside from the caching behavior plus they always emit regardless of caching state. When required data is missing, a partial structure where only an id is guaranteed will be given (the `.partial` property is not guaranteed to exist on all partials). From 523a463a58e3cf9f6f850daa1a48076f227f78e3 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 17:58:25 +0100 Subject: [PATCH 104/120] Added a last connected property to session data to avoid unnecessary identifies Sessions timeout after about 1-2 minutes so we should check for when the session was last connected and if it is more than a minute old ( therefore probably stale ) we shouldn't try and reconnect as the identify will probably fail. --- client.js | 3 ++- init.js | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/client.js b/client.js index 848fb02..309e022 100644 --- a/client.js +++ b/client.js @@ -134,7 +134,8 @@ Discord.Client = class Client extends Discord.Client { return this.ws.shards.reduce((a, s) => { a[s.id] = { id: s.sessionID, - sequence: s.sequence + sequence: s.sequence, + lastConnected: Date.now() }; return a; }, {}); diff --git a/init.js b/init.js index ae087da..3577a70 100644 --- a/init.js +++ b/init.js @@ -53,8 +53,8 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const data = hotReload.sessionData?.[this.id] || this.manager.client._loadSessions(this.id) - if(data?.id && !this.sessionID) { + const data = (hotReload.sessionData || this.manager.client._loadSessions(this.id))?.[this.id] + if(data?.id && data?.sequence && !this.sessionID && data.lastConnected + 60000 > Date.now()) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } From 0d7bd76852ac3d9230907e9fdae8d5da9cfb50cf Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 18:03:26 +0100 Subject: [PATCH 105/120] Updated cache and session types. Maybe remove from readme --- README.md | 24 +++++++++++++++++++----- client.d.ts | 7 ++++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6c7749c..4057c63 100644 --- a/README.md +++ b/README.md @@ -153,19 +153,33 @@ Messages are cached only if the Channel they belong to is cached. Message cachin **THIS FEATURE IS CURRENTLY EXPERIMENTAL USE AT YOUR OWN RISK!** -When developing bots you will likely do lots of trial and error which often requires restartig your bot. +When developing bots you will likely do lots of trial and error which often requires restarting your bot. Each restart requires reconnecting to the Discord gateway on every shard which can often take a long time and eat up your daily identifies. Hot reloading provides a way to simply resume the previous gateway sessions on startup rather than creating new ones. This is done by storing the process data outside the process right before it exits and reloading the data into it when it starts. - -This option can be overriden with a few custom methods: + +This option can be overridden with a few custom methods: ```js +const cache = { + guilds: { + "581072557512458241": {} // Discord api type + } +} + +const sessions = { + "0": { + id: "3e961ec59a7c24b91198d07dd3189fa0", // Session ID + sequence: 19, // The close sequence of the session + lastConnected: 1617382560867 // Time the session was closed at + } +} + new Discord.Client({ hotReload: { - cacheData: cache // user-supplied cache data. if not present, cache will be loaded from disk - sessionData: sessions // user-supplied session data. if not present, sessions will be loaded from disk + cacheData: cache, // user-supplied cache data. if not present, cache will be loaded from disk + sessionData: sessions, // user-supplied session data. if not present, sessions will be loaded from disk onExit: (sessions, cache) => {} // user-supplied data storing function. if not present, data will be stored to disk. This function can be async if the process is exited by using SIGINT (Ctrl+C), any other ways of exit are sync-only. } }) diff --git a/client.d.ts b/client.d.ts index 808f46d..3ebd5a6 100644 --- a/client.d.ts +++ b/client.d.ts @@ -52,6 +52,7 @@ type SessionData = { [shardID: string]: { id: string sequence: number + lastConnected: number } } @@ -62,9 +63,9 @@ type CacheData = { } type HotReloadOptions = { - sessionData?: SessionData cacheData?: CacheData - onUnload?: Function + sessionData?: SessionData + onExit?: Function } declare module "discord.js-light" { @@ -76,8 +77,8 @@ declare module "discord.js-light" { cacheOverwrites?:boolean cacheEmojis?:boolean cacheMembers?:boolean - disabledEvents?: Array hotReload?: boolean | HotReloadOptions + disabledEvents?: Array } interface ClientEvents { rest:[{path:string,method:string,response?:Promise}] From c6b21de82025541b9dedb6df8e1b02059af38cdb Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 18:33:11 +0100 Subject: [PATCH 106/120] Small fixes and checks --- client.js | 4 ++-- init.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client.js b/client.js index 309e022..27bf0ae 100644 --- a/client.js +++ b/client.js @@ -173,7 +173,7 @@ Discord.Client = class Client extends Discord.Client { try { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); } catch (e) { /* no-op */ } - if(id) { + if(id && files.length) { const file = files.find(file => Number(file.slice(0, -5) === id)); if(file) { try { @@ -181,7 +181,7 @@ Discord.Client = class Client extends Discord.Client { data = JSON.parse(json); } catch(e) { /* no-op */ } } - } else { + } else if (files.length) { for(const file of files) { try { const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); diff --git a/init.js b/init.js index 3577a70..5dad715 100644 --- a/init.js +++ b/init.js @@ -54,7 +54,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { let hotReload = this.manager.client.options.hotReload; if(hotReload) { const data = (hotReload.sessionData || this.manager.client._loadSessions(this.id))?.[this.id] - if(data?.id && data?.sequence && !this.sessionID && data.lastConnected + 60000 > Date.now()) { + if(data?.id && data.sequence > 0 && !this.sessionID && data.lastConnected + 60000 > Date.now()) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } @@ -69,7 +69,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { } else { const guilds = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards)); for(const guild of Object.values(guilds)) { - this.manager.guilds.add(guild); + this.manager.client.guilds.add(guild); } } } From 0ce72a130a137cf7e08ca59029bfff684dca1ba0 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Fri, 2 Apr 2021 19:03:45 +0100 Subject: [PATCH 107/120] Fixed _loadSession + now returns data in line with user supplied data Spend ages trying to figure out why shard 0 wasn't being treated as truthy... Of course JS treats the number 0 falsy xD --- client.js | 8 ++++---- init.js | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/client.js b/client.js index 27bf0ae..de5dc3e 100644 --- a/client.js +++ b/client.js @@ -167,18 +167,18 @@ Discord.Client = class Client extends Discord.Client { * Loads the selected stored sessions on disk into memory * @private */ - _loadSessions(id) { + _loadSession(id) { let data = {}; let files = []; try { files = fs.readdirSync(`${process.cwd()}/.sessions/websocket`).filter(file => file.endsWith(".json")); } catch (e) { /* no-op */ } - if(id && files.length) { - const file = files.find(file => Number(file.slice(0, -5) === id)); + if(Number(id) >= 0 && files.length) { + const file = files.find(file => Number(file.slice(0, -5)) === id); if(file) { try { const json = fs.readFileSync(`${process.cwd()}/.sessions/websocket/${file}`, "utf8"); - data = JSON.parse(json); + data[id] = JSON.parse(json); } catch(e) { /* no-op */ } } } else if (files.length) { diff --git a/init.js b/init.js index 5dad715..e7e82da 100644 --- a/init.js +++ b/init.js @@ -53,7 +53,9 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const data = (hotReload.sessionData || this.manager.client._loadSessions(this.id))?.[this.id] + const t = hotReload.sessionData || this.manager.client._loadSession(this.id) + console.log(t); + const data = (t)?.[this.id] if(data?.id && data.sequence > 0 && !this.sessionID && data.lastConnected + 60000 > Date.now()) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; From c91567be11c73d65e9dc3ee525835740c3422663 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:20:01 +0100 Subject: [PATCH 108/120] Made data more consistent Different data types were being assigned to the same variables change the loadcache method to reflect the same data that the user will supply as it makes it easier to work with. Did a hard compare for filter to stop any falsy value returns ( like shard id 0 being wrongly filtered because the number 0 is falsy ) --- client.js | 7 +++---- init.js | 15 ++++++--------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/client.js b/client.js index de5dc3e..e911db4 100644 --- a/client.js +++ b/client.js @@ -154,14 +154,13 @@ Discord.Client = class Client extends Discord.Client { } catch(e) { /* no-op */ } for(const file of files) { let name = file.slice(0, -5); - if(typeof filter === "function" && !filter(name)) { continue; } + if(typeof filter === "function" && filter(name) === false) { continue; } try { const json = fs.readFileSync(`${process.cwd()}/.sessions/${cacheType}/${file}`, "utf8"); - const obj = JSON.parse(json); - cache[name] = obj; + cache[name] = JSON.parse(json); } catch(e) { /* no-op */ } } - return cache; + return { [cacheType]: cache}; } /** * Loads the selected stored sessions on disk into memory diff --git a/init.js b/init.js index e7e82da..8242fd9 100644 --- a/init.js +++ b/init.js @@ -53,23 +53,20 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const t = hotReload.sessionData || this.manager.client._loadSession(this.id) - console.log(t); - const data = (t)?.[this.id] + const data = (hotReload.sessionData || this.manager.client._loadSession(this.id))?.[this.id] if(data?.id && data.sequence > 0 && !this.sessionID && data.lastConnected + 60000 > Date.now()) { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } - const cache = this.manager.client.options.hotReload.cacheData; - if(cache?.guilds && typeof cache.guilds === "object") { - const keys = Object.keys(cache.guilds); - for(const id of keys) { + const { guilds } = this.manager.client.options.hotReload.cacheData; + if(guilds) { + for(const [id, guild] of Object.entries(guilds)) { if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { - this.manager.client.guilds.add(cache.guilds[id]); + this.manager.client.guilds.add(guild); } } } else { - const guilds = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards)); + const { guilds } = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id); for(const guild of Object.values(guilds)) { this.manager.client.guilds.add(guild); } From c9660588fe91d2cbdf8ebaf60a4bd936e8aecddd Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:30:11 +0100 Subject: [PATCH 109/120] Identify without loading cache if creating a new session If session resume checks fail the client will try to identify and not resume. If they are identifying they will receive new data so we don't need to restore old data --- init.js | 1 + 1 file changed, 1 insertion(+) diff --git a/init.js b/init.js index 8242fd9..9b34683 100644 --- a/init.js +++ b/init.js @@ -58,6 +58,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } + else { return super.identify(); } const { guilds } = this.manager.client.options.hotReload.cacheData; if(guilds) { for(const [id, guild] of Object.entries(guilds)) { From 76c52aacb605069870b1dfb0b5ad1d4467b9e756 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:32:38 +0100 Subject: [PATCH 110/120] Update init.js --- init.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/init.js b/init.js index 9b34683..5b8537d 100644 --- a/init.js +++ b/init.js @@ -59,9 +59,9 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { this.closeSequence = this.sequence = data.sequence; } else { return super.identify(); } - const { guilds } = this.manager.client.options.hotReload.cacheData; - if(guilds) { - for(const [id, guild] of Object.entries(guilds)) { + const cache = this.manager.client.options.hotReload.cacheData; + if(cache?.guilds) { + for(const [id, guild] of Object.entries(cache.guilds)) { if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { this.manager.client.guilds.add(guild); } From 74868b9b815ea4080d1a6c372940d55fcd148bd2 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:52:13 +0100 Subject: [PATCH 111/120] Moved loading of cache into resume event I kept getting 4003 Not authenticated errors from the gateway meaning we were sending packets before identifying. There must be some WS connection in our load cache method so now we wait until after the session has resumed before loading cache --- init.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/init.js b/init.js index 5b8537d..0f113f8 100644 --- a/init.js +++ b/init.js @@ -58,20 +58,21 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { this.sessionID = data.id; this.closeSequence = this.sequence = data.sequence; } - else { return super.identify(); } - const cache = this.manager.client.options.hotReload.cacheData; - if(cache?.guilds) { - for(const [id, guild] of Object.entries(cache.guilds)) { - if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { + this.once(Constants.ShardEvents.RESUMED, () => { + const cache = this.manager.client.options.hotReload.cacheData; + if(cache?.guilds) { + for(const [id, guild] of Object.entries(cache.guilds)) { + if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { + this.manager.client.guilds.add(guild); + } + } + } else { + const { guilds } = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id); + for(const guild of Object.values(guilds)) { this.manager.client.guilds.add(guild); } } - } else { - const { guilds } = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id); - for(const guild of Object.values(guilds)) { - this.manager.client.guilds.add(guild); - } - } + }) } return super.identify(); } From f9cf35b8f15a32330734049f7e1dfef3e7294f9f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 00:53:39 +0100 Subject: [PATCH 112/120] Temporary fix until we load client user --- classes.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classes.js b/classes.js index 070a988..d6184e4 100644 --- a/classes.js +++ b/classes.js @@ -288,8 +288,8 @@ Discord.Structures.extend("Guild", G => { this.members.add(member); } } - if(!this.members.cache.has(this.client.user.id)) { - this.members.fetch(this.client.user.id).catch(() => {}); + if(!this.members.cache.has(this.client.user?.id)) { + this.members.fetch(this.client.user?.id).catch(() => {}); } } if(Array.isArray(data.presences)) { From 8189e36dab87d238461efdc6141b479c5f33bfe6 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 01:38:19 +0100 Subject: [PATCH 113/120] Checks the close sequence of the last shard and doesn't wait if it exists Fixes https://github.com/timotejroiko/discord.js-light/pull/41#issuecomment-812631299 --- init.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init.js b/init.js index 0f113f8..bcdd185 100644 --- a/init.js +++ b/init.js @@ -161,7 +161,7 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { } } // If we have multiple shards add a 5s delay if identifying or no delay if resuming - if (this.shardQueue.size && Object.keys(this._hotreload).length) { + if (this.shardQueue.size && this.shards.last().closeSequence) { this.debug(`Shard Queue Size: ${this.shardQueue.size} with sessions; continuing immediately`); return this.createShards(); } else if (this.shardQueue.size) { From 28a3e19f917d793e18f976d624656d0a3cca105f Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 12:15:32 +0100 Subject: [PATCH 114/120] Moved session data to createShards First loops through all shards and requeues them if they need to identify --- init.js | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/init.js b/init.js index bcdd185..ddeb6f5 100644 --- a/init.js +++ b/init.js @@ -53,11 +53,6 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { identify() { let hotReload = this.manager.client.options.hotReload; if(hotReload) { - const data = (hotReload.sessionData || this.manager.client._loadSession(this.id))?.[this.id] - if(data?.id && data.sequence > 0 && !this.sessionID && data.lastConnected + 60000 > Date.now()) { - this.sessionID = data.id; - this.closeSequence = this.sequence = data.sequence; - } this.once(Constants.ShardEvents.RESUMED, () => { const cache = this.manager.client.options.hotReload.cacheData; if(cache?.guilds) { @@ -90,6 +85,23 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { const [shard] = this.shardQueue; + // Pushes shards that require reidentifying to the back of the queue + const hotReload = this.client.options.hotReload; + if (hotReload) { + const data = (hotReload.sessionData || this.client._loadSession(shard.id))?.[shard.id] + if(data?.id && data.sequence > 0 && !shard.sessionID && data.lastConnected + 60000 > Date.now()) { + shard.sessionID = data.id; + shard.closeSequence = shard.sequence = data.sequence; + } + else if (this.shardQueue.size > 1 && !shard.requeued) { + shard.requeued = true; + this.shardQueue.delete(shard); + this.shardQueue.add(shard); + this.debug("Shard required to identify, pushed to the back of the queue", shard); + return this.createShards(); + } + } + this.shardQueue.delete(shard); if (!shard.eventsAttached) { From 9f7df3b9c81e1b40568865415c96e2aa89aaaad1 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 12:34:28 +0100 Subject: [PATCH 115/120] Stores shardCount in session object Checks if the session is required to reidentify. Added a few extra debug logs --- client.d.ts | 1 + client.js | 4 +++- init.js | 8 +++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/client.d.ts b/client.d.ts index 3ebd5a6..897caaa 100644 --- a/client.d.ts +++ b/client.d.ts @@ -53,6 +53,7 @@ type SessionData = { id: string sequence: number lastConnected: number + shardCount: number } } diff --git a/client.js b/client.js index e911db4..5c2c32d 100644 --- a/client.js +++ b/client.js @@ -77,6 +77,7 @@ Discord.Client = class Client extends Discord.Client { for(const eventType of ["exit", "uncaughtException", "SIGINT", "SIGTERM"]) { process.on(eventType, async (...args) => { if(!dumped) { + this.ws.debug(`${eventType} Exit event. Storing shard sessions and cache.`); dumped = true; const cache = this.dumpCache(); const sessions = this.dumpSessions(); @@ -135,7 +136,8 @@ Discord.Client = class Client extends Discord.Client { a[s.id] = { id: s.sessionID, sequence: s.sequence, - lastConnected: Date.now() + lastConnected: Date.now(), + shardCount: this.ws.shards.size }; return a; }, {}); diff --git a/init.js b/init.js index ddeb6f5..9d21b2a 100644 --- a/init.js +++ b/init.js @@ -54,6 +54,7 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { let hotReload = this.manager.client.options.hotReload; if(hotReload) { this.once(Constants.ShardEvents.RESUMED, () => { + this.debug("Shard session resumed. Restoring cache"); const cache = this.manager.client.options.hotReload.cacheData; if(cache?.guilds) { for(const [id, guild] of Object.entries(cache.guilds)) { @@ -89,15 +90,16 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { const hotReload = this.client.options.hotReload; if (hotReload) { const data = (hotReload.sessionData || this.client._loadSession(shard.id))?.[shard.id] - if(data?.id && data.sequence > 0 && !shard.sessionID && data.lastConnected + 60000 > Date.now()) { + if(data?.id && data.sequence > 0 && !shard.sessionID && data.shardCount === this.totalShards && data.lastConnected + 60000 > Date.now()) { shard.sessionID = data.id; shard.closeSequence = shard.sequence = data.sequence; + this.debug("Loaded sessions from cache, resuming previous session.", shard); } else if (this.shardQueue.size > 1 && !shard.requeued) { shard.requeued = true; this.shardQueue.delete(shard); this.shardQueue.add(shard); - this.debug("Shard required to identify, pushed to the back of the queue", shard); + this.debug("Shard required to identify, pushed to the back of the queue.", shard); return this.createShards(); } } @@ -174,7 +176,7 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { } // If we have multiple shards add a 5s delay if identifying or no delay if resuming if (this.shardQueue.size && this.shards.last().closeSequence) { - this.debug(`Shard Queue Size: ${this.shardQueue.size} with sessions; continuing immediately`); + this.debug(`Shard Queue Size: ${this.shardQueue.size} with sessions; continuing immediately.`); return this.createShards(); } else if (this.shardQueue.size) { this.debug(`Shard Queue Size: ${this.shardQueue.size}; continuing in 5s seconds...`); From 46d3d2e31d1774e28d72ba42be63d5f314e17c47 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 13:35:29 +0100 Subject: [PATCH 116/120] shards.size -> totalShards --- client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client.js b/client.js index 5c2c32d..ed494ce 100644 --- a/client.js +++ b/client.js @@ -137,7 +137,7 @@ Discord.Client = class Client extends Discord.Client { id: s.sessionID, sequence: s.sequence, lastConnected: Date.now(), - shardCount: this.ws.shards.size + shardCount: this.ws.totalShards }; return a; }, {}); From fd20123312e2b75e6f8f77e1e89048ed9e3f69bc Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 13:57:29 +0100 Subject: [PATCH 117/120] Moved cache patching and added 15s timeout for resuming Moved cache patching to WebSocketManager and added timeout of 15s to remove the loadCache event. Now only attaches the event if a session ID is present to stop the event being attached on an identify connection attempt I don't know the best way to remove the event listener in the timeout so let me know how you would do it. --- init.js | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/init.js b/init.js index 9d21b2a..e477445 100644 --- a/init.js +++ b/init.js @@ -50,28 +50,6 @@ require.cache[SHPath].exports = class WebSocketShard extends SH { this.emitReady(); }, 15000); } - identify() { - let hotReload = this.manager.client.options.hotReload; - if(hotReload) { - this.once(Constants.ShardEvents.RESUMED, () => { - this.debug("Shard session resumed. Restoring cache"); - const cache = this.manager.client.options.hotReload.cacheData; - if(cache?.guilds) { - for(const [id, guild] of Object.entries(cache.guilds)) { - if(ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id) { - this.manager.client.guilds.add(guild); - } - } - } else { - const { guilds } = this.manager.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.manager.totalShards) === this.id); - for(const guild of Object.values(guilds)) { - this.manager.client.guilds.add(guild); - } - } - }) - } - return super.identify(); - } }; const SHMPath = resolve(require.resolve("discord.js").replace("index.js", "/client/websocket/WebSocketManager.js")); @@ -156,6 +134,34 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { this.reconnect(); }); + const hotReload = this.client.options.hotReload; + if(hotReload && shard.sessionID) { + shard.once(Constants.ShardEvents.RESUMED, () => { + this.debug("Shard session resumed. Restoring cache", shard); + shard.loadCacheTimeout = null; + this.client.clearTimeout(this.loadCacheTimeout); + const cache = hotReload.cacheData; + if(cache?.guilds) { + for(const [id, guild] of Object.entries(cache.guilds)) { + if(ShardClientUtil.shardIDForGuildID(id, this.totalShards) === shard.id) { + this.client.guilds.add(guild); + } + } + } else { + const { guilds } = this.client._loadCache("guilds", id => ShardClientUtil.shardIDForGuildID(id, this.totalShards) === shard.id); + for(const guild of Object.values(guilds)) { + this.client.guilds.add(guild); + } + } + }) + + shard.loadCacheTimeout = this.client.setTimeout(() => { + this.debug("Shard cache was never loaded as the session didn't resume in 15s", shard); + shard.loadCacheTimeout = null; + shard.removeEventListener(Constants.ShardEvents.RESUMED) // Remove the event in a better way? + }, 15000); + } + shard.eventsAttached = true; } From f738d16aaf8860b0cf8328e88e9dfc9aeaee8fa3 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 14:09:08 +0100 Subject: [PATCH 118/120] Event listener removed --- init.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init.js b/init.js index e477445..035bfcf 100644 --- a/init.js +++ b/init.js @@ -138,8 +138,8 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { if(hotReload && shard.sessionID) { shard.once(Constants.ShardEvents.RESUMED, () => { this.debug("Shard session resumed. Restoring cache", shard); + this.client.clearTimeout(shard.loadCacheTimeout); shard.loadCacheTimeout = null; - this.client.clearTimeout(this.loadCacheTimeout); const cache = hotReload.cacheData; if(cache?.guilds) { for(const [id, guild] of Object.entries(cache.guilds)) { @@ -158,7 +158,7 @@ require.cache[SHMPath].exports = class WebSocketManager extends SHM { shard.loadCacheTimeout = this.client.setTimeout(() => { this.debug("Shard cache was never loaded as the session didn't resume in 15s", shard); shard.loadCacheTimeout = null; - shard.removeEventListener(Constants.ShardEvents.RESUMED) // Remove the event in a better way? + shard.removeListener(Constants.ShardEvents.RESUMED, shard.listeners(Constants.ShardEvents.RESUMED)[0]) // Remove the event in a better way? }, 15000); } From 062778977892921e71cb2fc5a4aaab85df5bd065 Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 16:28:08 +0100 Subject: [PATCH 119/120] Cleanup --- init.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/init.js b/init.js index 035bfcf..6d3e622 100644 --- a/init.js +++ b/init.js @@ -7,8 +7,6 @@ const APIMessage = require(resolve(require.resolve("discord.js").replace("index. const Util = require(resolve(require.resolve("discord.js").replace("index.js", "/util/Util.js"))); const { Error: DJSError } = require(resolve(require.resolve("discord.js").replace("index.js", "/errors"))); const ShardClientUtil = require(resolve(require.resolve("discord.js").replace("index.js", "/sharding/ShardClientUtil.js"))); -const Util = require(resolve(require.resolve("discord.js").replace("index.js", "/util/Util.js"))); -const { Error: DJSError } = require(resolve(require.resolve("discord.js").replace("index.js", "/errors"))); const RHPath = resolve(require.resolve("discord.js").replace("index.js", "/rest/APIRequest.js")); const RH = require(RHPath); From b4ee875cd74ec008305b206f56e7d4f8ad71068e Mon Sep 17 00:00:00 2001 From: Euan <56805259+Ortovoxx@users.noreply.github.com> Date: Sat, 3 Apr 2021 17:05:53 +0100 Subject: [PATCH 120/120] Fixes from rebase Swapped Channel around and added optional chaining to nullable properties --- classes.js | 6 ++--- init.js | 76 ++++++++++++++++++++++++++++-------------------------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/classes.js b/classes.js index d6184e4..a3e692e 100644 --- a/classes.js +++ b/classes.js @@ -159,7 +159,7 @@ Discord.Structures.extend("Message", M => { party_id: this.activity.partyID, type: this.activity.type } : void 0, - member: this.member._unpatch(), + member: this.member?._unpatch(), flags: this.flags.valueOf(), message_reference: this.reference ? { channel_id: this.reference.channelID, @@ -343,7 +343,7 @@ Discord.Structures.extend("Guild", G => { mfa_level: this.mfaLevel, joinedTimestamp: this.joinedTimestamp, default_message_notifications: this.defaultMessageNotifications, - system_channel_flags: this.systemChannelFlags.valueOf(), + system_channel_flags: this.systemChannelFlags?.valueOf(), max_members: this.maximumMembers, max_presences: this.maximumPresences, approximate_member_count: this.approximateMemberCount, @@ -361,7 +361,7 @@ Discord.Structures.extend("Guild", G => { owner_id: this.ownerID, presences: this.presences.cache.map(x => x._unpatch()), voice_states: this.voiceStates.cache.map(x => x._unpatch()), - emojis: this.emojis.cache.map(x => x._unpatch()) + emojis: this.emojis?.cache.map(x => x._unpatch()) }; } get nameAcronym() { diff --git a/init.js b/init.js index 6d3e622..c46e9e5 100644 --- a/init.js +++ b/init.js @@ -246,6 +246,44 @@ require.cache[ALPath].exports = class GuildAuditLogs extends AL { } }; +const CPath = resolve(require.resolve("discord.js").replace("index.js", "/structures/Channel.js")); +const C = require(CPath); +require.cache[CPath].exports = class Channel extends C { + _unpatch() { + let obj = { + type: Constants.ChannelTypes[this.type.toUpperCase()], + id: this.id + }; + if(this.messages) { + obj.last_message_id = this.lastMessageID; + obj.last_pin_timestamp = this.lastPinTimestamp; + } + switch(this.type) { + case "dm": { + obj.recipients = [this.recipient._unpatch()]; + break; + } + case "text": case "news": { + obj.nsfw = this.nsfw; + obj.topic = this.topic; + obj.rate_limit_per_user = this.rateLimitPerUser; + obj.messages = this.messages.cache.map(x => x._unpatch()); + break; + } + case "voice": { + obj.bitrate = this.bitrate; + obj.user_limit = this.userLimit + break; + } + case "store": { + obj.nsfw = this.nsfw; + break; + } + } + return obj; + } +}; + const TXPath = resolve(require.resolve("discord.js").replace("index.js", "/structures/interfaces/TextBasedChannel.js")); const TX = require(TXPath); require.cache[TXPath].exports = class TextBasedChannel extends TX { @@ -314,43 +352,7 @@ require.cache[GCPath].exports = class GuildChannel extends GC { } }; -const CPath = resolve(require.resolve("discord.js").replace("index.js", "/structures/Channel.js")); -const C = require(CPath); -require.cache[CPath].exports = class Channel extends C { - _unpatch() { - let obj = { - type: Constants.ChannelTypes[this.type.toUpperCase()], - id: this.id - }; - if(this.messages) { - obj.last_message_id = this.lastMessageID; - obj.last_pin_timestamp = this.lastPinTimestamp; - } - switch(this.type) { - case "dm": { - obj.recipients = [this.recipient._unpatch()]; - break; - } - case "text": case "news": { - obj.nsfw = this.nsfw; - obj.topic = this.topic; - obj.rate_limit_per_user = this.rateLimitPerUser; - obj.messages = this.messages.cache.map(x => x._unpatch()); - break; - } - case "voice": { - obj.bitrate = this.bitrate; - obj.user_limit = this.userLimit - break; - } - case "store": { - obj.nsfw = this.nsfw; - break; - } - } - return obj; - } -}; + const Action = require(resolve(require.resolve("discord.js").replace("index.js", "/client/actions/Action.js"))); Action.prototype.getPayload = function(data, manager, id, partialType, cache) {