From fb2f1ff96ae9560d71907ded00005f565d7ebdf8 Mon Sep 17 00:00:00 2001 From: TheArcaneBrony Date: Mon, 24 Apr 2023 17:20:39 +0200 Subject: [PATCH 01/14] [Squashed] Add nix flake, pg/mysql as deps a: nix test Working build Remove some unused assets Some path fixes Make flake work Add more build targets Update flake.nix Update flake.nix Rebase and update inputs rebase? Add pg dependency Rebase and add pg/mysql --- .gitignore | 3 +- assets/inline-plugins/.gitkeep | 0 assets/plugins/.gitkeep | 0 assets/preload-plugins/.gitkeep | 0 assets/preload-plugins/checkLocale.js | 38 ----- assets/preload-plugins/oauth2.js | 9 -- assets/public/fosscord-login.css | 68 -------- assets/public/fosscord.css | 92 ----------- assets/public/user.css | 1 - assets/webrtc.js | 82 ---------- flake.lock | 61 +++++++ flake.nix | 67 ++++++++ flake.template.nix | 67 ++++++++ nix-build-test.sh | 3 + nix-rebuild-flake.sh | 4 + package-lock.json | 198 +++++++++++++++++++++++ package.json | 2 + src/cdn/routes/embed.ts | 4 +- src/util/connections/ConnectionLoader.ts | 2 +- 19 files changed, 407 insertions(+), 294 deletions(-) delete mode 100644 assets/inline-plugins/.gitkeep delete mode 100644 assets/plugins/.gitkeep delete mode 100644 assets/preload-plugins/.gitkeep delete mode 100644 assets/preload-plugins/checkLocale.js delete mode 100644 assets/preload-plugins/oauth2.js delete mode 100644 assets/public/fosscord-login.css delete mode 100644 assets/public/fosscord.css delete mode 100644 assets/public/user.css delete mode 100644 assets/webrtc.js create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 flake.template.nix create mode 100755 nix-build-test.sh create mode 100755 nix-rebuild-flake.sh diff --git a/.gitignore b/.gitignore index bc780d640..e62c03d60 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ build *.log.ansi *.tmp tmp/ -dump/ \ No newline at end of file +dump/ +result diff --git a/assets/inline-plugins/.gitkeep b/assets/inline-plugins/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/assets/plugins/.gitkeep b/assets/plugins/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/assets/preload-plugins/.gitkeep b/assets/preload-plugins/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/assets/preload-plugins/checkLocale.js b/assets/preload-plugins/checkLocale.js deleted file mode 100644 index 1b5e7b719..000000000 --- a/assets/preload-plugins/checkLocale.js +++ /dev/null @@ -1,38 +0,0 @@ -const supportedLocales = [ - "bg", - "cs", - "da", - "de", - "el", - "en-GB", - "es-ES", - "fi", - "fr", - "hi", - "hr", - "hu", - "it", - "ja", - "ko", - "lt", - "nl", - "no", - "pl", - "pt-BR", - "ro", - "ru", - "sv-SE", - "th", - "tr", - "uk", - "vi", - "zh-CN", - "zh-TW" -]; - -const settings = JSON.parse(window.localStorage.getItem("UserSettingsStore")); -if (settings && !supportedLocales.includes(settings.locale)) { - // fix client locale wrong and client not loading at all - settings.locale = "en-US"; - window.localStorage.setItem("UserSettingsStore", JSON.stringify(settings)); -} \ No newline at end of file diff --git a/assets/preload-plugins/oauth2.js b/assets/preload-plugins/oauth2.js deleted file mode 100644 index 5b78ec83b..000000000 --- a/assets/preload-plugins/oauth2.js +++ /dev/null @@ -1,9 +0,0 @@ -// Fixes /oauth2 endpoints not requesting a CSS file - -if (location.pathname.startsWith("/oauth2/")) { - const link = document.createElement("link"); - link.rel = "stylesheet" - link.type = "text/css" - link.href = "/assets/40532.f7b1e10347ef10e790ac.css" - document.head.appendChild(link) -} \ No newline at end of file diff --git a/assets/public/fosscord-login.css b/assets/public/fosscord-login.css deleted file mode 100644 index e66e70a00..000000000 --- a/assets/public/fosscord-login.css +++ /dev/null @@ -1,68 +0,0 @@ -/* replace tos acceptance popup */ -#app-mount > div:nth-child(7) > div > div > div.tooltipContent-bqVLWK { - visibility: hidden; -} -#app-mount > div:nth-child(7) > div > div > div.tooltipContent-bqVLWK::after { - visibility: visible; - display: block; - content: "You need to agree to this instance's rules to continue"; - margin-top: -32px; -} -/* replace login header */ -#app-mount > div.app-1q1i1E > div > div > div > div > form > div > div > div.mainLoginContainer-1ddwnR > h3 { - visibility: hidden; -} -h3.title-jXR8lp.marginBottom8-AtZOdT.base-1x0h_U.size24-RIRrxO::after { - margin-top: -32px; - content: "Welcome to Spacebar!"; - visibility: visible; - display: block; -} - -/* Logo in top left when bg removed */ -#app-mount > div.app-1q1i1E > div > a { - /* replace me: original dimensions: 130x36 */ - background: url(https://raw.githubusercontent.com/spacebarchat/spacebarchat/master/branding/svg/Spacebar__Logo-Blue.svg); - width: 130px; - height: 23px; - background-size: contain; -} - -/* replace TOS text */ - -#app-mount - > div.app-1q1i1E - > div - > div - > div - > form - > div - > div - > div.flex-1xMQg5.flex-1O1GKY.horizontal-1ae9ci.horizontal-2EEEnY.flex-1O1GKY.directionRow-3v3tfG.justifyStart-2NDFzi.alignCenter-1dQNNs.noWrap-3jynv6.marginTop20-3TxNs6 - > label - > div.label-cywgfr.labelClickable-11AuB8.labelForward-1wfipV - > * { - visibility: hidden; -} - -#app-mount - > div.app-1q1i1E - > div - > div - > div - > form - > div - > div - > div.flex-1xMQg5.flex-1O1GKY.horizontal-1ae9ci.horizontal-2EEEnY.flex-1O1GKY.directionRow-3v3tfG.justifyStart-2NDFzi.alignCenter-1dQNNs.noWrap-3jynv6.marginTop20-3TxNs6 - > label - > div.label-cywgfr.labelClickable-11AuB8.labelForward-1wfipV::after { - visibility: visible; - content: "I have read and agree with the rules set by this instance."; - display: block; - margin-top: -16px; -} - -/* shrink login box to same size as register */ -.authBoxExpanded-2jqaBe { - width: 480px !important; -} \ No newline at end of file diff --git a/assets/public/fosscord.css b/assets/public/fosscord.css deleted file mode 100644 index 53ffd41b3..000000000 --- a/assets/public/fosscord.css +++ /dev/null @@ -1,92 +0,0 @@ -/* loading spinner */ -#app-mount > div.app-1q1i1E > div.container-16j22k.fixClipping-3qAKRb > div.content-1-zrf2 > video { - filter: opacity(1); - background: url("http://www.clipartbest.com/cliparts/7ca/6Rr/7ca6RrLAi.gif"); - background-size: contain; - /* width: 64px; - height: 64px; */ - padding-bottom: 64px; - background-repeat: no-repeat; -} - -/* home button icon */ -#app-mount - > div.app-1q1i1E - > div - > div.layers-3iHuyZ.layers-3q14ss - > div - > div - > nav - > ul - > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih - > div.tutorialContainer-2sGCg9 - > div - > div.listItemWrapper-KhRmzM - > div - > svg - > foreignObject - > div - > div { - background-image: url(https://raw.githubusercontent.com/spacebarchat/spacebarchat/master/branding/svg/Spacebar__Icon-Rounded-Subtract.svg); - background-size: contain; - border-radius: 50%; -} - -#app-mount - > div.app-1q1i1E - > div - > div.layers-3iHuyZ.layers-3q14ss - > div - > div - > nav - > ul - > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih - > div.tutorialContainer-2sGCg9 - > div - > div.listItemWrapper-KhRmzM - > div - > svg - > foreignObject - > div - > div, -#app-mount - > div.app-1q1i1E - > div - > div.layers-3iHuyZ.layers-3q14ss - > div - > div - > nav - > ul - > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih - > div.tutorialContainer-2sGCg9 - > div - > div.listItemWrapper-KhRmzM - > div - > svg - > foreignObject - > div - > div:hover { - background-color: white; -} -/* Login QR */ -#app-mount > div.app-1q1i1E > div > div > div > div > form > div > div > div.transitionGroup-aR7y1d.qrLogin-1AOZMt, -#app-mount > div.app-1q1i1E > div > div > div > div > form > div > div > div.verticalSeparator-3huAjp, -/* Remove login bg */ -#app-mount > div.app-1q1i1E > div > svg, -/* Download bar */ -#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > div > div.notice-3bPHh-.colorDefault-22HBa0, -/* Connection problem links */ -#app-mount > div.app-1q1i1E > div.container-16j22k.fixClipping-3qAKRb > div.problems-3mgf6w.slideIn-sCvzGz > div:nth-child(2), -/* Downloads button */ -#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > nav > ul > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih > div:nth-child(7) > div.listItemWrapper-KhRmzM > div > svg > foreignObject > div, -#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > nav > ul > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih > div:nth-child(6) > div, -/* help button */ -#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > div > div.content-98HsJk > div.chat-3bRxxu > section > div.toolbar-1t6TWx > a, -/* download button start of guild */ -#chat-messages-899316648933185083 > div > div > div:nth-child(5), -/* Thread permissions etc popups */ -#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > div > div.content-98HsJk > div.sidebar-2K8pFh.hasNotice-1XRy4h > nav > div.container-3O_wAf, -/* home button icon */ -#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > nav > ul > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih > div.tutorialContainer-2sGCg9 > div > div.listItemWrapper-KhRmzM > div > svg > foreignObject > div > div > svg { - display: none; -} \ No newline at end of file diff --git a/assets/public/user.css b/assets/public/user.css deleted file mode 100644 index a7e5c4f3d..000000000 --- a/assets/public/user.css +++ /dev/null @@ -1 +0,0 @@ -/* Your custom CSS goes here, enjoy! */ \ No newline at end of file diff --git a/assets/webrtc.js b/assets/webrtc.js deleted file mode 100644 index b56e41c4f..000000000 --- a/assets/webrtc.js +++ /dev/null @@ -1,82 +0,0 @@ -/* - This file is used to patch client version 134842 ( and probably a lot more ) to send additional info when using webrtc. - If you want to use it, throw it into the `preload-plugins` folder. - TODO: Make it so this file is not required for webrtc. - - Do note that webrtc, as of 17/12/2022, is not implemented yet in spacebarchat/server. -*/ - -(this.webpackChunkdiscord_app = this.webpackChunkdiscord_app || []).push([ - [[228974]], - { - 632540: (module, exports, req) => { - window.find = (filter, options = {}) => { - const { cacheOnly = false } = options; - for (let i in req.c) { - if (req.c.hasOwnProperty(i)) { - let m = req.c[i].exports; - if (m && m.__esModule && m.default && filter(m.default)) return m.default; - if (m && filter(m)) return m; - } - } - if (cacheOnly) { - console.warn("Cannot find loaded module in cache"); - return null; - } - console.warn("Cannot find loaded module in cache. Loading all modules may have unexpected side effects"); - for (let i = 0; i < req.m.length; ++i) { - let m = req(i); - if (m && m.__esModule && m.default && filter(m.default)) return m.default; - if (m && filter(m)) return m; - } - console.warn("Cannot find module"); - return null; - }; - window.findByUniqueProperties = (propNames, options) => - find((module) => propNames.every((prop) => module[prop] !== undefined), options); - window.findByDisplayName = (displayName, options) => find((module) => module.displayName === displayName, options); - window.req = req; - - init(); - } - }, - (t) => t(632540) -]); - -function retry(callback) { - return new Promise((resolve) => { - const interval = setInterval(() => { - const mod = callback(); - if (!mod) return; - - clearInterval(interval); - resolve(mod); - }, 50); - }); -} - -async function init() { - const SDP = await retry(() => findByUniqueProperties(["truncateSDP"])); - const StringManipulator = findByUniqueProperties(["uniq"]); - - const truncateSDP = SDP.truncateSDP; - SDP.truncateSDP = (e) => { - const result = truncateSDP(e); - const i = result.codecs.find((x) => x.name === "VP8"); - const a = new RegExp("^a=ice|a=extmap|opus|VP8|fingerprint|" + i?.rtxPayloadType + " rtx", "i"); - return { - sdp: StringManipulator(e) - .split(/\r\n/) - .filter(function (e) { - return a.test(e); - }) - .uniq() - .join("\n"), - codecs: result.codecs - }; - }; - // SDP.generateUnifiedSessionDescription = (e) => { - // console.log(e); - // return new RTCSessionDescription({ sdp: e.baseSDP.replace(/sendonly/g, "recvonly"), type: "answer" }); - // }; -} \ No newline at end of file diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..126832bed --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1685518550, + "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1685655444, + "narHash": "sha256-6EujQNAeaUkWvpEZZcVF8qSfQrNVWFNNGbUJxv/A5a8=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "e635192892f5abbc2289eaac3a73cdb249abaefd", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..a6550a849 --- /dev/null +++ b/flake.nix @@ -0,0 +1,67 @@ +{ + description = "Spacebar server, written in Typescript."; + + inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachSystem flake-utils.lib.allSystems (system: + let + pkgs = import nixpkgs { + inherit system; + }; + in rec { + packages.default = pkgs.buildNpmPackage { + pname = "spacebar-server-ts"; + src = ./.; + name = "spacebar-server-ts"; + nativeBuildInputs = with pkgs; [ python3 ]; + npmDepsHash = "sha256-sjKGxaSPm/Uzi8viOySNhEY122GUdFRGZuyyVMVqCtY="; + makeCacheWritable = true; + postPatch = '' + substituteInPlace package.json --replace 'npx patch-package' '${pkgs.nodePackages.patch-package}/bin/patch-package' + ''; + installPhase = '' + runHook preInstall + set -x + #remove packages not needed for production, or at least try to... + npm prune --omit dev --no-save $npmInstallFlags "''${npmInstallFlagsArray[@]}" $npmFlags "''${npmFlagsArray[@]}" + find node_modules -maxdepth 1 -type d -empty -delete + + mkdir -p $out/node_modules/ + cp -r node_modules/* $out/node_modules/ + cp -r dist/ $out/node_modules/@spacebar + for i in dist/**/start.js + do + makeWrapper ${pkgs.nodejs-slim}/bin/node $out/bin/start-`dirname ''${i/dist\//}` --prefix NODE_PATH : $out/node_modules --add-flags $out/node_modules/@spacebar`dirname ''${i/dist/}`/start.js + done + set +x + substituteInPlace package.json --replace 'dist/' 'node_modules/@spacebar/' + find $out/node_modules/@spacebar/ -type f -name "*.js" | while read srcFile; do + echo Patching imports in ''${srcFile/$out\/node_modules\/@spacebar//}... + substituteInPlace $srcFile --replace 'require("./' 'require(__dirname + "/' + substituteInPlace $srcFile --replace 'require("../' 'require(__dirname + "/../' + substituteInPlace $srcFile --replace ', "assets"' ', "..", "assets"' + #substituteInPlace $srcFile --replace 'require("@spacebar/' 'require(" + done + set -x + cp -r assets/ $out/ + cp package.json $out/ + rm -v $out/assets/openapi.json + #rm -v $out/assets/schemas.json + + #debug utils: + #cp $out/node_modules/@spacebar/ $out/build_output -r + set +x + runHook postInstall + ''; + }; + devShell = pkgs.mkShell { + buildInputs = with pkgs; [ + nodejs + nodePackages.typescript + ]; + }; + } + ); +} diff --git a/flake.template.nix b/flake.template.nix new file mode 100644 index 000000000..be4261009 --- /dev/null +++ b/flake.template.nix @@ -0,0 +1,67 @@ +{ + description = "Spacebar server, written in Typescript."; + + inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachSystem flake-utils.lib.allSystems (system: + let + pkgs = import nixpkgs { + inherit system; + }; + in rec { + packages.default = pkgs.buildNpmPackage { + pname = "spacebar-server-ts"; + src = ./.; + name = "spacebar-server-ts"; + nativeBuildInputs = with pkgs; [ python3 ]; + npmDepsHash = "$NPM_HASH"; + makeCacheWritable = true; + postPatch = '' + substituteInPlace package.json --replace 'npx patch-package' '${pkgs.nodePackages.patch-package}/bin/patch-package' + ''; + installPhase = '' + runHook preInstall + set -x + #remove packages not needed for production, or at least try to... + npm prune --omit dev --no-save $npmInstallFlags "''${npmInstallFlagsArray[@]}" $npmFlags "''${npmFlagsArray[@]}" + find node_modules -maxdepth 1 -type d -empty -delete + + mkdir -p $out/node_modules/ + cp -r node_modules/* $out/node_modules/ + cp -r dist/ $out/node_modules/@spacebar + for i in dist/**/start.js + do + makeWrapper ${pkgs.nodejs-slim}/bin/node $out/bin/start-`dirname ''${i/dist\//}` --prefix NODE_PATH : $out/node_modules --add-flags $out/node_modules/@spacebar`dirname ''${i/dist/}`/start.js + done + set +x + substituteInPlace package.json --replace 'dist/' 'node_modules/@spacebar/' + find $out/node_modules/@spacebar/ -type f -name "*.js" | while read srcFile; do + echo Patching imports in ''${srcFile/$out\/node_modules\/@spacebar//}... + substituteInPlace $srcFile --replace 'require("./' 'require(__dirname + "/' + substituteInPlace $srcFile --replace 'require("../' 'require(__dirname + "/../' + substituteInPlace $srcFile --replace ', "assets"' ', "..", "assets"' + #substituteInPlace $srcFile --replace 'require("@spacebar/' 'require(" + done + set -x + cp -r assets/ $out/ + cp package.json $out/ + rm -v $out/assets/openapi.json + #rm -v $out/assets/schemas.json + + #debug utils: + #cp $out/node_modules/@spacebar/ $out/build_output -r + set +x + runHook postInstall + ''; + }; + devShell = pkgs.mkShell { + buildInputs = with pkgs; [ + nodejs + nodePackages.typescript + ]; + }; + } + ); +} diff --git a/nix-build-test.sh b/nix-build-test.sh new file mode 100755 index 000000000..5c6914620 --- /dev/null +++ b/nix-build-test.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +#nix build --update-input pnpm2nix --debugger --ignore-try +nix build --debugger --ignore-try --print-out-paths --print-build-logs --http2 "$@" diff --git a/nix-rebuild-flake.sh b/nix-rebuild-flake.sh new file mode 100755 index 000000000..08f55840b --- /dev/null +++ b/nix-rebuild-flake.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i "bash -x" -p bash +DEPS_HASH=`nix run nixpkgs#prefetch-npm-deps -- package-lock.json` +sed 's/$NPM_HASH/'${DEPS_HASH/\//\\\/}'/g' flake.template.nix > flake.nix diff --git a/package-lock.json b/package-lock.json index bc58336ff..a6aedf3ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,10 +38,12 @@ "morgan": "^1.10.0", "multer": "^1.4.5-lts.1", "murmurhash-js": "^1.0.0", + "mysql": "^2.18.1", "node-2fa": "^2.0.3", "node-fetch": "^2.6.12", "node-os-utils": "^1.3.7", "nodemailer": "^6.9.4", + "pg": "^8.11.3", "picocolors": "^1.0.0", "probe-image-size": "^7.2.3", "proxy-agent": "^6.3.0", @@ -2996,6 +2998,14 @@ "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -5724,6 +5734,55 @@ "resolved": "https://registry.npmjs.org/murmurhash-js/-/murmurhash-js-1.0.0.tgz", "integrity": "sha512-TvmkNhkv8yct0SVBSy+o8wYzXjE4Zz3PCesbfs8HiCXXdcTuocApFv11UWlNFWKYsP2okqrhb7JNlSm9InBhIw==" }, + "node_modules/mysql": { + "version": "2.18.1", + "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz", + "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==", + "dependencies": { + "bignumber.js": "9.0.0", + "readable-stream": "2.3.7", + "safe-buffer": "5.1.2", + "sqlstring": "2.3.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mysql/node_modules/bignumber.js": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", + "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==", + "engines": { + "node": "*" + } + }, + "node_modules/mysql/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "node_modules/mysql/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/mysql/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -6276,6 +6335,11 @@ "node": ">= 14" } }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -6371,6 +6435,89 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/pg": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.1.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", @@ -6403,6 +6550,41 @@ "node": ">=12.0.0" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -7105,6 +7287,14 @@ "source-map": "^0.6.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sqlite3": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.1.6.tgz", @@ -7134,6 +7324,14 @@ "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==", "optional": true }, + "node_modules/sqlstring": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", + "integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/ssri": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz", diff --git a/package.json b/package.json index a0db1d881..beebc2ea9 100644 --- a/package.json +++ b/package.json @@ -93,10 +93,12 @@ "morgan": "^1.10.0", "multer": "^1.4.5-lts.1", "murmurhash-js": "^1.0.0", + "mysql": "^2.18.1", "node-2fa": "^2.0.3", "node-fetch": "^2.6.12", "node-os-utils": "^1.3.7", "nodemailer": "^6.9.4", + "pg": "^8.11.3", "picocolors": "^1.0.0", "probe-image-size": "^7.2.3", "proxy-agent": "^6.3.0", diff --git a/src/cdn/routes/embed.ts b/src/cdn/routes/embed.ts index 9d3469cd4..a545a6f8d 100644 --- a/src/cdn/routes/embed.ts +++ b/src/cdn/routes/embed.ts @@ -63,7 +63,7 @@ router.get("/avatars/:id", async (req: Request, res: Response) => { id = id.split(".")[0]; // remove .file extension const hash = defaultAvatarHashMap.get(id); if (!hash) throw new HTTPError("not found", 404); - const path = join(process.cwd(), "assets", "public", `${hash}.png`); + const path = join(__dirname, "..", "..", "..", "assets", "public", `${hash}.png`); const file = await getFile(path); if (!file) throw new HTTPError("not found", 404); @@ -80,7 +80,7 @@ router.get("/group-avatars/:id", async (req: Request, res: Response) => { id = id.split(".")[0]; // remove .file extension const hash = defaultGroupDMAvatarHashMap.get(id); if (!hash) throw new HTTPError("not found", 404); - const path = join(process.cwd(), "assets", "public", `${hash}.png`); + const path = join(__dirname, "..", "..", "..", "assets", "public", `${hash}.png`); const file = await getFile(path); if (!file) throw new HTTPError("not found", 404); diff --git a/src/util/connections/ConnectionLoader.ts b/src/util/connections/ConnectionLoader.ts index e9dc6973e..31b8e04b2 100644 --- a/src/util/connections/ConnectionLoader.ts +++ b/src/util/connections/ConnectionLoader.ts @@ -22,7 +22,7 @@ import path from "path"; import { ConnectionConfig } from "./ConnectionConfig"; import { ConnectionStore } from "./ConnectionStore"; -const root = "dist/connections"; +const root = path.join(__dirname, "..", "..", "connections"); const connectionsLoaded = false; export class ConnectionLoader { From c71708eff705f7f40b92a66ca6ae9cc84cabde1b Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 07:51:51 +0100 Subject: [PATCH 02/14] Update nixpkgs, format inputs as a block --- flake.lock | 12 ++++++------ flake.nix | 8 +++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 126832bed..ae5e8b230 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1685518550, - "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", "owner": "numtide", "repo": "flake-utils", - "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1685655444, - "narHash": "sha256-6EujQNAeaUkWvpEZZcVF8qSfQrNVWFNNGbUJxv/A5a8=", + "lastModified": 1708118438, + "narHash": "sha256-kk9/0nuVgA220FcqH/D2xaN6uGyHp/zoxPNUmPCMmEE=", "owner": "nixos", "repo": "nixpkgs", - "rev": "e635192892f5abbc2289eaac3a73cdb249abaefd", + "rev": "5863c27340ba4de8f83e7e3c023b9599c3cb3c80", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index a6550a849..d90cabc3c 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,10 @@ { description = "Spacebar server, written in Typescript."; - inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; - inputs.flake-utils.url = "github:numtide/flake-utils"; + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; outputs = { self, nixpkgs, flake-utils }: flake-utils.lib.eachSystem flake-utils.lib.allSystems (system: @@ -16,7 +18,7 @@ src = ./.; name = "spacebar-server-ts"; nativeBuildInputs = with pkgs; [ python3 ]; - npmDepsHash = "sha256-sjKGxaSPm/Uzi8viOySNhEY122GUdFRGZuyyVMVqCtY="; + npmDepsHash = "sha256-fuW15WgfDaKPVDQx8OhRAa253J+SQDUr35rKt42KsTc="; makeCacheWritable = true; postPatch = '' substituteInPlace package.json --replace 'npx patch-package' '${pkgs.nodePackages.patch-package}/bin/patch-package' From e4e2f842676e52ce7170b24ae3ac970a79e79fbd Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 08:20:20 +0100 Subject: [PATCH 03/14] fixup! [Squashed] Add nix flake, pg/mysql as deps --- src/cdn/routes/embed.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/cdn/routes/embed.ts b/src/cdn/routes/embed.ts index a545a6f8d..b99396ebe 100644 --- a/src/cdn/routes/embed.ts +++ b/src/cdn/routes/embed.ts @@ -63,7 +63,15 @@ router.get("/avatars/:id", async (req: Request, res: Response) => { id = id.split(".")[0]; // remove .file extension const hash = defaultAvatarHashMap.get(id); if (!hash) throw new HTTPError("not found", 404); - const path = join(__dirname, "..", "..", "..", "assets", "public", `${hash}.png`); + const path = join( + __dirname, + "..", + "..", + "..", + "assets", + "public", + `${hash}.png`, + ); const file = await getFile(path); if (!file) throw new HTTPError("not found", 404); @@ -80,7 +88,15 @@ router.get("/group-avatars/:id", async (req: Request, res: Response) => { id = id.split(".")[0]; // remove .file extension const hash = defaultGroupDMAvatarHashMap.get(id); if (!hash) throw new HTTPError("not found", 404); - const path = join(__dirname, "..", "..", "..", "assets", "public", `${hash}.png`); + const path = join( + __dirname, + "..", + "..", + "..", + "assets", + "public", + `${hash}.png`, + ); const file = await getFile(path); if (!file) throw new HTTPError("not found", 404); From fee04d24b5b88ad7715c761c5e20f1d279f27f15 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 08:34:17 +0100 Subject: [PATCH 04/14] Cleanup --- flake.nix | 3 +- flake.template.nix | 67 -------------------------------------------- hashes.json | 4 +++ nix-build-test.sh | 3 -- nix-rebuild-flake.sh | 4 --- nix-update.sh | 6 ++++ 6 files changed, 12 insertions(+), 75 deletions(-) delete mode 100644 flake.template.nix create mode 100644 hashes.json delete mode 100755 nix-build-test.sh delete mode 100755 nix-rebuild-flake.sh create mode 100755 nix-update.sh diff --git a/flake.nix b/flake.nix index d90cabc3c..542de19aa 100644 --- a/flake.nix +++ b/flake.nix @@ -12,13 +12,14 @@ pkgs = import nixpkgs { inherit system; }; + hashesFile = builtins.fromJSON (builtins.readFile ./hashes.json); in rec { packages.default = pkgs.buildNpmPackage { pname = "spacebar-server-ts"; src = ./.; name = "spacebar-server-ts"; nativeBuildInputs = with pkgs; [ python3 ]; - npmDepsHash = "sha256-fuW15WgfDaKPVDQx8OhRAa253J+SQDUr35rKt42KsTc="; + npmDepsHash = hashesFile.npm_deps_hash; makeCacheWritable = true; postPatch = '' substituteInPlace package.json --replace 'npx patch-package' '${pkgs.nodePackages.patch-package}/bin/patch-package' diff --git a/flake.template.nix b/flake.template.nix deleted file mode 100644 index be4261009..000000000 --- a/flake.template.nix +++ /dev/null @@ -1,67 +0,0 @@ -{ - description = "Spacebar server, written in Typescript."; - - inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; - inputs.flake-utils.url = "github:numtide/flake-utils"; - - outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.eachSystem flake-utils.lib.allSystems (system: - let - pkgs = import nixpkgs { - inherit system; - }; - in rec { - packages.default = pkgs.buildNpmPackage { - pname = "spacebar-server-ts"; - src = ./.; - name = "spacebar-server-ts"; - nativeBuildInputs = with pkgs; [ python3 ]; - npmDepsHash = "$NPM_HASH"; - makeCacheWritable = true; - postPatch = '' - substituteInPlace package.json --replace 'npx patch-package' '${pkgs.nodePackages.patch-package}/bin/patch-package' - ''; - installPhase = '' - runHook preInstall - set -x - #remove packages not needed for production, or at least try to... - npm prune --omit dev --no-save $npmInstallFlags "''${npmInstallFlagsArray[@]}" $npmFlags "''${npmFlagsArray[@]}" - find node_modules -maxdepth 1 -type d -empty -delete - - mkdir -p $out/node_modules/ - cp -r node_modules/* $out/node_modules/ - cp -r dist/ $out/node_modules/@spacebar - for i in dist/**/start.js - do - makeWrapper ${pkgs.nodejs-slim}/bin/node $out/bin/start-`dirname ''${i/dist\//}` --prefix NODE_PATH : $out/node_modules --add-flags $out/node_modules/@spacebar`dirname ''${i/dist/}`/start.js - done - set +x - substituteInPlace package.json --replace 'dist/' 'node_modules/@spacebar/' - find $out/node_modules/@spacebar/ -type f -name "*.js" | while read srcFile; do - echo Patching imports in ''${srcFile/$out\/node_modules\/@spacebar//}... - substituteInPlace $srcFile --replace 'require("./' 'require(__dirname + "/' - substituteInPlace $srcFile --replace 'require("../' 'require(__dirname + "/../' - substituteInPlace $srcFile --replace ', "assets"' ', "..", "assets"' - #substituteInPlace $srcFile --replace 'require("@spacebar/' 'require(" - done - set -x - cp -r assets/ $out/ - cp package.json $out/ - rm -v $out/assets/openapi.json - #rm -v $out/assets/schemas.json - - #debug utils: - #cp $out/node_modules/@spacebar/ $out/build_output -r - set +x - runHook postInstall - ''; - }; - devShell = pkgs.mkShell { - buildInputs = with pkgs; [ - nodejs - nodePackages.typescript - ]; - }; - } - ); -} diff --git a/hashes.json b/hashes.json new file mode 100644 index 000000000..56228e98b --- /dev/null +++ b/hashes.json @@ -0,0 +1,4 @@ +{ + "npm_deps_hash": "sha256-fuW15WgfDaKPVDQx8OhRAa253J+SQDUr35rKt42KsTc=", + "tmp": "sha256-fuW15WgfDaKPVDQx8OhRAa253J+SQDUr35rKt42KsTc=" +} diff --git a/nix-build-test.sh b/nix-build-test.sh deleted file mode 100755 index 5c6914620..000000000 --- a/nix-build-test.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -#nix build --update-input pnpm2nix --debugger --ignore-try -nix build --debugger --ignore-try --print-out-paths --print-build-logs --http2 "$@" diff --git a/nix-rebuild-flake.sh b/nix-rebuild-flake.sh deleted file mode 100755 index 08f55840b..000000000 --- a/nix-rebuild-flake.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i "bash -x" -p bash -DEPS_HASH=`nix run nixpkgs#prefetch-npm-deps -- package-lock.json` -sed 's/$NPM_HASH/'${DEPS_HASH/\//\\\/}'/g' flake.template.nix > flake.nix diff --git a/nix-update.sh b/nix-update.sh new file mode 100755 index 000000000..c0092232c --- /dev/null +++ b/nix-update.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i "bash -x" -p bash prefetch-npm-deps jq +DEPS_HASH=`prefetch-npm-deps package-lock.json` +TMPFILE=$(mktemp) +jq '.npm_deps_hash = "'$DEPS_HASH'"' hashes.json > $TMPFILE +mv -- "$TMPFILE" hashes.json \ No newline at end of file From 09dacacf2edf351bed0ac6852ab632e9718c8dd2 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 08:43:24 +0100 Subject: [PATCH 05/14] Move pg and mysql to optional deps --- hashes.json | 2 +- package-lock.json | 34 ++++++++++++++++++++++++++++------ package.json | 4 ++-- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/hashes.json b/hashes.json index 56228e98b..1317ed225 100644 --- a/hashes.json +++ b/hashes.json @@ -1,4 +1,4 @@ { - "npm_deps_hash": "sha256-fuW15WgfDaKPVDQx8OhRAa253J+SQDUr35rKt42KsTc=", + "npm_deps_hash": "sha256-fZNDN2/fNy6Nu7tbr0RhQ8j4BP7X1Yhrh/fSTH7hbJc=", "tmp": "sha256-fuW15WgfDaKPVDQx8OhRAa253J+SQDUr35rKt42KsTc=" } diff --git a/package-lock.json b/package-lock.json index a6aedf3ed..875aba1b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,12 +38,12 @@ "morgan": "^1.10.0", "multer": "^1.4.5-lts.1", "murmurhash-js": "^1.0.0", - "mysql": "^2.18.1", + "mysql": "*", "node-2fa": "^2.0.3", "node-fetch": "^2.6.12", "node-os-utils": "^1.3.7", "nodemailer": "^6.9.4", - "pg": "^8.11.3", + "pg": "*", "picocolors": "^1.0.0", "probe-image-size": "^7.2.3", "proxy-agent": "^6.3.0", @@ -85,9 +85,11 @@ }, "optionalDependencies": { "erlpack": "^0.1.4", + "mysql": "^2.18.1", "nodemailer-mailgun-transport": "^2.1.5", "nodemailer-mailjet-transport": "github:n0script22/nodemailer-mailjet-transport", "nodemailer-sendgrid-transport": "github:Maria-Golomb/nodemailer-sendgrid-transport", + "pg": "^8.11.3", "sqlite3": "^5.1.6" } }, @@ -3002,6 +3004,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "optional": true, "engines": { "node": ">=4" } @@ -5738,6 +5741,7 @@ "version": "2.18.1", "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz", "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==", + "optional": true, "dependencies": { "bignumber.js": "9.0.0", "readable-stream": "2.3.7", @@ -5752,6 +5756,7 @@ "version": "9.0.0", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==", + "optional": true, "engines": { "node": "*" } @@ -5759,12 +5764,14 @@ "node_modules/mysql/node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "optional": true }, "node_modules/mysql/node_modules/readable-stream": { "version": "2.3.7", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "optional": true, "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -5779,6 +5786,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "optional": true, "dependencies": { "safe-buffer": "~5.1.0" } @@ -6338,7 +6346,8 @@ "node_modules/packet-reader": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", - "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==", + "optional": true }, "node_modules/parent-module": { "version": "1.0.1", @@ -6439,6 +6448,7 @@ "version": "8.11.3", "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "optional": true, "dependencies": { "buffer-writer": "2.0.0", "packet-reader": "1.0.0", @@ -6472,12 +6482,14 @@ "node_modules/pg-connection-string": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", - "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==", + "optional": true }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "optional": true, "engines": { "node": ">=4.0.0" } @@ -6486,6 +6498,7 @@ "version": "3.6.1", "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "optional": true, "peerDependencies": { "pg": ">=8.0" } @@ -6493,12 +6506,14 @@ "node_modules/pg-protocol": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", - "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==", + "optional": true }, "node_modules/pg-types": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "optional": true, "dependencies": { "pg-int8": "1.0.1", "postgres-array": "~2.0.0", @@ -6514,6 +6529,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "optional": true, "dependencies": { "split2": "^4.1.0" } @@ -6554,6 +6570,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "optional": true, "engines": { "node": ">=4" } @@ -6562,6 +6579,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "optional": true, "engines": { "node": ">=0.10.0" } @@ -6570,6 +6588,7 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "optional": true, "engines": { "node": ">=0.10.0" } @@ -6578,6 +6597,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "optional": true, "dependencies": { "xtend": "^4.0.0" }, @@ -7291,6 +7311,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "optional": true, "engines": { "node": ">= 10.x" } @@ -7328,6 +7349,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", "integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ==", + "optional": true, "engines": { "node": ">= 0.6" } diff --git a/package.json b/package.json index beebc2ea9..f506e0be5 100644 --- a/package.json +++ b/package.json @@ -93,12 +93,10 @@ "morgan": "^1.10.0", "multer": "^1.4.5-lts.1", "murmurhash-js": "^1.0.0", - "mysql": "^2.18.1", "node-2fa": "^2.0.3", "node-fetch": "^2.6.12", "node-os-utils": "^1.3.7", "nodemailer": "^6.9.4", - "pg": "^8.11.3", "picocolors": "^1.0.0", "probe-image-size": "^7.2.3", "proxy-agent": "^6.3.0", @@ -118,9 +116,11 @@ }, "optionalDependencies": { "erlpack": "^0.1.4", + "mysql": "^2.18.1", "nodemailer-mailgun-transport": "^2.1.5", "nodemailer-mailjet-transport": "github:n0script22/nodemailer-mailjet-transport", "nodemailer-sendgrid-transport": "github:Maria-Golomb/nodemailer-sendgrid-transport", + "pg": "^8.11.3", "sqlite3": "^5.1.6" } } From f952ab757713a9eca120b12c63f1e7b880769963 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 10:23:15 +0100 Subject: [PATCH 06/14] Add nix to git hook --- .husky/pre-commit | 22 +++++++++++++++++++++- nix-update.sh | 8 ++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.husky/pre-commit b/.husky/pre-commit index 1bc5a8c6d..6eec4741a 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1,24 @@ #!/usr/bin/env sh +#!nix-shell -i "bash" -p bash prefetch-npm-deps jq nodejs nix-output-monitor . "$(dirname -- "$0")/_/husky.sh" -npx -y lint-staged \ No newline at end of file +if [ -x "$(/usr/bin/env which nix-shell 2>/dev/null)" ]; then + if [ ! "$HOOK_NIX_SHELL" ]; then + echo "Nix is available, updating nix flake..." + export HOOK_NIX_SHELL=1 + nix-shell $0 + exit $? + else + # run ./nix-update.sh if package lock has changed and has no unstaged changes + if [ -n "$(git status --porcelain=v1 2>/dev/null | grep -E '^(M| M) package-lock.json')" ]; then + echo "package-lock.json has unstaged changes. Skipping update of nix dependencies." + else + ./nix-update.sh || exit $? + fi + fi +else + echo "You do not appear to have nix installed. Skipping update of nix dependencies." +fi + +npx -y lint-staged + diff --git a/nix-update.sh b/nix-update.sh index c0092232c..4413e6e0f 100755 --- a/nix-update.sh +++ b/nix-update.sh @@ -1,6 +1,10 @@ #!/usr/bin/env nix-shell -#!nix-shell -i "bash -x" -p bash prefetch-npm-deps jq +#!nix-shell -i "bash -x" -p bash prefetch-npm-deps jq git nix-output-monitor +nix flake update DEPS_HASH=`prefetch-npm-deps package-lock.json` TMPFILE=$(mktemp) jq '.npm_deps_hash = "'$DEPS_HASH'"' hashes.json > $TMPFILE -mv -- "$TMPFILE" hashes.json \ No newline at end of file +mv -- "$TMPFILE" hashes.json + +nom build .# || exit $? +git add hashes.json flake.lock flake.nix \ No newline at end of file From b39747640e836af505ea6ccd0453994f532b94ab Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 10:25:29 +0100 Subject: [PATCH 07/14] Prettier hashes file From c940cfb3554ad597960e41138bf10443254cbe46 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 10:42:00 +0100 Subject: [PATCH 08/14] Dont recalculate package hash and build if package-lock isnt updated --- .husky/pre-commit | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.husky/pre-commit b/.husky/pre-commit index 6eec4741a..702231b5c 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -2,16 +2,21 @@ #!nix-shell -i "bash" -p bash prefetch-npm-deps jq nodejs nix-output-monitor . "$(dirname -- "$0")/_/husky.sh" +# Check if nix is available if [ -x "$(/usr/bin/env which nix-shell 2>/dev/null)" ]; then + # Check if we haven't re-executed ourselves yet if [ ! "$HOOK_NIX_SHELL" ]; then echo "Nix is available, updating nix flake..." export HOOK_NIX_SHELL=1 nix-shell $0 exit $? else + nix flake update # run ./nix-update.sh if package lock has changed and has no unstaged changes - if [ -n "$(git status --porcelain=v1 2>/dev/null | grep -E '^(M| M) package-lock.json')" ]; then + if [ -n "$(git status --porcelain=v1 2>/dev/null | grep -E '^(MM| M) package-lock.json')" ]; then echo "package-lock.json has unstaged changes. Skipping update of nix dependencies." + elif [ ! -n "$(git status --porcelain=v1 2>/dev/null | grep -E '^M package-lock.json')" ]; then + echo "package-lock.json has no changes. Skipping update of nix dependencies." else ./nix-update.sh || exit $? fi @@ -21,4 +26,3 @@ else fi npx -y lint-staged - From 26c8eca37e7d759b426ace7c7f6266545d3c6fa6 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 10:44:55 +0100 Subject: [PATCH 09/14] Manually pretty-fi hashes.json, because apparently prettier wont... --- hashes.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hashes.json b/hashes.json index 1317ed225..39393f035 100644 --- a/hashes.json +++ b/hashes.json @@ -1,4 +1,3 @@ { - "npm_deps_hash": "sha256-fZNDN2/fNy6Nu7tbr0RhQ8j4BP7X1Yhrh/fSTH7hbJc=", - "tmp": "sha256-fuW15WgfDaKPVDQx8OhRAa253J+SQDUr35rKt42KsTc=" + "npm_deps_hash": "sha256-fZNDN2/fNy6Nu7tbr0RhQ8j4BP7X1Yhrh/fSTH7hbJc=" } From 357395b2541c1f3625f5a85852a1bb0eab66d22b Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 10:47:13 +0100 Subject: [PATCH 10/14] Use camel case naming in hashes.json --- flake.nix | 2 +- hashes.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 542de19aa..00a18f640 100644 --- a/flake.nix +++ b/flake.nix @@ -19,7 +19,7 @@ src = ./.; name = "spacebar-server-ts"; nativeBuildInputs = with pkgs; [ python3 ]; - npmDepsHash = hashesFile.npm_deps_hash; + npmDepsHash = hashesFile.npmDepsHash; makeCacheWritable = true; postPatch = '' substituteInPlace package.json --replace 'npx patch-package' '${pkgs.nodePackages.patch-package}/bin/patch-package' diff --git a/hashes.json b/hashes.json index 39393f035..dd55b81da 100644 --- a/hashes.json +++ b/hashes.json @@ -1,3 +1,3 @@ { - "npm_deps_hash": "sha256-fZNDN2/fNy6Nu7tbr0RhQ8j4BP7X1Yhrh/fSTH7hbJc=" + "npmDepsHash": "sha256-fZNDN2/fNy6Nu7tbr0RhQ8j4BP7X1Yhrh/fSTH7hbJc=" } From cde9239e55d728f8fcf2eb5c1db6cb74eb2f4d56 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 10:52:16 +0100 Subject: [PATCH 11/14] Add codeowners for core nix files - we& want to be notified of change proposals --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..b55d76cd5 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# Nix stuff is owned by Rory& - we& want to be notified if these are changed. +/flake.nix root@rory.gay +/nix-update.sh root@rory.gay \ No newline at end of file From 59b6f0ce32e5e9591eed099ce21ad75a016da549 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 10:58:51 +0100 Subject: [PATCH 12/14] Add nix build CI --- .github/workflows/nix-build.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/nix-build.yml diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml new file mode 100644 index 000000000..a2689a428 --- /dev/null +++ b/.github/workflows/nix-build.yml @@ -0,0 +1,18 @@ +name: Nix build + +on: + push: + pull_request: + +jobs: + build-nix: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-unstable + - uses: DeterminateSystems/magic-nix-cache-action@v2 + - run: nix build + - run: nix-shell --run "echo OK" From b41fc6d304d53ef35d99a43cf5ef250c4aa06986 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 11:03:39 +0100 Subject: [PATCH 13/14] Fix nix CI --- .github/workflows/nix-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index a2689a428..066934ba8 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -15,4 +15,4 @@ jobs: nix_path: nixpkgs=channel:nixos-unstable - uses: DeterminateSystems/magic-nix-cache-action@v2 - run: nix build - - run: nix-shell --run "echo OK" + - run: nix develop --command echo OK From 84a1e3d2ed5499b59af62eebfab2d47ffae0c934 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Mon, 19 Feb 2024 11:05:46 +0100 Subject: [PATCH 14/14] Nix CI logs --- .github/workflows/nix-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 066934ba8..ae9afa2ed 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -14,5 +14,5 @@ jobs: with: nix_path: nixpkgs=channel:nixos-unstable - uses: DeterminateSystems/magic-nix-cache-action@v2 - - run: nix build + - run: nix build -L - run: nix develop --command echo OK