From 772dd0940fbe09e54d50345ebc8615726828de87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 9 Apr 2024 10:58:19 +0200
Subject: [PATCH 01/86] Initial setup

---
 package.json                               |  1 +
 packages/cli/benchmark/package.json        | 11 +++++++++++
 packages/cli/benchmark/src/main.ts         | 17 +++++++++++++++++
 packages/cli/benchmark/tsconfig.build.json |  8 ++++++++
 packages/cli/benchmark/tsconfig.json       |  8 ++++++++
 packages/cli/tsconfig.json                 |  1 +
 pnpm-workspace.yaml                        |  1 +
 7 files changed, 47 insertions(+)
 create mode 100644 packages/cli/benchmark/package.json
 create mode 100644 packages/cli/benchmark/src/main.ts
 create mode 100644 packages/cli/benchmark/tsconfig.build.json
 create mode 100644 packages/cli/benchmark/tsconfig.json

diff --git a/package.json b/package.json
index 265951c9a1111..5c63615a252ac 100644
--- a/package.json
+++ b/package.json
@@ -10,6 +10,7 @@
   "packageManager": "pnpm@8.14.3",
   "scripts": {
     "preinstall": "node scripts/block-npm-install.js",
+    "benchmark": "pnpm --filter=n8n-benchmark benchmark",
     "build": "turbo run build",
     "build:backend": "pnpm --filter=!@n8n/chat --filter=!n8n-design-system --filter=!n8n-editor-ui build",
     "build:frontend": "pnpm --filter=@n8n/chat --filter=n8n-design-system --filter=n8n-editor-ui build",
diff --git a/packages/cli/benchmark/package.json b/packages/cli/benchmark/package.json
new file mode 100644
index 0000000000000..f132e75b461da
--- /dev/null
+++ b/packages/cli/benchmark/package.json
@@ -0,0 +1,11 @@
+{
+  "name": "n8n-benchmark",
+  "private": true,
+  "scripts": {
+    "benchmark": "tsc -p tsconfig.build.json && node dist/main.js"
+  },
+  "dependencies": {
+    "@codspeed/tinybench-plugin": "^3.1.0",
+    "tinybench": "^2.6.0"
+  }
+}
diff --git a/packages/cli/benchmark/src/main.ts b/packages/cli/benchmark/src/main.ts
new file mode 100644
index 0000000000000..bbe4f1290355d
--- /dev/null
+++ b/packages/cli/benchmark/src/main.ts
@@ -0,0 +1,17 @@
+import { Bench } from 'tinybench';
+import { withCodSpeed } from '@codspeed/tinybench-plugin';
+
+async function main() {
+	const bench = withCodSpeed(new Bench());
+
+	bench.add('some benchmark', () => {
+		console.log(Math.random());
+	});
+
+	await bench.warmup();
+	await bench.run();
+
+	console.table(bench.table());
+}
+
+void main();
diff --git a/packages/cli/benchmark/tsconfig.build.json b/packages/cli/benchmark/tsconfig.build.json
new file mode 100644
index 0000000000000..a4e1b4f611676
--- /dev/null
+++ b/packages/cli/benchmark/tsconfig.build.json
@@ -0,0 +1,8 @@
+{
+	"extends": ["./tsconfig.json", "../../../tsconfig.build.json"],
+	"compilerOptions": {
+		"rootDir": "src",
+		"outDir": "dist",
+		"tsBuildInfoFile": "dist/build.tsbuildinfo"
+	}
+}
diff --git a/packages/cli/benchmark/tsconfig.json b/packages/cli/benchmark/tsconfig.json
new file mode 100644
index 0000000000000..34069dcc0646a
--- /dev/null
+++ b/packages/cli/benchmark/tsconfig.json
@@ -0,0 +1,8 @@
+{
+	"extends": "../../../tsconfig.json",
+	"compilerOptions": {
+		"rootDir": ".",
+		"tsBuildInfoFile": "dist/typecheck.tsbuildinfo"
+	},
+	"include": ["src/**/*.ts"]
+}
diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json
index 86b8d550e813a..0f7b08eabc74e 100644
--- a/packages/cli/tsconfig.json
+++ b/packages/cli/tsconfig.json
@@ -16,6 +16,7 @@
 		"useUnknownInCatchVariables": false
 	},
 	"include": ["src/**/*.ts", "test/**/*.ts", "src/sso/saml/saml-schema-metadata-2.0.xsd"],
+	"exclude": ["benchmark/**"],
 	"references": [
 		{ "path": "../workflow/tsconfig.build.json" },
 		{ "path": "../core/tsconfig.build.json" },
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index ce18506b3820a..4c0e86359e8c6 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -1,4 +1,5 @@
 packages:
   - packages/*
+  - packages/cli/benchmark
   - packages/@n8n/*
   - packages/@n8n_io/*

From 762754b26e65b3c102cf4eefa47be04356597755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 9 Apr 2024 11:42:25 +0200
Subject: [PATCH 02/86] Add CI action

---
 .github/workflows/benchmark.yml | 43 +++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 .github/workflows/benchmark.yml

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 0000000000000..84b5d41555753
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,43 @@
+name: benchmark
+
+on:
+  push:
+    branches:
+      - 'master'
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.1.1
+        with:
+          repository: n8n-io/n8n
+          ref: ${{ inputs.ref }}
+
+      - run: corepack enable
+      - name: Use Node.js ${{ inputs.nodeVersion }}
+        uses: actions/setup-node@v4.0.1
+        with:
+          node-version: ${{ inputs.nodeVersion }}
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build
+        if: ${{ inputs.cacheKey == '' }}
+        run: pnpm build
+
+      - name: Restore cached build artifacts
+        if: ${{ inputs.cacheKey != '' }}
+        uses: actions/cache/restore@v4.0.0
+        with:
+          path: ./packages/**/dist
+          key: ${{ inputs.cacheKey }}
+
+      - name: Benchmark
+        uses: CodSpeedHQ/action@v2
+        with:
+          run: pnpm benchmark

From c6d07e8a5b72bfbe530158f908174fa4a165b74a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 9 Apr 2024 12:20:02 +0200
Subject: [PATCH 03/86] Refactor into `packages/cli/src/benchmark`

---
 packages/cli/benchmark/package.json           | 11 ---
 packages/cli/benchmark/tsconfig.build.json    |  8 --
 packages/cli/benchmark/tsconfig.json          |  8 --
 packages/cli/package.json                     |  3 +
 packages/cli/src/benchmark/example.bm.ts      |  7 ++
 .../{benchmark/src => src/benchmark}/main.ts  | 11 ++-
 packages/cli/tsconfig.benchmark.json          | 10 +++
 packages/cli/tsconfig.build.json              |  2 +-
 packages/cli/tsconfig.json                    |  1 -
 pnpm-lock.yaml                                | 80 +++++++++++++++++--
 10 files changed, 101 insertions(+), 40 deletions(-)
 delete mode 100644 packages/cli/benchmark/package.json
 delete mode 100644 packages/cli/benchmark/tsconfig.build.json
 delete mode 100644 packages/cli/benchmark/tsconfig.json
 create mode 100644 packages/cli/src/benchmark/example.bm.ts
 rename packages/cli/{benchmark/src => src/benchmark}/main.ts (56%)
 create mode 100644 packages/cli/tsconfig.benchmark.json

diff --git a/packages/cli/benchmark/package.json b/packages/cli/benchmark/package.json
deleted file mode 100644
index f132e75b461da..0000000000000
--- a/packages/cli/benchmark/package.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-  "name": "n8n-benchmark",
-  "private": true,
-  "scripts": {
-    "benchmark": "tsc -p tsconfig.build.json && node dist/main.js"
-  },
-  "dependencies": {
-    "@codspeed/tinybench-plugin": "^3.1.0",
-    "tinybench": "^2.6.0"
-  }
-}
diff --git a/packages/cli/benchmark/tsconfig.build.json b/packages/cli/benchmark/tsconfig.build.json
deleted file mode 100644
index a4e1b4f611676..0000000000000
--- a/packages/cli/benchmark/tsconfig.build.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-	"extends": ["./tsconfig.json", "../../../tsconfig.build.json"],
-	"compilerOptions": {
-		"rootDir": "src",
-		"outDir": "dist",
-		"tsBuildInfoFile": "dist/build.tsbuildinfo"
-	}
-}
diff --git a/packages/cli/benchmark/tsconfig.json b/packages/cli/benchmark/tsconfig.json
deleted file mode 100644
index 34069dcc0646a..0000000000000
--- a/packages/cli/benchmark/tsconfig.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-	"extends": "../../../tsconfig.json",
-	"compilerOptions": {
-		"rootDir": ".",
-		"tsBuildInfoFile": "dist/typecheck.tsbuildinfo"
-	},
-	"include": ["src/**/*.ts"]
-}
diff --git a/packages/cli/package.json b/packages/cli/package.json
index e9738aadaa152..ccb539cb0bf74 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,6 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
+    "benchmark": "tsc -p tsconfig.benchmark.json && node dist/benchmark/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
@@ -60,6 +61,7 @@
     "!dist/**/e2e.*"
   ],
   "devDependencies": {
+    "@codspeed/tinybench-plugin": "^3.1.0",
     "@redocly/cli": "^1.6.0",
     "@types/aws4": "^1.5.1",
     "@types/basic-auth": "^1.1.3",
@@ -88,6 +90,7 @@
     "chokidar": "^3.5.2",
     "concurrently": "^8.2.0",
     "ioredis-mock": "^8.8.1",
+    "tinybench": "^2.6.0",
     "ts-essentials": "^7.0.3"
   },
   "dependencies": {
diff --git a/packages/cli/src/benchmark/example.bm.ts b/packages/cli/src/benchmark/example.bm.ts
new file mode 100644
index 0000000000000..5f426034623bc
--- /dev/null
+++ b/packages/cli/src/benchmark/example.bm.ts
@@ -0,0 +1,7 @@
+import type Bench from 'tinybench';
+
+export function example(bench: Bench) {
+	bench.add('Example', () => {
+		console.log(Math.random());
+	});
+}
diff --git a/packages/cli/benchmark/src/main.ts b/packages/cli/src/benchmark/main.ts
similarity index 56%
rename from packages/cli/benchmark/src/main.ts
rename to packages/cli/src/benchmark/main.ts
index bbe4f1290355d..0ed09f17a15bf 100644
--- a/packages/cli/benchmark/src/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,12 +1,17 @@
+/* eslint-disable import/no-extraneous-dependencies */ // @TODO
+
 import { Bench } from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
+import { example } from './example.bm';
+
+function registerBenchmarks(bench: Bench) {
+	example(bench);
+}
 
 async function main() {
 	const bench = withCodSpeed(new Bench());
 
-	bench.add('some benchmark', () => {
-		console.log(Math.random());
-	});
+	registerBenchmarks(bench);
 
 	await bench.warmup();
 	await bench.run();
diff --git a/packages/cli/tsconfig.benchmark.json b/packages/cli/tsconfig.benchmark.json
new file mode 100644
index 0000000000000..ce92313cf2c38
--- /dev/null
+++ b/packages/cli/tsconfig.benchmark.json
@@ -0,0 +1,10 @@
+{
+	"extends": ["./tsconfig.json", "../../tsconfig.build.json"],
+	"compilerOptions": {
+		"rootDir": "src",
+		"outDir": "dist",
+		"tsBuildInfoFile": "dist/benchmark.tsbuildinfo"
+	},
+	"include": ["src/**/*.ts"],
+	"exclude": ["test/**"]
+}
diff --git a/packages/cli/tsconfig.build.json b/packages/cli/tsconfig.build.json
index 1e8a2ff7fa476..f95c8d7ff965a 100644
--- a/packages/cli/tsconfig.build.json
+++ b/packages/cli/tsconfig.build.json
@@ -6,5 +6,5 @@
 		"tsBuildInfoFile": "dist/build.tsbuildinfo"
 	},
 	"include": ["src/**/*.ts"],
-	"exclude": ["test/**"]
+	"exclude": ["test/**", "src/benchmark/**"]
 }
diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json
index 0f7b08eabc74e..86b8d550e813a 100644
--- a/packages/cli/tsconfig.json
+++ b/packages/cli/tsconfig.json
@@ -16,7 +16,6 @@
 		"useUnknownInCatchVariables": false
 	},
 	"include": ["src/**/*.ts", "test/**/*.ts", "src/sso/saml/saml-schema-metadata-2.0.xsd"],
-	"exclude": ["benchmark/**"],
 	"references": [
 		{ "path": "../workflow/tsconfig.build.json" },
 		{ "path": "../core/tsconfig.build.json" },
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 901856b834ce6..bd92ab5ff2b1f 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -706,6 +706,9 @@ importers:
         specifier: 0.3.0
         version: 0.3.0
     devDependencies:
+      '@codspeed/tinybench-plugin':
+        specifier: ^3.1.0
+        version: 3.1.0(tinybench@2.6.0)
       '@redocly/cli':
         specifier: ^1.6.0
         version: 1.6.0
@@ -790,10 +793,22 @@ importers:
       ioredis-mock:
         specifier: ^8.8.1
         version: 8.8.1(@types/ioredis-mock@8.2.2)(ioredis@5.3.2)
+      tinybench:
+        specifier: ^2.6.0
+        version: 2.6.0
       ts-essentials:
         specifier: ^7.0.3
         version: 7.0.3(typescript@5.4.2)
 
+  packages/cli/benchmark:
+    dependencies:
+      '@codspeed/tinybench-plugin':
+        specifier: ^3.1.0
+        version: 3.1.0(tinybench@2.6.0)
+      tinybench:
+        specifier: ^2.6.0
+        version: 2.6.0
+
   packages/core:
     dependencies:
       '@n8n/client-oauth2':
@@ -4670,6 +4685,27 @@ packages:
       w3c-keyname: 2.2.6
     dev: false
 
+  /@codspeed/core@3.1.0:
+    resolution: {integrity: sha512-oYd7X46QhnRkgRbZkqAoX9i3Fwm17FpunK4Ee5RdrvRYR0Xr93ewH8/O5g6uyTPDOOqDEv1v2KRYtWhVgN+2VQ==}
+    dependencies:
+      axios: 1.6.7
+      find-up: 6.3.0
+      form-data: 4.0.0
+      node-gyp-build: 4.7.0
+    transitivePeerDependencies:
+      - debug
+
+  /@codspeed/tinybench-plugin@3.1.0(tinybench@2.6.0):
+    resolution: {integrity: sha512-yl0WzzUGIXkZzWaw7+2U+xGkuIal1Rs9hS09DtlDZGGAcGRoMMU5d2vyCS8nBrna4hrPQZ5Sx/hIKerO+lqWaw==}
+    peerDependencies:
+      tinybench: ^2.3.0
+    dependencies:
+      '@codspeed/core': 3.1.0
+      stack-trace: 1.0.0-pre2
+      tinybench: 2.6.0
+    transitivePeerDependencies:
+      - debug
+
   /@colors/colors@1.5.0:
     resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
     engines: {node: '>=0.1.90'}
@@ -11493,7 +11529,6 @@ packages:
       proxy-from-env: 1.1.0
     transitivePeerDependencies:
       - debug
-    dev: false
 
   /axios@1.6.7(debug@3.2.7):
     resolution: {integrity: sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==}
@@ -15210,6 +15245,13 @@ packages:
       path-exists: 4.0.0
     dev: true
 
+  /find-up@6.3.0:
+    resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      locate-path: 7.2.0
+      path-exists: 5.0.0
+
   /findup-sync@2.0.0:
     resolution: {integrity: sha512-vs+3unmJT45eczmcAZ6zMJtxN3l/QXeccaXQx5cu/MeJMhewVfoWZqibRkOxPnmoR59+Zy5hjabfQc6JLSah4g==}
     engines: {node: '>= 0.10'}
@@ -15296,7 +15338,6 @@ packages:
         optional: true
     dependencies:
       debug: 3.2.7(supports-color@5.5.0)
-    dev: false
 
   /follow-redirects@1.15.6(debug@4.3.4):
     resolution: {integrity: sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==}
@@ -18875,6 +18916,12 @@ packages:
       p-locate: 5.0.0
     dev: true
 
+  /locate-path@7.2.0:
+    resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      p-locate: 6.0.0
+
   /lodash-es@4.17.21:
     resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==}
     dev: false
@@ -20058,7 +20105,6 @@ packages:
   /node-gyp-build@4.7.0:
     resolution: {integrity: sha512-PbZERfeFdrHQOOXiAKOY0VPbykZy90ndPKk0d+CFDegTKmWp1VgOTz2xACVbr1BjCWxrQp68CXtvNsveFhqDJg==}
     hasBin: true
-    dev: false
 
   /node-gyp@8.4.1:
     resolution: {integrity: sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==}
@@ -20679,6 +20725,12 @@ packages:
     dependencies:
       yocto-queue: 0.1.0
 
+  /p-limit@4.0.0:
+    resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      yocto-queue: 1.0.0
+
   /p-limit@5.0.0:
     resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==}
     engines: {node: '>=18'}
@@ -20707,6 +20759,12 @@ packages:
       p-limit: 3.1.0
     dev: true
 
+  /p-locate@6.0.0:
+    resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      p-limit: 4.0.0
+
   /p-map@4.0.0:
     resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==}
     engines: {node: '>=10'}
@@ -20883,6 +20941,10 @@ packages:
     engines: {node: '>=8'}
     dev: true
 
+  /path-exists@5.0.0:
+    resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+
   /path-is-absolute@1.0.1:
     resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}
     engines: {node: '>=0.10.0'}
@@ -23611,6 +23673,10 @@ packages:
   /stack-trace@0.0.10:
     resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==}
 
+  /stack-trace@1.0.0-pre2:
+    resolution: {integrity: sha512-2ztBJRek8IVofG9DBJqdy2N5kulaacX30Nz7xmkYF6ale9WBVmIy6mFBchvGX7Vx/MyjBhx+Rcxqrj+dbOnQ6A==}
+    engines: {node: '>=16'}
+
   /stack-utils@2.0.6:
     resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==}
     engines: {node: '>=10'}
@@ -24315,9 +24381,8 @@ packages:
     resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==}
     dev: true
 
-  /tinybench@2.5.1:
-    resolution: {integrity: sha512-65NKvSuAVDP/n4CqH+a9w2kTlLReS9vhsAP06MWx+/89nMinJyB2icyl58RIcqCmIggpojIGeuJGhjU1aGMBSg==}
-    dev: true
+  /tinybench@2.6.0:
+    resolution: {integrity: sha512-N8hW3PG/3aOoZAN5V/NSAEDz0ZixDSSt5b/a05iqtpgfLWMSVuCo7w0k2vVvEjdrIoeGqZzweX2WlyioNIHchA==}
 
   /tinypool@0.8.2:
     resolution: {integrity: sha512-SUszKYe5wgsxnNOVlBYO6IC+8VGWdVGZWAqUxp3UErNBtptZvWbwyUOyzNL59zigz2rCA92QiL3wvG+JDSdJdQ==}
@@ -25553,7 +25618,7 @@ packages:
       picocolors: 1.0.0
       std-env: 3.6.0
       strip-literal: 2.0.0
-      tinybench: 2.5.1
+      tinybench: 2.6.0
       tinypool: 0.8.2
       vite: 5.1.6(sass@1.64.1)
       vite-node: 1.3.1
@@ -26446,7 +26511,6 @@ packages:
   /yocto-queue@1.0.0:
     resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==}
     engines: {node: '>=12.20'}
-    dev: true
 
   /yup@0.32.11:
     resolution: {integrity: sha512-Z2Fe1bn+eLstG8DRR6FTavGD+MeAwyfmouhHsIUgaADz8jvFKbO/fXc2trJKZg+5EBjh4gGm3iU/t3onKlXHIg==}

From 1a6045afe12c9ca8aed001f9a36867ed89b5c9d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 9 Apr 2024 13:04:05 +0200
Subject: [PATCH 04/86] Add start cmd

---
 packages/cli/package.json              |  2 +-
 packages/cli/src/benchmark/main.ts     |  8 ++++++--
 packages/cli/src/benchmark/start.bm.ts | 12 ++++++++++++
 3 files changed, 19 insertions(+), 3 deletions(-)
 create mode 100644 packages/cli/src/benchmark/start.bm.ts

diff --git a/packages/cli/package.json b/packages/cli/package.json
index ccb539cb0bf74..069acb7d4cdb1 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,7 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "tsc -p tsconfig.benchmark.json && node dist/benchmark/main.js",
+    "benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node dist/benchmark/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 0ed09f17a15bf..4a45f5286b66f 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,11 +1,15 @@
+import 'reflect-metadata';
+
 /* eslint-disable import/no-extraneous-dependencies */ // @TODO
 
 import { Bench } from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
-import { example } from './example.bm';
+// import { example } from './example.bm';
+import { start } from './start.bm';
 
 function registerBenchmarks(bench: Bench) {
-	example(bench);
+	// example(bench);
+	start(bench);
 }
 
 async function main() {
diff --git a/packages/cli/src/benchmark/start.bm.ts b/packages/cli/src/benchmark/start.bm.ts
new file mode 100644
index 0000000000000..430e055e95a25
--- /dev/null
+++ b/packages/cli/src/benchmark/start.bm.ts
@@ -0,0 +1,12 @@
+import { Start } from '../commands/start';
+import { Config } from '@oclif/core';
+import type Bench from 'tinybench';
+
+export function start(bench: Bench) {
+	bench.add('`start` command', async () => {
+		const args: string[] = [];
+		const config = new Config({ root: __dirname });
+
+		await new Start(args, config).run();
+	});
+}

From b0832cb5e3076555913e630eea58809e68ba0389 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 9 Apr 2024 13:14:47 +0200
Subject: [PATCH 05/86] Fix script ref

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 5c63615a252ac..823bbd4ba123b 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,7 @@
   "packageManager": "pnpm@8.14.3",
   "scripts": {
     "preinstall": "node scripts/block-npm-install.js",
-    "benchmark": "pnpm --filter=n8n-benchmark benchmark",
+    "benchmark": "pnpm --filter=n8n benchmark",
     "build": "turbo run build",
     "build:backend": "pnpm --filter=!@n8n/chat --filter=!n8n-design-system --filter=!n8n-editor-ui build",
     "build:frontend": "pnpm --filter=@n8n/chat --filter=n8n-design-system --filter=n8n-editor-ui build",

From a1e19fbfe8755865e2794a31ff85764908f61c5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 9 Apr 2024 13:15:00 +0200
Subject: [PATCH 06/86] Init and run start cmd

---
 packages/cli/src/benchmark/start.bm.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/cli/src/benchmark/start.bm.ts b/packages/cli/src/benchmark/start.bm.ts
index 430e055e95a25..4a658b081d256 100644
--- a/packages/cli/src/benchmark/start.bm.ts
+++ b/packages/cli/src/benchmark/start.bm.ts
@@ -7,6 +7,9 @@ export function start(bench: Bench) {
 		const args: string[] = [];
 		const config = new Config({ root: __dirname });
 
-		await new Start(args, config).run();
+		const startCommand = new Start(args, config);
+
+		await startCommand.init();
+		await startCommand.run();
 	});
 }

From c974ec2c01e5f8c4d31ffe73964e12e0e3295860 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Wed, 10 Apr 2024 10:34:49 +0200
Subject: [PATCH 07/86] Update lockfile

---
 pnpm-lock.yaml | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index bd92ab5ff2b1f..c420e37bf4269 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -800,15 +800,6 @@ importers:
         specifier: ^7.0.3
         version: 7.0.3(typescript@5.4.2)
 
-  packages/cli/benchmark:
-    dependencies:
-      '@codspeed/tinybench-plugin':
-        specifier: ^3.1.0
-        version: 3.1.0(tinybench@2.6.0)
-      tinybench:
-        specifier: ^2.6.0
-        version: 2.6.0
-
   packages/core:
     dependencies:
       '@n8n/client-oauth2':
@@ -4694,6 +4685,7 @@ packages:
       node-gyp-build: 4.7.0
     transitivePeerDependencies:
       - debug
+    dev: true
 
   /@codspeed/tinybench-plugin@3.1.0(tinybench@2.6.0):
     resolution: {integrity: sha512-yl0WzzUGIXkZzWaw7+2U+xGkuIal1Rs9hS09DtlDZGGAcGRoMMU5d2vyCS8nBrna4hrPQZ5Sx/hIKerO+lqWaw==}
@@ -4705,6 +4697,7 @@ packages:
       tinybench: 2.6.0
     transitivePeerDependencies:
       - debug
+    dev: true
 
   /@colors/colors@1.5.0:
     resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
@@ -15251,6 +15244,7 @@ packages:
     dependencies:
       locate-path: 7.2.0
       path-exists: 5.0.0
+    dev: true
 
   /findup-sync@2.0.0:
     resolution: {integrity: sha512-vs+3unmJT45eczmcAZ6zMJtxN3l/QXeccaXQx5cu/MeJMhewVfoWZqibRkOxPnmoR59+Zy5hjabfQc6JLSah4g==}
@@ -18921,6 +18915,7 @@ packages:
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
     dependencies:
       p-locate: 6.0.0
+    dev: true
 
   /lodash-es@4.17.21:
     resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==}
@@ -20730,6 +20725,7 @@ packages:
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
     dependencies:
       yocto-queue: 1.0.0
+    dev: true
 
   /p-limit@5.0.0:
     resolution: {integrity: sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==}
@@ -20764,6 +20760,7 @@ packages:
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
     dependencies:
       p-limit: 4.0.0
+    dev: true
 
   /p-map@4.0.0:
     resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==}
@@ -20944,6 +20941,7 @@ packages:
   /path-exists@5.0.0:
     resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dev: true
 
   /path-is-absolute@1.0.1:
     resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}
@@ -23676,6 +23674,7 @@ packages:
   /stack-trace@1.0.0-pre2:
     resolution: {integrity: sha512-2ztBJRek8IVofG9DBJqdy2N5kulaacX30Nz7xmkYF6ale9WBVmIy6mFBchvGX7Vx/MyjBhx+Rcxqrj+dbOnQ6A==}
     engines: {node: '>=16'}
+    dev: true
 
   /stack-utils@2.0.6:
     resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==}
@@ -24383,6 +24382,7 @@ packages:
 
   /tinybench@2.6.0:
     resolution: {integrity: sha512-N8hW3PG/3aOoZAN5V/NSAEDz0ZixDSSt5b/a05iqtpgfLWMSVuCo7w0k2vVvEjdrIoeGqZzweX2WlyioNIHchA==}
+    dev: true
 
   /tinypool@0.8.2:
     resolution: {integrity: sha512-SUszKYe5wgsxnNOVlBYO6IC+8VGWdVGZWAqUxp3UErNBtptZvWbwyUOyzNL59zigz2rCA92QiL3wvG+JDSdJdQ==}
@@ -26511,6 +26511,7 @@ packages:
   /yocto-queue@1.0.0:
     resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==}
     engines: {node: '>=12.20'}
+    dev: true
 
   /yup@0.32.11:
     resolution: {integrity: sha512-Z2Fe1bn+eLstG8DRR6FTavGD+MeAwyfmouhHsIUgaADz8jvFKbO/fXc2trJKZg+5EBjh4gGm3iU/t3onKlXHIg==}

From f8127b8e4d5c28c4cf39b0b72944bb6c0aa5c951 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Wed, 10 Apr 2024 10:35:01 +0200
Subject: [PATCH 08/86] Limit CI action to BE PRs

---
 .github/workflows/benchmark.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 84b5d41555753..a0ee3d3000f6f 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -3,8 +3,12 @@ name: benchmark
 on:
   push:
     branches:
-      - 'master'
+      - '**'
   pull_request:
+    paths:
+      - 'packages/cli/**'
+      - 'packages/core/**'
+      - 'packages/workflow/**'
   workflow_dispatch:
 
 jobs:

From 168ccb088002ead06a7298a71748cf309c351025 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 10:40:15 +0200
Subject: [PATCH 09/86] Cleanup

---
 .github/workflows/benchmark.yml          |  2 +-
 packages/cli/src/benchmark/example.bm.ts |  7 -------
 packages/cli/src/benchmark/main.ts       | 19 +++++++++++--------
 packages/cli/src/benchmark/start.bm.ts   | 15 ---------------
 packages/cli/src/benchmark/webhook.bm.ts | 24 ++++++++++++++++++++++++
 5 files changed, 36 insertions(+), 31 deletions(-)
 delete mode 100644 packages/cli/src/benchmark/example.bm.ts
 delete mode 100644 packages/cli/src/benchmark/start.bm.ts
 create mode 100644 packages/cli/src/benchmark/webhook.bm.ts

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index a0ee3d3000f6f..943d1740f2c41 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -1,4 +1,4 @@
-name: benchmark
+name: Benchmarks
 
 on:
   push:
diff --git a/packages/cli/src/benchmark/example.bm.ts b/packages/cli/src/benchmark/example.bm.ts
deleted file mode 100644
index 5f426034623bc..0000000000000
--- a/packages/cli/src/benchmark/example.bm.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import type Bench from 'tinybench';
-
-export function example(bench: Bench) {
-	bench.add('Example', () => {
-		console.log(Math.random());
-	});
-}
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 4a45f5286b66f..a934c66aa002e 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,23 +1,26 @@
-import 'reflect-metadata';
+/* eslint-disable import/no-extraneous-dependencies */ // @TODO: Remove
 
-/* eslint-disable import/no-extraneous-dependencies */ // @TODO
+import 'reflect-metadata';
 
 import { Bench } from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
-// import { example } from './example.bm';
-import { start } from './start.bm';
+
+import { webhook } from './webhook.bm';
 
 function registerBenchmarks(bench: Bench) {
-	// example(bench);
-	start(bench);
+	webhook(bench);
 }
 
 async function main() {
-	const bench = withCodSpeed(new Bench());
+	const bench = withCodSpeed(
+		new Bench(
+			{ time: 0, iterations: 1 }, // @TEMP: Remove
+		),
+	);
 
 	registerBenchmarks(bench);
 
-	await bench.warmup();
+	// await bench.warmup(); // @TODO: Restore
 	await bench.run();
 
 	console.table(bench.table());
diff --git a/packages/cli/src/benchmark/start.bm.ts b/packages/cli/src/benchmark/start.bm.ts
deleted file mode 100644
index 4a658b081d256..0000000000000
--- a/packages/cli/src/benchmark/start.bm.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-import { Start } from '../commands/start';
-import { Config } from '@oclif/core';
-import type Bench from 'tinybench';
-
-export function start(bench: Bench) {
-	bench.add('`start` command', async () => {
-		const args: string[] = [];
-		const config = new Config({ root: __dirname });
-
-		const startCommand = new Start(args, config);
-
-		await startCommand.init();
-		await startCommand.run();
-	});
-}
diff --git a/packages/cli/src/benchmark/webhook.bm.ts b/packages/cli/src/benchmark/webhook.bm.ts
new file mode 100644
index 0000000000000..d8313cdced92c
--- /dev/null
+++ b/packages/cli/src/benchmark/webhook.bm.ts
@@ -0,0 +1,24 @@
+import type Bench from 'tinybench';
+// import { init } from './init';
+
+export const testDbPrefix = 'n8n_test_';
+
+export function webhook(bench: Bench) {
+	bench.add(
+		'`start` command',
+		async () => {
+			// console.log('ended');
+			console.log(bench.iterations);
+		},
+		{
+			beforeAll: async () => {
+				console.log('beforeAll start');
+
+				// await init.startCommand();
+				// await init.database();
+
+				console.log('beforeAll end');
+			},
+		},
+	);
+}

From 24042a12cab439d54c876fee992e1fcdb59007b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 11:34:15 +0200
Subject: [PATCH 10/86] Move benchmarks inside `cli/test`

---
 packages/cli/package.json                       |  2 +-
 packages/cli/test/benchmark/init.ts             | 17 +++++++++++++++++
 packages/cli/{src => test}/benchmark/main.ts    |  0
 .../cli/{src => test}/benchmark/webhook.bm.ts   |  5 ++---
 packages/cli/tsconfig.benchmark.json            |  7 +++----
 packages/cli/tsconfig.build.json                |  2 +-
 6 files changed, 24 insertions(+), 9 deletions(-)
 create mode 100644 packages/cli/test/benchmark/init.ts
 rename packages/cli/{src => test}/benchmark/main.ts (100%)
 rename packages/cli/{src => test}/benchmark/webhook.bm.ts (75%)

diff --git a/packages/cli/package.json b/packages/cli/package.json
index 980413d39a9fe..9a8cbbbf890a3 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,7 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node dist/benchmark/main.js",
+    "benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node dist/test/benchmark/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
diff --git a/packages/cli/test/benchmark/init.ts b/packages/cli/test/benchmark/init.ts
new file mode 100644
index 0000000000000..aaf58e9019107
--- /dev/null
+++ b/packages/cli/test/benchmark/init.ts
@@ -0,0 +1,17 @@
+import { Config } from '@oclif/core';
+import { Start } from '@/commands/start';
+
+export async function startCommand() {
+	const args: string[] = [];
+	const _config = new Config({ root: __dirname });
+
+	const cmd = new Start(args, _config);
+
+	await cmd.init();
+	await cmd.run();
+}
+
+export const init = {
+	// database: testDb.init,
+	startCommand,
+};
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/test/benchmark/main.ts
similarity index 100%
rename from packages/cli/src/benchmark/main.ts
rename to packages/cli/test/benchmark/main.ts
diff --git a/packages/cli/src/benchmark/webhook.bm.ts b/packages/cli/test/benchmark/webhook.bm.ts
similarity index 75%
rename from packages/cli/src/benchmark/webhook.bm.ts
rename to packages/cli/test/benchmark/webhook.bm.ts
index d8313cdced92c..575a1b33aec47 100644
--- a/packages/cli/src/benchmark/webhook.bm.ts
+++ b/packages/cli/test/benchmark/webhook.bm.ts
@@ -1,7 +1,6 @@
 import type Bench from 'tinybench';
-// import { init } from './init';
 
-export const testDbPrefix = 'n8n_test_';
+import { init } from './init';
 
 export function webhook(bench: Bench) {
 	bench.add(
@@ -14,7 +13,7 @@ export function webhook(bench: Bench) {
 			beforeAll: async () => {
 				console.log('beforeAll start');
 
-				// await init.startCommand();
+				await init.startCommand();
 				// await init.database();
 
 				console.log('beforeAll end');
diff --git a/packages/cli/tsconfig.benchmark.json b/packages/cli/tsconfig.benchmark.json
index ce92313cf2c38..930bb9c97389d 100644
--- a/packages/cli/tsconfig.benchmark.json
+++ b/packages/cli/tsconfig.benchmark.json
@@ -1,10 +1,9 @@
 {
 	"extends": ["./tsconfig.json", "../../tsconfig.build.json"],
 	"compilerOptions": {
-		"rootDir": "src",
+		"rootDir": ".",
 		"outDir": "dist",
 		"tsBuildInfoFile": "dist/benchmark.tsbuildinfo"
-	},
-	"include": ["src/**/*.ts"],
-	"exclude": ["test/**"]
+	}
+	// "include": ["test/benchmark/**/*.ts"]
 }
diff --git a/packages/cli/tsconfig.build.json b/packages/cli/tsconfig.build.json
index f95c8d7ff965a..1e8a2ff7fa476 100644
--- a/packages/cli/tsconfig.build.json
+++ b/packages/cli/tsconfig.build.json
@@ -6,5 +6,5 @@
 		"tsBuildInfoFile": "dist/build.tsbuildinfo"
 	},
 	"include": ["src/**/*.ts"],
-	"exclude": ["test/**", "src/benchmark/**"]
+	"exclude": ["test/**"]
 }

From cd99c6e124885a1674ff29a5aa531f5a24e9b781 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 11:50:01 +0200
Subject: [PATCH 11/86] Add `ts-ignore` to allow `test` to compile

---
 .../cli/test/integration/auth.api.test.ts     |  1 +
 packages/cli/test/integration/auth.mw.test.ts |  2 +
 .../integration/publicApi/executions.test.ts  |  1 +
 .../test/integration/publicApi/tags.test.ts   |  1 +
 .../integration/publicApi/workflows.test.ts   |  1 +
 .../test/integration/shared/utils/index.ts    |  1 +
 .../integration/shared/utils/testServer.ts    |  3 ++
 .../cli/test/integration/webhooks.api.test.ts |  4 +-
 packages/cli/test/unit/Telemetry.test.ts      |  4 +-
 .../test/unit/services/events.service.test.ts |  4 +-
 .../services/orchestration.service.test.ts    |  3 ++
 .../test/unit/services/redis.service.test.ts  | 42 -------------------
 packages/cli/test/unit/webhooks.test.ts       |  4 +-
 13 files changed, 22 insertions(+), 49 deletions(-)

diff --git a/packages/cli/test/integration/auth.api.test.ts b/packages/cli/test/integration/auth.api.test.ts
index 9435fa7a7dfcf..618c75a209cb4 100644
--- a/packages/cli/test/integration/auth.api.test.ts
+++ b/packages/cli/test/integration/auth.api.test.ts
@@ -158,6 +158,7 @@ describe('GET /login', () => {
 	});
 
 	test('should return 401 Unauthorized if invalid cookie', async () => {
+		// @ts-ignore
 		testServer.authlessAgent.jar.setCookie(`${AUTH_COOKIE_NAME}=invalid`);
 
 		const response = await testServer.authlessAgent.get('/login');
diff --git a/packages/cli/test/integration/auth.mw.test.ts b/packages/cli/test/integration/auth.mw.test.ts
index 8f40759f965c6..add424a03823d 100644
--- a/packages/cli/test/integration/auth.mw.test.ts
+++ b/packages/cli/test/integration/auth.mw.test.ts
@@ -29,6 +29,7 @@ describe('Auth Middleware', () => {
 		ROUTES_REQUIRING_AUTHENTICATION.concat(ROUTES_REQUIRING_AUTHORIZATION).forEach(
 			([method, endpoint]) => {
 				test(`${method} ${endpoint} should return 401 Unauthorized if no cookie`, async () => {
+					// @ts-ignore
 					const { statusCode } = await testServer.authlessAgent[method.toLowerCase()](endpoint);
 					expect(statusCode).toBe(401);
 				});
@@ -45,6 +46,7 @@ describe('Auth Middleware', () => {
 
 		ROUTES_REQUIRING_AUTHORIZATION.forEach(async ([method, endpoint]) => {
 			test(`${method} ${endpoint} should return 403 Forbidden for member`, async () => {
+				// @ts-ignore
 				const { statusCode } = await authMemberAgent[method.toLowerCase()](endpoint);
 				expect(statusCode).toBe(403);
 			});
diff --git a/packages/cli/test/integration/publicApi/executions.test.ts b/packages/cli/test/integration/publicApi/executions.test.ts
index 012519df66580..92966d5862886 100644
--- a/packages/cli/test/integration/publicApi/executions.test.ts
+++ b/packages/cli/test/integration/publicApi/executions.test.ts
@@ -61,6 +61,7 @@ afterEach(async () => {
 
 const testWithAPIKey =
 	(method: 'get' | 'post' | 'put' | 'delete', url: string, apiKey: string | null) => async () => {
+		// @ts-ignore
 		void authOwnerAgent.set({ 'X-N8N-API-KEY': apiKey });
 		const response = await authOwnerAgent[method](url);
 		expect(response.statusCode).toBe(401);
diff --git a/packages/cli/test/integration/publicApi/tags.test.ts b/packages/cli/test/integration/publicApi/tags.test.ts
index 7e8fcacdead0f..9a88b11d9a7a2 100644
--- a/packages/cli/test/integration/publicApi/tags.test.ts
+++ b/packages/cli/test/integration/publicApi/tags.test.ts
@@ -37,6 +37,7 @@ beforeEach(async () => {
 
 const testWithAPIKey =
 	(method: 'get' | 'post' | 'put' | 'delete', url: string, apiKey: string | null) => async () => {
+		// @ts-ignore
 		void authOwnerAgent.set({ 'X-N8N-API-KEY': apiKey });
 		const response = await authOwnerAgent[method](url);
 		expect(response.statusCode).toBe(401);
diff --git a/packages/cli/test/integration/publicApi/workflows.test.ts b/packages/cli/test/integration/publicApi/workflows.test.ts
index c91178750cc18..309030736d29a 100644
--- a/packages/cli/test/integration/publicApi/workflows.test.ts
+++ b/packages/cli/test/integration/publicApi/workflows.test.ts
@@ -67,6 +67,7 @@ afterEach(async () => {
 
 const testWithAPIKey =
 	(method: 'get' | 'post' | 'put' | 'delete', url: string, apiKey: string | null) => async () => {
+		// @ts-ignore
 		void authOwnerAgent.set({ 'X-N8N-API-KEY': apiKey });
 		const response = await authOwnerAgent[method](url);
 		expect(response.statusCode).toBe(401);
diff --git a/packages/cli/test/integration/shared/utils/index.ts b/packages/cli/test/integration/shared/utils/index.ts
index 2558c461ac9c8..e79db27f97d14 100644
--- a/packages/cli/test/integration/shared/utils/index.ts
+++ b/packages/cli/test/integration/shared/utils/index.ts
@@ -96,6 +96,7 @@ export async function initBinaryDataService(mode: 'default' | 'filesystem' = 'de
  * Extract the value (token) of the auth cookie in a response.
  */
 export function getAuthToken(response: request.Response, authCookieName = AUTH_COOKIE_NAME) {
+	// @ts-ignore
 	const cookies: string[] = response.headers['set-cookie'];
 
 	if (!cookies) return undefined;
diff --git a/packages/cli/test/integration/shared/utils/testServer.ts b/packages/cli/test/integration/shared/utils/testServer.ts
index d6b9aa40efd04..161ada8bcc43f 100644
--- a/packages/cli/test/integration/shared/utils/testServer.ts
+++ b/packages/cli/test/integration/shared/utils/testServer.ts
@@ -89,8 +89,11 @@ export const setupTestServer = ({
 	const testServer: TestServer = {
 		app,
 		httpServer: app.listen(0),
+		// @ts-ignore
 		authAgentFor: (user: User) => createAgent(app, { auth: true, user }),
+		// @ts-ignore
 		authlessAgent: createAgent(app),
+		// @ts-ignore
 		publicApiAgentFor: (user) => publicApiAgent(app, { user }),
 		license: new LicenseMocker(),
 	};
diff --git a/packages/cli/test/integration/webhooks.api.test.ts b/packages/cli/test/integration/webhooks.api.test.ts
index 37862be6aa237..e4f7aed11b924 100644
--- a/packages/cli/test/integration/webhooks.api.test.ts
+++ b/packages/cli/test/integration/webhooks.api.test.ts
@@ -1,5 +1,5 @@
 import { readFileSync } from 'fs';
-import type { SuperAgentTest } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { agent as testAgent } from 'supertest';
 import type { INodeType, INodeTypeDescription, IWebhookFunctions } from 'n8n-workflow';
 
@@ -21,7 +21,7 @@ describe('Webhook API', () => {
 	mockInstance(InternalHooks);
 	mockInstance(Push);
 
-	let agent: SuperAgentTest;
+	let agent: TestAgent;
 
 	beforeAll(async () => {
 		await testDb.init();
diff --git a/packages/cli/test/unit/Telemetry.test.ts b/packages/cli/test/unit/Telemetry.test.ts
index 25f4a808675b0..846c838ea72ff 100644
--- a/packages/cli/test/unit/Telemetry.test.ts
+++ b/packages/cli/test/unit/Telemetry.test.ts
@@ -15,8 +15,10 @@ describe('Telemetry', () => {
 	const spyTrack = jest.spyOn(Telemetry.prototype, 'track').mockName('track');
 
 	const mockRudderStack: Pick<RudderStack, 'flush' | 'identify' | 'track'> = {
-		flush: (resolve) => resolve?.(),
+		flush: async (resolve) => resolve?.(),
+		// @ts-ignore
 		identify: (data, resolve) => resolve?.(),
+		// @ts-ignore
 		track: (data, resolve) => resolve?.(),
 	};
 
diff --git a/packages/cli/test/unit/services/events.service.test.ts b/packages/cli/test/unit/services/events.service.test.ts
index afdd4091d3186..37cb6f368896a 100644
--- a/packages/cli/test/unit/services/events.service.test.ts
+++ b/packages/cli/test/unit/services/events.service.test.ts
@@ -107,7 +107,7 @@ describe('EventsService', () => {
 			};
 			const runData: IRun = {
 				finished: false,
-				status: 'failed',
+				status: 'error',
 				data: { resultData: { runData: {} } },
 				mode: 'internal' as WorkflowExecuteMode,
 				startedAt: new Date(),
@@ -193,7 +193,7 @@ describe('EventsService', () => {
 
 		test('should not send metrics for entries that already have the flag set', async () => {
 			// Fetch data for workflow 2 which is set up to not be altered in the mocks
-			entityManager.insert.mockRejectedValueOnce(new QueryFailedError('', undefined, ''));
+			entityManager.insert.mockRejectedValueOnce(new QueryFailedError('', undefined, new Error()));
 			const workflowId = '1';
 			const node = {
 				id: 'abcde',
diff --git a/packages/cli/test/unit/services/orchestration.service.test.ts b/packages/cli/test/unit/services/orchestration.service.test.ts
index 35ad3a53bef57..25b0da2c50827 100644
--- a/packages/cli/test/unit/services/orchestration.service.test.ts
+++ b/packages/cli/test/unit/services/orchestration.service.test.ts
@@ -147,7 +147,10 @@ describe('Orchestration Service', () => {
 			}),
 		);
 		expect(helpers.debounceMessageReceiver).toHaveBeenCalledTimes(2);
+
+		// @ts-ignore
 		expect(res1!.payload).toBeUndefined();
+		// @ts-ignore
 		expect(res2!.payload!.result).toEqual('debounced');
 	});
 
diff --git a/packages/cli/test/unit/services/redis.service.test.ts b/packages/cli/test/unit/services/redis.service.test.ts
index 04fb980db6797..f6f932a4b80f2 100644
--- a/packages/cli/test/unit/services/redis.service.test.ts
+++ b/packages/cli/test/unit/services/redis.service.test.ts
@@ -54,46 +54,4 @@ describe('RedisService', () => {
 		await sub.destroy();
 		await pub.destroy();
 	});
-
-	// NOTE: This test is failing because the mock Redis client does not support streams apparently
-	// eslint-disable-next-line n8n-local-rules/no-skipped-tests
-	test.skip('should create stream producer and consumer', async () => {
-		const consumer = await redisService.getStreamConsumer();
-		const producer = await redisService.getStreamProducer();
-
-		expect(consumer).toBeDefined();
-		expect(producer).toBeDefined();
-
-		const mockHandler = jest.fn();
-		mockHandler.mockImplementation((stream: string, id: string, message: string[]) => {
-			console.log('Received message', stream, id, message);
-		});
-		consumer.addMessageHandler('some handler', mockHandler);
-
-		await consumer.setPollingInterval(STREAM_CHANNEL, 50);
-		await consumer.listenToStream(STREAM_CHANNEL);
-
-		let timeout;
-		await new Promise((resolve) => {
-			timeout = setTimeout(async () => {
-				await producer.add(STREAM_CHANNEL, ['message', 'testMessage', 'event', 'testEveny']);
-				resolve(0);
-			}, 50);
-		});
-
-		await new Promise((resolve) =>
-			setTimeout(async () => {
-				resolve(0);
-			}, 100),
-		);
-
-		clearInterval(timeout);
-
-		consumer.stopListeningToStream(STREAM_CHANNEL);
-
-		expect(mockHandler).toHaveBeenCalled();
-
-		await consumer.destroy();
-		await producer.destroy();
-	});
 });
diff --git a/packages/cli/test/unit/webhooks.test.ts b/packages/cli/test/unit/webhooks.test.ts
index 3070a5d4781a1..3f25eedcc13a8 100644
--- a/packages/cli/test/unit/webhooks.test.ts
+++ b/packages/cli/test/unit/webhooks.test.ts
@@ -1,4 +1,4 @@
-import type { SuperAgentTest } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { agent as testAgent } from 'supertest';
 import { mock } from 'jest-mock-extended';
 
@@ -14,7 +14,7 @@ import type { IResponseCallbackData } from '@/Interfaces';
 
 import { mockInstance } from '../shared/mocking';
 
-let agent: SuperAgentTest;
+let agent: TestAgent;
 
 describe('WebhookServer', () => {
 	mockInstance(ExternalHooks);

From 7e358971193db07d3a462eafd856f60c81bde036 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 12:10:52 +0200
Subject: [PATCH 12/86] Cleanup

---
 packages/cli/package.json                     |  2 +-
 packages/cli/src/commands/start.ts            | 58 ++++++++++---------
 packages/cli/test/benchmark/init.ts           | 17 ------
 packages/cli/test/benchmarks/init.ts          | 18 ++++++
 .../test/{benchmark => benchmarks}/main.ts    | 17 ++----
 packages/cli/test/benchmarks/register.ts      |  6 ++
 .../webhook.benchmark.ts}                     | 10 ++--
 7 files changed, 65 insertions(+), 63 deletions(-)
 delete mode 100644 packages/cli/test/benchmark/init.ts
 create mode 100644 packages/cli/test/benchmarks/init.ts
 rename packages/cli/test/{benchmark => benchmarks}/main.ts (58%)
 create mode 100644 packages/cli/test/benchmarks/register.ts
 rename packages/cli/test/{benchmark/webhook.bm.ts => benchmarks/webhook.benchmark.ts} (66%)

diff --git a/packages/cli/package.json b/packages/cli/package.json
index 9a8cbbbf890a3..ca82e51af1ba8 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,7 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node dist/test/benchmark/main.js",
+    "benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node dist/test/benchmarks/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts
index 3b7ee763a3244..6a2284ef148e0 100644
--- a/packages/cli/src/commands/start.ts
+++ b/packages/cli/src/commands/start.ts
@@ -289,35 +289,37 @@ export class Start extends BaseCommand {
 		const editorUrl = Container.get(UrlService).baseUrl;
 		this.log(`\nEditor is now accessible via:\n${editorUrl}`);
 
-		// Allow to open n8n editor by pressing "o"
-		if (Boolean(process.stdout.isTTY) && process.stdin.setRawMode) {
-			process.stdin.setRawMode(true);
-			process.stdin.resume();
-			process.stdin.setEncoding('utf8');
+		console.log('hello');
 
-			if (flags.open) {
-				this.openBrowser();
-			}
-			this.log('\nPress "o" to open in Browser.');
-			process.stdin.on('data', (key: string) => {
-				if (key === 'o') {
-					this.openBrowser();
-				} else if (key.charCodeAt(0) === 3) {
-					// Ctrl + c got pressed
-					void this.stopProcess();
-				} else {
-					// When anything else got pressed, record it and send it on enter into the child process
-
-					if (key.charCodeAt(0) === 13) {
-						// send to child process and print in terminal
-						process.stdout.write('\n');
-					} else {
-						// record it and write into terminal
-						process.stdout.write(key);
-					}
-				}
-			});
-		}
+		// Allow to open n8n editor by pressing "o"
+		// if (Boolean(process.stdout.isTTY) && process.stdin.setRawMode) {
+		// 	process.stdin.setRawMode(true);
+		// 	process.stdin.resume();
+		// 	process.stdin.setEncoding('utf8');
+
+		// 	if (flags.open) {
+		// 		this.openBrowser();
+		// 	}
+		// 	this.log('\nPress "o" to open in Browser.');
+		// 	process.stdin.on('data', (key: string) => {
+		// 		if (key === 'o') {
+		// 			this.openBrowser();
+		// 		} else if (key.charCodeAt(0) === 3) {
+		// 			// Ctrl + c got pressed
+		// 			void this.stopProcess();
+		// 		} else {
+		// 			// When anything else got pressed, record it and send it on enter into the child process
+
+		// 			if (key.charCodeAt(0) === 13) {
+		// 				// send to child process and print in terminal
+		// 				process.stdout.write('\n');
+		// 			} else {
+		// 				// record it and write into terminal
+		// 				process.stdout.write(key);
+		// 			}
+		// 		}
+		// 	});
+		// }
 	}
 
 	async initPruning() {
diff --git a/packages/cli/test/benchmark/init.ts b/packages/cli/test/benchmark/init.ts
deleted file mode 100644
index aaf58e9019107..0000000000000
--- a/packages/cli/test/benchmark/init.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import { Config } from '@oclif/core';
-import { Start } from '@/commands/start';
-
-export async function startCommand() {
-	const args: string[] = [];
-	const _config = new Config({ root: __dirname });
-
-	const cmd = new Start(args, _config);
-
-	await cmd.init();
-	await cmd.run();
-}
-
-export const init = {
-	// database: testDb.init,
-	startCommand,
-};
diff --git a/packages/cli/test/benchmarks/init.ts b/packages/cli/test/benchmarks/init.ts
new file mode 100644
index 0000000000000..0bedc3587f71e
--- /dev/null
+++ b/packages/cli/test/benchmarks/init.ts
@@ -0,0 +1,18 @@
+import { Config } from '@oclif/core';
+import { Start } from '@/commands/start';
+import * as testDb from '../integration/shared/testDb';
+
+async function mainProcess() {
+	const args: string[] = [];
+	const _config = new Config({ root: __dirname });
+
+	const main = new Start(args, _config);
+
+	await main.init();
+	await main.run();
+}
+
+export const init = {
+	database: testDb.init,
+	mainProcess,
+};
diff --git a/packages/cli/test/benchmark/main.ts b/packages/cli/test/benchmarks/main.ts
similarity index 58%
rename from packages/cli/test/benchmark/main.ts
rename to packages/cli/test/benchmarks/main.ts
index a934c66aa002e..c1087be436172 100644
--- a/packages/cli/test/benchmark/main.ts
+++ b/packages/cli/test/benchmarks/main.ts
@@ -1,24 +1,15 @@
 /* eslint-disable import/no-extraneous-dependencies */ // @TODO: Remove
 
 import 'reflect-metadata';
-
 import { Bench } from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
-
-import { webhook } from './webhook.bm';
-
-function registerBenchmarks(bench: Bench) {
-	webhook(bench);
-}
+import { register } from './register';
 
 async function main() {
-	const bench = withCodSpeed(
-		new Bench(
-			{ time: 0, iterations: 1 }, // @TEMP: Remove
-		),
-	);
+	const _bench = new Bench({ time: 0, iterations: 1 }); // @TEMP: Remove arg
+	const bench = withCodSpeed(_bench);
 
-	registerBenchmarks(bench);
+	register(bench);
 
 	// await bench.warmup(); // @TODO: Restore
 	await bench.run();
diff --git a/packages/cli/test/benchmarks/register.ts b/packages/cli/test/benchmarks/register.ts
new file mode 100644
index 0000000000000..ead9d52506526
--- /dev/null
+++ b/packages/cli/test/benchmarks/register.ts
@@ -0,0 +1,6 @@
+import type Bench from 'tinybench';
+import { webhook } from './webhook.benchmark';
+
+export function register(bench: Bench) {
+	webhook(bench);
+}
diff --git a/packages/cli/test/benchmark/webhook.bm.ts b/packages/cli/test/benchmarks/webhook.benchmark.ts
similarity index 66%
rename from packages/cli/test/benchmark/webhook.bm.ts
rename to packages/cli/test/benchmarks/webhook.benchmark.ts
index 575a1b33aec47..02afda62e6d10 100644
--- a/packages/cli/test/benchmark/webhook.bm.ts
+++ b/packages/cli/test/benchmarks/webhook.benchmark.ts
@@ -6,18 +6,20 @@ export function webhook(bench: Bench) {
 	bench.add(
 		'`start` command',
 		async () => {
-			// console.log('ended');
-			console.log(bench.iterations);
+			console.log('iteration');
 		},
 		{
 			beforeAll: async () => {
 				console.log('beforeAll start');
 
-				await init.startCommand();
-				// await init.database();
+				await init.database();
+				await init.mainProcess();
 
 				console.log('beforeAll end');
 			},
+			afterAll: () => {
+				// stop process // @TODO
+			},
 		},
 	);
 }

From 4414eeec4c6d1bc10e6725af01a182e7c6481e10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 14:10:46 +0200
Subject: [PATCH 13/86] Set up test n8n dir

---
 packages/cli/test/benchmarks/init.ts          | 32 +++++++++++++++++++
 packages/cli/test/benchmarks/main.ts          |  2 ++
 .../cli/test/benchmarks/webhook.benchmark.ts  |  5 +--
 packages/core/src/InstanceSettings.ts         |  2 +-
 4 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/packages/cli/test/benchmarks/init.ts b/packages/cli/test/benchmarks/init.ts
index 0bedc3587f71e..c47f310b780b5 100644
--- a/packages/cli/test/benchmarks/init.ts
+++ b/packages/cli/test/benchmarks/init.ts
@@ -1,7 +1,38 @@
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
+
 import { Config } from '@oclif/core';
 import { Start } from '@/commands/start';
 import * as testDb from '../integration/shared/testDb';
+import Container from 'typedi';
+import { InstanceSettings } from 'n8n-core';
+
+/** Set up a temp `.n8n` dir for benchmarks to use. */
+function n8nDir() {
+	const baseDir = join(tmpdir(), 'n8n-benchmarks/');
+
+	mkdirSync(baseDir, { recursive: true });
+
+	const subDir = mkdtempSync(baseDir);
+	const n8nDir = join(subDir, '.n8n');
+
+	mkdirSync(n8nDir);
+
+	writeFileSync(
+		join(n8nDir, 'config'),
+		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: '123' }),
+		'utf-8',
+	);
+
+	const instanceSettings = Container.get(InstanceSettings);
+	instanceSettings.n8nFolder = n8nDir;
+	Container.set(InstanceSettings, instanceSettings);
+
+	console.log('n8nDir', n8nDir);
+}
 
+/** Initialize an n8n main process for benchmarking. */
 async function mainProcess() {
 	const args: string[] = [];
 	const _config = new Config({ root: __dirname });
@@ -13,6 +44,7 @@ async function mainProcess() {
 }
 
 export const init = {
+	n8nDir,
 	database: testDb.init,
 	mainProcess,
 };
diff --git a/packages/cli/test/benchmarks/main.ts b/packages/cli/test/benchmarks/main.ts
index c1087be436172..aea43c4b81670 100644
--- a/packages/cli/test/benchmarks/main.ts
+++ b/packages/cli/test/benchmarks/main.ts
@@ -9,6 +9,8 @@ async function main() {
 	const _bench = new Bench({ time: 0, iterations: 1 }); // @TEMP: Remove arg
 	const bench = withCodSpeed(_bench);
 
+	process.env.NODE_ENV = 'test';
+
 	register(bench);
 
 	// await bench.warmup(); // @TODO: Restore
diff --git a/packages/cli/test/benchmarks/webhook.benchmark.ts b/packages/cli/test/benchmarks/webhook.benchmark.ts
index 02afda62e6d10..1af538e388539 100644
--- a/packages/cli/test/benchmarks/webhook.benchmark.ts
+++ b/packages/cli/test/benchmarks/webhook.benchmark.ts
@@ -12,13 +12,14 @@ export function webhook(bench: Bench) {
 			beforeAll: async () => {
 				console.log('beforeAll start');
 
-				await init.database();
+				init.n8nDir();
+				await init.database(); // @TODO: Test with Postgres
 				await init.mainProcess();
 
 				console.log('beforeAll end');
 			},
 			afterAll: () => {
-				// stop process // @TODO
+				// @TODO stop main process gracefully
 			},
 		},
 	);
diff --git a/packages/core/src/InstanceSettings.ts b/packages/core/src/InstanceSettings.ts
index e8ab9aa553223..2a55445d04ff0 100644
--- a/packages/core/src/InstanceSettings.ts
+++ b/packages/core/src/InstanceSettings.ts
@@ -21,7 +21,7 @@ export class InstanceSettings {
 	private readonly userHome = this.getUserHome();
 
 	/** The path to the n8n folder in which all n8n related data gets saved */
-	readonly n8nFolder = path.join(this.userHome, '.n8n');
+	n8nFolder = path.join(this.userHome, '.n8n');
 
 	/** The path to the folder where all generated static assets are copied to */
 	readonly staticCacheDir = path.join(this.userHome, '.cache/n8n/public');

From 98df748fcfdc6f004d48f8e5ad5cce1d1c51214d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 14:17:58 +0200
Subject: [PATCH 14/86] Reload constants

---
 packages/cli/test/benchmarks/webhook.benchmark.ts | 2 ++
 packages/core/src/InstanceSettings.ts             | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/cli/test/benchmarks/webhook.benchmark.ts b/packages/cli/test/benchmarks/webhook.benchmark.ts
index 1af538e388539..aff402fdb4215 100644
--- a/packages/cli/test/benchmarks/webhook.benchmark.ts
+++ b/packages/cli/test/benchmarks/webhook.benchmark.ts
@@ -12,6 +12,8 @@ export function webhook(bench: Bench) {
 			beforeAll: async () => {
 				console.log('beforeAll start');
 
+				await import('../../src/constants');
+
 				init.n8nDir();
 				await init.database(); // @TODO: Test with Postgres
 				await init.mainProcess();
diff --git a/packages/core/src/InstanceSettings.ts b/packages/core/src/InstanceSettings.ts
index 2a55445d04ff0..66f127b2d9900 100644
--- a/packages/core/src/InstanceSettings.ts
+++ b/packages/core/src/InstanceSettings.ts
@@ -21,7 +21,7 @@ export class InstanceSettings {
 	private readonly userHome = this.getUserHome();
 
 	/** The path to the n8n folder in which all n8n related data gets saved */
-	n8nFolder = path.join(this.userHome, '.n8n');
+	n8nFolder = path.join(this.userHome, '.n8n'); // @TODO: Solution that keeps this readonly
 
 	/** The path to the folder where all generated static assets are copied to */
 	readonly staticCacheDir = path.join(this.userHome, '.cache/n8n/public');

From 155ff50a4812c57ee0fa6d22852e891e31e02cec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 14:44:38 +0200
Subject: [PATCH 15/86] Suite class

---
 .../cli/test/benchmarks/benchmark-suite.ts    | 37 +++++++++++++++++++
 packages/cli/test/benchmarks/main.ts          | 20 ++++------
 packages/cli/test/benchmarks/register.ts      |  6 ---
 .../cli/test/benchmarks/webhook.benchmark.ts  |  2 +-
 4 files changed, 46 insertions(+), 19 deletions(-)
 create mode 100644 packages/cli/test/benchmarks/benchmark-suite.ts
 delete mode 100644 packages/cli/test/benchmarks/register.ts

diff --git a/packages/cli/test/benchmarks/benchmark-suite.ts b/packages/cli/test/benchmarks/benchmark-suite.ts
new file mode 100644
index 0000000000000..fba6ebb7eec54
--- /dev/null
+++ b/packages/cli/test/benchmarks/benchmark-suite.ts
@@ -0,0 +1,37 @@
+import { withCodSpeed } from '@codspeed/tinybench-plugin';
+import { ApplicationError } from 'n8n-workflow';
+import Bench from 'tinybench';
+
+export class BenchmarkSuite {
+	private readonly suite: Bench;
+
+	constructor() {
+		this.suite = withCodSpeed(new Bench({ time: 0, iterations: 1 })); // @TEMP: Update constructor arg
+	}
+
+	register(registerFn: (bench: Bench) => void) {
+		registerFn(this.suite);
+
+		return this;
+	}
+
+	async warmup() {
+		this.assertBenchmarks();
+
+		await this.suite.warmup();
+	}
+
+	async run() {
+		this.assertBenchmarks();
+
+		await this.suite.run();
+	}
+
+	logResults() {
+		console.table(this.suite.table());
+	}
+
+	private assertBenchmarks() {
+		if (this.suite._tasks.size === 0) throw new ApplicationError('No benchmarks found');
+	}
+}
diff --git a/packages/cli/test/benchmarks/main.ts b/packages/cli/test/benchmarks/main.ts
index aea43c4b81670..9ac511901a955 100644
--- a/packages/cli/test/benchmarks/main.ts
+++ b/packages/cli/test/benchmarks/main.ts
@@ -1,22 +1,18 @@
-/* eslint-disable import/no-extraneous-dependencies */ // @TODO: Remove
-
 import 'reflect-metadata';
-import { Bench } from 'tinybench';
-import { withCodSpeed } from '@codspeed/tinybench-plugin';
-import { register } from './register';
+import { BenchmarkSuite } from './benchmark-suite';
+import { webhook } from './webhook.benchmark';
 
 async function main() {
-	const _bench = new Bench({ time: 0, iterations: 1 }); // @TEMP: Remove arg
-	const bench = withCodSpeed(_bench);
-
 	process.env.NODE_ENV = 'test';
 
-	register(bench);
+	const webhookSuite = new BenchmarkSuite().register(webhook);
+
+	process.env.NODE_ENV = 'test';
 
-	// await bench.warmup(); // @TODO: Restore
-	await bench.run();
+	// await suite.warmup(); // @TODO: Restore
+	await webhookSuite.run();
 
-	console.table(bench.table());
+	webhookSuite.logResults();
 }
 
 void main();
diff --git a/packages/cli/test/benchmarks/register.ts b/packages/cli/test/benchmarks/register.ts
deleted file mode 100644
index ead9d52506526..0000000000000
--- a/packages/cli/test/benchmarks/register.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-import type Bench from 'tinybench';
-import { webhook } from './webhook.benchmark';
-
-export function register(bench: Bench) {
-	webhook(bench);
-}
diff --git a/packages/cli/test/benchmarks/webhook.benchmark.ts b/packages/cli/test/benchmarks/webhook.benchmark.ts
index aff402fdb4215..19d6e84355a07 100644
--- a/packages/cli/test/benchmarks/webhook.benchmark.ts
+++ b/packages/cli/test/benchmarks/webhook.benchmark.ts
@@ -15,7 +15,7 @@ export function webhook(bench: Bench) {
 				await import('../../src/constants');
 
 				init.n8nDir();
-				await init.database(); // @TODO: Test with Postgres
+				// await init.database(); // @TODO: Test with Postgres
 				await init.mainProcess();
 
 				console.log('beforeAll end');

From 1d1a6ec67c3a51cda01772412dc0ac397e6ad506 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 14:48:30 +0200
Subject: [PATCH 16/86] Comment out constants reload

---
 packages/cli/test/benchmarks/webhook.benchmark.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/cli/test/benchmarks/webhook.benchmark.ts b/packages/cli/test/benchmarks/webhook.benchmark.ts
index 19d6e84355a07..285ad43e7cc04 100644
--- a/packages/cli/test/benchmarks/webhook.benchmark.ts
+++ b/packages/cli/test/benchmarks/webhook.benchmark.ts
@@ -12,7 +12,7 @@ export function webhook(bench: Bench) {
 			beforeAll: async () => {
 				console.log('beforeAll start');
 
-				await import('../../src/constants');
+				// await import('../../src/constants');
 
 				init.n8nDir();
 				// await init.database(); // @TODO: Test with Postgres

From ff2d09c70e006f8c545f23dc9ca45eaf28bbea3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 15:12:54 +0200
Subject: [PATCH 17/86] Create BenchmarkSetup

---
 .../cli/test/benchmarks/benchmark-setup.ts    | 52 +++++++++++++++++++
 packages/cli/test/benchmarks/main.ts          |  2 -
 .../cli/test/benchmarks/webhook.benchmark.ts  | 15 +-----
 3 files changed, 54 insertions(+), 15 deletions(-)
 create mode 100644 packages/cli/test/benchmarks/benchmark-setup.ts

diff --git a/packages/cli/test/benchmarks/benchmark-setup.ts b/packages/cli/test/benchmarks/benchmark-setup.ts
new file mode 100644
index 0000000000000..f5dbaeb2921b8
--- /dev/null
+++ b/packages/cli/test/benchmarks/benchmark-setup.ts
@@ -0,0 +1,52 @@
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
+
+import { Config } from '@oclif/core';
+import { Start } from '@/commands/start';
+import Container from 'typedi';
+import { InstanceSettings } from 'n8n-core';
+
+export class BenchmarkSetup {
+	/** Setup to run before all iterations of a single benchmark. */
+	static beforeAll() {
+		return async () => {
+			this.n8nDir();
+
+			await this.mainProcess();
+		};
+	}
+
+	static n8nDir() {
+		const baseDir = path.join(tmpdir(), 'n8n-benchmarks/');
+
+		mkdirSync(baseDir, { recursive: true });
+
+		const subDir = mkdtempSync(baseDir);
+		const n8nDir = path.join(subDir, '.n8n');
+
+		mkdirSync(n8nDir);
+
+		writeFileSync(
+			path.join(n8nDir, 'config'),
+			JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: '123' }),
+			'utf-8',
+		);
+
+		const instanceSettings = Container.get(InstanceSettings);
+		instanceSettings.n8nFolder = n8nDir;
+		Container.set(InstanceSettings, instanceSettings);
+
+		console.log('n8nDir', n8nDir);
+	}
+
+	static async mainProcess() {
+		const args: string[] = [];
+		const _config = new Config({ root: __dirname });
+
+		const main = new Start(args, _config);
+
+		await main.init();
+		await main.run();
+	}
+}
diff --git a/packages/cli/test/benchmarks/main.ts b/packages/cli/test/benchmarks/main.ts
index 9ac511901a955..6d478204c136d 100644
--- a/packages/cli/test/benchmarks/main.ts
+++ b/packages/cli/test/benchmarks/main.ts
@@ -7,8 +7,6 @@ async function main() {
 
 	const webhookSuite = new BenchmarkSuite().register(webhook);
 
-	process.env.NODE_ENV = 'test';
-
 	// await suite.warmup(); // @TODO: Restore
 	await webhookSuite.run();
 
diff --git a/packages/cli/test/benchmarks/webhook.benchmark.ts b/packages/cli/test/benchmarks/webhook.benchmark.ts
index 285ad43e7cc04..d471b4257b472 100644
--- a/packages/cli/test/benchmarks/webhook.benchmark.ts
+++ b/packages/cli/test/benchmarks/webhook.benchmark.ts
@@ -1,7 +1,6 @@
+import { BenchmarkSetup } from './benchmark-setup';
 import type Bench from 'tinybench';
 
-import { init } from './init';
-
 export function webhook(bench: Bench) {
 	bench.add(
 		'`start` command',
@@ -9,17 +8,7 @@ export function webhook(bench: Bench) {
 			console.log('iteration');
 		},
 		{
-			beforeAll: async () => {
-				console.log('beforeAll start');
-
-				// await import('../../src/constants');
-
-				init.n8nDir();
-				// await init.database(); // @TODO: Test with Postgres
-				await init.mainProcess();
-
-				console.log('beforeAll end');
-			},
+			beforeAll: BenchmarkSetup.beforeAll(),
 			afterAll: () => {
 				// @TODO stop main process gracefully
 			},

From f4e76d6f8526b09e1273c96eedda4bd976e31b4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 15:14:57 +0200
Subject: [PATCH 18/86] Remove unused `init.ts`

---
 packages/cli/test/benchmarks/init.ts | 50 ----------------------------
 1 file changed, 50 deletions(-)
 delete mode 100644 packages/cli/test/benchmarks/init.ts

diff --git a/packages/cli/test/benchmarks/init.ts b/packages/cli/test/benchmarks/init.ts
deleted file mode 100644
index c47f310b780b5..0000000000000
--- a/packages/cli/test/benchmarks/init.ts
+++ /dev/null
@@ -1,50 +0,0 @@
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
-
-import { Config } from '@oclif/core';
-import { Start } from '@/commands/start';
-import * as testDb from '../integration/shared/testDb';
-import Container from 'typedi';
-import { InstanceSettings } from 'n8n-core';
-
-/** Set up a temp `.n8n` dir for benchmarks to use. */
-function n8nDir() {
-	const baseDir = join(tmpdir(), 'n8n-benchmarks/');
-
-	mkdirSync(baseDir, { recursive: true });
-
-	const subDir = mkdtempSync(baseDir);
-	const n8nDir = join(subDir, '.n8n');
-
-	mkdirSync(n8nDir);
-
-	writeFileSync(
-		join(n8nDir, 'config'),
-		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: '123' }),
-		'utf-8',
-	);
-
-	const instanceSettings = Container.get(InstanceSettings);
-	instanceSettings.n8nFolder = n8nDir;
-	Container.set(InstanceSettings, instanceSettings);
-
-	console.log('n8nDir', n8nDir);
-}
-
-/** Initialize an n8n main process for benchmarking. */
-async function mainProcess() {
-	const args: string[] = [];
-	const _config = new Config({ root: __dirname });
-
-	const main = new Start(args, _config);
-
-	await main.init();
-	await main.run();
-}
-
-export const init = {
-	n8nDir,
-	database: testDb.init,
-	mainProcess,
-};

From 2382e65a0870f949ae2d6cf6177886b80029a982 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 16:11:45 +0200
Subject: [PATCH 19/86] Restructuring

---
 .../cli/test/benchmarks/benchmark-suite.ts    | 12 ++++-----
 packages/cli/test/benchmarks/main.ts          |  6 +++--
 .../{benchmark-setup.ts => setup-utils.ts}    | 14 ++--------
 packages/cli/test/benchmarks/types.ts         |  8 ++++++
 .../cli/test/benchmarks/webhook.benchmark.ts  | 17 ------------
 .../cli/test/benchmarks/webhook.benchmarks.ts | 26 +++++++++++++++++++
 6 files changed, 46 insertions(+), 37 deletions(-)
 rename packages/cli/test/benchmarks/{benchmark-setup.ts => setup-utils.ts} (84%)
 create mode 100644 packages/cli/test/benchmarks/types.ts
 delete mode 100644 packages/cli/test/benchmarks/webhook.benchmark.ts
 create mode 100644 packages/cli/test/benchmarks/webhook.benchmarks.ts

diff --git a/packages/cli/test/benchmarks/benchmark-suite.ts b/packages/cli/test/benchmarks/benchmark-suite.ts
index fba6ebb7eec54..54a9964a5333a 100644
--- a/packages/cli/test/benchmarks/benchmark-suite.ts
+++ b/packages/cli/test/benchmarks/benchmark-suite.ts
@@ -1,16 +1,16 @@
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
 import { ApplicationError } from 'n8n-workflow';
 import Bench from 'tinybench';
+import type { Benchmark } from './types';
 
 export class BenchmarkSuite {
-	private readonly suite: Bench;
+	private suite: Bench;
 
-	constructor() {
-		this.suite = withCodSpeed(new Bench({ time: 0, iterations: 1 })); // @TEMP: Update constructor arg
-	}
+	add({ register, setup, teardown }: Benchmark) {
+		// @TEMP: Update time and iterations
+		this.suite = withCodSpeed(new Bench({ time: 0, iterations: 1, setup, teardown }));
 
-	register(registerFn: (bench: Bench) => void) {
-		registerFn(this.suite);
+		register(this.suite);
 
 		return this;
 	}
diff --git a/packages/cli/test/benchmarks/main.ts b/packages/cli/test/benchmarks/main.ts
index 6d478204c136d..7d70fa70ba177 100644
--- a/packages/cli/test/benchmarks/main.ts
+++ b/packages/cli/test/benchmarks/main.ts
@@ -1,11 +1,13 @@
 import 'reflect-metadata';
 import { BenchmarkSuite } from './benchmark-suite';
-import { webhook } from './webhook.benchmark';
+import { webhook } from './webhook.benchmarks';
 
 async function main() {
 	process.env.NODE_ENV = 'test';
 
-	const webhookSuite = new BenchmarkSuite().register(webhook);
+	const webhookSuite = new BenchmarkSuite().add(webhook);
+
+	// const webhookSuite = new BenchmarkSuite({ setup, register });
 
 	// await suite.warmup(); // @TODO: Restore
 	await webhookSuite.run();
diff --git a/packages/cli/test/benchmarks/benchmark-setup.ts b/packages/cli/test/benchmarks/setup-utils.ts
similarity index 84%
rename from packages/cli/test/benchmarks/benchmark-setup.ts
rename to packages/cli/test/benchmarks/setup-utils.ts
index f5dbaeb2921b8..99cb8f831bae5 100644
--- a/packages/cli/test/benchmarks/benchmark-setup.ts
+++ b/packages/cli/test/benchmarks/setup-utils.ts
@@ -1,22 +1,12 @@
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
-
 import { Config } from '@oclif/core';
+import { InstanceSettings } from 'n8n-core';
 import { Start } from '@/commands/start';
 import Container from 'typedi';
-import { InstanceSettings } from 'n8n-core';
-
-export class BenchmarkSetup {
-	/** Setup to run before all iterations of a single benchmark. */
-	static beforeAll() {
-		return async () => {
-			this.n8nDir();
-
-			await this.mainProcess();
-		};
-	}
 
+export class SetupUtils {
 	static n8nDir() {
 		const baseDir = path.join(tmpdir(), 'n8n-benchmarks/');
 
diff --git a/packages/cli/test/benchmarks/types.ts b/packages/cli/test/benchmarks/types.ts
new file mode 100644
index 0000000000000..92bf29c438b1e
--- /dev/null
+++ b/packages/cli/test/benchmarks/types.ts
@@ -0,0 +1,8 @@
+import type { Hook } from 'tinybench';
+import type Bench from 'tinybench';
+
+export type Benchmark = {
+	setup: Hook;
+	teardown: Hook;
+	register: (bench: Bench) => void;
+};
diff --git a/packages/cli/test/benchmarks/webhook.benchmark.ts b/packages/cli/test/benchmarks/webhook.benchmark.ts
deleted file mode 100644
index d471b4257b472..0000000000000
--- a/packages/cli/test/benchmarks/webhook.benchmark.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import { BenchmarkSetup } from './benchmark-setup';
-import type Bench from 'tinybench';
-
-export function webhook(bench: Bench) {
-	bench.add(
-		'`start` command',
-		async () => {
-			console.log('iteration');
-		},
-		{
-			beforeAll: BenchmarkSetup.beforeAll(),
-			afterAll: () => {
-				// @TODO stop main process gracefully
-			},
-		},
-	);
-}
diff --git a/packages/cli/test/benchmarks/webhook.benchmarks.ts b/packages/cli/test/benchmarks/webhook.benchmarks.ts
new file mode 100644
index 0000000000000..21167074c848e
--- /dev/null
+++ b/packages/cli/test/benchmarks/webhook.benchmarks.ts
@@ -0,0 +1,26 @@
+import { SetupUtils } from './setup-utils';
+import type Bench from 'tinybench';
+
+export const webhook = {
+	async setup() {
+		SetupUtils.n8nDir();
+
+		await SetupUtils.mainProcess();
+	},
+
+	async teardown() {
+		// ...
+	},
+
+	register(bench: Bench) {
+		bench.add('Some description', async () => {
+			console.log('Something happening...');
+		});
+	},
+};
+
+/**
+ * benchmark('Some description', async () => {
+ *	console.log('Something happening...');
+ * });
+ */

From 648d5b83bbcbe35d9d25f74e0110df276ae9f12a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 19:01:56 +0200
Subject: [PATCH 20/86] refactor(core): Fix type errors in BE tests
 (no-changelog)

---
 .../ExternalSecrets/externalSecrets.api.test.ts   |  7 ++++---
 packages/cli/test/integration/auth.api.test.ts    |  6 +++---
 packages/cli/test/integration/auth.mw.test.ts     |  5 +++--
 .../cli/test/integration/binaryData.api.test.ts   |  5 +++--
 .../integration/community-packages.api.test.ts    |  5 +++--
 .../cli/test/integration/credentials.ee.test.ts   |  7 ++++---
 packages/cli/test/integration/credentials.test.ts |  7 ++++---
 .../cli/test/integration/debug.controller.test.ts |  5 +++--
 .../environments/SourceControl.test.ts            |  5 +++--
 packages/cli/test/integration/eventbus.ee.test.ts |  5 +++--
 packages/cli/test/integration/eventbus.test.ts    |  5 +++--
 .../cli/test/integration/ldap/ldap.api.test.ts    |  5 +++--
 packages/cli/test/integration/license.api.test.ts |  7 ++++---
 packages/cli/test/integration/me.api.test.ts      |  7 ++++---
 .../integration/publicApi/credentials.test.ts     |  7 ++++---
 .../test/integration/publicApi/executions.test.ts |  9 +++++----
 .../cli/test/integration/publicApi/tags.test.ts   |  8 ++++----
 .../test/integration/publicApi/users.ee.test.ts   |  5 +++--
 .../test/integration/publicApi/workflows.test.ts  |  7 ++++---
 .../cli/test/integration/saml/saml.api.test.ts    |  7 ++++---
 packages/cli/test/integration/shared/types.ts     |  9 +++++----
 .../test/integration/shared/utils/testServer.ts   |  3 ---
 packages/cli/test/integration/tags.api.test.ts    |  5 +++--
 packages/cli/test/integration/users.api.test.ts   | 15 ++++++++-------
 packages/cli/test/integration/variables.test.ts   |  7 ++++---
 .../test/integration/workflowHistory.api.test.ts  |  7 ++++---
 .../workflows/workflows.controller.ee.test.ts     |  9 +++++----
 .../workflows/workflows.controller.test.ts        |  5 +++--
 28 files changed, 103 insertions(+), 81 deletions(-)

diff --git a/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts b/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts
index 868c695f4a38a..3d47653038ca8 100644
--- a/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts
+++ b/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { License } from '@/License';
 import type { ExternalSecretsSettings, SecretsProviderState } from '@/Interfaces';
 import { Cipher } from 'n8n-core';
@@ -21,8 +22,8 @@ import {
 	TestFailProvider,
 } from '../../shared/ExternalSecrets/utils';
 
-let authOwnerAgent: SuperAgentTest;
-let authMemberAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authMemberAgent: TestAgent<Test>;
 
 const mockProvidersInstance = new MockProviders();
 mockInstance(ExternalSecretsProviders, mockProvidersInstance);
diff --git a/packages/cli/test/integration/auth.api.test.ts b/packages/cli/test/integration/auth.api.test.ts
index 618c75a209cb4..6a41b7c560b13 100644
--- a/packages/cli/test/integration/auth.api.test.ts
+++ b/packages/cli/test/integration/auth.api.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { Container } from 'typedi';
 import validator from 'validator';
 import config from '@/config';
@@ -13,7 +14,7 @@ import { UserRepository } from '@db/repositories/user.repository';
 import { MfaService } from '@/Mfa/mfa.service';
 
 let owner: User;
-let authOwnerAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
 const ownerPassword = randomValidPassword();
 
 const testServer = utils.setupTestServer({ endpointGroups: ['auth'] });
@@ -158,7 +159,6 @@ describe('GET /login', () => {
 	});
 
 	test('should return 401 Unauthorized if invalid cookie', async () => {
-		// @ts-ignore
 		testServer.authlessAgent.jar.setCookie(`${AUTH_COOKIE_NAME}=invalid`);
 
 		const response = await testServer.authlessAgent.get('/login');
diff --git a/packages/cli/test/integration/auth.mw.test.ts b/packages/cli/test/integration/auth.mw.test.ts
index add424a03823d..9414f21802d54 100644
--- a/packages/cli/test/integration/auth.mw.test.ts
+++ b/packages/cli/test/integration/auth.mw.test.ts
@@ -1,6 +1,7 @@
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import * as utils from './shared/utils/';
 import { createUser } from './shared/db/users';
 import { mockInstance } from '../shared/mocking';
@@ -38,7 +39,7 @@ describe('Auth Middleware', () => {
 	});
 
 	describe('Routes requiring Authorization', () => {
-		let authMemberAgent: SuperAgentTest;
+		let authMemberAgent: TestAgent<Test>;
 		beforeAll(async () => {
 			const member = await createUser({ role: 'global:member' });
 			authMemberAgent = testServer.authAgentFor(member);
diff --git a/packages/cli/test/integration/binaryData.api.test.ts b/packages/cli/test/integration/binaryData.api.test.ts
index 6a3a314c737fa..e1dc79672ad28 100644
--- a/packages/cli/test/integration/binaryData.api.test.ts
+++ b/packages/cli/test/integration/binaryData.api.test.ts
@@ -1,7 +1,8 @@
 import fsp from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { BinaryDataService, FileNotFoundError } from 'n8n-core';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 
 import { mockInstance } from '../shared/mocking';
 import { setupTestServer } from './shared/utils';
@@ -15,7 +16,7 @@ const throwFileNotFound = () => {
 
 const binaryDataService = mockInstance(BinaryDataService);
 let testServer = setupTestServer({ endpointGroups: ['binaryData'] });
-let authOwnerAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
 
 beforeAll(async () => {
 	const owner = await createOwner();
diff --git a/packages/cli/test/integration/community-packages.api.test.ts b/packages/cli/test/integration/community-packages.api.test.ts
index 03fd43f456824..cc1378dc9c571 100644
--- a/packages/cli/test/integration/community-packages.api.test.ts
+++ b/packages/cli/test/integration/community-packages.api.test.ts
@@ -1,5 +1,6 @@
 import path from 'path';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 
 import type { InstalledPackages } from '@db/entities/InstalledPackages';
 import type { InstalledNodes } from '@db/entities/InstalledNodes';
@@ -30,7 +31,7 @@ const parsedNpmPackageName = {
 	rawString: 'test',
 };
 
-let authAgent: SuperAgentTest;
+let authAgent: TestAgent<Test>;
 
 beforeAll(async () => {
 	const ownerShell = await createOwner();
diff --git a/packages/cli/test/integration/credentials.ee.test.ts b/packages/cli/test/integration/credentials.ee.test.ts
index 279f33c019dde..2abc483df0e64 100644
--- a/packages/cli/test/integration/credentials.ee.test.ts
+++ b/packages/cli/test/integration/credentials.ee.test.ts
@@ -1,5 +1,6 @@
 import { Container } from 'typedi';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { In } from '@n8n/typeorm';
 import type { IUser } from 'n8n-workflow';
 
@@ -26,8 +27,8 @@ const testServer = utils.setupTestServer({
 let owner: User;
 let member: User;
 let anotherMember: User;
-let authOwnerAgent: SuperAgentTest;
-let authAnotherMemberAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authAnotherMemberAgent: TestAgent<Test>;
 let saveCredential: SaveCredentialFunction;
 const mailer = mockInstance(UserManagementMailer);
 
diff --git a/packages/cli/test/integration/credentials.test.ts b/packages/cli/test/integration/credentials.test.ts
index aa40616ad8681..9ca946b63b1e1 100644
--- a/packages/cli/test/integration/credentials.test.ts
+++ b/packages/cli/test/integration/credentials.test.ts
@@ -1,5 +1,6 @@
 import { Container } from 'typedi';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 
 import config from '@/config';
 import type { ListQuery } from '@/requests';
@@ -23,8 +24,8 @@ const testServer = utils.setupTestServer({ endpointGroups: ['credentials'] });
 let owner: User;
 let member: User;
 let secondMember: User;
-let authOwnerAgent: SuperAgentTest;
-let authMemberAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authMemberAgent: TestAgent<Test>;
 let saveCredential: SaveCredentialFunction;
 
 beforeAll(async () => {
diff --git a/packages/cli/test/integration/debug.controller.test.ts b/packages/cli/test/integration/debug.controller.test.ts
index 9acd6a39931d4..468e9dc2c8142 100644
--- a/packages/cli/test/integration/debug.controller.test.ts
+++ b/packages/cli/test/integration/debug.controller.test.ts
@@ -5,7 +5,8 @@ import { randomName } from './shared/random';
 import { generateNanoId } from '@/databases/utils/generators';
 import type { WorkflowEntity } from '@/databases/entities/WorkflowEntity';
 import { setupTestServer } from './shared/utils';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { createOwner } from './shared/db/users';
 import { OrchestrationService } from '@/services/orchestration.service';
 import { MultiMainSetup } from '@/services/orchestration/main/MultiMainSetup.ee';
@@ -15,7 +16,7 @@ describe('DebugController', () => {
 	const activeWorkflowRunner = mockInstance(ActiveWorkflowRunner);
 
 	let testServer = setupTestServer({ endpointGroups: ['debug'] });
-	let ownerAgent: SuperAgentTest;
+	let ownerAgent: TestAgent<Test>;
 
 	beforeAll(async () => {
 		const owner = await createOwner();
diff --git a/packages/cli/test/integration/environments/SourceControl.test.ts b/packages/cli/test/integration/environments/SourceControl.test.ts
index f1da8d0672f25..3886dcb7651a6 100644
--- a/packages/cli/test/integration/environments/SourceControl.test.ts
+++ b/packages/cli/test/integration/environments/SourceControl.test.ts
@@ -1,5 +1,6 @@
 import { Container } from 'typedi';
-import type { SuperAgentTest } from 'supertest';
+import type Test from 'supertest/lib/test';
+import type TestAgent from 'supertest/lib/agent';
 
 import type { User } from '@db/entities/User';
 import config from '@/config';
@@ -10,7 +11,7 @@ import type { SourceControlledFile } from '@/environments/sourceControl/types/so
 import * as utils from '../shared/utils/';
 import { createUser } from '../shared/db/users';
 
-let authOwnerAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
 let owner: User;
 
 const testServer = utils.setupTestServer({
diff --git a/packages/cli/test/integration/eventbus.ee.test.ts b/packages/cli/test/integration/eventbus.ee.test.ts
index f486f39abe12b..dbde24a100ade 100644
--- a/packages/cli/test/integration/eventbus.ee.test.ts
+++ b/packages/cli/test/integration/eventbus.ee.test.ts
@@ -3,7 +3,8 @@ import config from '@/config';
 import axios from 'axios';
 import syslog from 'syslog-client';
 import { v4 as uuid } from 'uuid';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import type {
 	MessageEventBusDestinationSentryOptions,
 	MessageEventBusDestinationSyslogOptions,
@@ -38,7 +39,7 @@ jest.mock('syslog-client');
 const mockedSyslog = syslog as jest.Mocked<typeof syslog>;
 
 let owner: User;
-let authOwnerAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
 
 const testSyslogDestination: MessageEventBusDestinationSyslogOptions = {
 	...defaultMessageEventBusDestinationSyslogOptions,
diff --git a/packages/cli/test/integration/eventbus.test.ts b/packages/cli/test/integration/eventbus.test.ts
index 9e3e27dacb916..d13e87bc27492 100644
--- a/packages/cli/test/integration/eventbus.test.ts
+++ b/packages/cli/test/integration/eventbus.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 
 import type { User } from '@db/entities/User';
 import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus';
@@ -15,7 +16,7 @@ import { mockInstance } from '../shared/mocking';
  */
 
 let owner: User;
-let authOwnerAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
 
 mockInstance(MessageEventBus);
 mockInstance(ExecutionDataRecoveryService);
diff --git a/packages/cli/test/integration/ldap/ldap.api.test.ts b/packages/cli/test/integration/ldap/ldap.api.test.ts
index 17cb3e7b543b2..37441764557bd 100644
--- a/packages/cli/test/integration/ldap/ldap.api.test.ts
+++ b/packages/cli/test/integration/ldap/ldap.api.test.ts
@@ -1,5 +1,6 @@
 import Container from 'typedi';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import type { Entry as LdapUser } from 'ldapts';
 import { Not } from '@n8n/typeorm';
 import { jsonParse } from 'n8n-workflow';
@@ -25,7 +26,7 @@ import { AuthProviderSyncHistoryRepository } from '@db/repositories/authProvider
 jest.mock('@/telemetry');
 
 let owner: User;
-let authOwnerAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
 
 const defaultLdapConfig = {
 	...LDAP_DEFAULT_CONFIGURATION,
diff --git a/packages/cli/test/integration/license.api.test.ts b/packages/cli/test/integration/license.api.test.ts
index 3d1fc4cda84a7..bab60456f9299 100644
--- a/packages/cli/test/integration/license.api.test.ts
+++ b/packages/cli/test/integration/license.api.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import config from '@/config';
 import type { User } from '@db/entities/User';
 import type { ILicensePostResponse, ILicenseReadResponse } from '@/Interfaces';
@@ -12,8 +13,8 @@ const MOCK_RENEW_OFFSET = 259200;
 
 let owner: User;
 let member: User;
-let authOwnerAgent: SuperAgentTest;
-let authMemberAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authMemberAgent: TestAgent<Test>;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['license'] });
 
diff --git a/packages/cli/test/integration/me.api.test.ts b/packages/cli/test/integration/me.api.test.ts
index 53ee82343029f..3f23b81c6c9dc 100644
--- a/packages/cli/test/integration/me.api.test.ts
+++ b/packages/cli/test/integration/me.api.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { IsNull } from '@n8n/typeorm';
 import validator from 'validator';
 import type { User } from '@db/entities/User';
@@ -24,7 +25,7 @@ beforeEach(async () => {
 
 describe('Owner shell', () => {
 	let ownerShell: User;
-	let authOwnerShellAgent: SuperAgentTest;
+	let authOwnerShellAgent: TestAgent<Test>;
 
 	beforeEach(async () => {
 		ownerShell = await createUserShell('global:owner');
@@ -161,7 +162,7 @@ describe('Owner shell', () => {
 describe('Member', () => {
 	const memberPassword = randomValidPassword();
 	let member: User;
-	let authMemberAgent: SuperAgentTest;
+	let authMemberAgent: TestAgent<Test>;
 
 	beforeEach(async () => {
 		member = await createUser({
diff --git a/packages/cli/test/integration/publicApi/credentials.test.ts b/packages/cli/test/integration/publicApi/credentials.test.ts
index d028834638ca0..fae1acfb14273 100644
--- a/packages/cli/test/integration/publicApi/credentials.test.ts
+++ b/packages/cli/test/integration/publicApi/credentials.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import type { User } from '@db/entities/User';
 
 import { randomApiKey, randomName, randomString } from '../shared/random';
@@ -13,8 +14,8 @@ import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.
 
 let owner: User;
 let member: User;
-let authOwnerAgent: SuperAgentTest;
-let authMemberAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authMemberAgent: TestAgent<Test>;
 
 let saveCredential: SaveCredentialFunction;
 
diff --git a/packages/cli/test/integration/publicApi/executions.test.ts b/packages/cli/test/integration/publicApi/executions.test.ts
index 92966d5862886..319ee86401b29 100644
--- a/packages/cli/test/integration/publicApi/executions.test.ts
+++ b/packages/cli/test/integration/publicApi/executions.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import type { User } from '@db/entities/User';
 import type { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 
@@ -21,9 +22,9 @@ import {
 let owner: User;
 let user1: User;
 let user2: User;
-let authOwnerAgent: SuperAgentTest;
-let authUser1Agent: SuperAgentTest;
-let authUser2Agent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authUser1Agent: TestAgent<Test>;
+let authUser2Agent: TestAgent<Test>;
 let workflowRunner: ActiveWorkflowRunner;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] });
diff --git a/packages/cli/test/integration/publicApi/tags.test.ts b/packages/cli/test/integration/publicApi/tags.test.ts
index 9a88b11d9a7a2..574568a06b41e 100644
--- a/packages/cli/test/integration/publicApi/tags.test.ts
+++ b/packages/cli/test/integration/publicApi/tags.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import Container from 'typedi';
 import type { User } from '@db/entities/User';
 import { TagRepository } from '@db/repositories/tag.repository';
@@ -11,8 +12,8 @@ import { createTag } from '../shared/db/tags';
 
 let owner: User;
 let member: User;
-let authOwnerAgent: SuperAgentTest;
-let authMemberAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authMemberAgent: TestAgent<Test>;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] });
 
@@ -37,7 +38,6 @@ beforeEach(async () => {
 
 const testWithAPIKey =
 	(method: 'get' | 'post' | 'put' | 'delete', url: string, apiKey: string | null) => async () => {
-		// @ts-ignore
 		void authOwnerAgent.set({ 'X-N8N-API-KEY': apiKey });
 		const response = await authOwnerAgent[method](url);
 		expect(response.statusCode).toBe(401);
diff --git a/packages/cli/test/integration/publicApi/users.ee.test.ts b/packages/cli/test/integration/publicApi/users.ee.test.ts
index 8dfae84625e87..a18ee3e4435cf 100644
--- a/packages/cli/test/integration/publicApi/users.ee.test.ts
+++ b/packages/cli/test/integration/publicApi/users.ee.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import validator from 'validator';
 import { v4 as uuid } from 'uuid';
 
@@ -202,7 +203,7 @@ describe('With license unlimited quota:users', () => {
 });
 
 describe('With license without quota:users', () => {
-	let authOwnerAgent: SuperAgentTest;
+	let authOwnerAgent: TestAgent<Test>;
 
 	beforeEach(async () => {
 		mockInstance(License, { getUsersLimit: jest.fn().mockReturnValue(null) });
diff --git a/packages/cli/test/integration/publicApi/workflows.test.ts b/packages/cli/test/integration/publicApi/workflows.test.ts
index 309030736d29a..8c67a87fc63c6 100644
--- a/packages/cli/test/integration/publicApi/workflows.test.ts
+++ b/packages/cli/test/integration/publicApi/workflows.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import config from '@/config';
 import Container from 'typedi';
 import type { INode } from 'n8n-workflow';
@@ -20,8 +21,8 @@ import { mockInstance } from '../../shared/mocking';
 
 let owner: User;
 let member: User;
-let authOwnerAgent: SuperAgentTest;
-let authMemberAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authMemberAgent: TestAgent<Test>;
 let workflowRunner: ActiveWorkflowRunner;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] });
diff --git a/packages/cli/test/integration/saml/saml.api.test.ts b/packages/cli/test/integration/saml/saml.api.test.ts
index c12f8da1752fc..199ad9f727836 100644
--- a/packages/cli/test/integration/saml/saml.api.test.ts
+++ b/packages/cli/test/integration/saml/saml.api.test.ts
@@ -1,5 +1,6 @@
 import { Container } from 'typedi';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import type { AuthenticationMethod } from 'n8n-workflow';
 import type { User } from '@db/entities/User';
 import { setSamlLoginEnabled } from '@/sso/saml/samlHelpers';
@@ -15,8 +16,8 @@ import { createOwner, createUser } from '../shared/db/users';
 
 let someUser: User;
 let owner: User;
-let authMemberAgent: SuperAgentTest;
-let authOwnerAgent: SuperAgentTest;
+let authMemberAgent: TestAgent<Test>;
+let authOwnerAgent: TestAgent<Test>;
 
 async function enableSaml(enable: boolean) {
 	await setSamlLoginEnabled(enable);
diff --git a/packages/cli/test/integration/shared/types.ts b/packages/cli/test/integration/shared/types.ts
index 8355d6f39c3f0..1195220fb1e9f 100644
--- a/packages/cli/test/integration/shared/types.ts
+++ b/packages/cli/test/integration/shared/types.ts
@@ -1,6 +1,7 @@
 import type { Application } from 'express';
 import type { ICredentialDataDecryptedObject } from 'n8n-workflow';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import type { Server } from 'http';
 
 import type { CredentialsEntity } from '@db/entities/CredentialsEntity';
@@ -43,9 +44,9 @@ export interface SetupProps {
 export interface TestServer {
 	app: Application;
 	httpServer: Server;
-	authAgentFor: (user: User) => SuperAgentTest;
-	publicApiAgentFor: (user: User) => SuperAgentTest;
-	authlessAgent: SuperAgentTest;
+	authAgentFor: (user: User) => TestAgent<Test>;
+	publicApiAgentFor: (user: User) => TestAgent<Test>;
+	authlessAgent: TestAgent<Test>;
 	license: LicenseMocker;
 }
 
diff --git a/packages/cli/test/integration/shared/utils/testServer.ts b/packages/cli/test/integration/shared/utils/testServer.ts
index 161ada8bcc43f..d6b9aa40efd04 100644
--- a/packages/cli/test/integration/shared/utils/testServer.ts
+++ b/packages/cli/test/integration/shared/utils/testServer.ts
@@ -89,11 +89,8 @@ export const setupTestServer = ({
 	const testServer: TestServer = {
 		app,
 		httpServer: app.listen(0),
-		// @ts-ignore
 		authAgentFor: (user: User) => createAgent(app, { auth: true, user }),
-		// @ts-ignore
 		authlessAgent: createAgent(app),
-		// @ts-ignore
 		publicApiAgentFor: (user) => publicApiAgent(app, { user }),
 		license: new LicenseMocker(),
 	};
diff --git a/packages/cli/test/integration/tags.api.test.ts b/packages/cli/test/integration/tags.api.test.ts
index 73be97a1c269f..5aa2d930c348c 100644
--- a/packages/cli/test/integration/tags.api.test.ts
+++ b/packages/cli/test/integration/tags.api.test.ts
@@ -1,11 +1,12 @@
 import * as utils from './shared/utils/';
 import * as testDb from './shared/testDb';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { TagRepository } from '@db/repositories/tag.repository';
 import Container from 'typedi';
 import { createUserShell } from './shared/db/users';
 
-let authOwnerAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
 const testServer = utils.setupTestServer({ endpointGroups: ['tags'] });
 
 beforeAll(async () => {
diff --git a/packages/cli/test/integration/users.api.test.ts b/packages/cli/test/integration/users.api.test.ts
index 5c9bdc2600a8e..3625490c16d1c 100644
--- a/packages/cli/test/integration/users.api.test.ts
+++ b/packages/cli/test/integration/users.api.test.ts
@@ -1,5 +1,6 @@
 import Container from 'typedi';
-import type { SuperAgentTest } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
+import type { Test } from 'supertest';
 
 import { UsersController } from '@/controllers/users.controller';
 import type { User } from '@db/entities/User';
@@ -28,7 +29,7 @@ const testServer = utils.setupTestServer({
 describe('GET /users', () => {
 	let owner: User;
 	let member: User;
-	let ownerAgent: SuperAgentTest;
+	let ownerAgent: TestAgent<Test>;
 
 	beforeAll(async () => {
 		await testDb.truncate(['User']);
@@ -230,7 +231,7 @@ describe('GET /users', () => {
 describe('DELETE /users/:id', () => {
 	let owner: User;
 	let member: User;
-	let ownerAgent: SuperAgentTest;
+	let ownerAgent: TestAgent<Test>;
 
 	beforeAll(async () => {
 		await testDb.truncate(['User']);
@@ -347,10 +348,10 @@ describe('PATCH /users/:id/role', () => {
 	let member: User;
 	let otherMember: User;
 
-	let ownerAgent: SuperAgentTest;
-	let adminAgent: SuperAgentTest;
-	let memberAgent: SuperAgentTest;
-	let authlessAgent: SuperAgentTest;
+	let ownerAgent: TestAgent<Test>;
+	let adminAgent: TestAgent<Test>;
+	let memberAgent: TestAgent<Test>;
+	let authlessAgent: TestAgent<Test>;
 
 	const { NO_ADMIN_ON_OWNER, NO_USER, NO_OWNER_ON_OWNER } =
 		UsersController.ERROR_MESSAGES.CHANGE_ROLE;
diff --git a/packages/cli/test/integration/variables.test.ts b/packages/cli/test/integration/variables.test.ts
index 8b84f67dcfd18..e775ff9936a37 100644
--- a/packages/cli/test/integration/variables.test.ts
+++ b/packages/cli/test/integration/variables.test.ts
@@ -1,5 +1,6 @@
 import Container from 'typedi';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import type { Variables } from '@db/entities/Variables';
 import { VariablesRepository } from '@db/repositories/variables.repository';
 import { generateNanoId } from '@db/utils/generators';
@@ -9,8 +10,8 @@ import * as testDb from './shared/testDb';
 import * as utils from './shared/utils/';
 import { createOwner, createUser } from './shared/db/users';
 
-let authOwnerAgent: SuperAgentTest;
-let authMemberAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authMemberAgent: TestAgent<Test>;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['variables'] });
 const license = testServer.license;
diff --git a/packages/cli/test/integration/workflowHistory.api.test.ts b/packages/cli/test/integration/workflowHistory.api.test.ts
index b9ccd33b2c143..c4b4779bde721 100644
--- a/packages/cli/test/integration/workflowHistory.api.test.ts
+++ b/packages/cli/test/integration/workflowHistory.api.test.ts
@@ -1,4 +1,5 @@
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import type { User } from '@db/entities/User';
 
 import * as testDb from './shared/testDb';
@@ -8,9 +9,9 @@ import { createWorkflow } from './shared/db/workflows';
 import { createWorkflowHistoryItem } from './shared/db/workflowHistory';
 
 let owner: User;
-let authOwnerAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
 let member: User;
-let authMemberAgent: SuperAgentTest;
+let authMemberAgent: TestAgent<Test>;
 
 const testServer = utils.setupTestServer({
 	endpointGroups: ['workflowHistory'],
diff --git a/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts b/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts
index ed3e13d9f063d..99c165f129136 100644
--- a/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts
+++ b/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts
@@ -1,5 +1,6 @@
 import Container from 'typedi';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { v4 as uuid } from 'uuid';
 import type { INode } from 'n8n-workflow';
 
@@ -24,9 +25,9 @@ import config from '@/config';
 let owner: User;
 let member: User;
 let anotherMember: User;
-let authOwnerAgent: SuperAgentTest;
-let authMemberAgent: SuperAgentTest;
-let authAnotherMemberAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
+let authMemberAgent: TestAgent<Test>;
+let authAnotherMemberAgent: TestAgent<Test>;
 let saveCredential: SaveCredentialFunction;
 
 const activeWorkflowRunner = mockInstance(ActiveWorkflowRunner);
diff --git a/packages/cli/test/integration/workflows/workflows.controller.test.ts b/packages/cli/test/integration/workflows/workflows.controller.test.ts
index d493dd9d48e1c..f1dc00ecad441 100644
--- a/packages/cli/test/integration/workflows/workflows.controller.test.ts
+++ b/packages/cli/test/integration/workflows/workflows.controller.test.ts
@@ -1,5 +1,6 @@
 import Container from 'typedi';
-import type { SuperAgentTest } from 'supertest';
+import type { Test } from 'supertest';
+import type TestAgent from 'supertest/lib/agent';
 import { v4 as uuid } from 'uuid';
 import type { INode, IPinData } from 'n8n-workflow';
 
@@ -23,7 +24,7 @@ import { createTag } from '../shared/db/tags';
 import { License } from '@/License';
 
 let owner: User;
-let authOwnerAgent: SuperAgentTest;
+let authOwnerAgent: TestAgent<Test>;
 
 jest.spyOn(License.prototype, 'isSharingEnabled').mockReturnValue(false);
 

From 3f38f88034a518946bd1d2a4f3953478bcbd964f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 11 Apr 2024 20:58:35 +0200
Subject: [PATCH 21/86] Simplify

---
 .../cli/test/benchmarks/benchmark-suite.ts    | 37 ------------
 packages/cli/test/benchmarks/hooks.ts         | 60 +++++++++++++++++++
 packages/cli/test/benchmarks/main.ts          | 56 ++++++++++++++---
 packages/cli/test/benchmarks/setup-utils.ts   | 42 -------------
 .../test/benchmarks/tasks/example.tasks.ts    |  9 +++
 .../test/benchmarks/tasks/webhook.tasks.ts    | 12 ++++
 packages/cli/test/benchmarks/types.ts         | 11 ++--
 .../cli/test/benchmarks/webhook.benchmarks.ts | 26 --------
 .../cli/test/integration/shared/db/users.ts   |  5 ++
 9 files changed, 138 insertions(+), 120 deletions(-)
 delete mode 100644 packages/cli/test/benchmarks/benchmark-suite.ts
 create mode 100644 packages/cli/test/benchmarks/hooks.ts
 delete mode 100644 packages/cli/test/benchmarks/setup-utils.ts
 create mode 100644 packages/cli/test/benchmarks/tasks/example.tasks.ts
 create mode 100644 packages/cli/test/benchmarks/tasks/webhook.tasks.ts
 delete mode 100644 packages/cli/test/benchmarks/webhook.benchmarks.ts

diff --git a/packages/cli/test/benchmarks/benchmark-suite.ts b/packages/cli/test/benchmarks/benchmark-suite.ts
deleted file mode 100644
index 54a9964a5333a..0000000000000
--- a/packages/cli/test/benchmarks/benchmark-suite.ts
+++ /dev/null
@@ -1,37 +0,0 @@
-import { withCodSpeed } from '@codspeed/tinybench-plugin';
-import { ApplicationError } from 'n8n-workflow';
-import Bench from 'tinybench';
-import type { Benchmark } from './types';
-
-export class BenchmarkSuite {
-	private suite: Bench;
-
-	add({ register, setup, teardown }: Benchmark) {
-		// @TEMP: Update time and iterations
-		this.suite = withCodSpeed(new Bench({ time: 0, iterations: 1, setup, teardown }));
-
-		register(this.suite);
-
-		return this;
-	}
-
-	async warmup() {
-		this.assertBenchmarks();
-
-		await this.suite.warmup();
-	}
-
-	async run() {
-		this.assertBenchmarks();
-
-		await this.suite.run();
-	}
-
-	logResults() {
-		console.table(this.suite.table());
-	}
-
-	private assertBenchmarks() {
-		if (this.suite._tasks.size === 0) throw new ApplicationError('No benchmarks found');
-	}
-}
diff --git a/packages/cli/test/benchmarks/hooks.ts b/packages/cli/test/benchmarks/hooks.ts
new file mode 100644
index 0000000000000..38d38f786afa0
--- /dev/null
+++ b/packages/cli/test/benchmarks/hooks.ts
@@ -0,0 +1,60 @@
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
+import { Config } from '@oclif/core';
+import { InstanceSettings } from 'n8n-core';
+import { Start } from '@/commands/start';
+import Container from 'typedi';
+import { createOwner, deleteOwnerShell } from '../integration/shared/db/users';
+
+function n8nDir() {
+	const baseDir = path.join(tmpdir(), 'n8n-benchmarks/');
+
+	mkdirSync(baseDir, { recursive: true });
+
+	const subDir = mkdtempSync(baseDir);
+	const n8nDir = path.join(subDir, '.n8n');
+
+	mkdirSync(n8nDir);
+
+	writeFileSync(
+		path.join(n8nDir, 'config'),
+		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: '123' }),
+		'utf-8',
+	);
+
+	const instanceSettings = Container.get(InstanceSettings);
+	instanceSettings.n8nFolder = n8nDir;
+	Container.set(InstanceSettings, instanceSettings);
+
+	console.log('n8nDir', n8nDir);
+}
+
+let main: Start;
+
+async function mainProcess() {
+	const args: string[] = [];
+	const _config = new Config({ root: __dirname });
+
+	main = new Start(args, _config);
+
+	await main.init();
+	await main.run();
+}
+
+async function setup() {
+	n8nDir();
+
+	await mainProcess();
+	// @TODO: Postgres?
+
+	await deleteOwnerShell();
+	await createOwner();
+}
+
+async function teardown() {
+	await main.stopProcess();
+}
+
+/** Lifecycle hooks to run once before and after all benchmarking tasks. */
+export const hooks = { setup, teardown };
diff --git a/packages/cli/test/benchmarks/main.ts b/packages/cli/test/benchmarks/main.ts
index 7d70fa70ba177..0fd935384a74d 100644
--- a/packages/cli/test/benchmarks/main.ts
+++ b/packages/cli/test/benchmarks/main.ts
@@ -1,18 +1,58 @@
 import 'reflect-metadata';
-import { BenchmarkSuite } from './benchmark-suite';
-import { webhook } from './webhook.benchmarks';
+import path from 'node:path';
+import glob from 'fast-glob';
+import Bench from 'tinybench';
+import { withCodSpeed } from '@codspeed/tinybench-plugin';
+import type { Task } from './types';
+import { hooks } from './hooks';
+
+const tasks: Task[] = [];
+
+export function task(description: string, operation: Task['operation']) {
+	tasks.push({ description, operation });
+}
+
+async function loadTasks() {
+	const files = await glob('**/*.tasks.js', {
+		cwd: path.join('dist', 'test', 'benchmarks'),
+		absolute: true,
+	});
+
+	for (const file of files) {
+		await import(file);
+	}
+}
 
 async function main() {
-	process.env.NODE_ENV = 'test';
+	await loadTasks();
+
+	if (tasks.length === 0) {
+		console.log('No benchmarking tasks found');
+		return;
+	}
+
+	await hooks.setup();
+
+	const _bench = new Bench({
+		time: 0, // @TODO: Temp value
+		iterations: 1, // @TODO: Temp value
+	});
+
+	const bench = process.env.CI === 'true' ? withCodSpeed(_bench) : _bench;
+
+	for (const task of tasks) {
+		bench.add(task.description, task.operation);
+	}
+
+	console.log(`Running ${tasks.length} benchmarking tasks...`);
 
-	const webhookSuite = new BenchmarkSuite().add(webhook);
+	// await this.warmup(); // @TODO: Restore
 
-	// const webhookSuite = new BenchmarkSuite({ setup, register });
+	await bench.run();
 
-	// await suite.warmup(); // @TODO: Restore
-	await webhookSuite.run();
+	console.table(bench.table());
 
-	webhookSuite.logResults();
+	await hooks.teardown();
 }
 
 void main();
diff --git a/packages/cli/test/benchmarks/setup-utils.ts b/packages/cli/test/benchmarks/setup-utils.ts
deleted file mode 100644
index 99cb8f831bae5..0000000000000
--- a/packages/cli/test/benchmarks/setup-utils.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
-import { Config } from '@oclif/core';
-import { InstanceSettings } from 'n8n-core';
-import { Start } from '@/commands/start';
-import Container from 'typedi';
-
-export class SetupUtils {
-	static n8nDir() {
-		const baseDir = path.join(tmpdir(), 'n8n-benchmarks/');
-
-		mkdirSync(baseDir, { recursive: true });
-
-		const subDir = mkdtempSync(baseDir);
-		const n8nDir = path.join(subDir, '.n8n');
-
-		mkdirSync(n8nDir);
-
-		writeFileSync(
-			path.join(n8nDir, 'config'),
-			JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: '123' }),
-			'utf-8',
-		);
-
-		const instanceSettings = Container.get(InstanceSettings);
-		instanceSettings.n8nFolder = n8nDir;
-		Container.set(InstanceSettings, instanceSettings);
-
-		console.log('n8nDir', n8nDir);
-	}
-
-	static async mainProcess() {
-		const args: string[] = [];
-		const _config = new Config({ root: __dirname });
-
-		const main = new Start(args, _config);
-
-		await main.init();
-		await main.run();
-	}
-}
diff --git a/packages/cli/test/benchmarks/tasks/example.tasks.ts b/packages/cli/test/benchmarks/tasks/example.tasks.ts
new file mode 100644
index 0000000000000..9dc5e7e390da2
--- /dev/null
+++ b/packages/cli/test/benchmarks/tasks/example.tasks.ts
@@ -0,0 +1,9 @@
+import { task } from '../main.js';
+
+task('[Example] Should do something', async () => {
+	console.log('Example task 1 executed');
+});
+
+task('[Example] Should do something else', async () => {
+	console.log('Example task 2 executed');
+});
diff --git a/packages/cli/test/benchmarks/tasks/webhook.tasks.ts b/packages/cli/test/benchmarks/tasks/webhook.tasks.ts
new file mode 100644
index 0000000000000..f69aef42bb39b
--- /dev/null
+++ b/packages/cli/test/benchmarks/tasks/webhook.tasks.ts
@@ -0,0 +1,12 @@
+import { task } from '../main.js';
+import { getAllUsers } from '../../integration/shared/db/users.js';
+
+task('Production workflow with webhook node that responds immediately', async () => {
+	const users = await getAllUsers();
+	console.log('users', users);
+	console.log('[first] Task 1 executed');
+});
+
+task('[first] Task 2 should do something else', async () => {
+	console.log('[first] Task 2 executed');
+});
diff --git a/packages/cli/test/benchmarks/types.ts b/packages/cli/test/benchmarks/types.ts
index 92bf29c438b1e..813ce426b40f0 100644
--- a/packages/cli/test/benchmarks/types.ts
+++ b/packages/cli/test/benchmarks/types.ts
@@ -1,8 +1,5 @@
-import type { Hook } from 'tinybench';
-import type Bench from 'tinybench';
-
-export type Benchmark = {
-	setup: Hook;
-	teardown: Hook;
-	register: (bench: Bench) => void;
+/** A benchmarking task, i.e. a single operation whose performance to measure. */
+export type Task = {
+	description: string;
+	operation: () => Promise<void>;
 };
diff --git a/packages/cli/test/benchmarks/webhook.benchmarks.ts b/packages/cli/test/benchmarks/webhook.benchmarks.ts
deleted file mode 100644
index 21167074c848e..0000000000000
--- a/packages/cli/test/benchmarks/webhook.benchmarks.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import { SetupUtils } from './setup-utils';
-import type Bench from 'tinybench';
-
-export const webhook = {
-	async setup() {
-		SetupUtils.n8nDir();
-
-		await SetupUtils.mainProcess();
-	},
-
-	async teardown() {
-		// ...
-	},
-
-	register(bench: Bench) {
-		bench.add('Some description', async () => {
-			console.log('Something happening...');
-		});
-	},
-};
-
-/**
- * benchmark('Some description', async () => {
- *	console.log('Something happening...');
- * });
- */
diff --git a/packages/cli/test/integration/shared/db/users.ts b/packages/cli/test/integration/shared/db/users.ts
index 4f4ed8af184b9..9e0df2eef0ffc 100644
--- a/packages/cli/test/integration/shared/db/users.ts
+++ b/packages/cli/test/integration/shared/db/users.ts
@@ -8,6 +8,7 @@ import { TOTPService } from '@/Mfa/totp.service';
 import { MfaService } from '@/Mfa/mfa.service';
 
 import { randomApiKey, randomEmail, randomName, randomValidPassword } from '../random';
+import { IsNull } from '@n8n/typeorm';
 
 // pre-computed bcrypt hash for the string 'password', using `await hash('password', 10)`
 const passwordHash = '$2a$10$njedH7S6V5898mj6p0Jr..IGY9Ms.qNwR7RbSzzX9yubJocKfvGGK';
@@ -133,3 +134,7 @@ export const getLdapIdentities = async () =>
 export async function getGlobalOwner() {
 	return await Container.get(UserRepository).findOneByOrFail({ role: 'global:owner' });
 }
+
+export async function deleteOwnerShell() {
+	await Container.get(UserRepository).delete({ role: 'global:owner', email: IsNull() });
+}

From 9e7911aed1362d4ba9dd5bdf431ea8ef30d49a70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 12:06:30 +0200
Subject: [PATCH 22/86] Back to cli/src

---
 packages/cli/package.json                     |  2 +-
 packages/cli/src/benchmarks/db/users.ts       | 48 +++++++++++++++++++
 .../cli/{test => src}/benchmarks/hooks.ts     | 12 ++---
 packages/cli/{test => src}/benchmarks/main.ts |  8 ++--
 .../benchmarks/tasks/example.tasks.ts         |  0
 .../benchmarks/tasks/webhook.tasks.ts         |  4 +-
 .../cli/{test => src}/benchmarks/types.ts     |  0
 7 files changed, 62 insertions(+), 12 deletions(-)
 create mode 100644 packages/cli/src/benchmarks/db/users.ts
 rename packages/cli/{test => src}/benchmarks/hooks.ts (82%)
 rename packages/cli/{test => src}/benchmarks/main.ts (82%)
 rename packages/cli/{test => src}/benchmarks/tasks/example.tasks.ts (100%)
 rename packages/cli/{test => src}/benchmarks/tasks/webhook.tasks.ts (76%)
 rename packages/cli/{test => src}/benchmarks/types.ts (100%)

diff --git a/packages/cli/package.json b/packages/cli/package.json
index ca82e51af1ba8..a2e12c434c2a4 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,7 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node dist/test/benchmarks/main.js",
+    "benchmark": "pnpm build && node dist/benchmarks/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
diff --git a/packages/cli/src/benchmarks/db/users.ts b/packages/cli/src/benchmarks/db/users.ts
new file mode 100644
index 0000000000000..d5b453d2dc3fe
--- /dev/null
+++ b/packages/cli/src/benchmarks/db/users.ts
@@ -0,0 +1,48 @@
+/**
+ * @TODO Deduplicate with packages/cli/test/integration/shared/db/users.ts
+ */
+
+import Container from 'typedi';
+import type { User } from '@db/entities/User';
+import { UserRepository } from '@db/repositories/user.repository';
+import { IsNull } from '@n8n/typeorm';
+
+// pre-computed bcrypt hash for the string 'password', using `await hash('password', 10)`
+const passwordHash = '$2a$10$njedH7S6V5898mj6p0Jr..IGY9Ms.qNwR7RbSzzX9yubJocKfvGGK';
+
+/** Store a new user object, defaulting to a `member` */
+export async function newUser(attributes: Partial<User> = {}): Promise<User> {
+	const { email, password, firstName, lastName, role, ...rest } = attributes;
+	return Container.get(UserRepository).create({
+		email: 'test@test.com',
+		password: passwordHash,
+		firstName: 'John',
+		lastName: 'Smith',
+		role: role ?? 'global:member',
+		...rest,
+	});
+}
+
+/** Store a user object in the DB */
+export async function createUser(attributes: Partial<User> = {}): Promise<User> {
+	const user = await newUser(attributes);
+	user.computeIsOwner();
+	return await Container.get(UserRepository).save(user);
+}
+
+export async function createOwner() {
+	return await createUser({ role: 'global:owner' });
+}
+
+export async function createMember() {
+	return await createUser({ role: 'global:member' });
+}
+
+export async function deleteOwnerShell() {
+	await Container.get(UserRepository).delete({ role: 'global:owner', email: IsNull() });
+}
+
+export const getAllUsers = async () =>
+	await Container.get(UserRepository).find({
+		relations: ['authIdentities'],
+	});
diff --git a/packages/cli/test/benchmarks/hooks.ts b/packages/cli/src/benchmarks/hooks.ts
similarity index 82%
rename from packages/cli/test/benchmarks/hooks.ts
rename to packages/cli/src/benchmarks/hooks.ts
index 38d38f786afa0..eff43180f146c 100644
--- a/packages/cli/test/benchmarks/hooks.ts
+++ b/packages/cli/src/benchmarks/hooks.ts
@@ -5,7 +5,7 @@ import { Config } from '@oclif/core';
 import { InstanceSettings } from 'n8n-core';
 import { Start } from '@/commands/start';
 import Container from 'typedi';
-import { createOwner, deleteOwnerShell } from '../integration/shared/db/users';
+import { createOwner, deleteOwnerShell } from './db/users';
 
 function n8nDir() {
 	const baseDir = path.join(tmpdir(), 'n8n-benchmarks/');
@@ -13,21 +13,21 @@ function n8nDir() {
 	mkdirSync(baseDir, { recursive: true });
 
 	const subDir = mkdtempSync(baseDir);
-	const n8nDir = path.join(subDir, '.n8n');
+	const _n8nDir = path.join(subDir, '.n8n');
 
-	mkdirSync(n8nDir);
+	mkdirSync(_n8nDir);
 
 	writeFileSync(
-		path.join(n8nDir, 'config'),
+		path.join(_n8nDir, 'config'),
 		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: '123' }),
 		'utf-8',
 	);
 
 	const instanceSettings = Container.get(InstanceSettings);
-	instanceSettings.n8nFolder = n8nDir;
+	instanceSettings.n8nFolder = _n8nDir;
 	Container.set(InstanceSettings, instanceSettings);
 
-	console.log('n8nDir', n8nDir);
+	console.log('n8nDir', _n8nDir);
 }
 
 let main: Start;
diff --git a/packages/cli/test/benchmarks/main.ts b/packages/cli/src/benchmarks/main.ts
similarity index 82%
rename from packages/cli/test/benchmarks/main.ts
rename to packages/cli/src/benchmarks/main.ts
index 0fd935384a74d..0067e7a37f7a7 100644
--- a/packages/cli/test/benchmarks/main.ts
+++ b/packages/cli/src/benchmarks/main.ts
@@ -1,7 +1,9 @@
 import 'reflect-metadata';
 import path from 'node:path';
 import glob from 'fast-glob';
+// eslint-disable-next-line import/no-extraneous-dependencies
 import Bench from 'tinybench';
+// eslint-disable-next-line import/no-extraneous-dependencies
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
 import type { Task } from './types';
 import { hooks } from './hooks';
@@ -14,7 +16,7 @@ export function task(description: string, operation: Task['operation']) {
 
 async function loadTasks() {
 	const files = await glob('**/*.tasks.js', {
-		cwd: path.join('dist', 'test', 'benchmarks'),
+		cwd: path.join('dist', 'benchmarks'),
 		absolute: true,
 	});
 
@@ -40,8 +42,8 @@ async function main() {
 
 	const bench = process.env.CI === 'true' ? withCodSpeed(_bench) : _bench;
 
-	for (const task of tasks) {
-		bench.add(task.description, task.operation);
+	for (const t of tasks) {
+		bench.add(t.description, t.operation);
 	}
 
 	console.log(`Running ${tasks.length} benchmarking tasks...`);
diff --git a/packages/cli/test/benchmarks/tasks/example.tasks.ts b/packages/cli/src/benchmarks/tasks/example.tasks.ts
similarity index 100%
rename from packages/cli/test/benchmarks/tasks/example.tasks.ts
rename to packages/cli/src/benchmarks/tasks/example.tasks.ts
diff --git a/packages/cli/test/benchmarks/tasks/webhook.tasks.ts b/packages/cli/src/benchmarks/tasks/webhook.tasks.ts
similarity index 76%
rename from packages/cli/test/benchmarks/tasks/webhook.tasks.ts
rename to packages/cli/src/benchmarks/tasks/webhook.tasks.ts
index f69aef42bb39b..eac3fe5f2886c 100644
--- a/packages/cli/test/benchmarks/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmarks/tasks/webhook.tasks.ts
@@ -1,9 +1,9 @@
+import { getAllUsers } from '../db/users.js';
 import { task } from '../main.js';
-import { getAllUsers } from '../../integration/shared/db/users.js';
 
 task('Production workflow with webhook node that responds immediately', async () => {
 	const users = await getAllUsers();
-	console.log('users', users);
+	// console.log('users', users);
 	console.log('[first] Task 1 executed');
 });
 
diff --git a/packages/cli/test/benchmarks/types.ts b/packages/cli/src/benchmarks/types.ts
similarity index 100%
rename from packages/cli/test/benchmarks/types.ts
rename to packages/cli/src/benchmarks/types.ts

From 0c39b4fc053749ad77b5dab0634e9800aa964be2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 13:00:12 +0200
Subject: [PATCH 23/86] Remove logging noise

---
 packages/cli/package.json                          |  2 +-
 packages/cli/src/benchmarks/hooks.ts               | 14 +++++++++-----
 packages/cli/src/benchmarks/tasks/webhook.tasks.ts |  2 +-
 packages/cli/src/commands/start.ts                 |  2 --
 packages/cli/src/constants.ts                      |  1 +
 .../common/1711390882123-MoveSshKeysToDatabase.ts  |  3 ++-
 .../cli/src/databases/utils/migrationHelpers.ts    |  6 +++---
 7 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/packages/cli/package.json b/packages/cli/package.json
index a2e12c434c2a4..c0487f596de50 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,7 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "pnpm build && node dist/benchmarks/main.js",
+    "benchmark": "pnpm build && NODE_ENV=benchmark node dist/benchmarks/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
diff --git a/packages/cli/src/benchmarks/hooks.ts b/packages/cli/src/benchmarks/hooks.ts
index eff43180f146c..ca53a0cc1e4fa 100644
--- a/packages/cli/src/benchmarks/hooks.ts
+++ b/packages/cli/src/benchmarks/hooks.ts
@@ -8,12 +8,13 @@ import Container from 'typedi';
 import { createOwner, deleteOwnerShell } from './db/users';
 
 function n8nDir() {
-	const baseDir = path.join(tmpdir(), 'n8n-benchmarks/');
+	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
 
-	mkdirSync(baseDir, { recursive: true });
+	mkdirSync(baseDirPath, { recursive: true });
 
-	const subDir = mkdtempSync(baseDir);
-	const _n8nDir = path.join(subDir, '.n8n');
+	const userDir = mkdtempSync(baseDirPath);
+
+	const _n8nDir = path.join(userDir, '.n8n');
 
 	mkdirSync(_n8nDir);
 
@@ -23,11 +24,14 @@ function n8nDir() {
 		'utf-8',
 	);
 
+	/**
+	 * @TODO Better approach? Setting N8N_USER_FOLDER has no effect
+	 */
 	const instanceSettings = Container.get(InstanceSettings);
 	instanceSettings.n8nFolder = _n8nDir;
 	Container.set(InstanceSettings, instanceSettings);
 
-	console.log('n8nDir', _n8nDir);
+	console.info('.n8n dir', _n8nDir);
 }
 
 let main: Start;
diff --git a/packages/cli/src/benchmarks/tasks/webhook.tasks.ts b/packages/cli/src/benchmarks/tasks/webhook.tasks.ts
index eac3fe5f2886c..24e229c2f6a01 100644
--- a/packages/cli/src/benchmarks/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmarks/tasks/webhook.tasks.ts
@@ -3,7 +3,7 @@ import { task } from '../main.js';
 
 task('Production workflow with webhook node that responds immediately', async () => {
 	const users = await getAllUsers();
-	// console.log('users', users);
+	console.log('users', users);
 	console.log('[first] Task 1 executed');
 });
 
diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts
index 6a2284ef148e0..71097ab2e44f6 100644
--- a/packages/cli/src/commands/start.ts
+++ b/packages/cli/src/commands/start.ts
@@ -289,8 +289,6 @@ export class Start extends BaseCommand {
 		const editorUrl = Container.get(UrlService).baseUrl;
 		this.log(`\nEditor is now accessible via:\n${editorUrl}`);
 
-		console.log('hello');
-
 		// Allow to open n8n editor by pressing "o"
 		// if (Boolean(process.stdout.isTTY) && process.stdin.setRawMode) {
 		// 	process.stdin.setRawMode(true);
diff --git a/packages/cli/src/constants.ts b/packages/cli/src/constants.ts
index aecd7cd1efad9..39a4afe46e211 100644
--- a/packages/cli/src/constants.ts
+++ b/packages/cli/src/constants.ts
@@ -7,6 +7,7 @@ const { NODE_ENV, E2E_TESTS } = process.env;
 export const inProduction = NODE_ENV === 'production';
 export const inDevelopment = !NODE_ENV || NODE_ENV === 'development';
 export const inTest = NODE_ENV === 'test';
+export const inBenchmark = NODE_ENV === 'benchmark';
 export const inE2ETests = E2E_TESTS === 'true';
 
 export const CUSTOM_API_CALL_NAME = 'Custom API Call';
diff --git a/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
index 30d7d87c4ac6a..99b27bfaa877c 100644
--- a/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
+++ b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
@@ -4,6 +4,7 @@ import Container from 'typedi';
 import { Cipher, InstanceSettings } from 'n8n-core';
 import { jsonParse } from 'n8n-workflow';
 import type { MigrationContext, ReversibleMigration } from '@db/types';
+import { inBenchmark } from '@/constants';
 
 /**
  * Move SSH key pair from file system to database, to enable SSH connections
@@ -31,7 +32,7 @@ export class MoveSshKeysToDatabase1711390882123 implements ReversibleMigration {
 				readFile(this.publicKeyPath, { encoding: 'utf8' }),
 			]);
 		} catch {
-			logger.info(`[${migrationName}] No SSH keys in filesystem, skipping`);
+			if (!inBenchmark) logger.info(`[${migrationName}] No SSH keys in filesystem, skipping`);
 			return;
 		}
 
diff --git a/packages/cli/src/databases/utils/migrationHelpers.ts b/packages/cli/src/databases/utils/migrationHelpers.ts
index f87076394a8aa..4e40b2755def5 100644
--- a/packages/cli/src/databases/utils/migrationHelpers.ts
+++ b/packages/cli/src/databases/utils/migrationHelpers.ts
@@ -5,7 +5,7 @@ import type { ObjectLiteral } from '@n8n/typeorm';
 import type { QueryRunner } from '@n8n/typeorm/query-runner/QueryRunner';
 import { ApplicationError, jsonParse } from 'n8n-workflow';
 import config from '@/config';
-import { inTest } from '@/constants';
+import { inBenchmark, inTest } from '@/constants';
 import type { BaseMigration, Migration, MigrationContext, MigrationFn } from '@db/types';
 import { createSchemaBuilder } from '@db/dsl';
 import { NodeTypes } from '@/NodeTypes';
@@ -44,7 +44,7 @@ function loadSurveyFromDisk(): string | null {
 let runningMigrations = false;
 
 function logMigrationStart(migrationName: string): void {
-	if (inTest) return;
+	if (inTest || inBenchmark) return;
 
 	const logger = Container.get(Logger);
 	if (!runningMigrations) {
@@ -56,7 +56,7 @@ function logMigrationStart(migrationName: string): void {
 }
 
 function logMigrationEnd(migrationName: string): void {
-	if (inTest) return;
+	if (inTest || inBenchmark) return;
 
 	const logger = Container.get(Logger);
 	logger.info(`Finished migration ${migrationName}`);

From a193bd363ab92258feeca811168f2ab9f97a73dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 13:15:01 +0200
Subject: [PATCH 24/86] Make naming consistent

---
 packages/cli/package.json                             |  3 ++-
 .../cli/src/{benchmarks => benchmark}/db/users.ts     |  0
 packages/cli/src/{benchmarks => benchmark}/hooks.ts   |  0
 packages/cli/src/{benchmarks => benchmark}/main.ts    | 11 ++++++-----
 .../{benchmarks => benchmark}/tasks/example.tasks.ts  |  0
 .../{benchmarks => benchmark}/tasks/webhook.tasks.ts  |  0
 packages/cli/src/{benchmarks => benchmark}/types.ts   |  0
 packages/cli/tsconfig.benchmark.json                  |  7 ++++---
 packages/cli/tsconfig.build.json                      |  2 +-
 9 files changed, 13 insertions(+), 10 deletions(-)
 rename packages/cli/src/{benchmarks => benchmark}/db/users.ts (100%)
 rename packages/cli/src/{benchmarks => benchmark}/hooks.ts (100%)
 rename packages/cli/src/{benchmarks => benchmark}/main.ts (87%)
 rename packages/cli/src/{benchmarks => benchmark}/tasks/example.tasks.ts (100%)
 rename packages/cli/src/{benchmarks => benchmark}/tasks/webhook.tasks.ts (100%)
 rename packages/cli/src/{benchmarks => benchmark}/types.ts (100%)

diff --git a/packages/cli/package.json b/packages/cli/package.json
index c0487f596de50..91faa4474897c 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,10 +20,11 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "pnpm build && NODE_ENV=benchmark node dist/benchmarks/main.js",
+    "benchmark": "pnpm build:benchmark && NODE_ENV=benchmark node dist/benchmarks/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
+    "build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs",
     "buildAndDev": "pnpm run build && pnpm run dev",
     "dev": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon\"",
     "dev:worker": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon worker\"",
diff --git a/packages/cli/src/benchmarks/db/users.ts b/packages/cli/src/benchmark/db/users.ts
similarity index 100%
rename from packages/cli/src/benchmarks/db/users.ts
rename to packages/cli/src/benchmark/db/users.ts
diff --git a/packages/cli/src/benchmarks/hooks.ts b/packages/cli/src/benchmark/hooks.ts
similarity index 100%
rename from packages/cli/src/benchmarks/hooks.ts
rename to packages/cli/src/benchmark/hooks.ts
diff --git a/packages/cli/src/benchmarks/main.ts b/packages/cli/src/benchmark/main.ts
similarity index 87%
rename from packages/cli/src/benchmarks/main.ts
rename to packages/cli/src/benchmark/main.ts
index 0067e7a37f7a7..bb320c1ecfe51 100644
--- a/packages/cli/src/benchmarks/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,13 +1,14 @@
 import 'reflect-metadata';
 import path from 'node:path';
 import glob from 'fast-glob';
-// eslint-disable-next-line import/no-extraneous-dependencies
-import Bench from 'tinybench';
-// eslint-disable-next-line import/no-extraneous-dependencies
-import { withCodSpeed } from '@codspeed/tinybench-plugin';
 import type { Task } from './types';
 import { hooks } from './hooks';
 
+/* eslint-disable import/no-extraneous-dependencies */
+import { withCodSpeed } from '@codspeed/tinybench-plugin';
+import Bench from 'tinybench';
+/* eslint-ensable import/no-extraneous-dependencies */
+
 const tasks: Task[] = [];
 
 export function task(description: string, operation: Task['operation']) {
@@ -16,7 +17,7 @@ export function task(description: string, operation: Task['operation']) {
 
 async function loadTasks() {
 	const files = await glob('**/*.tasks.js', {
-		cwd: path.join('dist', 'benchmarks'),
+		cwd: path.join('dist', 'benchmark'),
 		absolute: true,
 	});
 
diff --git a/packages/cli/src/benchmarks/tasks/example.tasks.ts b/packages/cli/src/benchmark/tasks/example.tasks.ts
similarity index 100%
rename from packages/cli/src/benchmarks/tasks/example.tasks.ts
rename to packages/cli/src/benchmark/tasks/example.tasks.ts
diff --git a/packages/cli/src/benchmarks/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
similarity index 100%
rename from packages/cli/src/benchmarks/tasks/webhook.tasks.ts
rename to packages/cli/src/benchmark/tasks/webhook.tasks.ts
diff --git a/packages/cli/src/benchmarks/types.ts b/packages/cli/src/benchmark/types.ts
similarity index 100%
rename from packages/cli/src/benchmarks/types.ts
rename to packages/cli/src/benchmark/types.ts
diff --git a/packages/cli/tsconfig.benchmark.json b/packages/cli/tsconfig.benchmark.json
index 930bb9c97389d..568df33449491 100644
--- a/packages/cli/tsconfig.benchmark.json
+++ b/packages/cli/tsconfig.benchmark.json
@@ -1,9 +1,10 @@
 {
 	"extends": ["./tsconfig.json", "../../tsconfig.build.json"],
 	"compilerOptions": {
-		"rootDir": ".",
+		"rootDir": "src",
 		"outDir": "dist",
 		"tsBuildInfoFile": "dist/benchmark.tsbuildinfo"
-	}
-	// "include": ["test/benchmark/**/*.ts"]
+	},
+	"include": ["src/**/*.ts", "src/benchmark/**/*.ts"],
+	"exclude": ["test/**"]
 }
diff --git a/packages/cli/tsconfig.build.json b/packages/cli/tsconfig.build.json
index 1e8a2ff7fa476..a48bf1fc7f018 100644
--- a/packages/cli/tsconfig.build.json
+++ b/packages/cli/tsconfig.build.json
@@ -6,5 +6,5 @@
 		"tsBuildInfoFile": "dist/build.tsbuildinfo"
 	},
 	"include": ["src/**/*.ts"],
-	"exclude": ["test/**"]
+	"exclude": ["test/**", "src/benchmarks/**"]
 }

From 42b67ecda0371d7f9343db2cf7561ccefb98bdd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 14:17:55 +0200
Subject: [PATCH 25/86] Implement `beforeEach``

---
 packages/cli/package.json                     |  2 +-
 packages/cli/src/benchmark/hooks.ts           |  2 +-
 packages/cli/src/benchmark/main.ts            | 60 +++++++++++++++----
 .../cli/src/benchmark/tasks/example.tasks.ts  | 10 +++-
 .../cli/src/benchmark/tasks/webhook.tasks.ts  |  6 +-
 5 files changed, 61 insertions(+), 19 deletions(-)

diff --git a/packages/cli/package.json b/packages/cli/package.json
index 91faa4474897c..e80fa6f4d9a52 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,7 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "pnpm build:benchmark && NODE_ENV=benchmark node dist/benchmarks/main.js",
+    "benchmark": "pnpm build:benchmark && NODE_ENV=benchmark node dist/benchmark/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
diff --git a/packages/cli/src/benchmark/hooks.ts b/packages/cli/src/benchmark/hooks.ts
index ca53a0cc1e4fa..c60b8d1cde89f 100644
--- a/packages/cli/src/benchmark/hooks.ts
+++ b/packages/cli/src/benchmark/hooks.ts
@@ -61,4 +61,4 @@ async function teardown() {
 }
 
 /** Lifecycle hooks to run once before and after all benchmarking tasks. */
-export const hooks = { setup, teardown };
+export const globalHooks = { setup, teardown };
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index bb320c1ecfe51..33a1f3d48a612 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,18 +1,42 @@
 import 'reflect-metadata';
 import path from 'node:path';
+import { assert } from 'n8n-workflow';
 import glob from 'fast-glob';
+import callsites from 'callsites';
 import type { Task } from './types';
-import { hooks } from './hooks';
+import { globalHooks } from './hooks';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
 import Bench from 'tinybench';
-/* eslint-ensable import/no-extraneous-dependencies */
+/* eslint-enable import/no-extraneous-dependencies */
 
-const tasks: Task[] = [];
+const map: {
+	[suiteFilepath: string]: {
+		hooks: Partial<{ beforeEach: () => Promise<void> }>;
+		tasks: Task[];
+	};
+} = {};
+
+function suiteFilePath() {
+	const filePath = callsites()
+		.map((site) => site.getFileName())
+		.filter((site): site is string => site !== null)
+		.find((site) => site.endsWith('.tasks.js'));
+
+	assert(filePath !== undefined);
+
+	return filePath;
+}
 
 export function task(description: string, operation: Task['operation']) {
-	tasks.push({ description, operation });
+	map[suiteFilePath()] ||= { hooks: {}, tasks: [] };
+	map[suiteFilePath()].tasks.push({ description, operation });
+}
+
+export function beforeEach(fn: () => Promise<void>) {
+	map[suiteFilePath()] ||= { hooks: {}, tasks: [] };
+	map[suiteFilePath()].hooks.beforeEach = fn;
 }
 
 async function loadTasks() {
@@ -29,12 +53,14 @@ async function loadTasks() {
 async function main() {
 	await loadTasks();
 
-	if (tasks.length === 0) {
-		console.log('No benchmarking tasks found');
+	const suitesCount = Object.keys(map).length;
+
+	if (suitesCount === 0) {
+		console.log('No benchmarking suites found');
 		return;
 	}
 
-	await hooks.setup();
+	await globalHooks.setup();
 
 	const _bench = new Bench({
 		time: 0, // @TODO: Temp value
@@ -43,19 +69,27 @@ async function main() {
 
 	const bench = process.env.CI === 'true' ? withCodSpeed(_bench) : _bench;
 
-	for (const t of tasks) {
-		bench.add(t.description, t.operation);
-	}
+	for (const filePath of Object.keys(map)) {
+		const { tasks, hooks } = map[filePath];
 
-	console.log(`Running ${tasks.length} benchmarking tasks...`);
+		for (const t of tasks) {
+			/**
+			 * `beforeAll` in tinybench is called once before all iterations of a single operation.
+			 * This is functionally equivalent to `beforeEach` in jest and vitest.
+			 */
+			const options = hooks.beforeEach ? { beforeAll: hooks.beforeEach } : {};
+
+			bench.add(t.description, t.operation, options);
+		}
+	}
 
-	// await this.warmup(); // @TODO: Restore
+	console.log(`Running ${suitesCount} benchmarking suites...`);
 
 	await bench.run();
 
 	console.table(bench.table());
 
-	await hooks.teardown();
+	await globalHooks.teardown();
 }
 
 void main();
diff --git a/packages/cli/src/benchmark/tasks/example.tasks.ts b/packages/cli/src/benchmark/tasks/example.tasks.ts
index 9dc5e7e390da2..b1bb77a386764 100644
--- a/packages/cli/src/benchmark/tasks/example.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/example.tasks.ts
@@ -1,9 +1,13 @@
-import { task } from '../main.js';
+import { task, beforeEach } from '../main.js';
 
-task('[Example] Should do something', async () => {
+beforeEach(async () => {
+	console.log('[[[beforeEach]]] for example.tasks.ts');
+});
+
+task('[example] Should do something', async () => {
 	console.log('Example task 1 executed');
 });
 
-task('[Example] Should do something else', async () => {
+task('[example] Should do something else', async () => {
 	console.log('Example task 2 executed');
 });
diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index 24e229c2f6a01..98d6b2741ede3 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -1,5 +1,9 @@
 import { getAllUsers } from '../db/users.js';
-import { task } from '../main.js';
+import { task, beforeEach } from '../main.js';
+
+beforeEach(async () => {
+	console.log('[[[beforeEach]]] for webhook.tasks.ts');
+});
 
 task('Production workflow with webhook node that responds immediately', async () => {
 	const users = await getAllUsers();

From 6dcb2e5f51eb3a8989feeb2a7165f5211457f577 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 14:38:22 +0200
Subject: [PATCH 26/86] Move to suites.ts

---
 packages/cli/src/benchmark/lib/suites.ts | 68 ++++++++++++++++++++++++
 packages/cli/src/benchmark/main.ts       | 67 +++--------------------
 packages/cli/src/benchmark/types.ts      |  2 +
 3 files changed, 77 insertions(+), 60 deletions(-)
 create mode 100644 packages/cli/src/benchmark/lib/suites.ts

diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
new file mode 100644
index 0000000000000..fae053774d747
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -0,0 +1,68 @@
+import 'reflect-metadata';
+import path from 'node:path';
+import type Bench from 'tinybench';
+import { assert } from 'n8n-workflow';
+import glob from 'fast-glob';
+import callsites from 'callsites';
+import type { Task } from '../types';
+
+export const suites: {
+	[suiteFilepath: string]: {
+		hooks: Partial<{ beforeEach: () => Promise<void> }>;
+		tasks: Task[];
+	};
+} = {};
+
+export function suiteCount() {
+	return Object.keys(suites).length;
+}
+
+function suiteFilePath() {
+	const filePath = callsites()
+		.map((site) => site.getFileName())
+		.filter((site): site is string => site !== null)
+		.find((site) => site.endsWith('.tasks.js'));
+
+	assert(filePath !== undefined);
+
+	return filePath;
+}
+
+export async function collectSuites() {
+	const files = await glob('**/*.tasks.js', {
+		cwd: path.join('dist', 'benchmark'),
+		absolute: true,
+	});
+
+	for (const file of files) {
+		await import(file);
+	}
+}
+
+export function registerSuites(bench: Bench) {
+	for (const { tasks, hooks } of Object.values(suites)) {
+		for (const t of tasks) {
+			/**
+			 * `beforeAll` in tinybench is called once before all iterations of a single operation.
+			 * This is functionally equivalent to `beforeEach` in jest and vitest.
+			 */
+			const options = hooks.beforeEach ? { beforeAll: hooks.beforeEach } : {};
+
+			bench.add(t.description, t.operation, options);
+		}
+	}
+}
+
+/**
+ * Benchmarking API
+ */
+
+export function task(description: string, operation: Task['operation']) {
+	suites[suiteFilePath()] ||= { hooks: {}, tasks: [] };
+	suites[suiteFilePath()].tasks.push({ description, operation });
+}
+
+export function beforeEach(fn: () => Promise<void>) {
+	suites[suiteFilePath()] ||= { hooks: {}, tasks: [] };
+	suites[suiteFilePath()].hooks.beforeEach = fn;
+}
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 33a1f3d48a612..7069cb209462a 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,61 +1,20 @@
 import 'reflect-metadata';
-import path from 'node:path';
-import { assert } from 'n8n-workflow';
-import glob from 'fast-glob';
-import callsites from 'callsites';
-import type { Task } from './types';
 import { globalHooks } from './hooks';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
 import Bench from 'tinybench';
+import { collectSuites, registerSuites, suiteCount } from './lib/suites';
 /* eslint-enable import/no-extraneous-dependencies */
 
-const map: {
-	[suiteFilepath: string]: {
-		hooks: Partial<{ beforeEach: () => Promise<void> }>;
-		tasks: Task[];
-	};
-} = {};
-
-function suiteFilePath() {
-	const filePath = callsites()
-		.map((site) => site.getFileName())
-		.filter((site): site is string => site !== null)
-		.find((site) => site.endsWith('.tasks.js'));
-
-	assert(filePath !== undefined);
-
-	return filePath;
-}
-
-export function task(description: string, operation: Task['operation']) {
-	map[suiteFilePath()] ||= { hooks: {}, tasks: [] };
-	map[suiteFilePath()].tasks.push({ description, operation });
-}
-
-export function beforeEach(fn: () => Promise<void>) {
-	map[suiteFilePath()] ||= { hooks: {}, tasks: [] };
-	map[suiteFilePath()].hooks.beforeEach = fn;
-}
-
-async function loadTasks() {
-	const files = await glob('**/*.tasks.js', {
-		cwd: path.join('dist', 'benchmark'),
-		absolute: true,
-	});
-
-	for (const file of files) {
-		await import(file);
-	}
-}
+export { beforeEach, task } from './lib/suites';
 
 async function main() {
-	await loadTasks();
+	await collectSuites();
 
-	const suitesCount = Object.keys(map).length;
+	const count = suiteCount();
 
-	if (suitesCount === 0) {
+	if (count === 0) {
 		console.log('No benchmarking suites found');
 		return;
 	}
@@ -69,21 +28,9 @@ async function main() {
 
 	const bench = process.env.CI === 'true' ? withCodSpeed(_bench) : _bench;
 
-	for (const filePath of Object.keys(map)) {
-		const { tasks, hooks } = map[filePath];
-
-		for (const t of tasks) {
-			/**
-			 * `beforeAll` in tinybench is called once before all iterations of a single operation.
-			 * This is functionally equivalent to `beforeEach` in jest and vitest.
-			 */
-			const options = hooks.beforeEach ? { beforeAll: hooks.beforeEach } : {};
-
-			bench.add(t.description, t.operation, options);
-		}
-	}
+	registerSuites(bench);
 
-	console.log(`Running ${suitesCount} benchmarking suites...`);
+	console.log(`Running ${count} benchmarking suites...`);
 
 	await bench.run();
 
diff --git a/packages/cli/src/benchmark/types.ts b/packages/cli/src/benchmark/types.ts
index 813ce426b40f0..2d961e4198008 100644
--- a/packages/cli/src/benchmark/types.ts
+++ b/packages/cli/src/benchmark/types.ts
@@ -3,3 +3,5 @@ export type Task = {
 	description: string;
 	operation: () => Promise<void>;
 };
+
+export type Callback = () => void | Promise<void>;

From 8775968d76821d370b35424c72a2e978e41b862a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 14:40:49 +0200
Subject: [PATCH 27/86] Cleanup

---
 packages/cli/src/benchmark/lib/suites.ts | 22 +++++++++++-----------
 packages/cli/src/benchmark/main.ts       |  4 ++--
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index fae053774d747..2e013ac4d754f 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -17,17 +17,6 @@ export function suiteCount() {
 	return Object.keys(suites).length;
 }
 
-function suiteFilePath() {
-	const filePath = callsites()
-		.map((site) => site.getFileName())
-		.filter((site): site is string => site !== null)
-		.find((site) => site.endsWith('.tasks.js'));
-
-	assert(filePath !== undefined);
-
-	return filePath;
-}
-
 export async function collectSuites() {
 	const files = await glob('**/*.tasks.js', {
 		cwd: path.join('dist', 'benchmark'),
@@ -53,6 +42,17 @@ export function registerSuites(bench: Bench) {
 	}
 }
 
+function suiteFilePath() {
+	const filePath = callsites()
+		.map((site) => site.getFileName())
+		.filter((site): site is string => site !== null)
+		.find((site) => site.endsWith('.tasks.js'));
+
+	assert(filePath !== undefined);
+
+	return filePath;
+}
+
 /**
  * Benchmarking API
  */
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 7069cb209462a..51693a27fc70f 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,10 +1,10 @@
 import 'reflect-metadata';
 import { globalHooks } from './hooks';
+import { collectSuites, registerSuites, suiteCount } from './lib/suites';
 
 /* eslint-disable import/no-extraneous-dependencies */
-import { withCodSpeed } from '@codspeed/tinybench-plugin';
 import Bench from 'tinybench';
-import { collectSuites, registerSuites, suiteCount } from './lib/suites';
+import { withCodSpeed } from '@codspeed/tinybench-plugin';
 /* eslint-enable import/no-extraneous-dependencies */
 
 export { beforeEach, task } from './lib/suites';

From a6bd9ea424f26693cbceb978448e75dc0ee7fb95 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 14:42:44 +0200
Subject: [PATCH 28/86] More cleanup

---
 packages/cli/src/benchmark/lib/suites.ts | 9 ++-------
 packages/cli/src/benchmark/types.ts      | 7 +++++++
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index 2e013ac4d754f..5b959d6a4a28d 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -4,14 +4,9 @@ import type Bench from 'tinybench';
 import { assert } from 'n8n-workflow';
 import glob from 'fast-glob';
 import callsites from 'callsites';
-import type { Task } from '../types';
+import type { Suites, Task } from '../types';
 
-export const suites: {
-	[suiteFilepath: string]: {
-		hooks: Partial<{ beforeEach: () => Promise<void> }>;
-		tasks: Task[];
-	};
-} = {};
+export const suites: Suites = {};
 
 export function suiteCount() {
 	return Object.keys(suites).length;
diff --git a/packages/cli/src/benchmark/types.ts b/packages/cli/src/benchmark/types.ts
index 2d961e4198008..b7b562d014516 100644
--- a/packages/cli/src/benchmark/types.ts
+++ b/packages/cli/src/benchmark/types.ts
@@ -1,3 +1,10 @@
+export type Suites = {
+	[suiteFilepath: string]: {
+		hooks: Partial<{ beforeEach: () => Promise<void> }>;
+		tasks: Task[];
+	};
+};
+
 /** A benchmarking task, i.e. a single operation whose performance to measure. */
 export type Task = {
 	description: string;

From 8282963b4266a540c342680b2634b102f874b197 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 14:44:20 +0200
Subject: [PATCH 29/86] Organize

---
 packages/cli/src/benchmark/{hooks.ts => lib/global-hooks.ts} | 2 +-
 packages/cli/src/benchmark/lib/suites.ts                     | 2 +-
 packages/cli/src/benchmark/{ => lib}/types.ts                | 0
 packages/cli/src/benchmark/main.ts                           | 2 +-
 4 files changed, 3 insertions(+), 3 deletions(-)
 rename packages/cli/src/benchmark/{hooks.ts => lib/global-hooks.ts} (95%)
 rename packages/cli/src/benchmark/{ => lib}/types.ts (100%)

diff --git a/packages/cli/src/benchmark/hooks.ts b/packages/cli/src/benchmark/lib/global-hooks.ts
similarity index 95%
rename from packages/cli/src/benchmark/hooks.ts
rename to packages/cli/src/benchmark/lib/global-hooks.ts
index c60b8d1cde89f..7008920f6494f 100644
--- a/packages/cli/src/benchmark/hooks.ts
+++ b/packages/cli/src/benchmark/lib/global-hooks.ts
@@ -5,7 +5,7 @@ import { Config } from '@oclif/core';
 import { InstanceSettings } from 'n8n-core';
 import { Start } from '@/commands/start';
 import Container from 'typedi';
-import { createOwner, deleteOwnerShell } from './db/users';
+import { createOwner, deleteOwnerShell } from '../db/users';
 
 function n8nDir() {
 	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index 5b959d6a4a28d..98dd0bb28cadf 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -4,7 +4,7 @@ import type Bench from 'tinybench';
 import { assert } from 'n8n-workflow';
 import glob from 'fast-glob';
 import callsites from 'callsites';
-import type { Suites, Task } from '../types';
+import type { Suites, Task } from './types';
 
 export const suites: Suites = {};
 
diff --git a/packages/cli/src/benchmark/types.ts b/packages/cli/src/benchmark/lib/types.ts
similarity index 100%
rename from packages/cli/src/benchmark/types.ts
rename to packages/cli/src/benchmark/lib/types.ts
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 51693a27fc70f..9d622f7267e90 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,5 +1,5 @@
 import 'reflect-metadata';
-import { globalHooks } from './hooks';
+import { globalHooks } from './lib/global-hooks';
 import { collectSuites, registerSuites, suiteCount } from './lib/suites';
 
 /* eslint-disable import/no-extraneous-dependencies */

From d24f41286a372bcb2ec2d2949ec3db09c3b43b49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 14:55:16 +0200
Subject: [PATCH 30/86] Prevent duplicate `beforeEach``

---
 .../src/benchmark/lib/duplicate-hook.error.ts | 10 ++++++++
 packages/cli/src/benchmark/lib/suites.ts      | 25 +++++++++++++------
 packages/cli/src/benchmark/lib/types.ts       |  4 +--
 3 files changed, 29 insertions(+), 10 deletions(-)
 create mode 100644 packages/cli/src/benchmark/lib/duplicate-hook.error.ts

diff --git a/packages/cli/src/benchmark/lib/duplicate-hook.error.ts b/packages/cli/src/benchmark/lib/duplicate-hook.error.ts
new file mode 100644
index 0000000000000..21730047e8be7
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/duplicate-hook.error.ts
@@ -0,0 +1,10 @@
+import { ApplicationError } from 'n8n-workflow';
+
+export class DuplicateHookError extends ApplicationError {
+	constructor(hookName: 'beforeEach', filePath: string) {
+		super(
+			`Duplicate \`${hookName}\` hook found in benchmarking suite \`${filePath}\`. Please define a single \`${hookName}\` hook for this suite.`,
+			{ level: 'warning' },
+		);
+	}
+}
diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index 98dd0bb28cadf..f18db31eee4c0 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -4,7 +4,8 @@ import type Bench from 'tinybench';
 import { assert } from 'n8n-workflow';
 import glob from 'fast-glob';
 import callsites from 'callsites';
-import type { Suites, Task } from './types';
+import type { Suites, Task, Callback } from './types';
+import { DuplicateHookError } from './duplicate-hook.error';
 
 export const suites: Suites = {};
 
@@ -18,8 +19,8 @@ export async function collectSuites() {
 		absolute: true,
 	});
 
-	for (const file of files) {
-		await import(file);
+	for (const f of files) {
+		await import(f);
 	}
 }
 
@@ -53,11 +54,19 @@ function suiteFilePath() {
  */
 
 export function task(description: string, operation: Task['operation']) {
-	suites[suiteFilePath()] ||= { hooks: {}, tasks: [] };
-	suites[suiteFilePath()].tasks.push({ description, operation });
+	const filePath = suiteFilePath();
+
+	suites[filePath] ||= { hooks: {}, tasks: [] };
+	suites[filePath].tasks.push({ description, operation });
 }
 
-export function beforeEach(fn: () => Promise<void>) {
-	suites[suiteFilePath()] ||= { hooks: {}, tasks: [] };
-	suites[suiteFilePath()].hooks.beforeEach = fn;
+export function beforeEach(fn: Callback) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]?.hooks.beforeEach) {
+		throw new DuplicateHookError('beforeEach', filePath);
+	}
+
+	suites[filePath] ||= { hooks: {}, tasks: [] };
+	suites[filePath].hooks.beforeEach = fn;
 }
diff --git a/packages/cli/src/benchmark/lib/types.ts b/packages/cli/src/benchmark/lib/types.ts
index b7b562d014516..ce3830dc0c3b7 100644
--- a/packages/cli/src/benchmark/lib/types.ts
+++ b/packages/cli/src/benchmark/lib/types.ts
@@ -1,6 +1,6 @@
 export type Suites = {
 	[suiteFilepath: string]: {
-		hooks: Partial<{ beforeEach: () => Promise<void> }>;
+		hooks: Partial<{ beforeEach: Callback }>;
 		tasks: Task[];
 	};
 };
@@ -8,7 +8,7 @@ export type Suites = {
 /** A benchmarking task, i.e. a single operation whose performance to measure. */
 export type Task = {
 	description: string;
-	operation: () => Promise<void>;
+	operation: Callback;
 };
 
 export type Callback = () => void | Promise<void>;

From 8a0f64b486850784e9f413291901be57005a906a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 15:10:07 +0200
Subject: [PATCH 31/86] Implement `afterEach`

---
 .../src/benchmark/lib/duplicate-hook.error.ts |  2 +-
 packages/cli/src/benchmark/lib/suites.ts      | 33 +++++++++++++++----
 packages/cli/src/benchmark/lib/types.ts       |  5 ++-
 packages/cli/src/benchmark/main.ts            |  2 +-
 .../cli/src/benchmark/tasks/example.tasks.ts  |  6 +++-
 5 files changed, 37 insertions(+), 11 deletions(-)

diff --git a/packages/cli/src/benchmark/lib/duplicate-hook.error.ts b/packages/cli/src/benchmark/lib/duplicate-hook.error.ts
index 21730047e8be7..a99464065002e 100644
--- a/packages/cli/src/benchmark/lib/duplicate-hook.error.ts
+++ b/packages/cli/src/benchmark/lib/duplicate-hook.error.ts
@@ -1,7 +1,7 @@
 import { ApplicationError } from 'n8n-workflow';
 
 export class DuplicateHookError extends ApplicationError {
-	constructor(hookName: 'beforeEach', filePath: string) {
+	constructor(hookName: 'beforeEach' | 'afterEach', filePath: string) {
 		super(
 			`Duplicate \`${hookName}\` hook found in benchmarking suite \`${filePath}\`. Please define a single \`${hookName}\` hook for this suite.`,
 			{ level: 'warning' },
diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index f18db31eee4c0..a555c2d9c0b67 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -25,14 +25,22 @@ export async function collectSuites() {
 }
 
 export function registerSuites(bench: Bench) {
-	for (const { tasks, hooks } of Object.values(suites)) {
-		for (const t of tasks) {
-			/**
-			 * `beforeAll` in tinybench is called once before all iterations of a single operation.
-			 * This is functionally equivalent to `beforeEach` in jest and vitest.
-			 */
-			const options = hooks.beforeEach ? { beforeAll: hooks.beforeEach } : {};
+	for (const { hooks, tasks } of Object.values(suites)) {
+		/**
+		 * In tinybench, `beforeAll` and `afterAll` refer to all iterations of
+		 * a single task, while `beforeEach` and `afterEach` refer to each iteration.
+		 *
+		 * In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
+		 * while `beforeEach` and `afterEach` refer to each individual test.
+		 *
+		 * The API renames tinybench's hooks to prevent confusion from this difference.
+		 */
+		const options: Record<string, Callback> = {};
+
+		if (hooks.beforeEach) options.beforeAll = hooks.beforeEach;
+		if (hooks.afterEach) options.afterAll = hooks.afterEach;
 
+		for (const t of tasks) {
 			bench.add(t.description, t.operation, options);
 		}
 	}
@@ -70,3 +78,14 @@ export function beforeEach(fn: Callback) {
 	suites[filePath] ||= { hooks: {}, tasks: [] };
 	suites[filePath].hooks.beforeEach = fn;
 }
+
+export function afterEach(fn: Callback) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]?.hooks.afterEach) {
+		throw new DuplicateHookError('afterEach', filePath);
+	}
+
+	suites[filePath] ||= { hooks: {}, tasks: [] };
+	suites[filePath].hooks.afterEach = fn;
+}
diff --git a/packages/cli/src/benchmark/lib/types.ts b/packages/cli/src/benchmark/lib/types.ts
index ce3830dc0c3b7..afb0f792deb2c 100644
--- a/packages/cli/src/benchmark/lib/types.ts
+++ b/packages/cli/src/benchmark/lib/types.ts
@@ -1,6 +1,9 @@
 export type Suites = {
 	[suiteFilepath: string]: {
-		hooks: Partial<{ beforeEach: Callback }>;
+		hooks: {
+			beforeEach?: Callback;
+			afterEach?: Callback;
+		};
 		tasks: Task[];
 	};
 };
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 9d622f7267e90..8f4837ba1d84b 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -7,7 +7,7 @@ import Bench from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
 /* eslint-enable import/no-extraneous-dependencies */
 
-export { beforeEach, task } from './lib/suites';
+export { beforeEach, afterEach, task } from './lib/suites';
 
 async function main() {
 	await collectSuites();
diff --git a/packages/cli/src/benchmark/tasks/example.tasks.ts b/packages/cli/src/benchmark/tasks/example.tasks.ts
index b1bb77a386764..1167f6c4e2dfd 100644
--- a/packages/cli/src/benchmark/tasks/example.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/example.tasks.ts
@@ -1,9 +1,13 @@
-import { task, beforeEach } from '../main.js';
+import { task, beforeEach, afterEach } from '../main.js';
 
 beforeEach(async () => {
 	console.log('[[[beforeEach]]] for example.tasks.ts');
 });
 
+afterEach(async () => {
+	console.log('[[[afterEach]]] for example.tasks.ts');
+});
+
 task('[example] Should do something', async () => {
 	console.log('Example task 1 executed');
 });

From fe4ce3c849415e02c1db700b4eab462a163c0bc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 19 Apr 2024 15:19:24 +0200
Subject: [PATCH 32/86] Cleanup

---
 packages/cli/src/benchmark/lib/global-hooks.ts | 11 ++++-------
 packages/cli/src/benchmark/main.ts             |  2 +-
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/packages/cli/src/benchmark/lib/global-hooks.ts b/packages/cli/src/benchmark/lib/global-hooks.ts
index 7008920f6494f..1ad0ac37df4e7 100644
--- a/packages/cli/src/benchmark/lib/global-hooks.ts
+++ b/packages/cli/src/benchmark/lib/global-hooks.ts
@@ -20,12 +20,12 @@ function n8nDir() {
 
 	writeFileSync(
 		path.join(_n8nDir, 'config'),
-		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: '123' }),
+		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: 'temp-123' }),
 		'utf-8',
 	);
 
 	/**
-	 * @TODO Better approach? Setting N8N_USER_FOLDER has no effect
+	 * @TODO Better approach than overriding? Setting N8N_USER_FOLDER has no effect
 	 */
 	const instanceSettings = Container.get(InstanceSettings);
 	instanceSettings.n8nFolder = _n8nDir;
@@ -46,7 +46,7 @@ async function mainProcess() {
 	await main.run();
 }
 
-async function setup() {
+export async function setup() {
 	n8nDir();
 
 	await mainProcess();
@@ -56,9 +56,6 @@ async function setup() {
 	await createOwner();
 }
 
-async function teardown() {
+export async function teardown() {
 	await main.stopProcess();
 }
-
-/** Lifecycle hooks to run once before and after all benchmarking tasks. */
-export const globalHooks = { setup, teardown };
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 8f4837ba1d84b..47117a72f194b 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,5 +1,5 @@
 import 'reflect-metadata';
-import { globalHooks } from './lib/global-hooks';
+import * as globalHooks from './lib/global-hooks';
 import { collectSuites, registerSuites, suiteCount } from './lib/suites';
 
 /* eslint-disable import/no-extraneous-dependencies */

From 86562dee11a8784601de066fd873d2d83005efb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Mon, 22 Apr 2024 10:37:58 +0200
Subject: [PATCH 33/86] Reduce diff

---
 .../1711390882123-MoveSshKeysToDatabase.ts    |  5 ++-
 .../externalSecrets.api.test.ts               |  7 ++--
 .../cli/test/integration/auth.api.test.ts     |  5 +--
 packages/cli/test/integration/auth.mw.test.ts |  7 +---
 .../test/integration/binaryData.api.test.ts   |  5 +--
 .../community-packages.api.test.ts            |  5 +--
 .../test/integration/credentials.ee.test.ts   |  7 ++--
 .../cli/test/integration/credentials.test.ts  |  7 ++--
 .../test/integration/debug.controller.test.ts |  5 +--
 .../environments/SourceControl.test.ts        |  5 +--
 .../cli/test/integration/eventbus.ee.test.ts  |  5 +--
 .../cli/test/integration/eventbus.test.ts     |  5 +--
 .../test/integration/ldap/ldap.api.test.ts    |  5 +--
 .../cli/test/integration/license.api.test.ts  |  7 ++--
 packages/cli/test/integration/me.api.test.ts  |  7 ++--
 .../integration/publicApi/credentials.test.ts |  7 ++--
 .../integration/publicApi/executions.test.ts  | 10 ++---
 .../test/integration/publicApi/tags.test.ts   |  7 ++--
 .../integration/publicApi/users.ee.test.ts    |  5 +--
 .../integration/publicApi/workflows.test.ts   |  8 ++--
 .../test/integration/saml/saml.api.test.ts    |  7 ++--
 .../cli/test/integration/shared/db/users.ts   |  5 ---
 packages/cli/test/integration/shared/types.ts |  9 ++--
 .../test/integration/shared/utils/index.ts    |  1 -
 .../cli/test/integration/tags.api.test.ts     |  5 +--
 .../cli/test/integration/users.api.test.ts    | 15 ++++---
 .../cli/test/integration/variables.test.ts    |  7 ++--
 .../cli/test/integration/webhooks.api.test.ts |  4 +-
 .../integration/workflowHistory.api.test.ts   |  7 ++--
 .../workflows/workflows.controller.ee.test.ts |  9 ++--
 .../workflows/workflows.controller.test.ts    |  5 +--
 packages/cli/test/unit/Telemetry.test.ts      |  4 +-
 .../test/unit/services/events.service.test.ts |  4 +-
 .../services/orchestration.service.test.ts    |  3 --
 .../test/unit/services/redis.service.test.ts  | 42 +++++++++++++++++++
 packages/cli/test/unit/webhooks.test.ts       |  4 +-
 pnpm-workspace.yaml                           |  1 -
 37 files changed, 128 insertions(+), 128 deletions(-)

diff --git a/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
index 99b27bfaa877c..fcadaf911bfa3 100644
--- a/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
+++ b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
@@ -4,7 +4,7 @@ import Container from 'typedi';
 import { Cipher, InstanceSettings } from 'n8n-core';
 import { jsonParse } from 'n8n-workflow';
 import type { MigrationContext, ReversibleMigration } from '@db/types';
-import { inBenchmark } from '@/constants';
+import { inTest, inBenchmark } from '@/constants';
 
 /**
  * Move SSH key pair from file system to database, to enable SSH connections
@@ -32,7 +32,8 @@ export class MoveSshKeysToDatabase1711390882123 implements ReversibleMigration {
 				readFile(this.publicKeyPath, { encoding: 'utf8' }),
 			]);
 		} catch {
-			if (!inBenchmark) logger.info(`[${migrationName}] No SSH keys in filesystem, skipping`);
+			if (!inTest && !inBenchmark)
+				logger.info(`[${migrationName}] No SSH keys in filesystem, skipping`);
 			return;
 		}
 
diff --git a/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts b/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts
index 3d47653038ca8..868c695f4a38a 100644
--- a/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts
+++ b/packages/cli/test/integration/ExternalSecrets/externalSecrets.api.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { License } from '@/License';
 import type { ExternalSecretsSettings, SecretsProviderState } from '@/Interfaces';
 import { Cipher } from 'n8n-core';
@@ -22,8 +21,8 @@ import {
 	TestFailProvider,
 } from '../../shared/ExternalSecrets/utils';
 
-let authOwnerAgent: TestAgent<Test>;
-let authMemberAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authMemberAgent: SuperAgentTest;
 
 const mockProvidersInstance = new MockProviders();
 mockInstance(ExternalSecretsProviders, mockProvidersInstance);
diff --git a/packages/cli/test/integration/auth.api.test.ts b/packages/cli/test/integration/auth.api.test.ts
index 6a41b7c560b13..9435fa7a7dfcf 100644
--- a/packages/cli/test/integration/auth.api.test.ts
+++ b/packages/cli/test/integration/auth.api.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { Container } from 'typedi';
 import validator from 'validator';
 import config from '@/config';
@@ -14,7 +13,7 @@ import { UserRepository } from '@db/repositories/user.repository';
 import { MfaService } from '@/Mfa/mfa.service';
 
 let owner: User;
-let authOwnerAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
 const ownerPassword = randomValidPassword();
 
 const testServer = utils.setupTestServer({ endpointGroups: ['auth'] });
diff --git a/packages/cli/test/integration/auth.mw.test.ts b/packages/cli/test/integration/auth.mw.test.ts
index 9414f21802d54..8f40759f965c6 100644
--- a/packages/cli/test/integration/auth.mw.test.ts
+++ b/packages/cli/test/integration/auth.mw.test.ts
@@ -1,7 +1,6 @@
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import * as utils from './shared/utils/';
 import { createUser } from './shared/db/users';
 import { mockInstance } from '../shared/mocking';
@@ -30,7 +29,6 @@ describe('Auth Middleware', () => {
 		ROUTES_REQUIRING_AUTHENTICATION.concat(ROUTES_REQUIRING_AUTHORIZATION).forEach(
 			([method, endpoint]) => {
 				test(`${method} ${endpoint} should return 401 Unauthorized if no cookie`, async () => {
-					// @ts-ignore
 					const { statusCode } = await testServer.authlessAgent[method.toLowerCase()](endpoint);
 					expect(statusCode).toBe(401);
 				});
@@ -39,7 +37,7 @@ describe('Auth Middleware', () => {
 	});
 
 	describe('Routes requiring Authorization', () => {
-		let authMemberAgent: TestAgent<Test>;
+		let authMemberAgent: SuperAgentTest;
 		beforeAll(async () => {
 			const member = await createUser({ role: 'global:member' });
 			authMemberAgent = testServer.authAgentFor(member);
@@ -47,7 +45,6 @@ describe('Auth Middleware', () => {
 
 		ROUTES_REQUIRING_AUTHORIZATION.forEach(async ([method, endpoint]) => {
 			test(`${method} ${endpoint} should return 403 Forbidden for member`, async () => {
-				// @ts-ignore
 				const { statusCode } = await authMemberAgent[method.toLowerCase()](endpoint);
 				expect(statusCode).toBe(403);
 			});
diff --git a/packages/cli/test/integration/binaryData.api.test.ts b/packages/cli/test/integration/binaryData.api.test.ts
index e1dc79672ad28..6a3a314c737fa 100644
--- a/packages/cli/test/integration/binaryData.api.test.ts
+++ b/packages/cli/test/integration/binaryData.api.test.ts
@@ -1,8 +1,7 @@
 import fsp from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { BinaryDataService, FileNotFoundError } from 'n8n-core';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 
 import { mockInstance } from '../shared/mocking';
 import { setupTestServer } from './shared/utils';
@@ -16,7 +15,7 @@ const throwFileNotFound = () => {
 
 const binaryDataService = mockInstance(BinaryDataService);
 let testServer = setupTestServer({ endpointGroups: ['binaryData'] });
-let authOwnerAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
 
 beforeAll(async () => {
 	const owner = await createOwner();
diff --git a/packages/cli/test/integration/community-packages.api.test.ts b/packages/cli/test/integration/community-packages.api.test.ts
index cc1378dc9c571..03fd43f456824 100644
--- a/packages/cli/test/integration/community-packages.api.test.ts
+++ b/packages/cli/test/integration/community-packages.api.test.ts
@@ -1,6 +1,5 @@
 import path from 'path';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 
 import type { InstalledPackages } from '@db/entities/InstalledPackages';
 import type { InstalledNodes } from '@db/entities/InstalledNodes';
@@ -31,7 +30,7 @@ const parsedNpmPackageName = {
 	rawString: 'test',
 };
 
-let authAgent: TestAgent<Test>;
+let authAgent: SuperAgentTest;
 
 beforeAll(async () => {
 	const ownerShell = await createOwner();
diff --git a/packages/cli/test/integration/credentials.ee.test.ts b/packages/cli/test/integration/credentials.ee.test.ts
index 2abc483df0e64..279f33c019dde 100644
--- a/packages/cli/test/integration/credentials.ee.test.ts
+++ b/packages/cli/test/integration/credentials.ee.test.ts
@@ -1,6 +1,5 @@
 import { Container } from 'typedi';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { In } from '@n8n/typeorm';
 import type { IUser } from 'n8n-workflow';
 
@@ -27,8 +26,8 @@ const testServer = utils.setupTestServer({
 let owner: User;
 let member: User;
 let anotherMember: User;
-let authOwnerAgent: TestAgent<Test>;
-let authAnotherMemberAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authAnotherMemberAgent: SuperAgentTest;
 let saveCredential: SaveCredentialFunction;
 const mailer = mockInstance(UserManagementMailer);
 
diff --git a/packages/cli/test/integration/credentials.test.ts b/packages/cli/test/integration/credentials.test.ts
index 9ca946b63b1e1..aa40616ad8681 100644
--- a/packages/cli/test/integration/credentials.test.ts
+++ b/packages/cli/test/integration/credentials.test.ts
@@ -1,6 +1,5 @@
 import { Container } from 'typedi';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 
 import config from '@/config';
 import type { ListQuery } from '@/requests';
@@ -24,8 +23,8 @@ const testServer = utils.setupTestServer({ endpointGroups: ['credentials'] });
 let owner: User;
 let member: User;
 let secondMember: User;
-let authOwnerAgent: TestAgent<Test>;
-let authMemberAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authMemberAgent: SuperAgentTest;
 let saveCredential: SaveCredentialFunction;
 
 beforeAll(async () => {
diff --git a/packages/cli/test/integration/debug.controller.test.ts b/packages/cli/test/integration/debug.controller.test.ts
index 468e9dc2c8142..9acd6a39931d4 100644
--- a/packages/cli/test/integration/debug.controller.test.ts
+++ b/packages/cli/test/integration/debug.controller.test.ts
@@ -5,8 +5,7 @@ import { randomName } from './shared/random';
 import { generateNanoId } from '@/databases/utils/generators';
 import type { WorkflowEntity } from '@/databases/entities/WorkflowEntity';
 import { setupTestServer } from './shared/utils';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { createOwner } from './shared/db/users';
 import { OrchestrationService } from '@/services/orchestration.service';
 import { MultiMainSetup } from '@/services/orchestration/main/MultiMainSetup.ee';
@@ -16,7 +15,7 @@ describe('DebugController', () => {
 	const activeWorkflowRunner = mockInstance(ActiveWorkflowRunner);
 
 	let testServer = setupTestServer({ endpointGroups: ['debug'] });
-	let ownerAgent: TestAgent<Test>;
+	let ownerAgent: SuperAgentTest;
 
 	beforeAll(async () => {
 		const owner = await createOwner();
diff --git a/packages/cli/test/integration/environments/SourceControl.test.ts b/packages/cli/test/integration/environments/SourceControl.test.ts
index 3886dcb7651a6..f1da8d0672f25 100644
--- a/packages/cli/test/integration/environments/SourceControl.test.ts
+++ b/packages/cli/test/integration/environments/SourceControl.test.ts
@@ -1,6 +1,5 @@
 import { Container } from 'typedi';
-import type Test from 'supertest/lib/test';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 
 import type { User } from '@db/entities/User';
 import config from '@/config';
@@ -11,7 +10,7 @@ import type { SourceControlledFile } from '@/environments/sourceControl/types/so
 import * as utils from '../shared/utils/';
 import { createUser } from '../shared/db/users';
 
-let authOwnerAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
 let owner: User;
 
 const testServer = utils.setupTestServer({
diff --git a/packages/cli/test/integration/eventbus.ee.test.ts b/packages/cli/test/integration/eventbus.ee.test.ts
index dbde24a100ade..f486f39abe12b 100644
--- a/packages/cli/test/integration/eventbus.ee.test.ts
+++ b/packages/cli/test/integration/eventbus.ee.test.ts
@@ -3,8 +3,7 @@ import config from '@/config';
 import axios from 'axios';
 import syslog from 'syslog-client';
 import { v4 as uuid } from 'uuid';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import type {
 	MessageEventBusDestinationSentryOptions,
 	MessageEventBusDestinationSyslogOptions,
@@ -39,7 +38,7 @@ jest.mock('syslog-client');
 const mockedSyslog = syslog as jest.Mocked<typeof syslog>;
 
 let owner: User;
-let authOwnerAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
 
 const testSyslogDestination: MessageEventBusDestinationSyslogOptions = {
 	...defaultMessageEventBusDestinationSyslogOptions,
diff --git a/packages/cli/test/integration/eventbus.test.ts b/packages/cli/test/integration/eventbus.test.ts
index d13e87bc27492..9e3e27dacb916 100644
--- a/packages/cli/test/integration/eventbus.test.ts
+++ b/packages/cli/test/integration/eventbus.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 
 import type { User } from '@db/entities/User';
 import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus';
@@ -16,7 +15,7 @@ import { mockInstance } from '../shared/mocking';
  */
 
 let owner: User;
-let authOwnerAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
 
 mockInstance(MessageEventBus);
 mockInstance(ExecutionDataRecoveryService);
diff --git a/packages/cli/test/integration/ldap/ldap.api.test.ts b/packages/cli/test/integration/ldap/ldap.api.test.ts
index 37441764557bd..17cb3e7b543b2 100644
--- a/packages/cli/test/integration/ldap/ldap.api.test.ts
+++ b/packages/cli/test/integration/ldap/ldap.api.test.ts
@@ -1,6 +1,5 @@
 import Container from 'typedi';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import type { Entry as LdapUser } from 'ldapts';
 import { Not } from '@n8n/typeorm';
 import { jsonParse } from 'n8n-workflow';
@@ -26,7 +25,7 @@ import { AuthProviderSyncHistoryRepository } from '@db/repositories/authProvider
 jest.mock('@/telemetry');
 
 let owner: User;
-let authOwnerAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
 
 const defaultLdapConfig = {
 	...LDAP_DEFAULT_CONFIGURATION,
diff --git a/packages/cli/test/integration/license.api.test.ts b/packages/cli/test/integration/license.api.test.ts
index bab60456f9299..3d1fc4cda84a7 100644
--- a/packages/cli/test/integration/license.api.test.ts
+++ b/packages/cli/test/integration/license.api.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import config from '@/config';
 import type { User } from '@db/entities/User';
 import type { ILicensePostResponse, ILicenseReadResponse } from '@/Interfaces';
@@ -13,8 +12,8 @@ const MOCK_RENEW_OFFSET = 259200;
 
 let owner: User;
 let member: User;
-let authOwnerAgent: TestAgent<Test>;
-let authMemberAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authMemberAgent: SuperAgentTest;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['license'] });
 
diff --git a/packages/cli/test/integration/me.api.test.ts b/packages/cli/test/integration/me.api.test.ts
index 3f23b81c6c9dc..53ee82343029f 100644
--- a/packages/cli/test/integration/me.api.test.ts
+++ b/packages/cli/test/integration/me.api.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { IsNull } from '@n8n/typeorm';
 import validator from 'validator';
 import type { User } from '@db/entities/User';
@@ -25,7 +24,7 @@ beforeEach(async () => {
 
 describe('Owner shell', () => {
 	let ownerShell: User;
-	let authOwnerShellAgent: TestAgent<Test>;
+	let authOwnerShellAgent: SuperAgentTest;
 
 	beforeEach(async () => {
 		ownerShell = await createUserShell('global:owner');
@@ -162,7 +161,7 @@ describe('Owner shell', () => {
 describe('Member', () => {
 	const memberPassword = randomValidPassword();
 	let member: User;
-	let authMemberAgent: TestAgent<Test>;
+	let authMemberAgent: SuperAgentTest;
 
 	beforeEach(async () => {
 		member = await createUser({
diff --git a/packages/cli/test/integration/publicApi/credentials.test.ts b/packages/cli/test/integration/publicApi/credentials.test.ts
index fae1acfb14273..d028834638ca0 100644
--- a/packages/cli/test/integration/publicApi/credentials.test.ts
+++ b/packages/cli/test/integration/publicApi/credentials.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import type { User } from '@db/entities/User';
 
 import { randomApiKey, randomName, randomString } from '../shared/random';
@@ -14,8 +13,8 @@ import { SharedCredentialsRepository } from '@db/repositories/sharedCredentials.
 
 let owner: User;
 let member: User;
-let authOwnerAgent: TestAgent<Test>;
-let authMemberAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authMemberAgent: SuperAgentTest;
 
 let saveCredential: SaveCredentialFunction;
 
diff --git a/packages/cli/test/integration/publicApi/executions.test.ts b/packages/cli/test/integration/publicApi/executions.test.ts
index 319ee86401b29..012519df66580 100644
--- a/packages/cli/test/integration/publicApi/executions.test.ts
+++ b/packages/cli/test/integration/publicApi/executions.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import type { User } from '@db/entities/User';
 import type { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 
@@ -22,9 +21,9 @@ import {
 let owner: User;
 let user1: User;
 let user2: User;
-let authOwnerAgent: TestAgent<Test>;
-let authUser1Agent: TestAgent<Test>;
-let authUser2Agent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authUser1Agent: SuperAgentTest;
+let authUser2Agent: SuperAgentTest;
 let workflowRunner: ActiveWorkflowRunner;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] });
@@ -62,7 +61,6 @@ afterEach(async () => {
 
 const testWithAPIKey =
 	(method: 'get' | 'post' | 'put' | 'delete', url: string, apiKey: string | null) => async () => {
-		// @ts-ignore
 		void authOwnerAgent.set({ 'X-N8N-API-KEY': apiKey });
 		const response = await authOwnerAgent[method](url);
 		expect(response.statusCode).toBe(401);
diff --git a/packages/cli/test/integration/publicApi/tags.test.ts b/packages/cli/test/integration/publicApi/tags.test.ts
index 574568a06b41e..7e8fcacdead0f 100644
--- a/packages/cli/test/integration/publicApi/tags.test.ts
+++ b/packages/cli/test/integration/publicApi/tags.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import Container from 'typedi';
 import type { User } from '@db/entities/User';
 import { TagRepository } from '@db/repositories/tag.repository';
@@ -12,8 +11,8 @@ import { createTag } from '../shared/db/tags';
 
 let owner: User;
 let member: User;
-let authOwnerAgent: TestAgent<Test>;
-let authMemberAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authMemberAgent: SuperAgentTest;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] });
 
diff --git a/packages/cli/test/integration/publicApi/users.ee.test.ts b/packages/cli/test/integration/publicApi/users.ee.test.ts
index a18ee3e4435cf..8dfae84625e87 100644
--- a/packages/cli/test/integration/publicApi/users.ee.test.ts
+++ b/packages/cli/test/integration/publicApi/users.ee.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import validator from 'validator';
 import { v4 as uuid } from 'uuid';
 
@@ -203,7 +202,7 @@ describe('With license unlimited quota:users', () => {
 });
 
 describe('With license without quota:users', () => {
-	let authOwnerAgent: TestAgent<Test>;
+	let authOwnerAgent: SuperAgentTest;
 
 	beforeEach(async () => {
 		mockInstance(License, { getUsersLimit: jest.fn().mockReturnValue(null) });
diff --git a/packages/cli/test/integration/publicApi/workflows.test.ts b/packages/cli/test/integration/publicApi/workflows.test.ts
index 8c67a87fc63c6..c91178750cc18 100644
--- a/packages/cli/test/integration/publicApi/workflows.test.ts
+++ b/packages/cli/test/integration/publicApi/workflows.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import config from '@/config';
 import Container from 'typedi';
 import type { INode } from 'n8n-workflow';
@@ -21,8 +20,8 @@ import { mockInstance } from '../../shared/mocking';
 
 let owner: User;
 let member: User;
-let authOwnerAgent: TestAgent<Test>;
-let authMemberAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authMemberAgent: SuperAgentTest;
 let workflowRunner: ActiveWorkflowRunner;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['publicApi'] });
@@ -68,7 +67,6 @@ afterEach(async () => {
 
 const testWithAPIKey =
 	(method: 'get' | 'post' | 'put' | 'delete', url: string, apiKey: string | null) => async () => {
-		// @ts-ignore
 		void authOwnerAgent.set({ 'X-N8N-API-KEY': apiKey });
 		const response = await authOwnerAgent[method](url);
 		expect(response.statusCode).toBe(401);
diff --git a/packages/cli/test/integration/saml/saml.api.test.ts b/packages/cli/test/integration/saml/saml.api.test.ts
index 199ad9f727836..c12f8da1752fc 100644
--- a/packages/cli/test/integration/saml/saml.api.test.ts
+++ b/packages/cli/test/integration/saml/saml.api.test.ts
@@ -1,6 +1,5 @@
 import { Container } from 'typedi';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import type { AuthenticationMethod } from 'n8n-workflow';
 import type { User } from '@db/entities/User';
 import { setSamlLoginEnabled } from '@/sso/saml/samlHelpers';
@@ -16,8 +15,8 @@ import { createOwner, createUser } from '../shared/db/users';
 
 let someUser: User;
 let owner: User;
-let authMemberAgent: TestAgent<Test>;
-let authOwnerAgent: TestAgent<Test>;
+let authMemberAgent: SuperAgentTest;
+let authOwnerAgent: SuperAgentTest;
 
 async function enableSaml(enable: boolean) {
 	await setSamlLoginEnabled(enable);
diff --git a/packages/cli/test/integration/shared/db/users.ts b/packages/cli/test/integration/shared/db/users.ts
index 9e0df2eef0ffc..4f4ed8af184b9 100644
--- a/packages/cli/test/integration/shared/db/users.ts
+++ b/packages/cli/test/integration/shared/db/users.ts
@@ -8,7 +8,6 @@ import { TOTPService } from '@/Mfa/totp.service';
 import { MfaService } from '@/Mfa/mfa.service';
 
 import { randomApiKey, randomEmail, randomName, randomValidPassword } from '../random';
-import { IsNull } from '@n8n/typeorm';
 
 // pre-computed bcrypt hash for the string 'password', using `await hash('password', 10)`
 const passwordHash = '$2a$10$njedH7S6V5898mj6p0Jr..IGY9Ms.qNwR7RbSzzX9yubJocKfvGGK';
@@ -134,7 +133,3 @@ export const getLdapIdentities = async () =>
 export async function getGlobalOwner() {
 	return await Container.get(UserRepository).findOneByOrFail({ role: 'global:owner' });
 }
-
-export async function deleteOwnerShell() {
-	await Container.get(UserRepository).delete({ role: 'global:owner', email: IsNull() });
-}
diff --git a/packages/cli/test/integration/shared/types.ts b/packages/cli/test/integration/shared/types.ts
index 1195220fb1e9f..8355d6f39c3f0 100644
--- a/packages/cli/test/integration/shared/types.ts
+++ b/packages/cli/test/integration/shared/types.ts
@@ -1,7 +1,6 @@
 import type { Application } from 'express';
 import type { ICredentialDataDecryptedObject } from 'n8n-workflow';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import type { Server } from 'http';
 
 import type { CredentialsEntity } from '@db/entities/CredentialsEntity';
@@ -44,9 +43,9 @@ export interface SetupProps {
 export interface TestServer {
 	app: Application;
 	httpServer: Server;
-	authAgentFor: (user: User) => TestAgent<Test>;
-	publicApiAgentFor: (user: User) => TestAgent<Test>;
-	authlessAgent: TestAgent<Test>;
+	authAgentFor: (user: User) => SuperAgentTest;
+	publicApiAgentFor: (user: User) => SuperAgentTest;
+	authlessAgent: SuperAgentTest;
 	license: LicenseMocker;
 }
 
diff --git a/packages/cli/test/integration/shared/utils/index.ts b/packages/cli/test/integration/shared/utils/index.ts
index e79db27f97d14..2558c461ac9c8 100644
--- a/packages/cli/test/integration/shared/utils/index.ts
+++ b/packages/cli/test/integration/shared/utils/index.ts
@@ -96,7 +96,6 @@ export async function initBinaryDataService(mode: 'default' | 'filesystem' = 'de
  * Extract the value (token) of the auth cookie in a response.
  */
 export function getAuthToken(response: request.Response, authCookieName = AUTH_COOKIE_NAME) {
-	// @ts-ignore
 	const cookies: string[] = response.headers['set-cookie'];
 
 	if (!cookies) return undefined;
diff --git a/packages/cli/test/integration/tags.api.test.ts b/packages/cli/test/integration/tags.api.test.ts
index 5aa2d930c348c..73be97a1c269f 100644
--- a/packages/cli/test/integration/tags.api.test.ts
+++ b/packages/cli/test/integration/tags.api.test.ts
@@ -1,12 +1,11 @@
 import * as utils from './shared/utils/';
 import * as testDb from './shared/testDb';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { TagRepository } from '@db/repositories/tag.repository';
 import Container from 'typedi';
 import { createUserShell } from './shared/db/users';
 
-let authOwnerAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
 const testServer = utils.setupTestServer({ endpointGroups: ['tags'] });
 
 beforeAll(async () => {
diff --git a/packages/cli/test/integration/users.api.test.ts b/packages/cli/test/integration/users.api.test.ts
index 3625490c16d1c..5c9bdc2600a8e 100644
--- a/packages/cli/test/integration/users.api.test.ts
+++ b/packages/cli/test/integration/users.api.test.ts
@@ -1,6 +1,5 @@
 import Container from 'typedi';
-import type TestAgent from 'supertest/lib/agent';
-import type { Test } from 'supertest';
+import type { SuperAgentTest } from 'supertest';
 
 import { UsersController } from '@/controllers/users.controller';
 import type { User } from '@db/entities/User';
@@ -29,7 +28,7 @@ const testServer = utils.setupTestServer({
 describe('GET /users', () => {
 	let owner: User;
 	let member: User;
-	let ownerAgent: TestAgent<Test>;
+	let ownerAgent: SuperAgentTest;
 
 	beforeAll(async () => {
 		await testDb.truncate(['User']);
@@ -231,7 +230,7 @@ describe('GET /users', () => {
 describe('DELETE /users/:id', () => {
 	let owner: User;
 	let member: User;
-	let ownerAgent: TestAgent<Test>;
+	let ownerAgent: SuperAgentTest;
 
 	beforeAll(async () => {
 		await testDb.truncate(['User']);
@@ -348,10 +347,10 @@ describe('PATCH /users/:id/role', () => {
 	let member: User;
 	let otherMember: User;
 
-	let ownerAgent: TestAgent<Test>;
-	let adminAgent: TestAgent<Test>;
-	let memberAgent: TestAgent<Test>;
-	let authlessAgent: TestAgent<Test>;
+	let ownerAgent: SuperAgentTest;
+	let adminAgent: SuperAgentTest;
+	let memberAgent: SuperAgentTest;
+	let authlessAgent: SuperAgentTest;
 
 	const { NO_ADMIN_ON_OWNER, NO_USER, NO_OWNER_ON_OWNER } =
 		UsersController.ERROR_MESSAGES.CHANGE_ROLE;
diff --git a/packages/cli/test/integration/variables.test.ts b/packages/cli/test/integration/variables.test.ts
index e775ff9936a37..8b84f67dcfd18 100644
--- a/packages/cli/test/integration/variables.test.ts
+++ b/packages/cli/test/integration/variables.test.ts
@@ -1,6 +1,5 @@
 import Container from 'typedi';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import type { Variables } from '@db/entities/Variables';
 import { VariablesRepository } from '@db/repositories/variables.repository';
 import { generateNanoId } from '@db/utils/generators';
@@ -10,8 +9,8 @@ import * as testDb from './shared/testDb';
 import * as utils from './shared/utils/';
 import { createOwner, createUser } from './shared/db/users';
 
-let authOwnerAgent: TestAgent<Test>;
-let authMemberAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authMemberAgent: SuperAgentTest;
 
 const testServer = utils.setupTestServer({ endpointGroups: ['variables'] });
 const license = testServer.license;
diff --git a/packages/cli/test/integration/webhooks.api.test.ts b/packages/cli/test/integration/webhooks.api.test.ts
index e4f7aed11b924..37862be6aa237 100644
--- a/packages/cli/test/integration/webhooks.api.test.ts
+++ b/packages/cli/test/integration/webhooks.api.test.ts
@@ -1,5 +1,5 @@
 import { readFileSync } from 'fs';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { agent as testAgent } from 'supertest';
 import type { INodeType, INodeTypeDescription, IWebhookFunctions } from 'n8n-workflow';
 
@@ -21,7 +21,7 @@ describe('Webhook API', () => {
 	mockInstance(InternalHooks);
 	mockInstance(Push);
 
-	let agent: TestAgent;
+	let agent: SuperAgentTest;
 
 	beforeAll(async () => {
 		await testDb.init();
diff --git a/packages/cli/test/integration/workflowHistory.api.test.ts b/packages/cli/test/integration/workflowHistory.api.test.ts
index c4b4779bde721..b9ccd33b2c143 100644
--- a/packages/cli/test/integration/workflowHistory.api.test.ts
+++ b/packages/cli/test/integration/workflowHistory.api.test.ts
@@ -1,5 +1,4 @@
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import type { User } from '@db/entities/User';
 
 import * as testDb from './shared/testDb';
@@ -9,9 +8,9 @@ import { createWorkflow } from './shared/db/workflows';
 import { createWorkflowHistoryItem } from './shared/db/workflowHistory';
 
 let owner: User;
-let authOwnerAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
 let member: User;
-let authMemberAgent: TestAgent<Test>;
+let authMemberAgent: SuperAgentTest;
 
 const testServer = utils.setupTestServer({
 	endpointGroups: ['workflowHistory'],
diff --git a/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts b/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts
index 99c165f129136..ed3e13d9f063d 100644
--- a/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts
+++ b/packages/cli/test/integration/workflows/workflows.controller.ee.test.ts
@@ -1,6 +1,5 @@
 import Container from 'typedi';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { v4 as uuid } from 'uuid';
 import type { INode } from 'n8n-workflow';
 
@@ -25,9 +24,9 @@ import config from '@/config';
 let owner: User;
 let member: User;
 let anotherMember: User;
-let authOwnerAgent: TestAgent<Test>;
-let authMemberAgent: TestAgent<Test>;
-let authAnotherMemberAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
+let authMemberAgent: SuperAgentTest;
+let authAnotherMemberAgent: SuperAgentTest;
 let saveCredential: SaveCredentialFunction;
 
 const activeWorkflowRunner = mockInstance(ActiveWorkflowRunner);
diff --git a/packages/cli/test/integration/workflows/workflows.controller.test.ts b/packages/cli/test/integration/workflows/workflows.controller.test.ts
index f1dc00ecad441..d493dd9d48e1c 100644
--- a/packages/cli/test/integration/workflows/workflows.controller.test.ts
+++ b/packages/cli/test/integration/workflows/workflows.controller.test.ts
@@ -1,6 +1,5 @@
 import Container from 'typedi';
-import type { Test } from 'supertest';
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { v4 as uuid } from 'uuid';
 import type { INode, IPinData } from 'n8n-workflow';
 
@@ -24,7 +23,7 @@ import { createTag } from '../shared/db/tags';
 import { License } from '@/License';
 
 let owner: User;
-let authOwnerAgent: TestAgent<Test>;
+let authOwnerAgent: SuperAgentTest;
 
 jest.spyOn(License.prototype, 'isSharingEnabled').mockReturnValue(false);
 
diff --git a/packages/cli/test/unit/Telemetry.test.ts b/packages/cli/test/unit/Telemetry.test.ts
index 846c838ea72ff..25f4a808675b0 100644
--- a/packages/cli/test/unit/Telemetry.test.ts
+++ b/packages/cli/test/unit/Telemetry.test.ts
@@ -15,10 +15,8 @@ describe('Telemetry', () => {
 	const spyTrack = jest.spyOn(Telemetry.prototype, 'track').mockName('track');
 
 	const mockRudderStack: Pick<RudderStack, 'flush' | 'identify' | 'track'> = {
-		flush: async (resolve) => resolve?.(),
-		// @ts-ignore
+		flush: (resolve) => resolve?.(),
 		identify: (data, resolve) => resolve?.(),
-		// @ts-ignore
 		track: (data, resolve) => resolve?.(),
 	};
 
diff --git a/packages/cli/test/unit/services/events.service.test.ts b/packages/cli/test/unit/services/events.service.test.ts
index 37cb6f368896a..afdd4091d3186 100644
--- a/packages/cli/test/unit/services/events.service.test.ts
+++ b/packages/cli/test/unit/services/events.service.test.ts
@@ -107,7 +107,7 @@ describe('EventsService', () => {
 			};
 			const runData: IRun = {
 				finished: false,
-				status: 'error',
+				status: 'failed',
 				data: { resultData: { runData: {} } },
 				mode: 'internal' as WorkflowExecuteMode,
 				startedAt: new Date(),
@@ -193,7 +193,7 @@ describe('EventsService', () => {
 
 		test('should not send metrics for entries that already have the flag set', async () => {
 			// Fetch data for workflow 2 which is set up to not be altered in the mocks
-			entityManager.insert.mockRejectedValueOnce(new QueryFailedError('', undefined, new Error()));
+			entityManager.insert.mockRejectedValueOnce(new QueryFailedError('', undefined, ''));
 			const workflowId = '1';
 			const node = {
 				id: 'abcde',
diff --git a/packages/cli/test/unit/services/orchestration.service.test.ts b/packages/cli/test/unit/services/orchestration.service.test.ts
index 25b0da2c50827..35ad3a53bef57 100644
--- a/packages/cli/test/unit/services/orchestration.service.test.ts
+++ b/packages/cli/test/unit/services/orchestration.service.test.ts
@@ -147,10 +147,7 @@ describe('Orchestration Service', () => {
 			}),
 		);
 		expect(helpers.debounceMessageReceiver).toHaveBeenCalledTimes(2);
-
-		// @ts-ignore
 		expect(res1!.payload).toBeUndefined();
-		// @ts-ignore
 		expect(res2!.payload!.result).toEqual('debounced');
 	});
 
diff --git a/packages/cli/test/unit/services/redis.service.test.ts b/packages/cli/test/unit/services/redis.service.test.ts
index f6f932a4b80f2..04fb980db6797 100644
--- a/packages/cli/test/unit/services/redis.service.test.ts
+++ b/packages/cli/test/unit/services/redis.service.test.ts
@@ -54,4 +54,46 @@ describe('RedisService', () => {
 		await sub.destroy();
 		await pub.destroy();
 	});
+
+	// NOTE: This test is failing because the mock Redis client does not support streams apparently
+	// eslint-disable-next-line n8n-local-rules/no-skipped-tests
+	test.skip('should create stream producer and consumer', async () => {
+		const consumer = await redisService.getStreamConsumer();
+		const producer = await redisService.getStreamProducer();
+
+		expect(consumer).toBeDefined();
+		expect(producer).toBeDefined();
+
+		const mockHandler = jest.fn();
+		mockHandler.mockImplementation((stream: string, id: string, message: string[]) => {
+			console.log('Received message', stream, id, message);
+		});
+		consumer.addMessageHandler('some handler', mockHandler);
+
+		await consumer.setPollingInterval(STREAM_CHANNEL, 50);
+		await consumer.listenToStream(STREAM_CHANNEL);
+
+		let timeout;
+		await new Promise((resolve) => {
+			timeout = setTimeout(async () => {
+				await producer.add(STREAM_CHANNEL, ['message', 'testMessage', 'event', 'testEveny']);
+				resolve(0);
+			}, 50);
+		});
+
+		await new Promise((resolve) =>
+			setTimeout(async () => {
+				resolve(0);
+			}, 100),
+		);
+
+		clearInterval(timeout);
+
+		consumer.stopListeningToStream(STREAM_CHANNEL);
+
+		expect(mockHandler).toHaveBeenCalled();
+
+		await consumer.destroy();
+		await producer.destroy();
+	});
 });
diff --git a/packages/cli/test/unit/webhooks.test.ts b/packages/cli/test/unit/webhooks.test.ts
index 3f25eedcc13a8..3070a5d4781a1 100644
--- a/packages/cli/test/unit/webhooks.test.ts
+++ b/packages/cli/test/unit/webhooks.test.ts
@@ -1,4 +1,4 @@
-import type TestAgent from 'supertest/lib/agent';
+import type { SuperAgentTest } from 'supertest';
 import { agent as testAgent } from 'supertest';
 import { mock } from 'jest-mock-extended';
 
@@ -14,7 +14,7 @@ import type { IResponseCallbackData } from '@/Interfaces';
 
 import { mockInstance } from '../shared/mocking';
 
-let agent: TestAgent;
+let agent: SuperAgentTest;
 
 describe('WebhookServer', () => {
 	mockInstance(ExternalHooks);
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 4c0e86359e8c6..ce18506b3820a 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -1,5 +1,4 @@
 packages:
   - packages/*
-  - packages/cli/benchmark
   - packages/@n8n/*
   - packages/@n8n_io/*

From 116b92d9a9587d4a2be1d492a03b77d5dfc7a5fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Mon, 22 Apr 2024 12:54:33 +0200
Subject: [PATCH 34/86] Fixtures setup

---
 .../cli/src/benchmark/benchmarks-index.md     |  7 +++
 packages/cli/src/benchmark/db/users.ts        | 48 -------------------
 packages/cli/src/benchmark/fixtures/1.1.json  | 25 ++++++++++
 .../cli/src/benchmark/lib/global-hooks.ts     | 19 ++++++--
 .../cli/src/benchmark/tasks/webhook.tasks.ts  | 26 ++++++++--
 .../databases/repositories/user.repository.ts | 34 +++++++++++++
 6 files changed, 103 insertions(+), 56 deletions(-)
 create mode 100644 packages/cli/src/benchmark/benchmarks-index.md
 delete mode 100644 packages/cli/src/benchmark/db/users.ts
 create mode 100644 packages/cli/src/benchmark/fixtures/1.1.json

diff --git a/packages/cli/src/benchmark/benchmarks-index.md b/packages/cli/src/benchmark/benchmarks-index.md
new file mode 100644
index 0000000000000..1555dfc38cc47
--- /dev/null
+++ b/packages/cli/src/benchmark/benchmarks-index.md
@@ -0,0 +1,7 @@
+# Benchmarks index
+
+## 1. Production workflow with webhook node
+
+1.1. "Respond immediately" mode
+1.2. "When last node finishes" mode
+1.3. "Using 'Respond to Webhook' node" mode
diff --git a/packages/cli/src/benchmark/db/users.ts b/packages/cli/src/benchmark/db/users.ts
deleted file mode 100644
index d5b453d2dc3fe..0000000000000
--- a/packages/cli/src/benchmark/db/users.ts
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * @TODO Deduplicate with packages/cli/test/integration/shared/db/users.ts
- */
-
-import Container from 'typedi';
-import type { User } from '@db/entities/User';
-import { UserRepository } from '@db/repositories/user.repository';
-import { IsNull } from '@n8n/typeorm';
-
-// pre-computed bcrypt hash for the string 'password', using `await hash('password', 10)`
-const passwordHash = '$2a$10$njedH7S6V5898mj6p0Jr..IGY9Ms.qNwR7RbSzzX9yubJocKfvGGK';
-
-/** Store a new user object, defaulting to a `member` */
-export async function newUser(attributes: Partial<User> = {}): Promise<User> {
-	const { email, password, firstName, lastName, role, ...rest } = attributes;
-	return Container.get(UserRepository).create({
-		email: 'test@test.com',
-		password: passwordHash,
-		firstName: 'John',
-		lastName: 'Smith',
-		role: role ?? 'global:member',
-		...rest,
-	});
-}
-
-/** Store a user object in the DB */
-export async function createUser(attributes: Partial<User> = {}): Promise<User> {
-	const user = await newUser(attributes);
-	user.computeIsOwner();
-	return await Container.get(UserRepository).save(user);
-}
-
-export async function createOwner() {
-	return await createUser({ role: 'global:owner' });
-}
-
-export async function createMember() {
-	return await createUser({ role: 'global:member' });
-}
-
-export async function deleteOwnerShell() {
-	await Container.get(UserRepository).delete({ role: 'global:owner', email: IsNull() });
-}
-
-export const getAllUsers = async () =>
-	await Container.get(UserRepository).find({
-		relations: ['authIdentities'],
-	});
diff --git a/packages/cli/src/benchmark/fixtures/1.1.json b/packages/cli/src/benchmark/fixtures/1.1.json
new file mode 100644
index 0000000000000..70454b967b872
--- /dev/null
+++ b/packages/cli/src/benchmark/fixtures/1.1.json
@@ -0,0 +1,25 @@
+{
+	"name": "1.1",
+	"nodes": [
+		{
+			"parameters": {
+				"path": "d58b0160-2370-417b-bc0e-f3050b0c7adf",
+				"options": {}
+			},
+			"id": "000012bb-d534-4e81-a7e4-e62edf816582",
+			"name": "Webhook",
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [580, 280],
+			"webhookId": "d58b0160-2370-417b-bc0e-f3050b0c7adf"
+		}
+	],
+	"pinData": {},
+	"connections": {},
+	"active": true,
+	"settings": {
+		"executionOrder": "v1"
+	},
+	"versionId": "0cea62e0-b011-46f1-bd03-0011f648d7e9",
+	"tags": []
+}
diff --git a/packages/cli/src/benchmark/lib/global-hooks.ts b/packages/cli/src/benchmark/lib/global-hooks.ts
index 1ad0ac37df4e7..f6c3e1bc5ac61 100644
--- a/packages/cli/src/benchmark/lib/global-hooks.ts
+++ b/packages/cli/src/benchmark/lib/global-hooks.ts
@@ -5,7 +5,11 @@ import { Config } from '@oclif/core';
 import { InstanceSettings } from 'n8n-core';
 import { Start } from '@/commands/start';
 import Container from 'typedi';
-import { createOwner, deleteOwnerShell } from '../db/users';
+import FixtureWorkflow2fZ from '../fixtures/1.1.json';
+import { WorkflowsController } from '@/workflows/workflows.controller';
+import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
+import { UserRepository } from '@/databases/repositories/user.repository';
+import type { User } from '@/databases/entities/User';
 
 function n8nDir() {
 	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
@@ -46,14 +50,23 @@ async function mainProcess() {
 	await main.run();
 }
 
+async function loadFixtures(owner: User) {
+	// @ts-ignore
+	await Container.get(WorkflowsController).create({ body: FixtureWorkflow2fZ, user: owner });
+
+	const allActive = await Container.get(WorkflowRepository).getAllActive();
+	console.log('allActive', allActive);
+}
+
 export async function setup() {
 	n8nDir();
 
 	await mainProcess();
 	// @TODO: Postgres?
 
-	await deleteOwnerShell();
-	await createOwner();
+	const owner = await Container.get(UserRepository).testOwner();
+
+	await loadFixtures(owner);
 }
 
 export async function teardown() {
diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index 98d6b2741ede3..72cb5e8735f4f 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -1,13 +1,29 @@
-import { getAllUsers } from '../db/users.js';
+import axios from 'axios';
 import { task, beforeEach } from '../main.js';
 
+async function login() {
+	const client = axios.create({
+		baseURL: 'http://localhost:5678/',
+	});
+
+	await client.post('/rest/login', {
+		email: 'test@test.com',
+		password: 'password',
+	});
+}
+
 beforeEach(async () => {
-	console.log('[[[beforeEach]]] for webhook.tasks.ts');
+	await login();
 });
 
-task('Production workflow with webhook node that responds immediately', async () => {
-	const users = await getAllUsers();
-	console.log('users', users);
+task('Production workflow - webhook node - no auth - respond immediately', async () => {
+	// @TODO: setup for this task
+
+	// const wf = Container.get(WorkflowRepository).create(
+	// 	ProductionWorkflow as unknown as WorkflowEntity,
+	// );
+	// await Container.get(WorkflowRepository).save(wf);
+	// console.log('wf', wf);
 	console.log('[first] Task 1 executed');
 });
 
diff --git a/packages/cli/src/databases/repositories/user.repository.ts b/packages/cli/src/databases/repositories/user.repository.ts
index 6b81f8984bff0..fa3b8ff6dfb43 100644
--- a/packages/cli/src/databases/repositories/user.repository.ts
+++ b/packages/cli/src/databases/repositories/user.repository.ts
@@ -104,4 +104,38 @@ export class UserRepository extends Repository<User> {
 			where: { id: In(userIds), password: Not(IsNull()) },
 		});
 	}
+
+	// ----------------------------------
+	//           test utils
+	// ----------------------------------
+
+	async createTestUser(attributes: Partial<User> = {}): Promise<User> {
+		const { email, password, firstName, lastName, role, ...rest } = attributes;
+
+		// pre-computed bcrypt hash for the string 'password', using `await hash('password', 10)`
+		const passwordHash = '$2a$10$njedH7S6V5898mj6p0Jr..IGY9Ms.qNwR7RbSzzX9yubJocKfvGGK';
+
+		return this.create({
+			email: 'test@test.com',
+			password: passwordHash,
+			firstName: 'John',
+			lastName: 'Smith',
+			role: role ?? 'global:member',
+			...rest,
+		});
+	}
+
+	async createUser(attributes: Partial<User> = {}): Promise<User> {
+		const user = await this.createTestUser(attributes);
+
+		user.computeIsOwner();
+
+		return await this.save(user);
+	}
+
+	async testOwner() {
+		await this.delete({ role: 'global:owner', email: IsNull() });
+
+		return await this.createUser({ role: 'global:owner' });
+	}
 }

From 7bb25ca3cabf8767802f582950fb83475d4ed6b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Mon, 22 Apr 2024 13:26:56 +0200
Subject: [PATCH 35/86] Pick up fixtures

---
 .../cli/src/benchmark/lib/global-hooks.ts     | 25 ++++++++++++++++---
 .../databases/repositories/user.repository.ts |  2 +-
 .../cli/src/workflows/workflow.request.ts     |  2 ++
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/packages/cli/src/benchmark/lib/global-hooks.ts b/packages/cli/src/benchmark/lib/global-hooks.ts
index f6c3e1bc5ac61..2c21a6360b18c 100644
--- a/packages/cli/src/benchmark/lib/global-hooks.ts
+++ b/packages/cli/src/benchmark/lib/global-hooks.ts
@@ -5,11 +5,14 @@ import { Config } from '@oclif/core';
 import { InstanceSettings } from 'n8n-core';
 import { Start } from '@/commands/start';
 import Container from 'typedi';
-import FixtureWorkflow2fZ from '../fixtures/1.1.json';
 import { WorkflowsController } from '@/workflows/workflows.controller';
 import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
 import { UserRepository } from '@/databases/repositories/user.repository';
 import type { User } from '@/databases/entities/User';
+import glob from 'fast-glob';
+import { jsonParse } from 'n8n-workflow';
+import { readFile } from 'fs/promises';
+import type { WorkflowRequest } from '@/workflows/workflow.request';
 
 function n8nDir() {
 	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
@@ -51,8 +54,22 @@ async function mainProcess() {
 }
 
 async function loadFixtures(owner: User) {
-	// @ts-ignore
-	await Container.get(WorkflowsController).create({ body: FixtureWorkflow2fZ, user: owner });
+	const files = await glob('fixtures/*.json', {
+		cwd: path.join('dist', 'benchmark'),
+		absolute: true,
+	});
+
+	const fixtures: WorkflowRequest.CreatePayload[] = [];
+
+	for (const file of files) {
+		const content = await readFile(file, 'utf8');
+		fixtures.push(jsonParse<WorkflowRequest.CreatePayload>(content));
+	}
+
+	for (const fixture of fixtures) {
+		// @ts-ignore @TODO Fix typing
+		await Container.get(WorkflowsController).create({ body: fixture, user: owner });
+	}
 
 	const allActive = await Container.get(WorkflowRepository).getAllActive();
 	console.log('allActive', allActive);
@@ -64,7 +81,7 @@ export async function setup() {
 	await mainProcess();
 	// @TODO: Postgres?
 
-	const owner = await Container.get(UserRepository).testOwner();
+	const owner = await Container.get(UserRepository).createTestOwner();
 
 	await loadFixtures(owner);
 }
diff --git a/packages/cli/src/databases/repositories/user.repository.ts b/packages/cli/src/databases/repositories/user.repository.ts
index fa3b8ff6dfb43..8bfd94d48cbdf 100644
--- a/packages/cli/src/databases/repositories/user.repository.ts
+++ b/packages/cli/src/databases/repositories/user.repository.ts
@@ -133,7 +133,7 @@ export class UserRepository extends Repository<User> {
 		return await this.save(user);
 	}
 
-	async testOwner() {
+	async createTestOwner() {
 		await this.delete({ role: 'global:owner', email: IsNull() });
 
 		return await this.createUser({ role: 'global:owner' });
diff --git a/packages/cli/src/workflows/workflow.request.ts b/packages/cli/src/workflows/workflow.request.ts
index 77d653a2a052c..44f837c9b3bcc 100644
--- a/packages/cli/src/workflows/workflow.request.ts
+++ b/packages/cli/src/workflows/workflow.request.ts
@@ -10,6 +10,8 @@ import type {
 } from 'n8n-workflow';
 
 export declare namespace WorkflowRequest {
+	export type CreatePayload = CreateUpdatePayload;
+
 	type CreateUpdatePayload = Partial<{
 		id: string; // delete if sent
 		name: string;

From c42016d8a13e118a576ad9e3d04fd8d87a8e2d5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Mon, 22 Apr 2024 15:05:21 +0200
Subject: [PATCH 36/86] First benchmark

---
 packages/cli/src/ActiveExecutions.ts          |  5 +++-
 .../cli/src/benchmark/benchmarks-index.md     |  8 +++---
 .../cli/src/benchmark/lib/global-hooks.ts     | 15 ++++++----
 .../cli/src/benchmark/tasks/webhook.tasks.ts  | 28 +++++++------------
 packages/cli/src/commands/start.ts            |  4 +--
 packages/cli/tsconfig.benchmark.json          |  2 +-
 6 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/packages/cli/src/ActiveExecutions.ts b/packages/cli/src/ActiveExecutions.ts
index e799dba13e65c..a215b03d494e7 100644
--- a/packages/cli/src/ActiveExecutions.ts
+++ b/packages/cli/src/ActiveExecutions.ts
@@ -18,6 +18,7 @@ import type {
 import { isWorkflowIdValid } from '@/utils';
 import { ExecutionRepository } from '@db/repositories/execution.repository';
 import { Logger } from '@/Logger';
+import { inBenchmark } from './constants';
 
 @Service()
 export class ActiveExecutions {
@@ -201,7 +202,9 @@ export class ActiveExecutions {
 		let count = 0;
 		while (executionIds.length !== 0) {
 			if (count++ % 4 === 0) {
-				this.logger.info(`Waiting for ${executionIds.length} active executions to finish...`);
+				if (!inBenchmark) {
+					this.logger.info(`Waiting for ${executionIds.length} active executions to finish...`);
+				}
 			}
 
 			await sleep(500);
diff --git a/packages/cli/src/benchmark/benchmarks-index.md b/packages/cli/src/benchmark/benchmarks-index.md
index 1555dfc38cc47..fb45de8686cc8 100644
--- a/packages/cli/src/benchmark/benchmarks-index.md
+++ b/packages/cli/src/benchmark/benchmarks-index.md
@@ -1,7 +1,7 @@
 # Benchmarks index
 
-## 1. Production workflow with webhook node
+## 1. Production workflow with authless webhook node
 
-1.1. "Respond immediately" mode
-1.2. "When last node finishes" mode
-1.3. "Using 'Respond to Webhook' node" mode
+1.1. with "Respond immediately" mode
+1.2. with "When last node finishes" mode
+1.3. with "Using 'Respond to Webhook' node" mode
diff --git a/packages/cli/src/benchmark/lib/global-hooks.ts b/packages/cli/src/benchmark/lib/global-hooks.ts
index 2c21a6360b18c..6eaa440cc6415 100644
--- a/packages/cli/src/benchmark/lib/global-hooks.ts
+++ b/packages/cli/src/benchmark/lib/global-hooks.ts
@@ -6,13 +6,13 @@ import { InstanceSettings } from 'n8n-core';
 import { Start } from '@/commands/start';
 import Container from 'typedi';
 import { WorkflowsController } from '@/workflows/workflows.controller';
-import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
 import { UserRepository } from '@/databases/repositories/user.repository';
 import type { User } from '@/databases/entities/User';
 import glob from 'fast-glob';
 import { jsonParse } from 'n8n-workflow';
 import { readFile } from 'fs/promises';
 import type { WorkflowRequest } from '@/workflows/workflow.request';
+import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 
 function n8nDir() {
 	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
@@ -67,12 +67,17 @@ async function loadFixtures(owner: User) {
 	}
 
 	for (const fixture of fixtures) {
-		// @ts-ignore @TODO Fix typing
-		await Container.get(WorkflowsController).create({ body: fixture, user: owner });
+		try {
+			// @ts-ignore @TODO Fix typing
+			await Container.get(WorkflowsController).create({ body: fixture, user: owner });
+			await Container.get(ActiveWorkflowRunner).add(fixture.id as string, 'activate');
+		} catch (e) {
+			console.log(e);
+		}
 	}
 
-	const allActive = await Container.get(WorkflowRepository).getAllActive();
-	console.log('allActive', allActive);
+	// const allActive = await Container.get(WorkflowRepository).getAllActive();
+	// console.log('allActive', allActive);
 }
 
 export async function setup() {
diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index 72cb5e8735f4f..2565c76b48159 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -1,31 +1,23 @@
 import axios from 'axios';
 import { task, beforeEach } from '../main.js';
 
-async function login() {
-	const client = axios.create({
-		baseURL: 'http://localhost:5678/',
-	});
+const client = axios.create({
+	baseURL: 'http://localhost:5678/',
+});
 
+beforeEach(async () => {
 	await client.post('/rest/login', {
 		email: 'test@test.com',
 		password: 'password',
 	});
-}
-
-beforeEach(async () => {
-	await login();
 });
 
-task('Production workflow - webhook node - no auth - respond immediately', async () => {
-	// @TODO: setup for this task
-
-	// const wf = Container.get(WorkflowRepository).create(
-	// 	ProductionWorkflow as unknown as WorkflowEntity,
-	// );
-	// await Container.get(WorkflowRepository).save(wf);
-	// console.log('wf', wf);
-	console.log('[first] Task 1 executed');
-});
+task(
+	'1.1. Production workflow with authless webhook node with "Respond immediately" mode',
+	async () => {
+		await client.get('/webhook/d58b0160-2370-417b-bc0e-f3050b0c7adf');
+	},
+);
 
 task('[first] Task 2 should do something else', async () => {
 	console.log('[first] Task 2 executed');
diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts
index 71097ab2e44f6..cc3432774f182 100644
--- a/packages/cli/src/commands/start.ts
+++ b/packages/cli/src/commands/start.ts
@@ -14,7 +14,7 @@ import config from '@/config';
 import { ActiveExecutions } from '@/ActiveExecutions';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 import { Server } from '@/Server';
-import { EDITOR_UI_DIST_DIR, LICENSE_FEATURES } from '@/constants';
+import { EDITOR_UI_DIST_DIR, LICENSE_FEATURES, inBenchmark } from '@/constants';
 import { MessageEventBus } from '@/eventbus/MessageEventBus/MessageEventBus';
 import { InternalHooks } from '@/InternalHooks';
 import { License } from '@/License';
@@ -88,7 +88,7 @@ export class Start extends BaseCommand {
 	 * get removed.
 	 */
 	async stopProcess() {
-		this.logger.info('\nStopping n8n...');
+		if (!inBenchmark) this.logger.info('\nStopping n8n...');
 
 		try {
 			// Stop with trying to activate workflows that could not be activated
diff --git a/packages/cli/tsconfig.benchmark.json b/packages/cli/tsconfig.benchmark.json
index 568df33449491..307359ecb4291 100644
--- a/packages/cli/tsconfig.benchmark.json
+++ b/packages/cli/tsconfig.benchmark.json
@@ -5,6 +5,6 @@
 		"outDir": "dist",
 		"tsBuildInfoFile": "dist/benchmark.tsbuildinfo"
 	},
-	"include": ["src/**/*.ts", "src/benchmark/**/*.ts"],
+	"include": ["src/**/*.ts", "src/benchmark/**/*.ts", "src/benchmark/fixtures/*.json"],
 	"exclude": ["test/**"]
 }

From 093e75c3172d38a2481072f0c6c64281cc5ab786 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 15:22:56 +0200
Subject: [PATCH 37/86] Phrasing

---
 packages/cli/src/benchmark/benchmarks-index.md    | 6 +++---
 packages/cli/src/benchmark/tasks/webhook.tasks.ts | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/cli/src/benchmark/benchmarks-index.md b/packages/cli/src/benchmark/benchmarks-index.md
index fb45de8686cc8..4f4d5a9371a27 100644
--- a/packages/cli/src/benchmark/benchmarks-index.md
+++ b/packages/cli/src/benchmark/benchmarks-index.md
@@ -2,6 +2,6 @@
 
 ## 1. Production workflow with authless webhook node
 
-1.1. with "Respond immediately" mode
-1.2. with "When last node finishes" mode
-1.3. with "Using 'Respond to Webhook' node" mode
+1.1. using "Respond immediately" mode
+1.2. using "When last node finishes" mode
+1.3. using "Using 'Respond to Webhook' node" mode
diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index 2565c76b48159..905d52054bb14 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -13,7 +13,7 @@ beforeEach(async () => {
 });
 
 task(
-	'1.1. Production workflow with authless webhook node with "Respond immediately" mode',
+	'1.1. Production workflow with authless webhook node using "Respond immediately" mode',
 	async () => {
 		await client.get('/webhook/d58b0160-2370-417b-bc0e-f3050b0c7adf');
 	},

From 3c36304df7fb07671b12b897b0db850a8e079842 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 16:01:12 +0200
Subject: [PATCH 38/86] Cleanup

---
 package.json                                  |  2 +-
 packages/cli/package.json                     |  2 +-
 .../cli/src/benchmark/benchmarks-index.md     |  7 --
 packages/cli/src/benchmark/benchmarks.md      | 21 ++++++
 .../lib/{ => errors}/duplicate-hook.error.ts  |  0
 .../lib/errors/unsupported-database.error.ts  |  7 ++
 .../lib/{global-hooks.ts => hooks.ts}         | 75 ++++++++++---------
 packages/cli/src/benchmark/lib/suites.ts      | 15 +++-
 packages/cli/src/benchmark/lib/types.ts       |  7 +-
 packages/cli/src/benchmark/main.ts            | 24 ++++--
 .../cli/src/benchmark/tasks/example.tasks.ts  | 17 -----
 .../{fixtures => workflows}/1.1.json          |  0
 packages/cli/tsconfig.benchmark.json          |  2 +-
 13 files changed, 106 insertions(+), 73 deletions(-)
 delete mode 100644 packages/cli/src/benchmark/benchmarks-index.md
 create mode 100644 packages/cli/src/benchmark/benchmarks.md
 rename packages/cli/src/benchmark/lib/{ => errors}/duplicate-hook.error.ts (100%)
 create mode 100644 packages/cli/src/benchmark/lib/errors/unsupported-database.error.ts
 rename packages/cli/src/benchmark/lib/{global-hooks.ts => hooks.ts} (62%)
 delete mode 100644 packages/cli/src/benchmark/tasks/example.tasks.ts
 rename packages/cli/src/benchmark/{fixtures => workflows}/1.1.json (100%)

diff --git a/package.json b/package.json
index 1e4363020f905..88bf17878daf4 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,7 @@
   "packageManager": "pnpm@8.14.3",
   "scripts": {
     "preinstall": "node scripts/block-npm-install.js",
-    "benchmark": "pnpm --filter=n8n benchmark",
+    "benchmark": "pnpm --filter=n8n benchmark:sqlite",
     "build": "turbo run build",
     "build:backend": "pnpm --filter=!@n8n/chat --filter=!n8n-design-system --filter=!n8n-editor-ui build",
     "build:frontend": "pnpm --filter=@n8n/chat --filter=n8n-design-system --filter=n8n-editor-ui build",
diff --git a/packages/cli/package.json b/packages/cli/package.json
index 45480164ffb14..00df739b8a90d 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,7 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "pnpm build:benchmark && NODE_ENV=benchmark node dist/benchmark/main.js",
+    "benchmark:sqlite": "pnpm build:benchmark && NODE_ENV=benchmark node dist/benchmark/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
diff --git a/packages/cli/src/benchmark/benchmarks-index.md b/packages/cli/src/benchmark/benchmarks-index.md
deleted file mode 100644
index 4f4d5a9371a27..0000000000000
--- a/packages/cli/src/benchmark/benchmarks-index.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# Benchmarks index
-
-## 1. Production workflow with authless webhook node
-
-1.1. using "Respond immediately" mode
-1.2. using "When last node finishes" mode
-1.3. using "Using 'Respond to Webhook' node" mode
diff --git a/packages/cli/src/benchmark/benchmarks.md b/packages/cli/src/benchmark/benchmarks.md
new file mode 100644
index 0000000000000..fb7b7e14080cc
--- /dev/null
+++ b/packages/cli/src/benchmark/benchmarks.md
@@ -0,0 +1,21 @@
+# Benchmarks
+
+To run benchmarks locally in `cli`:
+
+```sh
+pnpm benchmark:sqlite
+```
+
+## Creating
+
+To create a benchmark, @TODO
+
+## Listing
+
+### 1. Production workflow with authless webhook node
+
+1.1. using "Respond immediately" mode
+1.2. using "When last node finishes" mode
+1.3. using "Using 'Respond to Webhook' node" mode
+
+All workflows with default settings.
diff --git a/packages/cli/src/benchmark/lib/duplicate-hook.error.ts b/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
similarity index 100%
rename from packages/cli/src/benchmark/lib/duplicate-hook.error.ts
rename to packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
diff --git a/packages/cli/src/benchmark/lib/errors/unsupported-database.error.ts b/packages/cli/src/benchmark/lib/errors/unsupported-database.error.ts
new file mode 100644
index 0000000000000..042318909e577
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/errors/unsupported-database.error.ts
@@ -0,0 +1,7 @@
+import { ApplicationError } from 'n8n-workflow';
+
+export class UnsupportedDatabaseError extends ApplicationError {
+	constructor() {
+		super('Currently only sqlite is supported for benchmarking', { level: 'warning' });
+	}
+}
diff --git a/packages/cli/src/benchmark/lib/global-hooks.ts b/packages/cli/src/benchmark/lib/hooks.ts
similarity index 62%
rename from packages/cli/src/benchmark/lib/global-hooks.ts
rename to packages/cli/src/benchmark/lib/hooks.ts
index 6eaa440cc6415..611af5e617a96 100644
--- a/packages/cli/src/benchmark/lib/global-hooks.ts
+++ b/packages/cli/src/benchmark/lib/hooks.ts
@@ -13,7 +13,11 @@ import { jsonParse } from 'n8n-workflow';
 import { readFile } from 'fs/promises';
 import type { WorkflowRequest } from '@/workflows/workflow.request';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
+import { Logger } from '@/Logger';
 
+/**
+ * Create a temp `.n8n` dir for encryption key, sqlite DB, etc.
+ */
 function n8nDir() {
 	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
 
@@ -31,66 +35,69 @@ function n8nDir() {
 		'utf-8',
 	);
 
-	/**
-	 * @TODO Better approach than overriding? Setting N8N_USER_FOLDER has no effect
-	 */
+	// @TODO: Find better approach than overriding like this
+	// Setting N8N_USER_FOLDER has no effect
 	const instanceSettings = Container.get(InstanceSettings);
 	instanceSettings.n8nFolder = _n8nDir;
 	Container.set(InstanceSettings, instanceSettings);
 
-	console.info('.n8n dir', _n8nDir);
+	Container.get(Logger).info(`Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
 }
 
-let main: Start;
-
-async function mainProcess() {
-	const args: string[] = [];
-	const _config = new Config({ root: __dirname });
-
-	main = new Start(args, _config);
-
-	await main.init();
-	await main.run();
-}
-
-async function loadFixtures(owner: User) {
-	const files = await glob('fixtures/*.json', {
+/**
+ * Load into DB and activate in memory all workflows to use in benchmarks.
+ */
+async function prepareWorkflows(owner: User) {
+	const files = await glob('workflows/*.json', {
 		cwd: path.join('dist', 'benchmark'),
 		absolute: true,
 	});
 
-	const fixtures: WorkflowRequest.CreatePayload[] = [];
+	const workflows: WorkflowRequest.CreatePayload[] = [];
 
 	for (const file of files) {
 		const content = await readFile(file, 'utf8');
-		fixtures.push(jsonParse<WorkflowRequest.CreatePayload>(content));
+		workflows.push(jsonParse<WorkflowRequest.CreatePayload>(content));
 	}
 
-	for (const fixture of fixtures) {
-		try {
-			// @ts-ignore @TODO Fix typing
-			await Container.get(WorkflowsController).create({ body: fixture, user: owner });
-			await Container.get(ActiveWorkflowRunner).add(fixture.id as string, 'activate');
-		} catch (e) {
-			console.log(e);
-		}
+	for (const workflow of workflows) {
+		// @ts-ignore @TODO Fix typing
+		await Container.get(WorkflowsController).create({ body: workflow, user: owner });
+		await Container.get(ActiveWorkflowRunner).add(workflow.id as string, 'activate');
 	}
+}
+
+let main: Start;
 
-	// const allActive = await Container.get(WorkflowRepository).getAllActive();
-	// console.log('allActive', allActive);
+/**
+ * Start the main n8n process to use in benchmarks.
+ */
+async function mainProcess() {
+	const args: string[] = [];
+	const _config = new Config({ root: __dirname });
+
+	main = new Start(args, _config);
+
+	await main.init();
+	await main.run();
 }
 
-export async function setup() {
+/**
+ * Setup to run before once all benchmarks.
+ */
+export async function globalSetup() {
 	n8nDir();
 
 	await mainProcess();
-	// @TODO: Postgres?
 
 	const owner = await Container.get(UserRepository).createTestOwner();
 
-	await loadFixtures(owner);
+	await prepareWorkflows(owner);
 }
 
-export async function teardown() {
+/**
+ * Teardown to run before after all benchmarks.
+ */
+export async function globalTeardown() {
 	await main.stopProcess();
 }
diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index a555c2d9c0b67..f4f088751eac5 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -5,7 +5,7 @@ import { assert } from 'n8n-workflow';
 import glob from 'fast-glob';
 import callsites from 'callsites';
 import type { Suites, Task, Callback } from './types';
-import { DuplicateHookError } from './duplicate-hook.error';
+import { DuplicateHookError } from './errors/duplicate-hook.error';
 
 export const suites: Suites = {};
 
@@ -33,7 +33,7 @@ export function registerSuites(bench: Bench) {
 		 * In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
 		 * while `beforeEach` and `afterEach` refer to each individual test.
 		 *
-		 * The API renames tinybench's hooks to prevent confusion from this difference.
+		 * We rename tinybench's hooks to prevent confusion from this difference.
 		 */
 		const options: Record<string, Callback> = {};
 
@@ -58,9 +58,8 @@ function suiteFilePath() {
 }
 
 /**
- * Benchmarking API
+ * Run a benchmarking task, i.e. a single operation whose performance to measure.
  */
-
 export function task(description: string, operation: Task['operation']) {
 	const filePath = suiteFilePath();
 
@@ -68,6 +67,10 @@ export function task(description: string, operation: Task['operation']) {
 	suites[filePath].tasks.push({ description, operation });
 }
 
+/**
+ * Setup step to run once before each benchmarking task in a suite.
+ * Only one `beforeEach` is allowed per suite.
+ */
 export function beforeEach(fn: Callback) {
 	const filePath = suiteFilePath();
 
@@ -79,6 +82,10 @@ export function beforeEach(fn: Callback) {
 	suites[filePath].hooks.beforeEach = fn;
 }
 
+/**
+ * Teardown step to run once after each benchmarking task in a suite.
+ * Only one `afterEach` is allowed per suite.
+ */
 export function afterEach(fn: Callback) {
 	const filePath = suiteFilePath();
 
diff --git a/packages/cli/src/benchmark/lib/types.ts b/packages/cli/src/benchmark/lib/types.ts
index afb0f792deb2c..b09624337e579 100644
--- a/packages/cli/src/benchmark/lib/types.ts
+++ b/packages/cli/src/benchmark/lib/types.ts
@@ -1,3 +1,6 @@
+/**
+ * Benchmarking suites, i.e. `*.tasks.ts` files containing benchmarking tasks.
+ */
 export type Suites = {
 	[suiteFilepath: string]: {
 		hooks: {
@@ -8,7 +11,9 @@ export type Suites = {
 	};
 };
 
-/** A benchmarking task, i.e. a single operation whose performance to measure. */
+/**
+ * A benchmarking task, i.e. a single operation whose performance to measure.
+ */
 export type Task = {
 	description: string;
 	operation: Callback;
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 47117a72f194b..be017ba9285fe 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,6 +1,10 @@
 import 'reflect-metadata';
-import * as globalHooks from './lib/global-hooks';
+import * as hooks from './lib/hooks';
 import { collectSuites, registerSuites, suiteCount } from './lib/suites';
+import config from '@/config';
+import { UnsupportedDatabaseError } from './lib/errors/unsupported-database.error';
+import { Logger } from '@/Logger';
+import Container from 'typedi';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';
@@ -10,16 +14,24 @@ import { withCodSpeed } from '@codspeed/tinybench-plugin';
 export { beforeEach, afterEach, task } from './lib/suites';
 
 async function main() {
+	const dbType = config.getEnv('database.type');
+
+	if (dbType !== 'sqlite') throw new UnsupportedDatabaseError();
+
 	await collectSuites();
 
 	const count = suiteCount();
 
+	const logger = Container.get(Logger);
+
 	if (count === 0) {
-		console.log('No benchmarking suites found');
+		logger.info('No benchmarking suites found. Exiting...');
 		return;
 	}
 
-	await globalHooks.setup();
+	logger.info(`Running ${count} benchmarking ${count === 1 ? 'suite' : 'suites'}...`);
+
+	await hooks.globalSetup();
 
 	const _bench = new Bench({
 		time: 0, // @TODO: Temp value
@@ -30,13 +42,11 @@ async function main() {
 
 	registerSuites(bench);
 
-	console.log(`Running ${count} benchmarking suites...`);
-
 	await bench.run();
 
-	console.table(bench.table());
+	console.table(bench.table()); // @TODO: Output properly? Ref. Codspeed
 
-	await globalHooks.teardown();
+	await hooks.globalTeardown();
 }
 
 void main();
diff --git a/packages/cli/src/benchmark/tasks/example.tasks.ts b/packages/cli/src/benchmark/tasks/example.tasks.ts
deleted file mode 100644
index 1167f6c4e2dfd..0000000000000
--- a/packages/cli/src/benchmark/tasks/example.tasks.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import { task, beforeEach, afterEach } from '../main.js';
-
-beforeEach(async () => {
-	console.log('[[[beforeEach]]] for example.tasks.ts');
-});
-
-afterEach(async () => {
-	console.log('[[[afterEach]]] for example.tasks.ts');
-});
-
-task('[example] Should do something', async () => {
-	console.log('Example task 1 executed');
-});
-
-task('[example] Should do something else', async () => {
-	console.log('Example task 2 executed');
-});
diff --git a/packages/cli/src/benchmark/fixtures/1.1.json b/packages/cli/src/benchmark/workflows/1.1.json
similarity index 100%
rename from packages/cli/src/benchmark/fixtures/1.1.json
rename to packages/cli/src/benchmark/workflows/1.1.json
diff --git a/packages/cli/tsconfig.benchmark.json b/packages/cli/tsconfig.benchmark.json
index 307359ecb4291..ac0a1462ee671 100644
--- a/packages/cli/tsconfig.benchmark.json
+++ b/packages/cli/tsconfig.benchmark.json
@@ -5,6 +5,6 @@
 		"outDir": "dist",
 		"tsBuildInfoFile": "dist/benchmark.tsbuildinfo"
 	},
-	"include": ["src/**/*.ts", "src/benchmark/**/*.ts", "src/benchmark/fixtures/*.json"],
+	"include": ["src/**/*.ts", "src/benchmark/**/*.ts", "src/benchmark/workflows/*.json"],
 	"exclude": ["test/**"]
 }

From 169c6278365233cf97fdfacd945b7027e68c973e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 16:06:53 +0200
Subject: [PATCH 39/86] Typo

---
 packages/cli/src/benchmark/lib/hooks.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/cli/src/benchmark/lib/hooks.ts b/packages/cli/src/benchmark/lib/hooks.ts
index 611af5e617a96..bded4b7fde093 100644
--- a/packages/cli/src/benchmark/lib/hooks.ts
+++ b/packages/cli/src/benchmark/lib/hooks.ts
@@ -96,7 +96,7 @@ export async function globalSetup() {
 }
 
 /**
- * Teardown to run before after all benchmarks.
+ * Teardown to run after all benchmarks.
  */
 export async function globalTeardown() {
 	await main.stopProcess();

From ae6ff9b4f203c46631c5b273b9e89a4298b14829 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 16:08:43 +0200
Subject: [PATCH 40/86] Add comment

---
 packages/cli/src/databases/repositories/user.repository.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/cli/src/databases/repositories/user.repository.ts b/packages/cli/src/databases/repositories/user.repository.ts
index 8bfd94d48cbdf..2ab41f37260a5 100644
--- a/packages/cli/src/databases/repositories/user.repository.ts
+++ b/packages/cli/src/databases/repositories/user.repository.ts
@@ -109,6 +109,8 @@ export class UserRepository extends Repository<User> {
 	//           test utils
 	// ----------------------------------
 
+	// @TODO: Deduplicate with /packages/cli/test/integration/shared/db/users.ts
+
 	async createTestUser(attributes: Partial<User> = {}): Promise<User> {
 		const { email, password, firstName, lastName, role, ...rest } = attributes;
 

From d776bbe40631ce7702a590c175154201c5721bde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 16:09:20 +0200
Subject: [PATCH 41/86] Add question

---
 packages/cli/src/benchmark/lib/hooks.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/cli/src/benchmark/lib/hooks.ts b/packages/cli/src/benchmark/lib/hooks.ts
index bded4b7fde093..b2f727b989340 100644
--- a/packages/cli/src/benchmark/lib/hooks.ts
+++ b/packages/cli/src/benchmark/lib/hooks.ts
@@ -92,7 +92,7 @@ export async function globalSetup() {
 
 	const owner = await Container.get(UserRepository).createTestOwner();
 
-	await prepareWorkflows(owner);
+	await prepareWorkflows(owner); // @TODO: Load all here or as part of each benchmark's `beforeEach`?
 }
 
 /**

From 403b4664838743ca7aeeb6979e944760dfc917a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 16:21:29 +0200
Subject: [PATCH 42/86] Add second benchmark

---
 .../cli/src/benchmark/tasks/webhook.tasks.ts  |  7 +++
 packages/cli/src/benchmark/workflows/1.2.json | 48 +++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 packages/cli/src/benchmark/workflows/1.2.json

diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index 905d52054bb14..696201f631222 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -19,6 +19,13 @@ task(
 	},
 );
 
+task(
+	'1.2. Production workflow with authless webhook node using "When last node finishes" mode',
+	async () => {
+		await client.get('/webhook/a669d108-7de3-4e72-b332-919a11b2328b');
+	},
+);
+
 task('[first] Task 2 should do something else', async () => {
 	console.log('[first] Task 2 executed');
 });
diff --git a/packages/cli/src/benchmark/workflows/1.2.json b/packages/cli/src/benchmark/workflows/1.2.json
new file mode 100644
index 0000000000000..5670949751101
--- /dev/null
+++ b/packages/cli/src/benchmark/workflows/1.2.json
@@ -0,0 +1,48 @@
+{
+	"name": "1.2",
+	"nodes": [
+		{
+			"parameters": {
+				"jsCode": "// Loop over input items and add a new field called 'myNewField' to the JSON of each one\nfor (const item of $input.all()) {\n  item.json.myNewField = 1;\n}\n\nreturn $input.all();"
+			},
+			"id": "81af5f75-b4d4-4a99-8f3c-f86e307d75a3",
+			"name": "Code",
+			"type": "n8n-nodes-base.code",
+			"typeVersion": 2,
+			"position": [780, 280]
+		},
+		{
+			"parameters": {
+				"path": "a669d108-7de3-4e72-b332-919a11b2328b",
+				"responseMode": "lastNode",
+				"options": {}
+			},
+			"id": "5d3512bd-6f6d-4b67-a01e-1e2d9735e4f1",
+			"name": "Webhook",
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [580, 280],
+			"webhookId": "a669d108-7de3-4e72-b332-919a11b2328b"
+		}
+	],
+	"pinData": {},
+	"connections": {
+		"Webhook": {
+			"main": [
+				[
+					{
+						"node": "Code",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		}
+	},
+	"active": false,
+	"settings": {
+		"executionOrder": "v1"
+	},
+	"versionId": "45e84048-db98-4754-80c2-0be5b141de0f",
+	"tags": []
+}

From b8c6b6208dcf9c0443e978701416d4ce11111533 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 17:10:48 +0200
Subject: [PATCH 43/86] Naming

---
 packages/cli/src/benchmark/{benchmarks.md => benchmark.md} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename packages/cli/src/benchmark/{benchmarks.md => benchmark.md} (96%)

diff --git a/packages/cli/src/benchmark/benchmarks.md b/packages/cli/src/benchmark/benchmark.md
similarity index 96%
rename from packages/cli/src/benchmark/benchmarks.md
rename to packages/cli/src/benchmark/benchmark.md
index fb7b7e14080cc..124037179c321 100644
--- a/packages/cli/src/benchmark/benchmarks.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -1,4 +1,4 @@
-# Benchmarks
+# Benchmark
 
 To run benchmarks locally in `cli`:
 

From 497966f20da31b4dece2db0dd091264fa4b4c900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 17:11:09 +0200
Subject: [PATCH 44/86] Add TODO

---
 packages/cli/src/benchmark/lib/hooks.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/cli/src/benchmark/lib/hooks.ts b/packages/cli/src/benchmark/lib/hooks.ts
index b2f727b989340..5edf2036b2477 100644
--- a/packages/cli/src/benchmark/lib/hooks.ts
+++ b/packages/cli/src/benchmark/lib/hooks.ts
@@ -64,6 +64,7 @@ async function prepareWorkflows(owner: User) {
 		// @ts-ignore @TODO Fix typing
 		await Container.get(WorkflowsController).create({ body: workflow, user: owner });
 		await Container.get(ActiveWorkflowRunner).add(workflow.id as string, 'activate');
+		// @TODO: Solve race condition when adding webhooks
 	}
 }
 

From 20069b55d623f7f3856b43188c3d071a51b2c143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 17:11:30 +0200
Subject: [PATCH 45/86] Add clarification

---
 packages/cli/src/benchmark/benchmark.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index 124037179c321..7517932409ffa 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -12,10 +12,10 @@ To create a benchmark, @TODO
 
 ## Listing
 
+All workflows with default settings unless otherwise specified.
+
 ### 1. Production workflow with authless webhook node
 
 1.1. using "Respond immediately" mode
 1.2. using "When last node finishes" mode
 1.3. using "Using 'Respond to Webhook' node" mode
-
-All workflows with default settings.

From facdac545ddafd62e640da0d99fb2b6524717398 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 17:35:51 +0200
Subject: [PATCH 46/86] Third benchmark

---
 packages/cli/src/benchmark/lib/hooks.ts       |  4 +-
 .../cli/src/benchmark/tasks/webhook.tasks.ts  |  9 ++--
 packages/cli/src/benchmark/workflows/1.1.json |  2 +-
 packages/cli/src/benchmark/workflows/1.2.json |  2 +-
 packages/cli/src/benchmark/workflows/1.3.json | 50 +++++++++++++++++++
 5 files changed, 60 insertions(+), 7 deletions(-)
 create mode 100644 packages/cli/src/benchmark/workflows/1.3.json

diff --git a/packages/cli/src/benchmark/lib/hooks.ts b/packages/cli/src/benchmark/lib/hooks.ts
index 5edf2036b2477..4cb5f3377af68 100644
--- a/packages/cli/src/benchmark/lib/hooks.ts
+++ b/packages/cli/src/benchmark/lib/hooks.ts
@@ -63,9 +63,9 @@ async function prepareWorkflows(owner: User) {
 	for (const workflow of workflows) {
 		// @ts-ignore @TODO Fix typing
 		await Container.get(WorkflowsController).create({ body: workflow, user: owner });
-		await Container.get(ActiveWorkflowRunner).add(workflow.id as string, 'activate');
-		// @TODO: Solve race condition when adding webhooks
 	}
+
+	await Container.get(ActiveWorkflowRunner).init();
 }
 
 let main: Start;
diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index 696201f631222..9d3aeeb1c24e8 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -26,6 +26,9 @@ task(
 	},
 );
 
-task('[first] Task 2 should do something else', async () => {
-	console.log('[first] Task 2 executed');
-});
+task(
+	'1.3. Production workflow with authless webhook node using "Using \'Respond to Webhook\' node" mode',
+	async () => {
+		await client.get('/webhook/c143e038-b0bd-46ca-9708-33b868499c61');
+	},
+);
diff --git a/packages/cli/src/benchmark/workflows/1.1.json b/packages/cli/src/benchmark/workflows/1.1.json
index 70454b967b872..3eab286978ace 100644
--- a/packages/cli/src/benchmark/workflows/1.1.json
+++ b/packages/cli/src/benchmark/workflows/1.1.json
@@ -1,5 +1,6 @@
 {
 	"name": "1.1",
+	"active": true,
 	"nodes": [
 		{
 			"parameters": {
@@ -16,7 +17,6 @@
 	],
 	"pinData": {},
 	"connections": {},
-	"active": true,
 	"settings": {
 		"executionOrder": "v1"
 	},
diff --git a/packages/cli/src/benchmark/workflows/1.2.json b/packages/cli/src/benchmark/workflows/1.2.json
index 5670949751101..a235a7e44a64d 100644
--- a/packages/cli/src/benchmark/workflows/1.2.json
+++ b/packages/cli/src/benchmark/workflows/1.2.json
@@ -1,5 +1,6 @@
 {
 	"name": "1.2",
+	"active": true,
 	"nodes": [
 		{
 			"parameters": {
@@ -39,7 +40,6 @@
 			]
 		}
 	},
-	"active": false,
 	"settings": {
 		"executionOrder": "v1"
 	},
diff --git a/packages/cli/src/benchmark/workflows/1.3.json b/packages/cli/src/benchmark/workflows/1.3.json
new file mode 100644
index 0000000000000..864c2b94aff04
--- /dev/null
+++ b/packages/cli/src/benchmark/workflows/1.3.json
@@ -0,0 +1,50 @@
+{
+	"name": "1.3",
+	"active": true,
+	"nodes": [
+		{
+			"parameters": {
+				"respondWith": "text",
+				"responseBody": "Responding from \"Respond to Webhook\" node",
+				"options": {}
+			},
+			"id": "5716bed4-c3b1-40f0-b5cb-62a14e622fcb",
+			"name": "Respond to Webhook",
+			"type": "n8n-nodes-base.respondToWebhook",
+			"typeVersion": 1.1,
+			"position": [700, 280]
+		},
+		{
+			"parameters": {
+				"path": "c143e038-b0bd-46ca-9708-33b868499c61",
+				"responseMode": "responseNode",
+				"options": {}
+			},
+			"id": "65892623-8bea-42ba-b7ed-d0298f498bce",
+			"name": "Webhook",
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [500, 280],
+			"webhookId": "c143e038-b0bd-46ca-9708-33b868499c61"
+		}
+	],
+	"pinData": {},
+	"connections": {
+		"Webhook": {
+			"main": [
+				[
+					{
+						"node": "Respond to Webhook",
+						"type": "main",
+						"index": 0
+					}
+				]
+			]
+		}
+	},
+	"settings": {
+		"executionOrder": "v1"
+	},
+	"versionId": "08698a35-a4e4-4630-8b3c-3949e78f8620",
+	"tags": []
+}

From 4157807633816cc33f14826edc2ddc0dbeac02d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 17:47:47 +0200
Subject: [PATCH 47/86] Simplify paths

---
 .../cli/src/benchmark/tasks/webhook.tasks.ts  | 19 +++++--------------
 packages/cli/src/benchmark/workflows/1.1.json |  2 +-
 packages/cli/src/benchmark/workflows/1.2.json |  2 +-
 packages/cli/src/benchmark/workflows/1.3.json |  2 +-
 4 files changed, 8 insertions(+), 17 deletions(-)

diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index 9d3aeeb1c24e8..d2935df8cb6bd 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -1,34 +1,25 @@
 import axios from 'axios';
-import { task, beforeEach } from '../main.js';
+import { task } from '../main.js';
 
-const client = axios.create({
-	baseURL: 'http://localhost:5678/',
-});
-
-beforeEach(async () => {
-	await client.post('/rest/login', {
-		email: 'test@test.com',
-		password: 'password',
-	});
-});
+const client = axios.create({ baseURL: 'http://localhost:5678/' });
 
 task(
 	'1.1. Production workflow with authless webhook node using "Respond immediately" mode',
 	async () => {
-		await client.get('/webhook/d58b0160-2370-417b-bc0e-f3050b0c7adf');
+		await client.get('/webhook/1.1');
 	},
 );
 
 task(
 	'1.2. Production workflow with authless webhook node using "When last node finishes" mode',
 	async () => {
-		await client.get('/webhook/a669d108-7de3-4e72-b332-919a11b2328b');
+		await client.get('/webhook/1.2');
 	},
 );
 
 task(
 	'1.3. Production workflow with authless webhook node using "Using \'Respond to Webhook\' node" mode',
 	async () => {
-		await client.get('/webhook/c143e038-b0bd-46ca-9708-33b868499c61');
+		await client.get('/webhook/1.3');
 	},
 );
diff --git a/packages/cli/src/benchmark/workflows/1.1.json b/packages/cli/src/benchmark/workflows/1.1.json
index 3eab286978ace..71888ff22cf16 100644
--- a/packages/cli/src/benchmark/workflows/1.1.json
+++ b/packages/cli/src/benchmark/workflows/1.1.json
@@ -4,7 +4,7 @@
 	"nodes": [
 		{
 			"parameters": {
-				"path": "d58b0160-2370-417b-bc0e-f3050b0c7adf",
+				"path": "1.1",
 				"options": {}
 			},
 			"id": "000012bb-d534-4e81-a7e4-e62edf816582",
diff --git a/packages/cli/src/benchmark/workflows/1.2.json b/packages/cli/src/benchmark/workflows/1.2.json
index a235a7e44a64d..838c5112612ce 100644
--- a/packages/cli/src/benchmark/workflows/1.2.json
+++ b/packages/cli/src/benchmark/workflows/1.2.json
@@ -14,7 +14,7 @@
 		},
 		{
 			"parameters": {
-				"path": "a669d108-7de3-4e72-b332-919a11b2328b",
+				"path": "1.2",
 				"responseMode": "lastNode",
 				"options": {}
 			},
diff --git a/packages/cli/src/benchmark/workflows/1.3.json b/packages/cli/src/benchmark/workflows/1.3.json
index 864c2b94aff04..9d1e8b055e981 100644
--- a/packages/cli/src/benchmark/workflows/1.3.json
+++ b/packages/cli/src/benchmark/workflows/1.3.json
@@ -16,7 +16,7 @@
 		},
 		{
 			"parameters": {
-				"path": "c143e038-b0bd-46ca-9708-33b868499c61",
+				"path": "1.3",
 				"responseMode": "responseNode",
 				"options": {}
 			},

From 82b3f787794b2c0d96e3043a0449562fb69f4517 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 18:12:36 +0200
Subject: [PATCH 48/86] Introduce `describe()`

---
 packages/cli/src/benchmark/benchmark.md       |  2 +-
 packages/cli/src/benchmark/lib/suites.ts      | 29 +++++++++++--------
 packages/cli/src/benchmark/lib/types.ts       |  1 +
 packages/cli/src/benchmark/main.ts            |  2 +-
 .../cli/src/benchmark/tasks/webhook.tasks.ts  | 29 ++++++++-----------
 5 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index 7517932409ffa..cbd17aca5a2c8 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -18,4 +18,4 @@ All workflows with default settings unless otherwise specified.
 
 1.1. using "Respond immediately" mode
 1.2. using "When last node finishes" mode
-1.3. using "Using 'Respond to Webhook' node" mode
+1.3. using 'Respond to Webhook' node mode
diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index f4f088751eac5..625743ae44563 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -57,19 +57,27 @@ function suiteFilePath() {
 	return filePath;
 }
 
-/**
- * Run a benchmarking task, i.e. a single operation whose performance to measure.
- */
-export function task(description: string, operation: Task['operation']) {
+export function describe(suiteName: string, suiteFn: () => void) {
 	const filePath = suiteFilePath();
 
-	suites[filePath] ||= { hooks: {}, tasks: [] };
-	suites[filePath].tasks.push({ description, operation });
+	suites[filePath] = { name: suiteName, hooks: {}, tasks: [] };
+
+	suiteFn();
 }
 
+export function task(taskName: string, operation: Task['operation']) {
+	const filePath = suiteFilePath();
+
+	suites[filePath].tasks.push({
+		description: suites[filePath].name + ' ' + taskName,
+		operation,
+	});
+}
+
+// @TODO: Rename next two utils to dismbiguate?
+
 /**
- * Setup step to run once before each benchmarking task in a suite.
- * Only one `beforeEach` is allowed per suite.
+ * Setup step to run once before all iterations of each benchmarking task in a suite.
  */
 export function beforeEach(fn: Callback) {
 	const filePath = suiteFilePath();
@@ -78,13 +86,11 @@ export function beforeEach(fn: Callback) {
 		throw new DuplicateHookError('beforeEach', filePath);
 	}
 
-	suites[filePath] ||= { hooks: {}, tasks: [] };
 	suites[filePath].hooks.beforeEach = fn;
 }
 
 /**
- * Teardown step to run once after each benchmarking task in a suite.
- * Only one `afterEach` is allowed per suite.
+ * Teardown step to run once after all iterations of each benchmarking task in a suite.
  */
 export function afterEach(fn: Callback) {
 	const filePath = suiteFilePath();
@@ -93,6 +99,5 @@ export function afterEach(fn: Callback) {
 		throw new DuplicateHookError('afterEach', filePath);
 	}
 
-	suites[filePath] ||= { hooks: {}, tasks: [] };
 	suites[filePath].hooks.afterEach = fn;
 }
diff --git a/packages/cli/src/benchmark/lib/types.ts b/packages/cli/src/benchmark/lib/types.ts
index b09624337e579..4d8c897b08257 100644
--- a/packages/cli/src/benchmark/lib/types.ts
+++ b/packages/cli/src/benchmark/lib/types.ts
@@ -3,6 +3,7 @@
  */
 export type Suites = {
 	[suiteFilepath: string]: {
+		name: string;
 		hooks: {
 			beforeEach?: Callback;
 			afterEach?: Callback;
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index be017ba9285fe..732194ffa7467 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -11,7 +11,7 @@ import Bench from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
 /* eslint-enable import/no-extraneous-dependencies */
 
-export { beforeEach, afterEach, task } from './lib/suites';
+export { describe, task, beforeEach, afterEach } from './lib/suites';
 
 async function main() {
 	const dbType = config.getEnv('database.type');
diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index d2935df8cb6bd..eb281682476c3 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -1,25 +1,20 @@
 import axios from 'axios';
-import { task } from '../main.js';
+import { task, describe } from '../main';
 
-const client = axios.create({ baseURL: 'http://localhost:5678/' });
+// @TODO: Rename file?
 
-task(
-	'1.1. Production workflow with authless webhook node using "Respond immediately" mode',
-	async () => {
+describe('1. Production workflow with authless webhook node', () => {
+	const client = axios.create({ baseURL: 'http://localhost:5678/' });
+
+	task('(1.1) using "Respond immediately" mode', async () => {
 		await client.get('/webhook/1.1');
-	},
-);
+	});
 
-task(
-	'1.2. Production workflow with authless webhook node using "When last node finishes" mode',
-	async () => {
+	task('(1.2) using "When last node finishes" mode', async () => {
 		await client.get('/webhook/1.2');
-	},
-);
+	});
 
-task(
-	'1.3. Production workflow with authless webhook node using "Using \'Respond to Webhook\' node" mode',
-	async () => {
+	task('(1.3) using "Respond to Webhook" node mode', async () => {
 		await client.get('/webhook/1.3');
-	},
-);
+	});
+});

From 421f23b17da5190fb3c68a55de9d5b60119f360d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 18:38:39 +0200
Subject: [PATCH 49/86] Prevent duplicate suites

---
 .../src/benchmark/lib/errors/duplicate-suite.error.ts    | 9 +++++++++
 packages/cli/src/benchmark/lib/suites.ts                 | 3 +++
 2 files changed, 12 insertions(+)
 create mode 100644 packages/cli/src/benchmark/lib/errors/duplicate-suite.error.ts

diff --git a/packages/cli/src/benchmark/lib/errors/duplicate-suite.error.ts b/packages/cli/src/benchmark/lib/errors/duplicate-suite.error.ts
new file mode 100644
index 0000000000000..65116fab18c49
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/errors/duplicate-suite.error.ts
@@ -0,0 +1,9 @@
+import { ApplicationError } from 'n8n-workflow';
+
+export class DuplicateSuiteError extends ApplicationError {
+	constructor(filePath: string) {
+		super(`Duplicate suite found at \`${filePath}\`. Please define a single suite for this file.`, {
+			level: 'warning',
+		});
+	}
+}
diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index 625743ae44563..a8625ba1ace42 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -6,6 +6,7 @@ import glob from 'fast-glob';
 import callsites from 'callsites';
 import type { Suites, Task, Callback } from './types';
 import { DuplicateHookError } from './errors/duplicate-hook.error';
+import { DuplicateSuiteError } from './errors/duplicate-suite.error';
 
 export const suites: Suites = {};
 
@@ -60,6 +61,8 @@ function suiteFilePath() {
 export function describe(suiteName: string, suiteFn: () => void) {
 	const filePath = suiteFilePath();
 
+	if (suites[filePath]) throw new DuplicateSuiteError(filePath);
+
 	suites[filePath] = { name: suiteName, hooks: {}, tasks: [] };
 
 	suiteFn();

From b12d254db7cd50d8e05bbeec18dbd1319c1656f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 18:41:13 +0200
Subject: [PATCH 50/86] `describe` -> `suite`

---
 packages/cli/src/benchmark/lib/suites.ts          | 2 +-
 packages/cli/src/benchmark/main.ts                | 2 +-
 packages/cli/src/benchmark/tasks/webhook.tasks.ts | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index a8625ba1ace42..c0365cc3faa32 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -58,7 +58,7 @@ function suiteFilePath() {
 	return filePath;
 }
 
-export function describe(suiteName: string, suiteFn: () => void) {
+export function suite(suiteName: string, suiteFn: () => void) {
 	const filePath = suiteFilePath();
 
 	if (suites[filePath]) throw new DuplicateSuiteError(filePath);
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 732194ffa7467..45544b2ff350a 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -11,7 +11,7 @@ import Bench from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
 /* eslint-enable import/no-extraneous-dependencies */
 
-export { describe, task, beforeEach, afterEach } from './lib/suites';
+export { suite, task, beforeEach, afterEach } from './lib/suites';
 
 async function main() {
 	const dbType = config.getEnv('database.type');
diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index eb281682476c3..37f5710df33d2 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -1,9 +1,9 @@
 import axios from 'axios';
-import { task, describe } from '../main';
+import { task, suite } from '../main';
 
 // @TODO: Rename file?
 
-describe('1. Production workflow with authless webhook node', () => {
+suite('1. Production workflow with authless webhook node', () => {
 	const client = axios.create({ baseURL: 'http://localhost:5678/' });
 
 	task('(1.1) using "Respond immediately" mode', async () => {

From 57724d50c63fc0a0577a16c2c1142175bab1a04a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 19:29:51 +0200
Subject: [PATCH 51/86] Script to document suites

---
 packages/cli/package.json                     |  2 +-
 packages/cli/src/benchmark/benchmark.md       | 18 +++++---
 packages/cli/src/benchmark/lib/index.ts       |  1 +
 packages/cli/src/benchmark/lib/suites.ts      |  8 ++--
 packages/cli/src/benchmark/main.ts            |  8 ++--
 .../src/benchmark/scripts/document-suites.ts  | 41 +++++++++++++++++++
 .../cli/src/benchmark/tasks/webhook.tasks.ts  |  2 +-
 7 files changed, 62 insertions(+), 18 deletions(-)
 create mode 100644 packages/cli/src/benchmark/lib/index.ts
 create mode 100644 packages/cli/src/benchmark/scripts/document-suites.ts

diff --git a/packages/cli/package.json b/packages/cli/package.json
index 00df739b8a90d..1dd4e4b77e2ce 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -24,7 +24,7 @@
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
-    "build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs",
+    "build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs && node dist/benchmark/scripts/document-suites.js",
     "buildAndDev": "pnpm run build && pnpm run dev",
     "dev": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon\"",
     "dev:worker": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon worker\"",
diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index cbd17aca5a2c8..521e53d67a9bc 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -1,6 +1,6 @@
 # Benchmark
 
-To run benchmarks locally in `cli`:
+To run benchmarks locally in sqlite:
 
 ```sh
 pnpm benchmark:sqlite
@@ -10,12 +10,18 @@ pnpm benchmark:sqlite
 
 To create a benchmark, @TODO
 
-## Listing
+## List
 
-All workflows with default settings unless otherwise specified.
+> **Note**: All workflows with default settings unless otherwise specified.
+
+<!-- BENCHMARK_SUITES_LIST -->
 
 ### 1. Production workflow with authless webhook node
 
-1.1. using "Respond immediately" mode
-1.2. using "When last node finishes" mode
-1.3. using 'Respond to Webhook' node mode
+Suite file: `webhook.tasks.js`
+
+(1.1) using "Respond immediately" mode
+(1.2) using "When last node finishes" mode
+(1.3) using "Respond to Webhook" node mode
+
+<!-- /BENCHMARK_SUITES_LIST -->
diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
new file mode 100644
index 0000000000000..fceaaa29a3025
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -0,0 +1 @@
+export { suite, collectSuites, task, beforeEach, afterEach } from './suites';
diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
index c0365cc3faa32..50b8ea4dc7a1a 100644
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ b/packages/cli/src/benchmark/lib/suites.ts
@@ -8,11 +8,7 @@ import type { Suites, Task, Callback } from './types';
 import { DuplicateHookError } from './errors/duplicate-hook.error';
 import { DuplicateSuiteError } from './errors/duplicate-suite.error';
 
-export const suites: Suites = {};
-
-export function suiteCount() {
-	return Object.keys(suites).length;
-}
+const suites: Suites = {};
 
 export async function collectSuites() {
 	const files = await glob('**/*.tasks.js', {
@@ -23,6 +19,8 @@ export async function collectSuites() {
 	for (const f of files) {
 		await import(f);
 	}
+
+	return suites;
 }
 
 export function registerSuites(bench: Bench) {
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 45544b2ff350a..e0a83133dbc98 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,6 +1,6 @@
 import 'reflect-metadata';
 import * as hooks from './lib/hooks';
-import { collectSuites, registerSuites, suiteCount } from './lib/suites';
+import { collectSuites, registerSuites } from './lib/suites';
 import config from '@/config';
 import { UnsupportedDatabaseError } from './lib/errors/unsupported-database.error';
 import { Logger } from '@/Logger';
@@ -11,16 +11,14 @@ import Bench from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
 /* eslint-enable import/no-extraneous-dependencies */
 
-export { suite, task, beforeEach, afterEach } from './lib/suites';
-
 async function main() {
 	const dbType = config.getEnv('database.type');
 
 	if (dbType !== 'sqlite') throw new UnsupportedDatabaseError();
 
-	await collectSuites();
+	const suites = await collectSuites();
 
-	const count = suiteCount();
+	const count = Object.keys(suites).length;
 
 	const logger = Container.get(Logger);
 
diff --git a/packages/cli/src/benchmark/scripts/document-suites.ts b/packages/cli/src/benchmark/scripts/document-suites.ts
new file mode 100644
index 0000000000000..ea69ab433cb15
--- /dev/null
+++ b/packages/cli/src/benchmark/scripts/document-suites.ts
@@ -0,0 +1,41 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { writeFileSync } from 'node:fs';
+import { collectSuites } from '../lib';
+import type { Suites } from '../lib/types';
+
+function toSuitesList(suites: Suites) {
+	let list = '';
+
+	for (const [fullPath, suite] of Object.entries(suites)) {
+		list += `\n### ${suite.name}` + '\n\n';
+
+		list += 'Suite file: `' + fullPath.split('/').pop() + '`\n\n';
+
+		for (const task of suite.tasks) {
+			list += task.description.replace(suite.name, '').trim() + '\n';
+		}
+	}
+
+	return list;
+}
+
+async function documentSuites() {
+	const filePath = path.resolve('src', 'benchmark', 'benchmark.md');
+	const oldDoc = await fs.readFile(filePath, 'utf8');
+
+	const MARK_START = '<!-- BENCHMARK_SUITES_LIST -->';
+	const MARK_END = '<!-- /BENCHMARK_SUITES_LIST -->';
+
+	const before = oldDoc.slice(0, oldDoc.indexOf(MARK_START) + MARK_START.length);
+	const after = oldDoc.slice(oldDoc.indexOf(MARK_END));
+
+	const suites = await collectSuites();
+	const suitesList = toSuitesList(suites);
+
+	const newDoc = [before, suitesList, after].join('\n');
+
+	writeFileSync(filePath, newDoc);
+}
+
+void documentSuites();
diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
index 37f5710df33d2..c8d1fd7c5b8ab 100644
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
@@ -1,5 +1,5 @@
 import axios from 'axios';
-import { task, suite } from '../main';
+import { task, suite } from '../lib';
 
 // @TODO: Rename file?
 

From d6d7efb961e39424388479dd8c5d6ebb42681c9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 23 Apr 2024 19:54:41 +0200
Subject: [PATCH 52/86] Cleanup

---
 packages/cli/package.json                     |   2 +-
 packages/cli/src/benchmark/benchmark.md       |  10 +-
 ...abase.error.ts => unsupported-db.error.ts} |   0
 packages/cli/src/benchmark/lib/hooks.ts       |  44 ++-----
 packages/cli/src/benchmark/lib/index.ts       | 109 +++++++++++++++++-
 packages/cli/src/benchmark/lib/suites.ts      | 104 -----------------
 packages/cli/src/benchmark/lib/types.ts       |   6 +-
 packages/cli/src/benchmark/main.ts            |  12 +-
 .../{document-suites.ts => list-suites.ts}    |  20 ++--
 ...ebhook-with-authless-webhook-node.suite.ts |  18 +++
 .../1.1.json => suites/workflows/001-1.json}  |   2 +-
 .../1.2.json => suites/workflows/001-2.json}  |   2 +-
 .../1.3.json => suites/workflows/001-3.json}  |   2 +-
 .../cli/src/benchmark/tasks/webhook.tasks.ts  |  20 ----
 packages/cli/tsconfig.benchmark.json          |   2 +-
 15 files changed, 169 insertions(+), 184 deletions(-)
 rename packages/cli/src/benchmark/lib/errors/{unsupported-database.error.ts => unsupported-db.error.ts} (100%)
 delete mode 100644 packages/cli/src/benchmark/lib/suites.ts
 rename packages/cli/src/benchmark/scripts/{document-suites.ts => list-suites.ts} (60%)
 create mode 100644 packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts
 rename packages/cli/src/benchmark/{workflows/1.1.json => suites/workflows/001-1.json} (95%)
 rename packages/cli/src/benchmark/{workflows/1.2.json => suites/workflows/001-2.json} (97%)
 rename packages/cli/src/benchmark/{workflows/1.3.json => suites/workflows/001-3.json} (97%)
 delete mode 100644 packages/cli/src/benchmark/tasks/webhook.tasks.ts

diff --git a/packages/cli/package.json b/packages/cli/package.json
index 1dd4e4b77e2ce..ac8b813b4b927 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -24,7 +24,7 @@
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
-    "build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs && node dist/benchmark/scripts/document-suites.js",
+    "build:benchmark": "tsc -p tsconfig.benchmark.json && tsc-alias -p tsconfig.benchmark.json && node scripts/build.mjs && node dist/benchmark/scripts/list-suites.js",
     "buildAndDev": "pnpm run build && pnpm run dev",
     "dev": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon\"",
     "dev:worker": "concurrently -k -n \"TypeScript,Node\" -c \"yellow.bold,cyan.bold\" \"npm run watch\" \"nodemon worker\"",
diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index 521e53d67a9bc..1d3f923c443c0 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -16,12 +16,10 @@ To create a benchmark, @TODO
 
 <!-- BENCHMARK_SUITES_LIST -->
 
-### 1. Production workflow with authless webhook node
+### 001 - Production workflow with authless webhook node
 
-Suite file: `webhook.tasks.js`
-
-(1.1) using "Respond immediately" mode
-(1.2) using "When last node finishes" mode
-(1.3) using "Respond to Webhook" node mode
+- [using "Respond immediately" mode](./suites/workflows/001-1.json)
+- [using "When last node finishes" mode](./suites/workflows/001-2.json)
+- [using "Respond to Webhook" node mode](./suites/workflows/001-3.json)
 
 <!-- /BENCHMARK_SUITES_LIST -->
diff --git a/packages/cli/src/benchmark/lib/errors/unsupported-database.error.ts b/packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts
similarity index 100%
rename from packages/cli/src/benchmark/lib/errors/unsupported-database.error.ts
rename to packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts
diff --git a/packages/cli/src/benchmark/lib/hooks.ts b/packages/cli/src/benchmark/lib/hooks.ts
index 4cb5f3377af68..044b61ee8db0c 100644
--- a/packages/cli/src/benchmark/lib/hooks.ts
+++ b/packages/cli/src/benchmark/lib/hooks.ts
@@ -15,10 +15,9 @@ import type { WorkflowRequest } from '@/workflows/workflow.request';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 import { Logger } from '@/Logger';
 
-/**
- * Create a temp `.n8n` dir for encryption key, sqlite DB, etc.
- */
-function n8nDir() {
+const logger = Container.get(Logger);
+
+function tempN8nDir() {
 	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
 
 	mkdirSync(baseDirPath, { recursive: true });
@@ -41,14 +40,11 @@ function n8nDir() {
 	instanceSettings.n8nFolder = _n8nDir;
 	Container.set(InstanceSettings, instanceSettings);
 
-	Container.get(Logger).info(`Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
+	logger.info(`[Benchmarking] Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
 }
 
-/**
- * Load into DB and activate in memory all workflows to use in benchmarks.
- */
-async function prepareWorkflows(owner: User) {
-	const files = await glob('workflows/*.json', {
+async function loadWorkflows(owner: User) {
+	const files = await glob('suites/workflows/*.json', {
 		cwd: path.join('dist', 'benchmark'),
 		absolute: true,
 	});
@@ -64,41 +60,25 @@ async function prepareWorkflows(owner: User) {
 		// @ts-ignore @TODO Fix typing
 		await Container.get(WorkflowsController).create({ body: workflow, user: owner });
 	}
-
-	await Container.get(ActiveWorkflowRunner).init();
 }
 
 let main: Start;
 
-/**
- * Start the main n8n process to use in benchmarks.
- */
-async function mainProcess() {
-	const args: string[] = [];
-	const _config = new Config({ root: __dirname });
+export async function globalSetup() {
+	tempN8nDir();
 
-	main = new Start(args, _config);
+	main = new Start([], new Config({ root: __dirname }));
 
 	await main.init();
 	await main.run();
-}
-
-/**
- * Setup to run before once all benchmarks.
- */
-export async function globalSetup() {
-	n8nDir();
-
-	await mainProcess();
 
 	const owner = await Container.get(UserRepository).createTestOwner();
 
-	await prepareWorkflows(owner); // @TODO: Load all here or as part of each benchmark's `beforeEach`?
+	await loadWorkflows(owner);
+
+	await Container.get(ActiveWorkflowRunner).init();
 }
 
-/**
- * Teardown to run after all benchmarks.
- */
 export async function globalTeardown() {
 	await main.stopProcess();
 }
diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
index fceaaa29a3025..02218edc851d9 100644
--- a/packages/cli/src/benchmark/lib/index.ts
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -1 +1,108 @@
-export { suite, collectSuites, task, beforeEach, afterEach } from './suites';
+import 'reflect-metadata';
+import path from 'node:path';
+import type Bench from 'tinybench';
+import { assert } from 'n8n-workflow';
+import glob from 'fast-glob';
+import callsites from 'callsites';
+import type { Suites, Task, Callback } from './types';
+import { DuplicateHookError } from './errors/duplicate-hook.error';
+import { DuplicateSuiteError } from './errors/duplicate-suite.error';
+
+const suites: Suites = {};
+
+export async function collectSuites() {
+	const files = await glob('**/*.suite.js', {
+		cwd: path.join('dist', 'benchmark'),
+		absolute: true,
+	});
+
+	for (const f of files) {
+		await import(f);
+	}
+
+	return suites;
+}
+
+export function registerSuites(bench: Bench) {
+	for (const { hooks, tasks } of Object.values(suites)) {
+		/**
+		 * In tinybench, `beforeAll` and `afterAll` refer to all iterations of
+		 * a single task, while `beforeEach` and `afterEach` refer to each iteration.
+		 *
+		 * In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
+		 * while `beforeEach` and `afterEach` refer to each individual test.
+		 *
+		 * We rename tinybench's hooks to prevent confusion from this difference.
+		 */
+		const options: Record<string, Callback> = {};
+
+		if (hooks.beforeEach) options.beforeAll = hooks.beforeEach;
+		if (hooks.afterEach) options.afterAll = hooks.afterEach;
+
+		for (const t of tasks) {
+			bench.add(t.name, t.operation, options);
+		}
+	}
+}
+
+function suiteFilePath() {
+	const filePath = callsites()
+		.map((site) => site.getFileName())
+		.filter((site): site is string => site !== null)
+		.find((site) => site.endsWith('.suite.js'));
+
+	assert(filePath !== undefined);
+
+	return filePath;
+}
+
+export function suite(suiteName: string, suiteFn: () => void) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]) throw new DuplicateSuiteError(filePath);
+
+	suites[filePath] = { name: suiteName, hooks: {}, tasks: [] };
+
+	suiteFn();
+}
+
+export function task(taskName: string, operation: Task['operation']) {
+	const filePath = suiteFilePath();
+
+	suites[filePath].tasks.push({
+		name: suites[filePath].name + ' ' + taskName,
+		operation,
+	});
+}
+
+// @TODO: Rename next two utils to dismbiguate?
+
+/**
+ * Setup step to run once before all iterations of each benchmarking task in a suite.
+ */
+export function beforeEach(fn: Callback) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]?.hooks.beforeEach) {
+		throw new DuplicateHookError('beforeEach', filePath);
+	}
+
+	suites[filePath].hooks.beforeEach = fn;
+}
+
+/**
+ * Teardown step to run once after all iterations of each benchmarking task in a suite.
+ */
+export function afterEach(fn: Callback) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]?.hooks.afterEach) {
+		throw new DuplicateHookError('afterEach', filePath);
+	}
+
+	suites[filePath].hooks.afterEach = fn;
+}
+
+export const BACKEND_BASE_URL = 'http://localhost:5678';
+
+export type { Suites } from './types';
diff --git a/packages/cli/src/benchmark/lib/suites.ts b/packages/cli/src/benchmark/lib/suites.ts
deleted file mode 100644
index 50b8ea4dc7a1a..0000000000000
--- a/packages/cli/src/benchmark/lib/suites.ts
+++ /dev/null
@@ -1,104 +0,0 @@
-import 'reflect-metadata';
-import path from 'node:path';
-import type Bench from 'tinybench';
-import { assert } from 'n8n-workflow';
-import glob from 'fast-glob';
-import callsites from 'callsites';
-import type { Suites, Task, Callback } from './types';
-import { DuplicateHookError } from './errors/duplicate-hook.error';
-import { DuplicateSuiteError } from './errors/duplicate-suite.error';
-
-const suites: Suites = {};
-
-export async function collectSuites() {
-	const files = await glob('**/*.tasks.js', {
-		cwd: path.join('dist', 'benchmark'),
-		absolute: true,
-	});
-
-	for (const f of files) {
-		await import(f);
-	}
-
-	return suites;
-}
-
-export function registerSuites(bench: Bench) {
-	for (const { hooks, tasks } of Object.values(suites)) {
-		/**
-		 * In tinybench, `beforeAll` and `afterAll` refer to all iterations of
-		 * a single task, while `beforeEach` and `afterEach` refer to each iteration.
-		 *
-		 * In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
-		 * while `beforeEach` and `afterEach` refer to each individual test.
-		 *
-		 * We rename tinybench's hooks to prevent confusion from this difference.
-		 */
-		const options: Record<string, Callback> = {};
-
-		if (hooks.beforeEach) options.beforeAll = hooks.beforeEach;
-		if (hooks.afterEach) options.afterAll = hooks.afterEach;
-
-		for (const t of tasks) {
-			bench.add(t.description, t.operation, options);
-		}
-	}
-}
-
-function suiteFilePath() {
-	const filePath = callsites()
-		.map((site) => site.getFileName())
-		.filter((site): site is string => site !== null)
-		.find((site) => site.endsWith('.tasks.js'));
-
-	assert(filePath !== undefined);
-
-	return filePath;
-}
-
-export function suite(suiteName: string, suiteFn: () => void) {
-	const filePath = suiteFilePath();
-
-	if (suites[filePath]) throw new DuplicateSuiteError(filePath);
-
-	suites[filePath] = { name: suiteName, hooks: {}, tasks: [] };
-
-	suiteFn();
-}
-
-export function task(taskName: string, operation: Task['operation']) {
-	const filePath = suiteFilePath();
-
-	suites[filePath].tasks.push({
-		description: suites[filePath].name + ' ' + taskName,
-		operation,
-	});
-}
-
-// @TODO: Rename next two utils to dismbiguate?
-
-/**
- * Setup step to run once before all iterations of each benchmarking task in a suite.
- */
-export function beforeEach(fn: Callback) {
-	const filePath = suiteFilePath();
-
-	if (suites[filePath]?.hooks.beforeEach) {
-		throw new DuplicateHookError('beforeEach', filePath);
-	}
-
-	suites[filePath].hooks.beforeEach = fn;
-}
-
-/**
- * Teardown step to run once after all iterations of each benchmarking task in a suite.
- */
-export function afterEach(fn: Callback) {
-	const filePath = suiteFilePath();
-
-	if (suites[filePath]?.hooks.afterEach) {
-		throw new DuplicateHookError('afterEach', filePath);
-	}
-
-	suites[filePath].hooks.afterEach = fn;
-}
diff --git a/packages/cli/src/benchmark/lib/types.ts b/packages/cli/src/benchmark/lib/types.ts
index 4d8c897b08257..6c3a643ada735 100644
--- a/packages/cli/src/benchmark/lib/types.ts
+++ b/packages/cli/src/benchmark/lib/types.ts
@@ -1,8 +1,8 @@
 /**
- * Benchmarking suites, i.e. `*.tasks.ts` files containing benchmarking tasks.
+ * Benchmarking suites, i.e. `*.suite.ts` files containing benchmarking tasks.
  */
 export type Suites = {
-	[suiteFilepath: string]: {
+	[suiteFilePath: string]: {
 		name: string;
 		hooks: {
 			beforeEach?: Callback;
@@ -16,7 +16,7 @@ export type Suites = {
  * A benchmarking task, i.e. a single operation whose performance to measure.
  */
 export type Task = {
-	description: string;
+	name: string;
 	operation: Callback;
 };
 
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index e0a83133dbc98..1e24ea4f82b5d 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,10 +1,10 @@
 import 'reflect-metadata';
-import * as hooks from './lib/hooks';
-import { collectSuites, registerSuites } from './lib/suites';
+import Container from 'typedi';
 import config from '@/config';
-import { UnsupportedDatabaseError } from './lib/errors/unsupported-database.error';
 import { Logger } from '@/Logger';
-import Container from 'typedi';
+import * as hooks from './lib/hooks';
+import { collectSuites, registerSuites } from './lib';
+import { UnsupportedDatabaseError } from './lib/errors/unsupported-db.error';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';
@@ -23,11 +23,11 @@ async function main() {
 	const logger = Container.get(Logger);
 
 	if (count === 0) {
-		logger.info('No benchmarking suites found. Exiting...');
+		logger.info('[Benchmarking] Found no suites. Exiting...');
 		return;
 	}
 
-	logger.info(`Running ${count} benchmarking ${count === 1 ? 'suite' : 'suites'}...`);
+	logger.info(`[Benchmarking] Running ${count} ${count === 1 ? 'suite' : 'suites'}...`);
 
 	await hooks.globalSetup();
 
diff --git a/packages/cli/src/benchmark/scripts/document-suites.ts b/packages/cli/src/benchmark/scripts/list-suites.ts
similarity index 60%
rename from packages/cli/src/benchmark/scripts/document-suites.ts
rename to packages/cli/src/benchmark/scripts/list-suites.ts
index ea69ab433cb15..1272fb6a53883 100644
--- a/packages/cli/src/benchmark/scripts/document-suites.ts
+++ b/packages/cli/src/benchmark/scripts/list-suites.ts
@@ -2,25 +2,31 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 import { writeFileSync } from 'node:fs';
 import { collectSuites } from '../lib';
-import type { Suites } from '../lib/types';
+import type { Suites } from '../lib';
 
 function toSuitesList(suites: Suites) {
 	let list = '';
 
 	for (const [fullPath, suite] of Object.entries(suites)) {
-		list += `\n### ${suite.name}` + '\n\n';
+		const suiteId = fullPath.split('/').pop()?.split('-').shift() ?? '';
 
-		list += 'Suite file: `' + fullPath.split('/').pop() + '`\n\n';
+		list += `\n### ${suiteId} - ${suite.name}\n\n`;
 
-		for (const task of suite.tasks) {
-			list += task.description.replace(suite.name, '').trim() + '\n';
+		for (let i = 0; i < suite.tasks.length; i++) {
+			const suiteName = suite.tasks[i].name.replace(suite.name, '').trim();
+			const workflowPath = `./suites/workflows/${suiteId}-${i + 1}.json`;
+
+			list += `- [${suiteName}](${workflowPath})\n`;
 		}
 	}
 
 	return list;
 }
 
-async function documentSuites() {
+/**
+ * Insert an auto-generated list of benchmarking suites into `benchmark.md`.
+ */
+async function listSuites() {
 	const filePath = path.resolve('src', 'benchmark', 'benchmark.md');
 	const oldDoc = await fs.readFile(filePath, 'utf8');
 
@@ -38,4 +44,4 @@ async function documentSuites() {
 	writeFileSync(filePath, newDoc);
 }
 
-void documentSuites();
+void listSuites();
diff --git a/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts b/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts
new file mode 100644
index 0000000000000..07068d635edc8
--- /dev/null
+++ b/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts
@@ -0,0 +1,18 @@
+import axios from 'axios';
+import { task, suite, BACKEND_BASE_URL } from '../lib';
+
+suite('Production workflow with authless webhook node', () => {
+	const client = axios.create({ baseURL: BACKEND_BASE_URL });
+
+	task('using "Respond immediately" mode', async () => {
+		await client.get('/webhook/001-1');
+	});
+
+	task('using "When last node finishes" mode', async () => {
+		await client.get('/webhook/001-2');
+	});
+
+	task('using "Respond to Webhook" node mode', async () => {
+		await client.get('/webhook/001-3');
+	});
+});
diff --git a/packages/cli/src/benchmark/workflows/1.1.json b/packages/cli/src/benchmark/suites/workflows/001-1.json
similarity index 95%
rename from packages/cli/src/benchmark/workflows/1.1.json
rename to packages/cli/src/benchmark/suites/workflows/001-1.json
index 71888ff22cf16..c6217ed2f2be0 100644
--- a/packages/cli/src/benchmark/workflows/1.1.json
+++ b/packages/cli/src/benchmark/suites/workflows/001-1.json
@@ -4,7 +4,7 @@
 	"nodes": [
 		{
 			"parameters": {
-				"path": "1.1",
+				"path": "001-1",
 				"options": {}
 			},
 			"id": "000012bb-d534-4e81-a7e4-e62edf816582",
diff --git a/packages/cli/src/benchmark/workflows/1.2.json b/packages/cli/src/benchmark/suites/workflows/001-2.json
similarity index 97%
rename from packages/cli/src/benchmark/workflows/1.2.json
rename to packages/cli/src/benchmark/suites/workflows/001-2.json
index 838c5112612ce..c40a1cba2b68f 100644
--- a/packages/cli/src/benchmark/workflows/1.2.json
+++ b/packages/cli/src/benchmark/suites/workflows/001-2.json
@@ -14,7 +14,7 @@
 		},
 		{
 			"parameters": {
-				"path": "1.2",
+				"path": "001-2",
 				"responseMode": "lastNode",
 				"options": {}
 			},
diff --git a/packages/cli/src/benchmark/workflows/1.3.json b/packages/cli/src/benchmark/suites/workflows/001-3.json
similarity index 97%
rename from packages/cli/src/benchmark/workflows/1.3.json
rename to packages/cli/src/benchmark/suites/workflows/001-3.json
index 9d1e8b055e981..46d75d027c891 100644
--- a/packages/cli/src/benchmark/workflows/1.3.json
+++ b/packages/cli/src/benchmark/suites/workflows/001-3.json
@@ -16,7 +16,7 @@
 		},
 		{
 			"parameters": {
-				"path": "1.3",
+				"path": "001-3",
 				"responseMode": "responseNode",
 				"options": {}
 			},
diff --git a/packages/cli/src/benchmark/tasks/webhook.tasks.ts b/packages/cli/src/benchmark/tasks/webhook.tasks.ts
deleted file mode 100644
index c8d1fd7c5b8ab..0000000000000
--- a/packages/cli/src/benchmark/tasks/webhook.tasks.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-import axios from 'axios';
-import { task, suite } from '../lib';
-
-// @TODO: Rename file?
-
-suite('1. Production workflow with authless webhook node', () => {
-	const client = axios.create({ baseURL: 'http://localhost:5678/' });
-
-	task('(1.1) using "Respond immediately" mode', async () => {
-		await client.get('/webhook/1.1');
-	});
-
-	task('(1.2) using "When last node finishes" mode', async () => {
-		await client.get('/webhook/1.2');
-	});
-
-	task('(1.3) using "Respond to Webhook" node mode', async () => {
-		await client.get('/webhook/1.3');
-	});
-});
diff --git a/packages/cli/tsconfig.benchmark.json b/packages/cli/tsconfig.benchmark.json
index ac0a1462ee671..1673d60bfddf9 100644
--- a/packages/cli/tsconfig.benchmark.json
+++ b/packages/cli/tsconfig.benchmark.json
@@ -5,6 +5,6 @@
 		"outDir": "dist",
 		"tsBuildInfoFile": "dist/benchmark.tsbuildinfo"
 	},
-	"include": ["src/**/*.ts", "src/benchmark/**/*.ts", "src/benchmark/workflows/*.json"],
+	"include": ["src/**/*.ts", "src/benchmark/**/*.ts", "src/benchmark/**/*.json"],
 	"exclude": ["test/**"]
 }

From d0be040ad3833f11291fb779cac4a04875ffb0e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Wed, 24 Apr 2024 09:53:56 +0200
Subject: [PATCH 53/86] Cleanup

---
 packages/cli/src/benchmark/lib/constants.ts   |   1 +
 .../lib/errors/duplicate-hook.error.ts        |   2 +-
 .../lib/errors/unsupported-db.error.ts        |   5 +-
 packages/cli/src/benchmark/lib/index.ts       | 109 +-----------------
 .../cli/src/benchmark/lib/registration.ts     | 105 +++++++++++++++++
 packages/cli/src/benchmark/main.ts            |   4 +-
 6 files changed, 116 insertions(+), 110 deletions(-)
 create mode 100644 packages/cli/src/benchmark/lib/constants.ts
 create mode 100644 packages/cli/src/benchmark/lib/registration.ts

diff --git a/packages/cli/src/benchmark/lib/constants.ts b/packages/cli/src/benchmark/lib/constants.ts
new file mode 100644
index 0000000000000..f1fb791dca864
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/constants.ts
@@ -0,0 +1 @@
+export const BACKEND_BASE_URL = 'http://localhost:5678';
diff --git a/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts b/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
index a99464065002e..7aa4692a0c0c8 100644
--- a/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
+++ b/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
@@ -3,7 +3,7 @@ import { ApplicationError } from 'n8n-workflow';
 export class DuplicateHookError extends ApplicationError {
 	constructor(hookName: 'beforeEach' | 'afterEach', filePath: string) {
 		super(
-			`Duplicate \`${hookName}\` hook found in benchmarking suite \`${filePath}\`. Please define a single \`${hookName}\` hook for this suite.`,
+			`Duplicate \`${hookName}\` hook found at \`${filePath}\`. Please define a single \`${hookName}\` hook for this file.`,
 			{ level: 'warning' },
 		);
 	}
diff --git a/packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts b/packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts
index 042318909e577..e1c96fc88e6ce 100644
--- a/packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts
+++ b/packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts
@@ -2,6 +2,9 @@ import { ApplicationError } from 'n8n-workflow';
 
 export class UnsupportedDatabaseError extends ApplicationError {
 	constructor() {
-		super('Currently only sqlite is supported for benchmarking', { level: 'warning' });
+		super(
+			'Currently only sqlite is supported for benchmarking. Please ensure DB_TYPE is set to `sqlite`',
+			{ level: 'warning' },
+		);
 	}
 }
diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
index 02218edc851d9..5d827a43c8b1b 100644
--- a/packages/cli/src/benchmark/lib/index.ts
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -1,108 +1,7 @@
 import 'reflect-metadata';
-import path from 'node:path';
-import type Bench from 'tinybench';
-import { assert } from 'n8n-workflow';
-import glob from 'fast-glob';
-import callsites from 'callsites';
-import type { Suites, Task, Callback } from './types';
-import { DuplicateHookError } from './errors/duplicate-hook.error';
-import { DuplicateSuiteError } from './errors/duplicate-suite.error';
-
-const suites: Suites = {};
-
-export async function collectSuites() {
-	const files = await glob('**/*.suite.js', {
-		cwd: path.join('dist', 'benchmark'),
-		absolute: true,
-	});
-
-	for (const f of files) {
-		await import(f);
-	}
-
-	return suites;
-}
-
-export function registerSuites(bench: Bench) {
-	for (const { hooks, tasks } of Object.values(suites)) {
-		/**
-		 * In tinybench, `beforeAll` and `afterAll` refer to all iterations of
-		 * a single task, while `beforeEach` and `afterEach` refer to each iteration.
-		 *
-		 * In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
-		 * while `beforeEach` and `afterEach` refer to each individual test.
-		 *
-		 * We rename tinybench's hooks to prevent confusion from this difference.
-		 */
-		const options: Record<string, Callback> = {};
-
-		if (hooks.beforeEach) options.beforeAll = hooks.beforeEach;
-		if (hooks.afterEach) options.afterAll = hooks.afterEach;
-
-		for (const t of tasks) {
-			bench.add(t.name, t.operation, options);
-		}
-	}
-}
-
-function suiteFilePath() {
-	const filePath = callsites()
-		.map((site) => site.getFileName())
-		.filter((site): site is string => site !== null)
-		.find((site) => site.endsWith('.suite.js'));
-
-	assert(filePath !== undefined);
-
-	return filePath;
-}
-
-export function suite(suiteName: string, suiteFn: () => void) {
-	const filePath = suiteFilePath();
-
-	if (suites[filePath]) throw new DuplicateSuiteError(filePath);
-
-	suites[filePath] = { name: suiteName, hooks: {}, tasks: [] };
-
-	suiteFn();
-}
-
-export function task(taskName: string, operation: Task['operation']) {
-	const filePath = suiteFilePath();
-
-	suites[filePath].tasks.push({
-		name: suites[filePath].name + ' ' + taskName,
-		operation,
-	});
-}
-
-// @TODO: Rename next two utils to dismbiguate?
-
-/**
- * Setup step to run once before all iterations of each benchmarking task in a suite.
- */
-export function beforeEach(fn: Callback) {
-	const filePath = suiteFilePath();
-
-	if (suites[filePath]?.hooks.beforeEach) {
-		throw new DuplicateHookError('beforeEach', filePath);
-	}
-
-	suites[filePath].hooks.beforeEach = fn;
-}
-
-/**
- * Teardown step to run once after all iterations of each benchmarking task in a suite.
- */
-export function afterEach(fn: Callback) {
-	const filePath = suiteFilePath();
-
-	if (suites[filePath]?.hooks.afterEach) {
-		throw new DuplicateHookError('afterEach', filePath);
-	}
-
-	suites[filePath].hooks.afterEach = fn;
-}
-
-export const BACKEND_BASE_URL = 'http://localhost:5678';
 
+export { BACKEND_BASE_URL } from './constants';
+export { suite, task, beforeEach, afterEach, collectSuites, registerSuites } from './registration';
+export * as hooks from './hooks';
 export type { Suites } from './types';
+export { UnsupportedDatabaseError } from './errors/unsupported-db.error';
diff --git a/packages/cli/src/benchmark/lib/registration.ts b/packages/cli/src/benchmark/lib/registration.ts
new file mode 100644
index 0000000000000..574bac1ff3eb5
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/registration.ts
@@ -0,0 +1,105 @@
+import 'reflect-metadata';
+import path from 'node:path';
+import type Bench from 'tinybench';
+import { assert } from 'n8n-workflow';
+import glob from 'fast-glob';
+import callsites from 'callsites';
+import type { Suites, Task, Callback } from './types';
+import { DuplicateHookError } from './errors/duplicate-hook.error';
+import { DuplicateSuiteError } from './errors/duplicate-suite.error';
+
+const suites: Suites = {};
+
+export async function collectSuites() {
+	const files = await glob('**/*.suite.js', {
+		cwd: path.join('dist', 'benchmark'),
+		absolute: true,
+	});
+
+	for (const f of files) {
+		await import(f);
+	}
+
+	return suites;
+}
+
+export function registerSuites(bench: Bench) {
+	for (const { hooks, tasks } of Object.values(suites)) {
+		/**
+		 * In tinybench, `beforeAll` and `afterAll` refer to all iterations of
+		 * a single task, while `beforeEach` and `afterEach` refer to each iteration.
+		 *
+		 * In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
+		 * while `beforeEach` and `afterEach` refer to each individual test.
+		 *
+		 * We rename tinybench's hooks to prevent confusion from this difference.
+		 */
+		const options: Record<string, Callback> = {};
+
+		if (hooks.beforeEach) options.beforeAll = hooks.beforeEach;
+		if (hooks.afterEach) options.afterAll = hooks.afterEach;
+
+		for (const t of tasks) {
+			bench.add(t.name, t.operation, options);
+		}
+	}
+}
+
+function suiteFilePath() {
+	const filePath = callsites()
+		.map((site) => site.getFileName())
+		.filter((site): site is string => site !== null)
+		.find((site) => site.endsWith('.suite.js'));
+
+	assert(filePath !== undefined);
+
+	return filePath;
+}
+
+// @TODO: Support async suiteFn
+export function suite(suiteName: string, suiteFn: () => void) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]) throw new DuplicateSuiteError(filePath);
+
+	suites[filePath] = { name: suiteName, hooks: {}, tasks: [] };
+
+	suiteFn();
+}
+
+export function task(taskName: string, operation: Task['operation']) {
+	const filePath = suiteFilePath();
+
+	suites[filePath].tasks.push({
+		name: suites[filePath].name + ' ' + taskName,
+		operation,
+	});
+}
+
+// @TODO: Rename `beforeEach` and `afteEach` to dismbiguate? e.g. `beforeEachTask`, `afterEachTask`
+
+/**
+ * Setup step to run once before all iterations of each benchmarking task in a suite.
+ */
+export function beforeEach(fn: Callback) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]?.hooks.beforeEach) {
+		throw new DuplicateHookError('beforeEach', filePath);
+	}
+
+	suites[filePath].hooks.beforeEach = fn;
+}
+
+/**
+ * Teardown step to run once after all iterations of each benchmarking task in a suite.
+ */
+export function afterEach(fn: Callback) {
+	const filePath = suiteFilePath();
+
+	if (suites[filePath]?.hooks.afterEach) {
+		throw new DuplicateHookError('afterEach', filePath);
+	}
+
+	suites[filePath].hooks.afterEach = fn;
+}
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 1e24ea4f82b5d..77f7ea93e6dba 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -2,9 +2,7 @@ import 'reflect-metadata';
 import Container from 'typedi';
 import config from '@/config';
 import { Logger } from '@/Logger';
-import * as hooks from './lib/hooks';
-import { collectSuites, registerSuites } from './lib';
-import { UnsupportedDatabaseError } from './lib/errors/unsupported-db.error';
+import { collectSuites, registerSuites, UnsupportedDatabaseError, hooks } from './lib';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';

From 8694d62cab857e80b0ce2b6830e10c82102319b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Wed, 24 Apr 2024 10:09:47 +0200
Subject: [PATCH 54/86] Account for no workflow referred to

---
 .../cli/src/benchmark/scripts/list-suites.ts  | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/packages/cli/src/benchmark/scripts/list-suites.ts b/packages/cli/src/benchmark/scripts/list-suites.ts
index 1272fb6a53883..dfc77d98ecce2 100644
--- a/packages/cli/src/benchmark/scripts/list-suites.ts
+++ b/packages/cli/src/benchmark/scripts/list-suites.ts
@@ -4,7 +4,18 @@ import { writeFileSync } from 'node:fs';
 import { collectSuites } from '../lib';
 import type { Suites } from '../lib';
 
-function toSuitesList(suites: Suites) {
+async function exists(filePath: string) {
+	const fullPath = path.resolve('src', 'benchmark', filePath);
+
+	try {
+		await fs.access(fullPath);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+async function toSuitesList(suites: Suites) {
 	let list = '';
 
 	for (const [fullPath, suite] of Object.entries(suites)) {
@@ -16,7 +27,9 @@ function toSuitesList(suites: Suites) {
 			const suiteName = suite.tasks[i].name.replace(suite.name, '').trim();
 			const workflowPath = `./suites/workflows/${suiteId}-${i + 1}.json`;
 
-			list += `- [${suiteName}](${workflowPath})\n`;
+			list += (await exists(workflowPath))
+				? `- [${suiteName}](${workflowPath})\n`
+				: `- ${suiteName}\n`;
 		}
 	}
 
@@ -37,7 +50,7 @@ async function listSuites() {
 	const after = oldDoc.slice(oldDoc.indexOf(MARK_END));
 
 	const suites = await collectSuites();
-	const suitesList = toSuitesList(suites);
+	const suitesList = await toSuitesList(suites);
 
 	const newDoc = [before, suitesList, after].join('\n');
 

From 2bb4f634599a352ec167a43821de81f2a3d7f29e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Wed, 24 Apr 2024 10:25:42 +0200
Subject: [PATCH 55/86] Docs

---
 packages/cli/src/benchmark/benchmark.md | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index 1d3f923c443c0..8b769a05bece1 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -1,16 +1,21 @@
 # Benchmark
 
-To run benchmarks locally in sqlite:
+To run all benchmark suites locally in sqlite:
 
 ```sh
 pnpm benchmark:sqlite
 ```
 
-## Creating
+## Creating a benchmark suite
 
-To create a benchmark, @TODO
+To create a benchmark suite:
 
-## List
+- Add a file to `./suites` following the pattern: `{id}-{description}`
+- Add workflows to `./suites/workflows`. These will all be loaded to the temp DB during setup. If a workflow is triggered by webhook, set the filename as its path for clarity.
+- Use `suite()` for the scenario to benchmark and `task()` for operations in that scenario. Ensure `task()` contains only the specific operation whose execution time will be measured. Move any setup and teardown to `beforeEachTask()` and `afterEachTask()` in the suite.
+- Run `build:benchmark` to add it to the list below.
+
+## Benchmark suites list
 
 > **Note**: All workflows with default settings unless otherwise specified.
 

From dcbd2b7249e99334ac4296716118b509aa3f9f9b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Wed, 24 Apr 2024 10:30:17 +0200
Subject: [PATCH 56/86] Disambiguate

---
 packages/cli/src/benchmark/lib/index.ts       |  9 +++++-
 .../cli/src/benchmark/lib/registration.ts     | 28 +++++++++----------
 packages/cli/src/benchmark/lib/types.ts       |  4 +--
 3 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
index 5d827a43c8b1b..03157d65a35a4 100644
--- a/packages/cli/src/benchmark/lib/index.ts
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -1,7 +1,14 @@
 import 'reflect-metadata';
 
 export { BACKEND_BASE_URL } from './constants';
-export { suite, task, beforeEach, afterEach, collectSuites, registerSuites } from './registration';
+export {
+	suite,
+	task,
+	beforeEachTask,
+	afterEachTask,
+	collectSuites,
+	registerSuites,
+} from './registration';
 export * as hooks from './hooks';
 export type { Suites } from './types';
 export { UnsupportedDatabaseError } from './errors/unsupported-db.error';
diff --git a/packages/cli/src/benchmark/lib/registration.ts b/packages/cli/src/benchmark/lib/registration.ts
index 574bac1ff3eb5..4ec668c4f3074 100644
--- a/packages/cli/src/benchmark/lib/registration.ts
+++ b/packages/cli/src/benchmark/lib/registration.ts
@@ -26,18 +26,18 @@ export async function collectSuites() {
 export function registerSuites(bench: Bench) {
 	for (const { hooks, tasks } of Object.values(suites)) {
 		/**
-		 * In tinybench, `beforeAll` and `afterAll` refer to all iterations of
-		 * a single task, while `beforeEach` and `afterEach` refer to each iteration.
+		 * In tinybench, `beforeAll` and `afterAll` refer to all _iterations_ of
+		 * a single task, while `beforeEach` and `afterEach` refer to each _iteration_.
 		 *
-		 * In jest and vitest, `beforeAll` and `afterAll` refer to all tests in a suite,
-		 * while `beforeEach` and `afterEach` refer to each individual test.
+		 * In jest and vitest, `beforeAll` and `afterAll` refer to all _tests_,
+		 * while `beforeEach` and `afterEach` refer to each _test_.
 		 *
-		 * We rename tinybench's hooks to prevent confusion from this difference.
+		 * We rename tinybench's hooks to prevent confusion from familiarity with jest.
 		 */
 		const options: Record<string, Callback> = {};
 
-		if (hooks.beforeEach) options.beforeAll = hooks.beforeEach;
-		if (hooks.afterEach) options.afterAll = hooks.afterEach;
+		if (hooks.beforeEachTask) options.beforeAll = hooks.beforeEachTask;
+		if (hooks.afterEachTask) options.afterAll = hooks.afterEachTask;
 
 		for (const t of tasks) {
 			bench.add(t.name, t.operation, options);
@@ -76,30 +76,28 @@ export function task(taskName: string, operation: Task['operation']) {
 	});
 }
 
-// @TODO: Rename `beforeEach` and `afteEach` to dismbiguate? e.g. `beforeEachTask`, `afterEachTask`
-
 /**
  * Setup step to run once before all iterations of each benchmarking task in a suite.
  */
-export function beforeEach(fn: Callback) {
+export function beforeEachTask(fn: Callback) {
 	const filePath = suiteFilePath();
 
-	if (suites[filePath]?.hooks.beforeEach) {
+	if (suites[filePath]?.hooks.beforeEachTask) {
 		throw new DuplicateHookError('beforeEach', filePath);
 	}
 
-	suites[filePath].hooks.beforeEach = fn;
+	suites[filePath].hooks.beforeEachTask = fn;
 }
 
 /**
  * Teardown step to run once after all iterations of each benchmarking task in a suite.
  */
-export function afterEach(fn: Callback) {
+export function afterEachTask(fn: Callback) {
 	const filePath = suiteFilePath();
 
-	if (suites[filePath]?.hooks.afterEach) {
+	if (suites[filePath]?.hooks.afterEachTask) {
 		throw new DuplicateHookError('afterEach', filePath);
 	}
 
-	suites[filePath].hooks.afterEach = fn;
+	suites[filePath].hooks.afterEachTask = fn;
 }
diff --git a/packages/cli/src/benchmark/lib/types.ts b/packages/cli/src/benchmark/lib/types.ts
index 6c3a643ada735..872149d57ea03 100644
--- a/packages/cli/src/benchmark/lib/types.ts
+++ b/packages/cli/src/benchmark/lib/types.ts
@@ -5,8 +5,8 @@ export type Suites = {
 	[suiteFilePath: string]: {
 		name: string;
 		hooks: {
-			beforeEach?: Callback;
-			afterEach?: Callback;
+			beforeEachTask?: Callback;
+			afterEachTask?: Callback;
 		};
 		tasks: Task[];
 	};

From fa9f59f5b06f63f69da88d09812b631e7e1bd29d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Wed, 24 Apr 2024 10:32:19 +0200
Subject: [PATCH 57/86] Clarification

---
 packages/cli/src/benchmark/lib/registration.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/cli/src/benchmark/lib/registration.ts b/packages/cli/src/benchmark/lib/registration.ts
index 4ec668c4f3074..3264c089e10ed 100644
--- a/packages/cli/src/benchmark/lib/registration.ts
+++ b/packages/cli/src/benchmark/lib/registration.ts
@@ -32,7 +32,7 @@ export function registerSuites(bench: Bench) {
 		 * In jest and vitest, `beforeAll` and `afterAll` refer to all _tests_,
 		 * while `beforeEach` and `afterEach` refer to each _test_.
 		 *
-		 * We rename tinybench's hooks to prevent confusion from familiarity with jest.
+		 * This API renames tinybench's hooks to prevent confusion from familiarity with jest.
 		 */
 		const options: Record<string, Callback> = {};
 

From 156a054a00a7a04f67856d7bf506f09ea5d40246 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Wed, 24 Apr 2024 10:51:56 +0200
Subject: [PATCH 58/86] Stop benchmarks on error

---
 packages/cli/src/benchmark/main.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 77f7ea93e6dba..74810e0505b95 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -32,6 +32,7 @@ async function main() {
 	const _bench = new Bench({
 		time: 0, // @TODO: Temp value
 		iterations: 1, // @TODO: Temp value
+		throws: true,
 	});
 
 	const bench = process.env.CI === 'true' ? withCodSpeed(_bench) : _bench;

From 133a2b1e448a0c41eabf18b7b19f2f6c81ab54da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 25 Apr 2024 11:25:51 +0200
Subject: [PATCH 59/86] Try 127.0.0.1

---
 packages/cli/src/benchmark/lib/constants.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/cli/src/benchmark/lib/constants.ts b/packages/cli/src/benchmark/lib/constants.ts
index f1fb791dca864..63cce5842eac9 100644
--- a/packages/cli/src/benchmark/lib/constants.ts
+++ b/packages/cli/src/benchmark/lib/constants.ts
@@ -1 +1,6 @@
-export const BACKEND_BASE_URL = 'http://localhost:5678';
+// localhost refuses connections on GH Actions runners
+export const BACKEND_BASE_URL = 'http://127.0.0.1:5678';
+
+// https://stackoverflow.com/a/72785693
+
+// https://stackoverflow.com/a/75881555

From ff53c576408697ea9335b39da98e6daf0f40c74f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 16:15:05 +0200
Subject: [PATCH 60/86] `client` -> `agent`

---
 packages/cli/src/benchmark/lib/agent.ts       | 22 +++++++++++++++++++
 packages/cli/src/benchmark/lib/constants.ts   |  6 ++---
 packages/cli/src/benchmark/lib/hooks.ts       | 14 ++++++------
 packages/cli/src/benchmark/lib/index.ts       |  1 +
 packages/cli/src/benchmark/main.ts            |  2 +-
 ...ebhook-with-authless-webhook-node.suite.ts | 11 ++++------
 6 files changed, 38 insertions(+), 18 deletions(-)
 create mode 100644 packages/cli/src/benchmark/lib/agent.ts

diff --git a/packages/cli/src/benchmark/lib/agent.ts b/packages/cli/src/benchmark/lib/agent.ts
new file mode 100644
index 0000000000000..540c215eea11b
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/agent.ts
@@ -0,0 +1,22 @@
+import axios from 'axios';
+import { BACKEND_BASE_URL, INSTANCE_ONWER_EMAIL, INSTANCE_ONWER_PASSWORD } from './constants';
+import { ApplicationError } from 'n8n-workflow';
+
+export const agent = axios.create({ baseURL: BACKEND_BASE_URL });
+
+export async function authenticateAgent() {
+	const response = await agent.post('/rest/login', {
+		email: INSTANCE_ONWER_EMAIL,
+		password: INSTANCE_ONWER_PASSWORD,
+	});
+
+	const cookies = response.headers['set-cookie'];
+
+	if (!cookies || cookies.length !== 1) {
+		throw new ApplicationError('Expected cookie', { level: 'warning' });
+	}
+
+	const [cookie] = cookies;
+
+	agent.defaults.headers.Cookie = cookie;
+}
diff --git a/packages/cli/src/benchmark/lib/constants.ts b/packages/cli/src/benchmark/lib/constants.ts
index 63cce5842eac9..b50c639af347d 100644
--- a/packages/cli/src/benchmark/lib/constants.ts
+++ b/packages/cli/src/benchmark/lib/constants.ts
@@ -1,6 +1,6 @@
-// localhost refuses connections on GH Actions runners
+// localhost on GH Actions runners refuses connections
 export const BACKEND_BASE_URL = 'http://127.0.0.1:5678';
 
-// https://stackoverflow.com/a/72785693
+export const INSTANCE_ONWER_EMAIL = 'test@test.com';
 
-// https://stackoverflow.com/a/75881555
+export const INSTANCE_ONWER_PASSWORD = 'password';
diff --git a/packages/cli/src/benchmark/lib/hooks.ts b/packages/cli/src/benchmark/lib/hooks.ts
index 044b61ee8db0c..dfbe554fc3c38 100644
--- a/packages/cli/src/benchmark/lib/hooks.ts
+++ b/packages/cli/src/benchmark/lib/hooks.ts
@@ -5,15 +5,14 @@ import { Config } from '@oclif/core';
 import { InstanceSettings } from 'n8n-core';
 import { Start } from '@/commands/start';
 import Container from 'typedi';
-import { WorkflowsController } from '@/workflows/workflows.controller';
 import { UserRepository } from '@/databases/repositories/user.repository';
-import type { User } from '@/databases/entities/User';
 import glob from 'fast-glob';
 import { jsonParse } from 'n8n-workflow';
 import { readFile } from 'fs/promises';
 import type { WorkflowRequest } from '@/workflows/workflow.request';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 import { Logger } from '@/Logger';
+import { agent, authenticateAgent } from './agent';
 
 const logger = Container.get(Logger);
 
@@ -43,7 +42,7 @@ function tempN8nDir() {
 	logger.info(`[Benchmarking] Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
 }
 
-async function loadWorkflows(owner: User) {
+async function loadWorkflows() {
 	const files = await glob('suites/workflows/*.json', {
 		cwd: path.join('dist', 'benchmark'),
 		absolute: true,
@@ -56,9 +55,10 @@ async function loadWorkflows(owner: User) {
 		workflows.push(jsonParse<WorkflowRequest.CreatePayload>(content));
 	}
 
+	await authenticateAgent();
+
 	for (const workflow of workflows) {
-		// @ts-ignore @TODO Fix typing
-		await Container.get(WorkflowsController).create({ body: workflow, user: owner });
+		await agent.post('/rest/workflows', workflow);
 	}
 }
 
@@ -72,9 +72,9 @@ export async function globalSetup() {
 	await main.init();
 	await main.run();
 
-	const owner = await Container.get(UserRepository).createTestOwner();
+	await Container.get(UserRepository).createTestOwner();
 
-	await loadWorkflows(owner);
+	await loadWorkflows();
 
 	await Container.get(ActiveWorkflowRunner).init();
 }
diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
index 03157d65a35a4..6580a17880bb4 100644
--- a/packages/cli/src/benchmark/lib/index.ts
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -10,5 +10,6 @@ export {
 	registerSuites,
 } from './registration';
 export * as hooks from './hooks';
+export { agent } from './agent';
 export type { Suites } from './types';
 export { UnsupportedDatabaseError } from './errors/unsupported-db.error';
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 74810e0505b95..bad4b8d27f4cc 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -41,7 +41,7 @@ async function main() {
 
 	await bench.run();
 
-	console.table(bench.table()); // @TODO: Output properly? Ref. Codspeed
+	if (process.env.CI !== 'true') console.table(bench.table());
 
 	await hooks.globalTeardown();
 }
diff --git a/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts b/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts
index 07068d635edc8..72b9bbe8aa73a 100644
--- a/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts
+++ b/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts
@@ -1,18 +1,15 @@
-import axios from 'axios';
-import { task, suite, BACKEND_BASE_URL } from '../lib';
+import { task, suite, agent } from '../lib';
 
 suite('Production workflow with authless webhook node', () => {
-	const client = axios.create({ baseURL: BACKEND_BASE_URL });
-
 	task('using "Respond immediately" mode', async () => {
-		await client.get('/webhook/001-1');
+		await agent.get('/webhook/001-1');
 	});
 
 	task('using "When last node finishes" mode', async () => {
-		await client.get('/webhook/001-2');
+		await agent.get('/webhook/001-2');
 	});
 
 	task('using "Respond to Webhook" node mode', async () => {
-		await client.get('/webhook/001-3');
+		await agent.get('/webhook/001-3');
 	});
 });

From ea2cc83f23fa21356e9b6acefdb7c6a23dd6a07e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 16:31:50 +0200
Subject: [PATCH 61/86] Deduplicate repository logic

---
 packages/cli/src/benchmark/lib/hooks.ts       |  5 +--
 .../repositories/user.benchmark-repository.ts | 26 ++++++++++++++
 packages/cli/src/benchmark/main.ts            |  1 +
 .../databases/repositories/user.repository.ts | 34 +------------------
 4 files changed, 31 insertions(+), 35 deletions(-)
 create mode 100644 packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts

diff --git a/packages/cli/src/benchmark/lib/hooks.ts b/packages/cli/src/benchmark/lib/hooks.ts
index dfbe554fc3c38..b169e770196a4 100644
--- a/packages/cli/src/benchmark/lib/hooks.ts
+++ b/packages/cli/src/benchmark/lib/hooks.ts
@@ -5,7 +5,6 @@ import { Config } from '@oclif/core';
 import { InstanceSettings } from 'n8n-core';
 import { Start } from '@/commands/start';
 import Container from 'typedi';
-import { UserRepository } from '@/databases/repositories/user.repository';
 import glob from 'fast-glob';
 import { jsonParse } from 'n8n-workflow';
 import { readFile } from 'fs/promises';
@@ -13,6 +12,7 @@ import type { WorkflowRequest } from '@/workflows/workflow.request';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 import { Logger } from '@/Logger';
 import { agent, authenticateAgent } from './agent';
+import { UserRepository } from './repositories/user.benchmark-repository';
 
 const logger = Container.get(Logger);
 
@@ -72,7 +72,8 @@ export async function globalSetup() {
 	await main.init();
 	await main.run();
 
-	await Container.get(UserRepository).createTestOwner();
+	await Container.get(UserRepository).deleteInstanceOwner();
+	await Container.get(UserRepository).createInstanceOwner();
 
 	await loadWorkflows();
 
diff --git a/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts b/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
new file mode 100644
index 0000000000000..00e5047d2cbda
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
@@ -0,0 +1,26 @@
+import { Service } from 'typedi';
+import { hash } from 'bcryptjs';
+import { UserRepository as ProductionUserRepository } from '@/databases/repositories/user.repository';
+import { INSTANCE_ONWER_EMAIL, INSTANCE_ONWER_PASSWORD } from '../constants';
+
+/**
+ * User repository for use in benchmarking suites only.
+ */
+@Service()
+export class UserRepository extends ProductionUserRepository {
+	async createInstanceOwner() {
+		const user = this.create({
+			email: INSTANCE_ONWER_EMAIL,
+			password: await hash(INSTANCE_ONWER_PASSWORD, 10),
+			firstName: 'John',
+			lastName: 'Smith',
+			role: 'global:owner',
+			apiKey:
+				'n8n_api_96a4804143bdeb044802273c93423c33e1582c89a764c645fd6304fd2df19f3b2c73f4b972e28194',
+		});
+
+		user.computeIsOwner();
+
+		return await this.save(user);
+	}
+}
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index bad4b8d27f4cc..cd78078bb0ca9 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -29,6 +29,7 @@ async function main() {
 
 	await hooks.globalSetup();
 
+	// @TODO: Make these values configurable
 	const _bench = new Bench({
 		time: 0, // @TODO: Temp value
 		iterations: 1, // @TODO: Temp value
diff --git a/packages/cli/src/databases/repositories/user.repository.ts b/packages/cli/src/databases/repositories/user.repository.ts
index 2ab41f37260a5..29ebdd59fbec4 100644
--- a/packages/cli/src/databases/repositories/user.repository.ts
+++ b/packages/cli/src/databases/repositories/user.repository.ts
@@ -105,39 +105,7 @@ export class UserRepository extends Repository<User> {
 		});
 	}
 
-	// ----------------------------------
-	//           test utils
-	// ----------------------------------
-
-	// @TODO: Deduplicate with /packages/cli/test/integration/shared/db/users.ts
-
-	async createTestUser(attributes: Partial<User> = {}): Promise<User> {
-		const { email, password, firstName, lastName, role, ...rest } = attributes;
-
-		// pre-computed bcrypt hash for the string 'password', using `await hash('password', 10)`
-		const passwordHash = '$2a$10$njedH7S6V5898mj6p0Jr..IGY9Ms.qNwR7RbSzzX9yubJocKfvGGK';
-
-		return this.create({
-			email: 'test@test.com',
-			password: passwordHash,
-			firstName: 'John',
-			lastName: 'Smith',
-			role: role ?? 'global:member',
-			...rest,
-		});
-	}
-
-	async createUser(attributes: Partial<User> = {}): Promise<User> {
-		const user = await this.createTestUser(attributes);
-
-		user.computeIsOwner();
-
-		return await this.save(user);
-	}
-
-	async createTestOwner() {
+	async deleteInstanceOwner() {
 		await this.delete({ role: 'global:owner', email: IsNull() });
-
-		return await this.createUser({ role: 'global:owner' });
 	}
 }

From 26c8e81f91301b9b119bca1bbd6c7d040d743e12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 16:49:23 +0200
Subject: [PATCH 62/86] Simplify setup and teardown

---
 .../lib/errors/duplicate-hook.error.ts        |  2 +-
 .../lib/errors/unsupported-db.error.ts        | 10 ---
 packages/cli/src/benchmark/lib/hooks.ts       | 85 -------------------
 .../cli/src/benchmark/lib/hooks/n8nDir.ts     | 34 ++++++++
 packages/cli/src/benchmark/lib/hooks/seed.ts  | 35 ++++++++
 .../benchmark/lib/hooks/setup-and-teardown.ts | 26 ++++++
 packages/cli/src/benchmark/lib/index.ts       |  7 +-
 .../cli/src/benchmark/lib/registration.ts     | 10 +--
 .../repositories/user.benchmark-repository.ts |  3 -
 packages/cli/src/benchmark/main.ts            | 11 +--
 10 files changed, 105 insertions(+), 118 deletions(-)
 delete mode 100644 packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts
 delete mode 100644 packages/cli/src/benchmark/lib/hooks.ts
 create mode 100644 packages/cli/src/benchmark/lib/hooks/n8nDir.ts
 create mode 100644 packages/cli/src/benchmark/lib/hooks/seed.ts
 create mode 100644 packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts

diff --git a/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts b/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
index 7aa4692a0c0c8..60aff18a50fdf 100644
--- a/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
+++ b/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
@@ -1,7 +1,7 @@
 import { ApplicationError } from 'n8n-workflow';
 
 export class DuplicateHookError extends ApplicationError {
-	constructor(hookName: 'beforeEach' | 'afterEach', filePath: string) {
+	constructor(hookName: 'beforeEachTask' | 'afterEachTask', filePath: string) {
 		super(
 			`Duplicate \`${hookName}\` hook found at \`${filePath}\`. Please define a single \`${hookName}\` hook for this file.`,
 			{ level: 'warning' },
diff --git a/packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts b/packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts
deleted file mode 100644
index e1c96fc88e6ce..0000000000000
--- a/packages/cli/src/benchmark/lib/errors/unsupported-db.error.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import { ApplicationError } from 'n8n-workflow';
-
-export class UnsupportedDatabaseError extends ApplicationError {
-	constructor() {
-		super(
-			'Currently only sqlite is supported for benchmarking. Please ensure DB_TYPE is set to `sqlite`',
-			{ level: 'warning' },
-		);
-	}
-}
diff --git a/packages/cli/src/benchmark/lib/hooks.ts b/packages/cli/src/benchmark/lib/hooks.ts
deleted file mode 100644
index b169e770196a4..0000000000000
--- a/packages/cli/src/benchmark/lib/hooks.ts
+++ /dev/null
@@ -1,85 +0,0 @@
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
-import { Config } from '@oclif/core';
-import { InstanceSettings } from 'n8n-core';
-import { Start } from '@/commands/start';
-import Container from 'typedi';
-import glob from 'fast-glob';
-import { jsonParse } from 'n8n-workflow';
-import { readFile } from 'fs/promises';
-import type { WorkflowRequest } from '@/workflows/workflow.request';
-import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
-import { Logger } from '@/Logger';
-import { agent, authenticateAgent } from './agent';
-import { UserRepository } from './repositories/user.benchmark-repository';
-
-const logger = Container.get(Logger);
-
-function tempN8nDir() {
-	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
-
-	mkdirSync(baseDirPath, { recursive: true });
-
-	const userDir = mkdtempSync(baseDirPath);
-
-	const _n8nDir = path.join(userDir, '.n8n');
-
-	mkdirSync(_n8nDir);
-
-	writeFileSync(
-		path.join(_n8nDir, 'config'),
-		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: 'temp-123' }),
-		'utf-8',
-	);
-
-	// @TODO: Find better approach than overriding like this
-	// Setting N8N_USER_FOLDER has no effect
-	const instanceSettings = Container.get(InstanceSettings);
-	instanceSettings.n8nFolder = _n8nDir;
-	Container.set(InstanceSettings, instanceSettings);
-
-	logger.info(`[Benchmarking] Temp .n8n dir location: ${instanceSettings.n8nFolder}`);
-}
-
-async function loadWorkflows() {
-	const files = await glob('suites/workflows/*.json', {
-		cwd: path.join('dist', 'benchmark'),
-		absolute: true,
-	});
-
-	const workflows: WorkflowRequest.CreatePayload[] = [];
-
-	for (const file of files) {
-		const content = await readFile(file, 'utf8');
-		workflows.push(jsonParse<WorkflowRequest.CreatePayload>(content));
-	}
-
-	await authenticateAgent();
-
-	for (const workflow of workflows) {
-		await agent.post('/rest/workflows', workflow);
-	}
-}
-
-let main: Start;
-
-export async function globalSetup() {
-	tempN8nDir();
-
-	main = new Start([], new Config({ root: __dirname }));
-
-	await main.init();
-	await main.run();
-
-	await Container.get(UserRepository).deleteInstanceOwner();
-	await Container.get(UserRepository).createInstanceOwner();
-
-	await loadWorkflows();
-
-	await Container.get(ActiveWorkflowRunner).init();
-}
-
-export async function globalTeardown() {
-	await main.stopProcess();
-}
diff --git a/packages/cli/src/benchmark/lib/hooks/n8nDir.ts b/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
new file mode 100644
index 0000000000000..b966c402cccd3
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
@@ -0,0 +1,34 @@
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
+import Container from 'typedi';
+import { InstanceSettings } from 'n8n-core';
+import { Logger } from '@/Logger';
+
+export function n8nDir() {
+	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
+
+	mkdirSync(baseDirPath, { recursive: true });
+
+	const userDir = mkdtempSync(baseDirPath);
+
+	const n8nDirPath = path.join(userDir, '.n8n');
+
+	mkdirSync(n8nDirPath);
+
+	writeFileSync(
+		path.join(n8nDirPath, 'config'),
+		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: 'temp-123' }),
+		'utf-8',
+	);
+
+	// @TODO: Find better approach than overriding like this
+	// Setting N8N_USER_FOLDER has no effect
+	const instanceSettings = Container.get(InstanceSettings);
+	instanceSettings.n8nFolder = n8nDirPath;
+	Container.set(InstanceSettings, instanceSettings);
+
+	Container.get(Logger).info(
+		`[Benchmarking] Temp .n8n dir location: ${instanceSettings.n8nFolder}`,
+	);
+}
diff --git a/packages/cli/src/benchmark/lib/hooks/seed.ts b/packages/cli/src/benchmark/lib/hooks/seed.ts
new file mode 100644
index 0000000000000..0f5a7500d7e17
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/hooks/seed.ts
@@ -0,0 +1,35 @@
+import path from 'node:path';
+import glob from 'fast-glob';
+import { jsonParse } from 'n8n-workflow';
+import { readFile } from 'fs/promises';
+import type { WorkflowRequest } from '@/workflows/workflow.request';
+import { agent, authenticateAgent } from '../agent';
+import Container from 'typedi';
+import { UserRepository } from '../repositories/user.benchmark-repository';
+// @TODO: @benchmark path
+
+export async function seedInstanceOwner() {
+	await Container.get(UserRepository).deleteInstanceOwner();
+	await Container.get(UserRepository).createInstanceOwner();
+}
+
+export async function seedWorkflows() {
+	const files = await glob('suites/workflows/*.json', {
+		cwd: path.join('dist', 'benchmark'),
+		absolute: true,
+	});
+
+	const workflowpayloads: WorkflowRequest.CreateUpdatePayload[] = [];
+
+	for (const file of files) {
+		const json = await readFile(file, 'utf8');
+		const payload = jsonParse<WorkflowRequest.CreatePayload>(json);
+		workflowpayloads.push(payload);
+	}
+
+	await authenticateAgent();
+
+	for (const workflow of workflowpayloads) {
+		await agent.post('/rest/workflows', workflow);
+	}
+}
diff --git a/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts b/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
new file mode 100644
index 0000000000000..9266ae263cb22
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
@@ -0,0 +1,26 @@
+import Container from 'typedi';
+import { Config } from '@oclif/core';
+import { Start } from '@/commands/start';
+import { n8nDir } from './n8nDir';
+import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
+import { seedInstanceOwner, seedWorkflows } from './seed';
+
+let main: Start;
+
+export async function setup() {
+	n8nDir();
+
+	main = new Start([], new Config({ root: __dirname }));
+
+	await main.init();
+	await main.run();
+
+	await seedInstanceOwner();
+	await seedWorkflows();
+
+	await Container.get(ActiveWorkflowRunner).init();
+}
+
+export async function teardown() {
+	await main.stopProcess();
+}
diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
index 6580a17880bb4..3d5de6cbd6f3e 100644
--- a/packages/cli/src/benchmark/lib/index.ts
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -1,6 +1,5 @@
 import 'reflect-metadata';
 
-export { BACKEND_BASE_URL } from './constants';
 export {
 	suite,
 	task,
@@ -9,7 +8,9 @@ export {
 	collectSuites,
 	registerSuites,
 } from './registration';
-export * as hooks from './hooks';
+
 export { agent } from './agent';
+
 export type { Suites } from './types';
-export { UnsupportedDatabaseError } from './errors/unsupported-db.error';
+
+export { setup, teardown } from './hooks/setup-and-teardown';
diff --git a/packages/cli/src/benchmark/lib/registration.ts b/packages/cli/src/benchmark/lib/registration.ts
index 3264c089e10ed..72f91ed97b396 100644
--- a/packages/cli/src/benchmark/lib/registration.ts
+++ b/packages/cli/src/benchmark/lib/registration.ts
@@ -76,27 +76,21 @@ export function task(taskName: string, operation: Task['operation']) {
 	});
 }
 
-/**
- * Setup step to run once before all iterations of each benchmarking task in a suite.
- */
 export function beforeEachTask(fn: Callback) {
 	const filePath = suiteFilePath();
 
 	if (suites[filePath]?.hooks.beforeEachTask) {
-		throw new DuplicateHookError('beforeEach', filePath);
+		throw new DuplicateHookError('beforeEachTask', filePath);
 	}
 
 	suites[filePath].hooks.beforeEachTask = fn;
 }
 
-/**
- * Teardown step to run once after all iterations of each benchmarking task in a suite.
- */
 export function afterEachTask(fn: Callback) {
 	const filePath = suiteFilePath();
 
 	if (suites[filePath]?.hooks.afterEachTask) {
-		throw new DuplicateHookError('afterEach', filePath);
+		throw new DuplicateHookError('afterEachTask', filePath);
 	}
 
 	suites[filePath].hooks.afterEachTask = fn;
diff --git a/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts b/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
index 00e5047d2cbda..b02a9fccb6771 100644
--- a/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
+++ b/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
@@ -3,9 +3,6 @@ import { hash } from 'bcryptjs';
 import { UserRepository as ProductionUserRepository } from '@/databases/repositories/user.repository';
 import { INSTANCE_ONWER_EMAIL, INSTANCE_ONWER_PASSWORD } from '../constants';
 
-/**
- * User repository for use in benchmarking suites only.
- */
 @Service()
 export class UserRepository extends ProductionUserRepository {
 	async createInstanceOwner() {
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index cd78078bb0ca9..029aca4edc79c 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,8 +1,7 @@
 import 'reflect-metadata';
 import Container from 'typedi';
-import config from '@/config';
 import { Logger } from '@/Logger';
-import { collectSuites, registerSuites, UnsupportedDatabaseError, hooks } from './lib';
+import { collectSuites, registerSuites, setup, teardown } from './lib';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';
@@ -10,10 +9,6 @@ import { withCodSpeed } from '@codspeed/tinybench-plugin';
 /* eslint-enable import/no-extraneous-dependencies */
 
 async function main() {
-	const dbType = config.getEnv('database.type');
-
-	if (dbType !== 'sqlite') throw new UnsupportedDatabaseError();
-
 	const suites = await collectSuites();
 
 	const count = Object.keys(suites).length;
@@ -27,7 +22,7 @@ async function main() {
 
 	logger.info(`[Benchmarking] Running ${count} ${count === 1 ? 'suite' : 'suites'}...`);
 
-	await hooks.globalSetup();
+	await setup();
 
 	// @TODO: Make these values configurable
 	const _bench = new Bench({
@@ -44,7 +39,7 @@ async function main() {
 
 	if (process.env.CI !== 'true') console.table(bench.table());
 
-	await hooks.globalTeardown();
+	await teardown();
 }
 
 void main();

From 54f5463b5f9b4de0139f120f890dbe14f0073148 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 16:52:04 +0200
Subject: [PATCH 63/86] `registration` -> `api`

---
 .../src/benchmark/lib/{registration.ts => api.ts}  |  0
 packages/cli/src/benchmark/lib/constants.ts        |  3 +--
 packages/cli/src/benchmark/lib/hooks/seed.ts       |  9 ++++-----
 packages/cli/src/benchmark/lib/index.ts            | 14 ++------------
 4 files changed, 7 insertions(+), 19 deletions(-)
 rename packages/cli/src/benchmark/lib/{registration.ts => api.ts} (100%)

diff --git a/packages/cli/src/benchmark/lib/registration.ts b/packages/cli/src/benchmark/lib/api.ts
similarity index 100%
rename from packages/cli/src/benchmark/lib/registration.ts
rename to packages/cli/src/benchmark/lib/api.ts
diff --git a/packages/cli/src/benchmark/lib/constants.ts b/packages/cli/src/benchmark/lib/constants.ts
index b50c639af347d..09eb199bf8170 100644
--- a/packages/cli/src/benchmark/lib/constants.ts
+++ b/packages/cli/src/benchmark/lib/constants.ts
@@ -1,5 +1,4 @@
-// localhost on GH Actions runners refuses connections
-export const BACKEND_BASE_URL = 'http://127.0.0.1:5678';
+export const BACKEND_BASE_URL = 'http://127.0.0.1:5678'; // localhost on GitHub Actions runners refuses connections
 
 export const INSTANCE_ONWER_EMAIL = 'test@test.com';
 
diff --git a/packages/cli/src/benchmark/lib/hooks/seed.ts b/packages/cli/src/benchmark/lib/hooks/seed.ts
index 0f5a7500d7e17..0d6e96667fbcd 100644
--- a/packages/cli/src/benchmark/lib/hooks/seed.ts
+++ b/packages/cli/src/benchmark/lib/hooks/seed.ts
@@ -6,7 +6,6 @@ import type { WorkflowRequest } from '@/workflows/workflow.request';
 import { agent, authenticateAgent } from '../agent';
 import Container from 'typedi';
 import { UserRepository } from '../repositories/user.benchmark-repository';
-// @TODO: @benchmark path
 
 export async function seedInstanceOwner() {
 	await Container.get(UserRepository).deleteInstanceOwner();
@@ -19,17 +18,17 @@ export async function seedWorkflows() {
 		absolute: true,
 	});
 
-	const workflowpayloads: WorkflowRequest.CreateUpdatePayload[] = [];
+	const payloads: WorkflowRequest.CreateUpdatePayload[] = [];
 
 	for (const file of files) {
 		const json = await readFile(file, 'utf8');
 		const payload = jsonParse<WorkflowRequest.CreatePayload>(json);
-		workflowpayloads.push(payload);
+		payloads.push(payload);
 	}
 
 	await authenticateAgent();
 
-	for (const workflow of workflowpayloads) {
-		await agent.post('/rest/workflows', workflow);
+	for (const p of payloads) {
+		await agent.post('/rest/workflows', p);
 	}
 }
diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
index 3d5de6cbd6f3e..497eaabc4da0c 100644
--- a/packages/cli/src/benchmark/lib/index.ts
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -1,16 +1,6 @@
 import 'reflect-metadata';
 
-export {
-	suite,
-	task,
-	beforeEachTask,
-	afterEachTask,
-	collectSuites,
-	registerSuites,
-} from './registration';
-
+export { suite, task, beforeEachTask, afterEachTask, collectSuites, registerSuites } from './api';
 export { agent } from './agent';
-
-export type { Suites } from './types';
-
 export { setup, teardown } from './hooks/setup-and-teardown';
+export type { Suites } from './types';

From 7a4be563b927274431098192e93861072f56a99d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 17:16:23 +0200
Subject: [PATCH 64/86] Make bench values configurable

---
 packages/cli/src/benchmark/main.ts | 13 ++++++++----
 packages/cli/src/config/schema.ts  | 33 ++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 029aca4edc79c..ac796d55dec4a 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -2,6 +2,7 @@ import 'reflect-metadata';
 import Container from 'typedi';
 import { Logger } from '@/Logger';
 import { collectSuites, registerSuites, setup, teardown } from './lib';
+import config from '@/config';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';
@@ -24,11 +25,15 @@ async function main() {
 
 	await setup();
 
-	// @TODO: Make these values configurable
 	const _bench = new Bench({
-		time: 0, // @TODO: Temp value
-		iterations: 1, // @TODO: Temp value
-		throws: true,
+		// @TODO: Temp values
+		time: 0,
+		iterations: 1,
+		// time: config.getEnv('benchmark.time'),
+		// iterations: config.getEnv('benchmark.iterations'),
+		throws: config.getEnv('benchmark.stopOnError'),
+		warmupTime: config.getEnv('benchmark.warmupTime'),
+		warmupIterations: config.getEnv('benchmark.warmupIterations'),
 	});
 
 	const bench = process.env.CI === 'true' ? withCodSpeed(_bench) : _bench;
diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts
index c4177548c6f01..a11eb072488e5 100644
--- a/packages/cli/src/config/schema.ts
+++ b/packages/cli/src/config/schema.ts
@@ -1436,4 +1436,37 @@ export const schema = {
 		env: 'N8N_PROXY_HOPS',
 		doc: 'Number of reverse-proxies n8n is running behind',
 	},
+
+	benchmark: {
+		time: {
+			doc: 'Length of time (ms) during which to repeatedly run a benchmarking task',
+			format: Number,
+			default: 500,
+			env: 'N8N_BENCHMARK_TIME',
+		},
+		iterations: {
+			doc: 'Number of times to run a benchmarking task, even if `N8N_BENCHMARK_TIME` is exceeded',
+			format: Number,
+			default: 10,
+			env: 'N8N_BENCHMARK_ITERATIONS',
+		},
+		stopOnError: {
+			doc: 'Whether to stop benchmarking if an error occurs in a task',
+			format: Boolean,
+			default: true,
+			env: 'N8N_BENCHMARK_STOP_ON_ERROR',
+		},
+		warmupTime: {
+			doc: 'Length of time (ms) during which to repeatedly run a benchmarking task for warmup',
+			format: Number,
+			default: 100,
+			env: 'N8N_BENCHMARK_WARMUP_TIME',
+		},
+		warmupIterations: {
+			doc: 'Number of times to run a benchmarking task for warmup, even if `N8N_BENCHMARK_WARMUP_TIME` is exceeded',
+			format: Number,
+			default: 5,
+			env: 'N8N_BENCHMARK_WARMUP_ITERATIONS',
+		},
+	},
 };

From 02f4809f91f67bf797b67c1450735f755c6ebc17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 17:18:48 +0200
Subject: [PATCH 65/86] Remove commented out start code

---
 packages/cli/src/commands/start.ts | 58 +++++++++++++++---------------
 1 file changed, 30 insertions(+), 28 deletions(-)

diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts
index cc3432774f182..3164d3c2b9e88 100644
--- a/packages/cli/src/commands/start.ts
+++ b/packages/cli/src/commands/start.ts
@@ -289,35 +289,37 @@ export class Start extends BaseCommand {
 		const editorUrl = Container.get(UrlService).baseUrl;
 		this.log(`\nEditor is now accessible via:\n${editorUrl}`);
 
+		if (inBenchmark) return;
+
 		// Allow to open n8n editor by pressing "o"
-		// if (Boolean(process.stdout.isTTY) && process.stdin.setRawMode) {
-		// 	process.stdin.setRawMode(true);
-		// 	process.stdin.resume();
-		// 	process.stdin.setEncoding('utf8');
-
-		// 	if (flags.open) {
-		// 		this.openBrowser();
-		// 	}
-		// 	this.log('\nPress "o" to open in Browser.');
-		// 	process.stdin.on('data', (key: string) => {
-		// 		if (key === 'o') {
-		// 			this.openBrowser();
-		// 		} else if (key.charCodeAt(0) === 3) {
-		// 			// Ctrl + c got pressed
-		// 			void this.stopProcess();
-		// 		} else {
-		// 			// When anything else got pressed, record it and send it on enter into the child process
-
-		// 			if (key.charCodeAt(0) === 13) {
-		// 				// send to child process and print in terminal
-		// 				process.stdout.write('\n');
-		// 			} else {
-		// 				// record it and write into terminal
-		// 				process.stdout.write(key);
-		// 			}
-		// 		}
-		// 	});
-		// }
+		if (Boolean(process.stdout.isTTY) && process.stdin.setRawMode) {
+			process.stdin.setRawMode(true);
+			process.stdin.resume();
+			process.stdin.setEncoding('utf8');
+
+			if (flags.open) {
+				this.openBrowser();
+			}
+			this.log('\nPress "o" to open in Browser.');
+			process.stdin.on('data', (key: string) => {
+				if (key === 'o') {
+					this.openBrowser();
+				} else if (key.charCodeAt(0) === 3) {
+					// Ctrl + c got pressed
+					void this.stopProcess();
+				} else {
+					// When anything else got pressed, record it and send it on enter into the child process
+
+					if (key.charCodeAt(0) === 13) {
+						// send to child process and print in terminal
+						process.stdout.write('\n');
+					} else {
+						// record it and write into terminal
+						process.stdout.write(key);
+					}
+				}
+			});
+		}
 	}
 
 	async initPruning() {

From 280caae19e8fe9f6a439700574eb3b870ffd8eb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 17:21:09 +0200
Subject: [PATCH 66/86] Remove outdated comment

---
 packages/cli/src/benchmark/lib/api.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/cli/src/benchmark/lib/api.ts b/packages/cli/src/benchmark/lib/api.ts
index 72f91ed97b396..4d2e9d5e66551 100644
--- a/packages/cli/src/benchmark/lib/api.ts
+++ b/packages/cli/src/benchmark/lib/api.ts
@@ -56,7 +56,6 @@ function suiteFilePath() {
 	return filePath;
 }
 
-// @TODO: Support async suiteFn
 export function suite(suiteName: string, suiteFn: () => void) {
 	const filePath = suiteFilePath();
 

From c8bf11b31d3db35b352fb3740f65da0424051ce3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 17:22:55 +0200
Subject: [PATCH 67/86] Typo

---
 .../001-production-webhook-with-authless-webhook-node.suite.ts  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts b/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts
index 72b9bbe8aa73a..a31f681298b53 100644
--- a/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts
+++ b/packages/cli/src/benchmark/suites/001-production-webhook-with-authless-webhook-node.suite.ts
@@ -9,7 +9,7 @@ suite('Production workflow with authless webhook node', () => {
 		await agent.get('/webhook/001-2');
 	});
 
-	task('using "Respond to Webhook" node mode', async () => {
+	task('using "Respond to Webhook node" mode', async () => {
 		await agent.get('/webhook/001-3');
 	});
 });

From b61f8c101377731191f9db4ca0ecd887bcdbc439 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 18:08:40 +0200
Subject: [PATCH 68/86] Better solution for `InstanceSettings` user home dir

---
 packages/cli/src/benchmark/benchmark.md       |  2 +-
 .../cli/src/benchmark/lib/hooks/n8nDir.ts     | 30 +++++++++----------
 packages/core/src/InstanceSettings.ts         |  2 +-
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index 8b769a05bece1..9063f96d46321 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -25,6 +25,6 @@ To create a benchmark suite:
 
 - [using "Respond immediately" mode](./suites/workflows/001-1.json)
 - [using "When last node finishes" mode](./suites/workflows/001-2.json)
-- [using "Respond to Webhook" node mode](./suites/workflows/001-3.json)
+- [using "Respond to Webhook node" mode](./suites/workflows/001-3.json)
 
 <!-- /BENCHMARK_SUITES_LIST -->
diff --git a/packages/cli/src/benchmark/lib/hooks/n8nDir.ts b/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
index b966c402cccd3..14b3944a93fec 100644
--- a/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
+++ b/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
@@ -3,32 +3,32 @@ import path from 'node:path';
 import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
 import Container from 'typedi';
 import { InstanceSettings } from 'n8n-core';
-import { Logger } from '@/Logger';
 
+/**
+ * Create a temp .n8n user dir for benchmarking.
+ */
 export function n8nDir() {
-	const baseDirPath = path.join(tmpdir(), 'n8n-benchmarks/');
+	const tempBaseDir = path.join(tmpdir(), 'n8n-benchmarks/');
 
-	mkdirSync(baseDirPath, { recursive: true });
+	mkdirSync(tempBaseDir, { recursive: true });
 
-	const userDir = mkdtempSync(baseDirPath);
+	const tempUserHomeDir = mkdtempSync(tempBaseDir);
 
-	const n8nDirPath = path.join(userDir, '.n8n');
+	const tempN8nDir = path.join(tempUserHomeDir, '.n8n');
 
-	mkdirSync(n8nDirPath);
+	mkdirSync(tempN8nDir);
 
 	writeFileSync(
-		path.join(n8nDirPath, 'config'),
+		path.join(tempN8nDir, 'config'),
 		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: 'temp-123' }),
 		'utf-8',
 	);
 
-	// @TODO: Find better approach than overriding like this
-	// Setting N8N_USER_FOLDER has no effect
-	const instanceSettings = Container.get(InstanceSettings);
-	instanceSettings.n8nFolder = n8nDirPath;
-	Container.set(InstanceSettings, instanceSettings);
+	process.env.N8N_USER_FOLDER = tempUserHomeDir;
 
-	Container.get(Logger).info(
-		`[Benchmarking] Temp .n8n dir location: ${instanceSettings.n8nFolder}`,
-	);
+	/**
+	 * `typedi` has already instantiated `InstanceSettings` using the default user home,
+	 * so re-instantiate it to ensure it picks up the temp user home dir path.
+	 */
+	Container.set(InstanceSettings, new InstanceSettings());
 }
diff --git a/packages/core/src/InstanceSettings.ts b/packages/core/src/InstanceSettings.ts
index 66f127b2d9900..b5aeb2d2175b6 100644
--- a/packages/core/src/InstanceSettings.ts
+++ b/packages/core/src/InstanceSettings.ts
@@ -21,7 +21,7 @@ export class InstanceSettings {
 	private readonly userHome = this.getUserHome();
 
 	/** The path to the n8n folder in which all n8n related data gets saved */
-	n8nFolder = path.join(this.userHome, '.n8n'); // @TODO: Solution that keeps this readonly
+	readonly n8nFolder = path.join(this.userHome, '.n8n'); // @TODO: Solution that keeps this readonly
 
 	/** The path to the folder where all generated static assets are copied to */
 	readonly staticCacheDir = path.join(this.userHome, '.cache/n8n/public');

From def86d24388b51a408a9507af072f3293598a24d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 18:19:39 +0200
Subject: [PATCH 69/86] Comment out unneeded steps

---
 .github/workflows/benchmark.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 943d1740f2c41..2c141d368ca8f 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -30,16 +30,16 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Build
-        if: ${{ inputs.cacheKey == '' }}
-        run: pnpm build
+      # - name: Build
+      #   if: ${{ inputs.cacheKey == '' }}
+      #   run: pnpm build
 
-      - name: Restore cached build artifacts
-        if: ${{ inputs.cacheKey != '' }}
-        uses: actions/cache/restore@v4.0.0
-        with:
-          path: ./packages/**/dist
-          key: ${{ inputs.cacheKey }}
+      # - name: Restore cached build artifacts
+      #   if: ${{ inputs.cacheKey != '' }}
+      #   uses: actions/cache/restore@v4.0.0
+      #   with:
+      #     path: ./packages/**/dist
+      #     key: ${{ inputs.cacheKey }}
 
       - name: Benchmark
         uses: CodSpeedHQ/action@v2

From d94d312374853f6de2ec0bfbe96720729e66b56a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 18:25:14 +0200
Subject: [PATCH 70/86] Remove outdated comment

---
 packages/core/src/InstanceSettings.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/core/src/InstanceSettings.ts b/packages/core/src/InstanceSettings.ts
index b5aeb2d2175b6..e8ab9aa553223 100644
--- a/packages/core/src/InstanceSettings.ts
+++ b/packages/core/src/InstanceSettings.ts
@@ -21,7 +21,7 @@ export class InstanceSettings {
 	private readonly userHome = this.getUserHome();
 
 	/** The path to the n8n folder in which all n8n related data gets saved */
-	readonly n8nFolder = path.join(this.userHome, '.n8n'); // @TODO: Solution that keeps this readonly
+	readonly n8nFolder = path.join(this.userHome, '.n8n');
 
 	/** The path to the folder where all generated static assets are copied to */
 	readonly staticCacheDir = path.join(this.userHome, '.cache/n8n/public');

From 732b358b60f08780727e57a25643f2a15fa9e4b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 18:25:53 +0200
Subject: [PATCH 71/86] Restore steps

---
 .github/workflows/benchmark.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 2c141d368ca8f..943d1740f2c41 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -30,16 +30,16 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
-      # - name: Build
-      #   if: ${{ inputs.cacheKey == '' }}
-      #   run: pnpm build
+      - name: Build
+        if: ${{ inputs.cacheKey == '' }}
+        run: pnpm build
 
-      # - name: Restore cached build artifacts
-      #   if: ${{ inputs.cacheKey != '' }}
-      #   uses: actions/cache/restore@v4.0.0
-      #   with:
-      #     path: ./packages/**/dist
-      #     key: ${{ inputs.cacheKey }}
+      - name: Restore cached build artifacts
+        if: ${{ inputs.cacheKey != '' }}
+        uses: actions/cache/restore@v4.0.0
+        with:
+          path: ./packages/**/dist
+          key: ${{ inputs.cacheKey }}
 
       - name: Benchmark
         uses: CodSpeedHQ/action@v2

From 3bef94cbdfab97e74605a0e8c5db9ecfa2806b02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 18:31:03 +0200
Subject: [PATCH 72/86] Add TODO

---
 .github/workflows/benchmark.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 943d1740f2c41..d6fda7b53276b 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -1,6 +1,7 @@
 name: Benchmarks
 
 on:
+  # @TODO: Do we need both push and pull_request?
   push:
     branches:
       - '**'

From dfbab140f2d39059fa316e9f95091355e8aa46ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 18:54:35 +0200
Subject: [PATCH 73/86] Cleanup

---
 packages/cli/src/benchmark/lib/agent.ts             |  6 +++---
 packages/cli/src/benchmark/lib/constants.ts         | 11 ++++++++---
 .../lib/repositories/user.benchmark-repository.ts   | 13 ++++++-------
 packages/core/src/InstanceSettings.ts               |  1 +
 4 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/packages/cli/src/benchmark/lib/agent.ts b/packages/cli/src/benchmark/lib/agent.ts
index 540c215eea11b..fb08d823bc6c1 100644
--- a/packages/cli/src/benchmark/lib/agent.ts
+++ b/packages/cli/src/benchmark/lib/agent.ts
@@ -1,13 +1,13 @@
 import axios from 'axios';
-import { BACKEND_BASE_URL, INSTANCE_ONWER_EMAIL, INSTANCE_ONWER_PASSWORD } from './constants';
+import { BACKEND_BASE_URL, INSTANCE_ONWER } from './constants';
 import { ApplicationError } from 'n8n-workflow';
 
 export const agent = axios.create({ baseURL: BACKEND_BASE_URL });
 
 export async function authenticateAgent() {
 	const response = await agent.post('/rest/login', {
-		email: INSTANCE_ONWER_EMAIL,
-		password: INSTANCE_ONWER_PASSWORD,
+		email: INSTANCE_ONWER.EMAIL,
+		password: INSTANCE_ONWER.PASSWORD,
 	});
 
 	const cookies = response.headers['set-cookie'];
diff --git a/packages/cli/src/benchmark/lib/constants.ts b/packages/cli/src/benchmark/lib/constants.ts
index 09eb199bf8170..07b44a5da40d2 100644
--- a/packages/cli/src/benchmark/lib/constants.ts
+++ b/packages/cli/src/benchmark/lib/constants.ts
@@ -1,5 +1,10 @@
 export const BACKEND_BASE_URL = 'http://127.0.0.1:5678'; // localhost on GitHub Actions runners refuses connections
 
-export const INSTANCE_ONWER_EMAIL = 'test@test.com';
-
-export const INSTANCE_ONWER_PASSWORD = 'password';
+export const INSTANCE_ONWER = {
+	EMAIL: 'john@smith.com',
+	PASSWORD: 'password',
+	FIRST_NAME: 'John',
+	LAST_NAME: 'Smith',
+	API_KEY:
+		'n8n_api_96a4804143bdeb044802273c93423c33e1582c89a764c645fd6304fd2df19f3b2c73f4b972e28194',
+};
diff --git a/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts b/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
index b02a9fccb6771..ac74a2d3a49f6 100644
--- a/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
+++ b/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
@@ -1,19 +1,18 @@
 import { Service } from 'typedi';
 import { hash } from 'bcryptjs';
 import { UserRepository as ProductionUserRepository } from '@/databases/repositories/user.repository';
-import { INSTANCE_ONWER_EMAIL, INSTANCE_ONWER_PASSWORD } from '../constants';
+import { INSTANCE_ONWER } from '../constants';
 
 @Service()
 export class UserRepository extends ProductionUserRepository {
 	async createInstanceOwner() {
 		const user = this.create({
-			email: INSTANCE_ONWER_EMAIL,
-			password: await hash(INSTANCE_ONWER_PASSWORD, 10),
-			firstName: 'John',
-			lastName: 'Smith',
+			email: INSTANCE_ONWER.EMAIL,
+			password: await hash(INSTANCE_ONWER.PASSWORD, 10),
+			firstName: INSTANCE_ONWER.FIRST_NAME,
+			lastName: INSTANCE_ONWER.LAST_NAME,
 			role: 'global:owner',
-			apiKey:
-				'n8n_api_96a4804143bdeb044802273c93423c33e1582c89a764c645fd6304fd2df19f3b2c73f4b972e28194',
+			apiKey: INSTANCE_ONWER.API_KEY,
 		});
 
 		user.computeIsOwner();
diff --git a/packages/core/src/InstanceSettings.ts b/packages/core/src/InstanceSettings.ts
index e8ab9aa553223..194da5cbc5005 100644
--- a/packages/core/src/InstanceSettings.ts
+++ b/packages/core/src/InstanceSettings.ts
@@ -71,6 +71,7 @@ export class InstanceSettings {
 				errorMessage: `Error parsing n8n-config file "${this.settingsFile}". It does not seem to be valid JSON.`,
 			});
 
+			// @TODO: Remove first two logs in benchmark run
 			if (!inTest) console.info(`User settings loaded from: ${this.settingsFile}`);
 
 			const { encryptionKey, tunnelSubdomain } = settings;

From f8cbdd8cb51bb61cf9a9b0145dd491e2f8252f99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Fri, 26 Apr 2024 21:41:58 +0200
Subject: [PATCH 74/86] More cleanup

---
 .github/workflows/benchmark.yml                    |  4 ----
 .../repository-extensions.ts}                      |  8 ++++++--
 packages/cli/src/benchmark/lib/hooks/seed.ts       |  6 +++---
 packages/cli/src/benchmark/main.ts                 | 14 ++++++++++----
 packages/cli/src/config/schema.ts                  |  1 +
 .../src/databases/repositories/user.repository.ts  |  4 ----
 6 files changed, 20 insertions(+), 17 deletions(-)
 rename packages/cli/src/benchmark/lib/{repositories/user.benchmark-repository.ts => hooks/repository-extensions.ts} (72%)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index d6fda7b53276b..de1fdbe4a0162 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -1,10 +1,6 @@
 name: Benchmarks
 
 on:
-  # @TODO: Do we need both push and pull_request?
-  push:
-    branches:
-      - '**'
   pull_request:
     paths:
       - 'packages/cli/**'
diff --git a/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts b/packages/cli/src/benchmark/lib/hooks/repository-extensions.ts
similarity index 72%
rename from packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
rename to packages/cli/src/benchmark/lib/hooks/repository-extensions.ts
index ac74a2d3a49f6..33cf62668cb83 100644
--- a/packages/cli/src/benchmark/lib/repositories/user.benchmark-repository.ts
+++ b/packages/cli/src/benchmark/lib/hooks/repository-extensions.ts
@@ -1,10 +1,14 @@
 import { Service } from 'typedi';
 import { hash } from 'bcryptjs';
-import { UserRepository as ProductionUserRepository } from '@/databases/repositories/user.repository';
+import { UserRepository } from '@/databases/repositories/user.repository';
 import { INSTANCE_ONWER } from '../constants';
 
 @Service()
-export class UserRepository extends ProductionUserRepository {
+export class UserRepositoryExtension extends UserRepository {
+	async deleteAll() {
+		await this.delete({});
+	}
+
 	async createInstanceOwner() {
 		const user = this.create({
 			email: INSTANCE_ONWER.EMAIL,
diff --git a/packages/cli/src/benchmark/lib/hooks/seed.ts b/packages/cli/src/benchmark/lib/hooks/seed.ts
index 0d6e96667fbcd..10abd2fd52d61 100644
--- a/packages/cli/src/benchmark/lib/hooks/seed.ts
+++ b/packages/cli/src/benchmark/lib/hooks/seed.ts
@@ -5,11 +5,11 @@ import { readFile } from 'fs/promises';
 import type { WorkflowRequest } from '@/workflows/workflow.request';
 import { agent, authenticateAgent } from '../agent';
 import Container from 'typedi';
-import { UserRepository } from '../repositories/user.benchmark-repository';
+import { UserRepositoryExtension } from './repository-extensions';
 
 export async function seedInstanceOwner() {
-	await Container.get(UserRepository).deleteInstanceOwner();
-	await Container.get(UserRepository).createInstanceOwner();
+	await Container.get(UserRepositoryExtension).deleteAll();
+	await Container.get(UserRepositoryExtension).createInstanceOwner();
 }
 
 export async function seedWorkflows() {
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index ac796d55dec4a..d87e620e157dc 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -23,7 +23,7 @@ async function main() {
 
 	logger.info(`[Benchmarking] Running ${count} ${count === 1 ? 'suite' : 'suites'}...`);
 
-	await setup();
+	await setup(); // 1. create a benchmark postgres DB
 
 	const _bench = new Bench({
 		// @TODO: Temp values
@@ -38,13 +38,19 @@ async function main() {
 
 	const bench = process.env.CI === 'true' ? withCodSpeed(_bench) : _bench;
 
-	registerSuites(bench);
+	registerSuites(bench); // 2. rename all suites to have a `[sqlite]` prefix
+
+	// 3. duplicate all suites
+	// - add a `[postgres]` prefix to each suite name
+	// - add a `beforeEachTask` hook to each new task, containing `config.set('database.type', 'postgresdb')`
+
+	// await bench.warmup(); // @TODO: Restore
 
 	await bench.run();
 
-	if (process.env.CI !== 'true') console.table(bench.table());
+	if (!process.env.CI) console.table(bench.table());
 
-	await teardown();
+	await teardown(); // 4. remove benchmark postgres DB
 }
 
 void main();
diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts
index a11eb072488e5..cba195da3b490 100644
--- a/packages/cli/src/config/schema.ts
+++ b/packages/cli/src/config/schema.ts
@@ -1437,6 +1437,7 @@ export const schema = {
 		doc: 'Number of reverse-proxies n8n is running behind',
 	},
 
+	// @TODO: Stay with tinybench's defaults?
 	benchmark: {
 		time: {
 			doc: 'Length of time (ms) during which to repeatedly run a benchmarking task',
diff --git a/packages/cli/src/databases/repositories/user.repository.ts b/packages/cli/src/databases/repositories/user.repository.ts
index 29ebdd59fbec4..6b81f8984bff0 100644
--- a/packages/cli/src/databases/repositories/user.repository.ts
+++ b/packages/cli/src/databases/repositories/user.repository.ts
@@ -104,8 +104,4 @@ export class UserRepository extends Repository<User> {
 			where: { id: In(userIds), password: Not(IsNull()) },
 		});
 	}
-
-	async deleteInstanceOwner() {
-		await this.delete({ role: 'global:owner', email: IsNull() });
-	}
 }

From d607d70fc0171469817a3af4d62296d457f2456e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Sat, 27 Apr 2024 13:51:09 +0200
Subject: [PATCH 75/86] Add display

---
 packages/cli/src/benchmark/lib/api.ts         |  36 +++---
 packages/cli/src/benchmark/lib/display.ts     | 110 ++++++++++++++++++
 .../lib/errors/duplicate-hook.error.ts        |   4 +-
 .../lib/errors/duplicate-suite.error.ts       |   4 +-
 .../benchmark/lib/hooks/setup-and-teardown.ts |   2 +-
 packages/cli/src/benchmark/lib/index.ts       |   1 +
 packages/cli/src/benchmark/lib/types.ts       |   6 +-
 packages/cli/src/benchmark/main.ts            |   8 +-
 8 files changed, 142 insertions(+), 29 deletions(-)
 create mode 100644 packages/cli/src/benchmark/lib/display.ts

diff --git a/packages/cli/src/benchmark/lib/api.ts b/packages/cli/src/benchmark/lib/api.ts
index 4d2e9d5e66551..7a3c479aed250 100644
--- a/packages/cli/src/benchmark/lib/api.ts
+++ b/packages/cli/src/benchmark/lib/api.ts
@@ -45,52 +45,52 @@ export function registerSuites(bench: Bench) {
 	}
 }
 
-function suiteFilePath() {
-	const filePath = callsites()
+function suiteKey() {
+	const key = callsites()
 		.map((site) => site.getFileName())
 		.filter((site): site is string => site !== null)
 		.find((site) => site.endsWith('.suite.js'));
 
-	assert(filePath !== undefined);
+	assert(key !== undefined);
 
-	return filePath;
+	return key.replace(/^.*benchmark\//, '').replace(/\.suite\.js$/, '');
 }
 
 export function suite(suiteName: string, suiteFn: () => void) {
-	const filePath = suiteFilePath();
+	const key = suiteKey();
 
-	if (suites[filePath]) throw new DuplicateSuiteError(filePath);
+	if (suites[key]) throw new DuplicateSuiteError(key);
 
-	suites[filePath] = { name: suiteName, hooks: {}, tasks: [] };
+	suites[key] = { name: suiteName, hooks: {}, tasks: [] };
 
 	suiteFn();
 }
 
 export function task(taskName: string, operation: Task['operation']) {
-	const filePath = suiteFilePath();
+	const key = suiteKey();
 
-	suites[filePath].tasks.push({
-		name: suites[filePath].name + ' ' + taskName,
+	suites[key].tasks.push({
+		name: taskName,
 		operation,
 	});
 }
 
 export function beforeEachTask(fn: Callback) {
-	const filePath = suiteFilePath();
+	const key = suiteKey();
 
-	if (suites[filePath]?.hooks.beforeEachTask) {
-		throw new DuplicateHookError('beforeEachTask', filePath);
+	if (suites[key]?.hooks.beforeEachTask) {
+		throw new DuplicateHookError('beforeEachTask', key);
 	}
 
-	suites[filePath].hooks.beforeEachTask = fn;
+	suites[key].hooks.beforeEachTask = fn;
 }
 
 export function afterEachTask(fn: Callback) {
-	const filePath = suiteFilePath();
+	const key = suiteKey();
 
-	if (suites[filePath]?.hooks.afterEachTask) {
-		throw new DuplicateHookError('afterEachTask', filePath);
+	if (suites[key]?.hooks.afterEachTask) {
+		throw new DuplicateHookError('afterEachTask', key);
 	}
 
-	suites[filePath].hooks.afterEachTask = fn;
+	suites[key].hooks.afterEachTask = fn;
 }
diff --git a/packages/cli/src/benchmark/lib/display.ts b/packages/cli/src/benchmark/lib/display.ts
new file mode 100644
index 0000000000000..5f82eb50c8c1c
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/display.ts
@@ -0,0 +1,110 @@
+import pico from 'picocolors';
+import type Bench from 'tinybench';
+import type { Suites } from './types';
+import { assert } from 'n8n-workflow';
+
+function truncate(n: number, decimalPlaces: number) {
+	const nStr = n.toString();
+	const index = nStr.indexOf('.');
+
+	let truncated = nStr.slice(0, index + decimalPlaces + 1);
+
+	while (truncated.length < index + decimalPlaces + 1) {
+		truncated += '0';
+	}
+
+	return truncated;
+}
+
+export function display(suites: Suites, results: Bench['results']) {
+	const INDENTATION = {
+		FIRST: ' ',
+		SECOND: ' '.repeat(4),
+		THIRD: ' '.repeat(6),
+	};
+
+	for (const [key, suite] of Object.entries(suites)) {
+		const segments = key.split('/');
+		const dirs = segments.slice(0, -1).join('/') + '/';
+		const [fileName] = segments.slice(-1);
+
+		console.log(
+			'\n',
+			pico.bgWhite(pico.black(' BENCHMARK ')),
+			pico.gray(dirs) + pico.bold(fileName + '.ts'),
+			'\n',
+		);
+
+		for (const task of suite.tasks) {
+			console.log(INDENTATION.FIRST, pico.white('•'), task.name);
+
+			const result = results.shift();
+
+			assert(result !== undefined);
+
+			const [p75, p99, p999] = [result.p75, result.p99, result.p999].map((n) => truncate(n, 3));
+
+			console.log(
+				INDENTATION.SECOND,
+				pico.dim('p75'),
+				pico.magenta(p75 + ' ms'),
+				'  ',
+				pico.dim('p99'),
+				pico.magenta(p99 + ' ms'),
+				'  ',
+				pico.dim('p999'),
+				pico.magenta(p999 + ' ms'),
+			);
+
+			const [min, max, mean] = [result.min, result.max, result.mean].map((n) => truncate(n, 3));
+
+			console.log(
+				INDENTATION.SECOND,
+				pico.dim('min'),
+				pico.magenta(min + ' ms'),
+				'  ',
+				pico.dim('max'),
+				pico.magenta(max + ' ms'),
+				'  ',
+				pico.dim('mean'),
+				pico.magenta(mean + ' ms'),
+			);
+
+			const { totalTime, samples, sd, hz, moe, sem, variance } = result;
+
+			console.log(
+				INDENTATION.SECOND,
+				pico.dim('details'),
+				'\n',
+				pico.dim(INDENTATION.THIRD + '├─'),
+				pico.dim('total time'),
+				pico.magenta(truncate(totalTime, 3) + ' ms'),
+				'\n',
+				pico.dim(INDENTATION.THIRD + '├─'),
+				pico.dim('iterations'),
+				pico.magenta(samples.length),
+				'\n',
+				pico.dim(INDENTATION.THIRD + '├─'),
+				pico.dim('throughput'),
+				pico.magenta(truncate(hz, 3) + ' ops/s'),
+				'\n',
+				pico.dim(INDENTATION.THIRD + '├─'),
+				pico.dim('variance'),
+				pico.magenta(variance + ' ms²'),
+				'\n',
+				pico.dim(INDENTATION.THIRD + '├─'),
+				pico.dim('margin of error'),
+				pico.magenta('±' + moe + '%'),
+				'\n',
+				pico.dim(INDENTATION.THIRD + '├─'),
+				pico.dim('standard deviation'),
+				pico.magenta(sd + ' ms'),
+				'\n',
+				pico.dim(INDENTATION.THIRD + '└─'),
+				pico.dim('standard error of the mean'),
+				pico.magenta(sem + ' ms'),
+				'\n',
+			);
+		}
+	}
+}
diff --git a/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts b/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
index 60aff18a50fdf..d9467b9f38987 100644
--- a/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
+++ b/packages/cli/src/benchmark/lib/errors/duplicate-hook.error.ts
@@ -1,9 +1,9 @@
 import { ApplicationError } from 'n8n-workflow';
 
 export class DuplicateHookError extends ApplicationError {
-	constructor(hookName: 'beforeEachTask' | 'afterEachTask', filePath: string) {
+	constructor(hookName: 'beforeEachTask' | 'afterEachTask', key: string) {
 		super(
-			`Duplicate \`${hookName}\` hook found at \`${filePath}\`. Please define a single \`${hookName}\` hook for this file.`,
+			`Duplicate \`${hookName}\` hook found at \`${key}\`. Please define a single \`${hookName}\` hook for this file.`,
 			{ level: 'warning' },
 		);
 	}
diff --git a/packages/cli/src/benchmark/lib/errors/duplicate-suite.error.ts b/packages/cli/src/benchmark/lib/errors/duplicate-suite.error.ts
index 65116fab18c49..566ac14c58a94 100644
--- a/packages/cli/src/benchmark/lib/errors/duplicate-suite.error.ts
+++ b/packages/cli/src/benchmark/lib/errors/duplicate-suite.error.ts
@@ -1,8 +1,8 @@
 import { ApplicationError } from 'n8n-workflow';
 
 export class DuplicateSuiteError extends ApplicationError {
-	constructor(filePath: string) {
-		super(`Duplicate suite found at \`${filePath}\`. Please define a single suite for this file.`, {
+	constructor(key: string) {
+		super(`Duplicate suite found at \`${key}\`. Please define a single suite for this file.`, {
 			level: 'warning',
 		});
 	}
diff --git a/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts b/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
index 9266ae263cb22..c559dd8659263 100644
--- a/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
+++ b/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
@@ -10,7 +10,7 @@ let main: Start;
 export async function setup() {
 	n8nDir();
 
-	main = new Start([], new Config({ root: __dirname }));
+	main = new Start([], new Config({ root: __dirname })); // @TODO: Silence stdout
 
 	await main.init();
 	await main.run();
diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
index 497eaabc4da0c..f2bf2ca428681 100644
--- a/packages/cli/src/benchmark/lib/index.ts
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -3,4 +3,5 @@ import 'reflect-metadata';
 export { suite, task, beforeEachTask, afterEachTask, collectSuites, registerSuites } from './api';
 export { agent } from './agent';
 export { setup, teardown } from './hooks/setup-and-teardown';
+export { display } from './display';
 export type { Suites } from './types';
diff --git a/packages/cli/src/benchmark/lib/types.ts b/packages/cli/src/benchmark/lib/types.ts
index 872149d57ea03..1629fa90874d3 100644
--- a/packages/cli/src/benchmark/lib/types.ts
+++ b/packages/cli/src/benchmark/lib/types.ts
@@ -1,8 +1,8 @@
 /**
- * Benchmarking suites, i.e. `*.suite.ts` files containing benchmarking tasks.
+ * Map of `*.suite.ts` files containing benchmarking tasks.
  */
 export type Suites = {
-	[suiteFilePath: string]: {
+	[key: string]: {
 		name: string;
 		hooks: {
 			beforeEachTask?: Callback;
@@ -13,7 +13,7 @@ export type Suites = {
 };
 
 /**
- * A benchmarking task, i.e. a single operation whose performance to measure.
+ * Single operation whose execution time to measure repeatedly.
  */
 export type Task = {
 	name: string;
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index d87e620e157dc..914aa2f7991cc 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,7 +1,7 @@
 import 'reflect-metadata';
 import Container from 'typedi';
 import { Logger } from '@/Logger';
-import { collectSuites, registerSuites, setup, teardown } from './lib';
+import { collectSuites, display, registerSuites, setup, teardown } from './lib';
 import config from '@/config';
 
 /* eslint-disable import/no-extraneous-dependencies */
@@ -10,7 +10,7 @@ import { withCodSpeed } from '@codspeed/tinybench-plugin';
 /* eslint-enable import/no-extraneous-dependencies */
 
 async function main() {
-	const suites = await collectSuites();
+	const suites = await collectSuites(); // @TODO: --filter suites
 
 	const count = Object.keys(suites).length;
 
@@ -48,7 +48,9 @@ async function main() {
 
 	await bench.run();
 
-	if (!process.env.CI) console.table(bench.table());
+	if (process.env.CI !== 'true') display(suites, bench.results);
+	// console.table(bench.table());
+	// console.log(bench.results);
 
 	await teardown(); // 4. remove benchmark postgres DB
 }

From 471470a19139b62791a577ae3abcca4da9854f29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Sat, 27 Apr 2024 14:19:11 +0200
Subject: [PATCH 76/86] Simplify logging

---
 package.json                                  |   2 +-
 packages/cli/package.json                     |   2 +-
 packages/cli/src/AbstractServer.ts            |   8 +-
 packages/cli/src/ActiveExecutions.ts          |   5 +-
 packages/cli/src/benchmark/benchmark.md       |   2 +-
 packages/cli/src/benchmark/lib/agent.ts       |   1 +
 packages/cli/src/benchmark/lib/api.ts         |   4 +-
 packages/cli/src/benchmark/lib/constants.ts   |   6 +-
 packages/cli/src/benchmark/lib/display.ts     | 110 --------------
 .../cli/src/benchmark/lib/hooks/n8nDir.ts     |   3 +
 packages/cli/src/benchmark/lib/hooks/seed.ts  |  18 ++-
 .../benchmark/lib/hooks/setup-and-teardown.ts |   5 +-
 packages/cli/src/benchmark/lib/index.ts       |   3 +-
 packages/cli/src/benchmark/lib/log.ts         | 137 ++++++++++++++++++
 packages/cli/src/benchmark/lib/types.ts       |   4 +-
 packages/cli/src/benchmark/main.ts            |  52 ++++---
 packages/cli/src/commands/start.ts            |   4 +-
 .../1711390882123-MoveSshKeysToDatabase.ts    |   5 +-
 .../src/databases/utils/migrationHelpers.ts   |   6 +-
 packages/core/src/InstanceSettings.ts         |   4 +-
 20 files changed, 213 insertions(+), 168 deletions(-)
 delete mode 100644 packages/cli/src/benchmark/lib/display.ts
 create mode 100644 packages/cli/src/benchmark/lib/log.ts

diff --git a/package.json b/package.json
index 88bf17878daf4..1e4363020f905 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,7 @@
   "packageManager": "pnpm@8.14.3",
   "scripts": {
     "preinstall": "node scripts/block-npm-install.js",
-    "benchmark": "pnpm --filter=n8n benchmark:sqlite",
+    "benchmark": "pnpm --filter=n8n benchmark",
     "build": "turbo run build",
     "build:backend": "pnpm --filter=!@n8n/chat --filter=!n8n-design-system --filter=!n8n-editor-ui build",
     "build:frontend": "pnpm --filter=@n8n/chat --filter=n8n-design-system --filter=n8n-editor-ui build",
diff --git a/packages/cli/package.json b/packages/cli/package.json
index ac8b813b4b927..13c82aab03725 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,7 +20,7 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark:sqlite": "pnpm build:benchmark && NODE_ENV=benchmark node dist/benchmark/main.js",
+    "benchmark": "pnpm build:benchmark && NODE_ENV=benchmark N8N_LOG_LEVEL=silent node dist/benchmark/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
diff --git a/packages/cli/src/AbstractServer.ts b/packages/cli/src/AbstractServer.ts
index cf622863a6ce4..c244b416deaec 100644
--- a/packages/cli/src/AbstractServer.ts
+++ b/packages/cli/src/AbstractServer.ts
@@ -149,7 +149,7 @@ export abstract class AbstractServer {
 
 		this.server.on('error', (error: Error & { code: string }) => {
 			if (error.code === 'EADDRINUSE') {
-				console.log(
+				this.logger.info(
 					`n8n's port ${PORT} is already in use. Do you have another instance of n8n running already?`,
 				);
 				process.exit(1);
@@ -162,7 +162,7 @@ export abstract class AbstractServer {
 
 		await this.setupHealthCheck();
 
-		console.log(`n8n ready on ${ADDRESS}, port ${PORT}`);
+		this.logger.info(`n8n ready on ${ADDRESS}, port ${PORT}`);
 	}
 
 	async start(): Promise<void> {
@@ -231,11 +231,11 @@ export abstract class AbstractServer {
 		await this.configure();
 
 		if (!inTest) {
-			console.log(`Version: ${N8N_VERSION}`);
+			this.logger.info(`Version: ${N8N_VERSION}`);
 
 			const defaultLocale = config.getEnv('defaultLocale');
 			if (defaultLocale !== 'en') {
-				console.log(`Locale: ${defaultLocale}`);
+				this.logger.info(`Locale: ${defaultLocale}`);
 			}
 
 			await this.externalHooks.run('n8n.ready', [this, config]);
diff --git a/packages/cli/src/ActiveExecutions.ts b/packages/cli/src/ActiveExecutions.ts
index a215b03d494e7..e799dba13e65c 100644
--- a/packages/cli/src/ActiveExecutions.ts
+++ b/packages/cli/src/ActiveExecutions.ts
@@ -18,7 +18,6 @@ import type {
 import { isWorkflowIdValid } from '@/utils';
 import { ExecutionRepository } from '@db/repositories/execution.repository';
 import { Logger } from '@/Logger';
-import { inBenchmark } from './constants';
 
 @Service()
 export class ActiveExecutions {
@@ -202,9 +201,7 @@ export class ActiveExecutions {
 		let count = 0;
 		while (executionIds.length !== 0) {
 			if (count++ % 4 === 0) {
-				if (!inBenchmark) {
-					this.logger.info(`Waiting for ${executionIds.length} active executions to finish...`);
-				}
+				this.logger.info(`Waiting for ${executionIds.length} active executions to finish...`);
 			}
 
 			await sleep(500);
diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index 9063f96d46321..9c00e4c843c9f 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -3,7 +3,7 @@
 To run all benchmark suites locally in sqlite:
 
 ```sh
-pnpm benchmark:sqlite
+pnpm benchmark
 ```
 
 ## Creating a benchmark suite
diff --git a/packages/cli/src/benchmark/lib/agent.ts b/packages/cli/src/benchmark/lib/agent.ts
index fb08d823bc6c1..62ffc3c63fda0 100644
--- a/packages/cli/src/benchmark/lib/agent.ts
+++ b/packages/cli/src/benchmark/lib/agent.ts
@@ -19,4 +19,5 @@ export async function authenticateAgent() {
 	const [cookie] = cookies;
 
 	agent.defaults.headers.Cookie = cookie;
+	agent.defaults.headers['x-n8n-api-key'] = INSTANCE_ONWER.API_KEY;
 }
diff --git a/packages/cli/src/benchmark/lib/api.ts b/packages/cli/src/benchmark/lib/api.ts
index 7a3c479aed250..e59c8a7db481a 100644
--- a/packages/cli/src/benchmark/lib/api.ts
+++ b/packages/cli/src/benchmark/lib/api.ts
@@ -53,7 +53,7 @@ function suiteKey() {
 
 	assert(key !== undefined);
 
-	return key.replace(/^.*benchmark\//, '').replace(/\.suite\.js$/, '');
+	return key.replace(/^.*benchmark\//, '').replace(/\.js$/, '.ts');
 }
 
 export function suite(suiteName: string, suiteFn: () => void) {
@@ -61,7 +61,7 @@ export function suite(suiteName: string, suiteFn: () => void) {
 
 	if (suites[key]) throw new DuplicateSuiteError(key);
 
-	suites[key] = { name: suiteName, hooks: {}, tasks: [] };
+	suites[key] = { name: suiteName, hooks: {}, tasks: [], db: 'sqlite' };
 
 	suiteFn();
 }
diff --git a/packages/cli/src/benchmark/lib/constants.ts b/packages/cli/src/benchmark/lib/constants.ts
index 07b44a5da40d2..ff1a276db194c 100644
--- a/packages/cli/src/benchmark/lib/constants.ts
+++ b/packages/cli/src/benchmark/lib/constants.ts
@@ -1,10 +1,10 @@
 export const BACKEND_BASE_URL = 'http://127.0.0.1:5678'; // localhost on GitHub Actions runners refuses connections
 
 export const INSTANCE_ONWER = {
-	EMAIL: 'john@smith.com',
+	EMAIL: 'instance@owner.com',
 	PASSWORD: 'password',
-	FIRST_NAME: 'John',
-	LAST_NAME: 'Smith',
+	FIRST_NAME: 'Instance',
+	LAST_NAME: 'Owner',
 	API_KEY:
 		'n8n_api_96a4804143bdeb044802273c93423c33e1582c89a764c645fd6304fd2df19f3b2c73f4b972e28194',
 };
diff --git a/packages/cli/src/benchmark/lib/display.ts b/packages/cli/src/benchmark/lib/display.ts
deleted file mode 100644
index 5f82eb50c8c1c..0000000000000
--- a/packages/cli/src/benchmark/lib/display.ts
+++ /dev/null
@@ -1,110 +0,0 @@
-import pico from 'picocolors';
-import type Bench from 'tinybench';
-import type { Suites } from './types';
-import { assert } from 'n8n-workflow';
-
-function truncate(n: number, decimalPlaces: number) {
-	const nStr = n.toString();
-	const index = nStr.indexOf('.');
-
-	let truncated = nStr.slice(0, index + decimalPlaces + 1);
-
-	while (truncated.length < index + decimalPlaces + 1) {
-		truncated += '0';
-	}
-
-	return truncated;
-}
-
-export function display(suites: Suites, results: Bench['results']) {
-	const INDENTATION = {
-		FIRST: ' ',
-		SECOND: ' '.repeat(4),
-		THIRD: ' '.repeat(6),
-	};
-
-	for (const [key, suite] of Object.entries(suites)) {
-		const segments = key.split('/');
-		const dirs = segments.slice(0, -1).join('/') + '/';
-		const [fileName] = segments.slice(-1);
-
-		console.log(
-			'\n',
-			pico.bgWhite(pico.black(' BENCHMARK ')),
-			pico.gray(dirs) + pico.bold(fileName + '.ts'),
-			'\n',
-		);
-
-		for (const task of suite.tasks) {
-			console.log(INDENTATION.FIRST, pico.white('•'), task.name);
-
-			const result = results.shift();
-
-			assert(result !== undefined);
-
-			const [p75, p99, p999] = [result.p75, result.p99, result.p999].map((n) => truncate(n, 3));
-
-			console.log(
-				INDENTATION.SECOND,
-				pico.dim('p75'),
-				pico.magenta(p75 + ' ms'),
-				'  ',
-				pico.dim('p99'),
-				pico.magenta(p99 + ' ms'),
-				'  ',
-				pico.dim('p999'),
-				pico.magenta(p999 + ' ms'),
-			);
-
-			const [min, max, mean] = [result.min, result.max, result.mean].map((n) => truncate(n, 3));
-
-			console.log(
-				INDENTATION.SECOND,
-				pico.dim('min'),
-				pico.magenta(min + ' ms'),
-				'  ',
-				pico.dim('max'),
-				pico.magenta(max + ' ms'),
-				'  ',
-				pico.dim('mean'),
-				pico.magenta(mean + ' ms'),
-			);
-
-			const { totalTime, samples, sd, hz, moe, sem, variance } = result;
-
-			console.log(
-				INDENTATION.SECOND,
-				pico.dim('details'),
-				'\n',
-				pico.dim(INDENTATION.THIRD + '├─'),
-				pico.dim('total time'),
-				pico.magenta(truncate(totalTime, 3) + ' ms'),
-				'\n',
-				pico.dim(INDENTATION.THIRD + '├─'),
-				pico.dim('iterations'),
-				pico.magenta(samples.length),
-				'\n',
-				pico.dim(INDENTATION.THIRD + '├─'),
-				pico.dim('throughput'),
-				pico.magenta(truncate(hz, 3) + ' ops/s'),
-				'\n',
-				pico.dim(INDENTATION.THIRD + '├─'),
-				pico.dim('variance'),
-				pico.magenta(variance + ' ms²'),
-				'\n',
-				pico.dim(INDENTATION.THIRD + '├─'),
-				pico.dim('margin of error'),
-				pico.magenta('±' + moe + '%'),
-				'\n',
-				pico.dim(INDENTATION.THIRD + '├─'),
-				pico.dim('standard deviation'),
-				pico.magenta(sd + ' ms'),
-				'\n',
-				pico.dim(INDENTATION.THIRD + '└─'),
-				pico.dim('standard error of the mean'),
-				pico.magenta(sem + ' ms'),
-				'\n',
-			);
-		}
-	}
-}
diff --git a/packages/cli/src/benchmark/lib/hooks/n8nDir.ts b/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
index 14b3944a93fec..40f4b9215117f 100644
--- a/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
+++ b/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
@@ -3,6 +3,7 @@ import path from 'node:path';
 import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
 import Container from 'typedi';
 import { InstanceSettings } from 'n8n-core';
+import { log } from '../log';
 
 /**
  * Create a temp .n8n user dir for benchmarking.
@@ -31,4 +32,6 @@ export function n8nDir() {
 	 * so re-instantiate it to ensure it picks up the temp user home dir path.
 	 */
 	Container.set(InstanceSettings, new InstanceSettings());
+
+	log('Created temp dir', tempN8nDir);
 }
diff --git a/packages/cli/src/benchmark/lib/hooks/seed.ts b/packages/cli/src/benchmark/lib/hooks/seed.ts
index 10abd2fd52d61..11eb0e8850d68 100644
--- a/packages/cli/src/benchmark/lib/hooks/seed.ts
+++ b/packages/cli/src/benchmark/lib/hooks/seed.ts
@@ -6,22 +6,26 @@ import type { WorkflowRequest } from '@/workflows/workflow.request';
 import { agent, authenticateAgent } from '../agent';
 import Container from 'typedi';
 import { UserRepositoryExtension } from './repository-extensions';
+import { log } from '../log';
 
 export async function seedInstanceOwner() {
 	await Container.get(UserRepositoryExtension).deleteAll();
-	await Container.get(UserRepositoryExtension).createInstanceOwner();
+
+	const user = await Container.get(UserRepositoryExtension).createInstanceOwner();
+
+	log('Seeded user', user.email);
 }
 
 export async function seedWorkflows() {
-	const files = await glob('suites/workflows/*.json', {
+	const _files = await glob('suites/workflows/*.json', {
 		cwd: path.join('dist', 'benchmark'),
 		absolute: true,
 	});
 
 	const payloads: WorkflowRequest.CreateUpdatePayload[] = [];
 
-	for (const file of files) {
-		const json = await readFile(file, 'utf8');
+	for (const f of _files) {
+		const json = await readFile(f, 'utf8');
 		const payload = jsonParse<WorkflowRequest.CreatePayload>(json);
 		payloads.push(payload);
 	}
@@ -31,4 +35,10 @@ export async function seedWorkflows() {
 	for (const p of payloads) {
 		await agent.post('/rest/workflows', p);
 	}
+
+	const files = _files.map((f) => f.replace(/.*workflows\//, '')).join(' ');
+
+	log('Seeded workflows', files);
+
+	return files;
 }
diff --git a/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts b/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
index c559dd8659263..d86ea0d6edcc4 100644
--- a/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
+++ b/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
@@ -4,6 +4,7 @@ import { Start } from '@/commands/start';
 import { n8nDir } from './n8nDir';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 import { seedInstanceOwner, seedWorkflows } from './seed';
+import { log } from '../log';
 
 let main: Start;
 
@@ -16,9 +17,11 @@ export async function setup() {
 	await main.run();
 
 	await seedInstanceOwner();
-	await seedWorkflows();
+	const files = await seedWorkflows();
 
 	await Container.get(ActiveWorkflowRunner).init();
+
+	log('Activated workflows', files);
 }
 
 export async function teardown() {
diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
index f2bf2ca428681..efcf4c96daa5c 100644
--- a/packages/cli/src/benchmark/lib/index.ts
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -3,5 +3,6 @@ import 'reflect-metadata';
 export { suite, task, beforeEachTask, afterEachTask, collectSuites, registerSuites } from './api';
 export { agent } from './agent';
 export { setup, teardown } from './hooks/setup-and-teardown';
-export { display } from './display';
 export type { Suites } from './types';
+
+export { log, logResults, toOneLineJson } from './log';
diff --git a/packages/cli/src/benchmark/lib/log.ts b/packages/cli/src/benchmark/lib/log.ts
new file mode 100644
index 0000000000000..ec0639e89b794
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/log.ts
@@ -0,0 +1,137 @@
+import pico from 'picocolors';
+import { assert } from 'n8n-workflow';
+import type Bench from 'tinybench';
+import type { Suites } from './types';
+
+export const toOneLineJson = (obj: object) =>
+	JSON.stringify(obj)
+		.replace(/:/g, ': ')
+		.replace(/,/g, ', ')
+		.replace(/"/g, '')
+		.replace(/^\{/, '{ ')
+		.replace(/\}$/, ' }');
+
+export function log(message: string, details?: string) {
+	const parts = [pico.magenta('[benchmarking]'), message];
+
+	if (details) {
+		parts[parts.length - 1] += ':';
+		parts.push(pico.dim(details));
+	}
+
+	console.log(parts.join(' '));
+}
+
+const indentation = {
+	first: ' ',
+	second: ' '.repeat(4),
+	third: ' '.repeat(6),
+};
+
+function truncate(n: number, decimalPlaces = 3) {
+	const nStr = n.toString();
+
+	return nStr.slice(0, nStr.indexOf('.') + decimalPlaces + 1);
+}
+
+const toDirsAndFileName = (key: string) => {
+	const segments = key.split('/');
+	const dirs = segments.slice(0, -1).join('/') + '/';
+	const [fileName] = segments.slice(-1);
+
+	return [dirs, fileName];
+};
+
+export function logResults(suites: Suites, results: Bench['results']) {
+	const columnDivider = pico.dim('·'.repeat(3));
+
+	for (const [key, suite] of Object.entries(suites)) {
+		const [dirs, fileName] = toDirsAndFileName(key);
+
+		const title = [
+			'\n',
+			pico.bgWhite(pico.black(' BENCHMARK ')),
+			pico.gray(dirs) + pico.bold(fileName),
+			pico.dim('[' + suite.db + ']'),
+			'\n',
+		].join(' ');
+
+		console.log(title);
+
+		for (const task of suite.tasks) {
+			console.log(indentation.first, pico.white('•'), task.name);
+
+			const result = results.shift();
+
+			assert(result !== undefined);
+
+			const { totalTime, samples, sd, hz, moe, sem, variance } = result;
+
+			const zerothLine = [
+				indentation.second + pico.dim('·'),
+				pico.dim('Ran'),
+				pico.magenta(samples.length),
+				pico.dim('iterations in'),
+				pico.magenta(truncate(totalTime) + ' ms'),
+				pico.dim('at'),
+				pico.magenta(truncate(hz) + ' op/s'),
+			].join(' ');
+
+			console.log(zerothLine);
+
+			const [p75, p99, p999] = [result.p75, result.p99, result.p999].map((n) => truncate(n));
+
+			const firstLine = [
+				indentation.second + pico.dim('·'),
+				pico.dim('p75'),
+				pico.magenta(p75 + ' ms'),
+				columnDivider,
+				pico.dim('p99'),
+				pico.magenta(p99 + ' ms'),
+				columnDivider,
+				pico.dim('p999'),
+				pico.magenta(p999 + ' ms'),
+			].join(' ');
+
+			console.log(firstLine);
+
+			const [min, max, mean] = [result.min, result.max, result.mean].map((n) => truncate(n));
+
+			const secondLine = [
+				indentation.second + pico.dim('·'),
+				pico.dim('min'),
+				pico.magenta(min + ' ms'),
+				columnDivider,
+				pico.dim('max'),
+				pico.magenta(max + ' ms'),
+				columnDivider,
+				pico.dim('mean'),
+				pico.magenta(mean + ' ms'),
+			].join(' ');
+
+			console.log(secondLine);
+
+			const thirdLine = [
+				indentation.second + pico.dim('·'),
+				pico.dim('margin of error'),
+				pico.magenta('±' + truncate(moe) + '%'),
+				columnDivider,
+				pico.dim('std error'),
+				pico.magenta(truncate(sem) + ' ms'),
+			].join(' ');
+
+			console.log(thirdLine);
+
+			const fourthLine = [
+				indentation.second + pico.dim('·'),
+				pico.dim('std deviation'),
+				pico.magenta(truncate(sd) + ' ms'),
+				columnDivider,
+				pico.dim('variance'),
+				pico.magenta(truncate(variance) + ' ms²'),
+			].join(' ');
+
+			console.log(fourthLine + '\n');
+		}
+	}
+}
diff --git a/packages/cli/src/benchmark/lib/types.ts b/packages/cli/src/benchmark/lib/types.ts
index 1629fa90874d3..b7db1b6132c34 100644
--- a/packages/cli/src/benchmark/lib/types.ts
+++ b/packages/cli/src/benchmark/lib/types.ts
@@ -1,9 +1,7 @@
-/**
- * Map of `*.suite.ts` files containing benchmarking tasks.
- */
 export type Suites = {
 	[key: string]: {
 		name: string;
+		db: 'sqlite' | 'postgres';
 		hooks: {
 			beforeEachTask?: Callback;
 			afterEachTask?: Callback;
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 914aa2f7991cc..543a06da1e787 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,8 +1,14 @@
 import 'reflect-metadata';
-import Container from 'typedi';
-import { Logger } from '@/Logger';
-import { collectSuites, display, registerSuites, setup, teardown } from './lib';
 import config from '@/config';
+import {
+	collectSuites,
+	log,
+	logResults,
+	registerSuites,
+	setup,
+	teardown,
+	toOneLineJson,
+} from './lib';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';
@@ -12,47 +18,47 @@ import { withCodSpeed } from '@codspeed/tinybench-plugin';
 async function main() {
 	const suites = await collectSuites(); // @TODO: --filter suites
 
-	const count = Object.keys(suites).length;
-
-	const logger = Container.get(Logger);
-
-	if (count === 0) {
-		logger.info('[Benchmarking] Found no suites. Exiting...');
-		return;
-	}
-
-	logger.info(`[Benchmarking] Running ${count} ${count === 1 ? 'suite' : 'suites'}...`);
+	log('Found suites', Object.keys(suites).join(' '));
 
 	await setup(); // 1. create a benchmark postgres DB
 
-	const _bench = new Bench({
-		// @TODO: Temp values
-		time: 0,
-		iterations: 1,
-		// time: config.getEnv('benchmark.time'),
-		// iterations: config.getEnv('benchmark.iterations'),
+	const benchConfig = {
+		time: config.getEnv('benchmark.time'),
+		iterations: config.getEnv('benchmark.iterations'),
 		throws: config.getEnv('benchmark.stopOnError'),
 		warmupTime: config.getEnv('benchmark.warmupTime'),
 		warmupIterations: config.getEnv('benchmark.warmupIterations'),
+	};
+
+	const _bench = new Bench({
+		// @TODO: Temp values
+		// time: 0,
+		// iterations: 1,
+		...benchConfig,
 	});
 
 	const bench = process.env.CI === 'true' ? withCodSpeed(_bench) : _bench;
 
-	registerSuites(bench); // 2. rename all suites to have a `[sqlite]` prefix
+	registerSuites(bench);
 
-	// 3. duplicate all suites
+	// 2. duplicate all suites
 	// - add a `[postgres]` prefix to each suite name
 	// - add a `beforeEachTask` hook to each new task, containing `config.set('database.type', 'postgresdb')`
 
 	// await bench.warmup(); // @TODO: Restore
 
+	log('Set config', toOneLineJson(benchConfig));
+	log('Running iterations, please wait...');
+
 	await bench.run();
 
-	if (process.env.CI !== 'true') display(suites, bench.results);
+	log('Iterations completed');
+
+	if (process.env.CI !== 'true') logResults(suites, bench.results);
 	// console.table(bench.table());
 	// console.log(bench.results);
 
-	await teardown(); // 4. remove benchmark postgres DB
+	await teardown(); // 3. remove benchmark postgres DB
 }
 
 void main();
diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts
index 3164d3c2b9e88..048ab9bf2cc05 100644
--- a/packages/cli/src/commands/start.ts
+++ b/packages/cli/src/commands/start.ts
@@ -88,7 +88,7 @@ export class Start extends BaseCommand {
 	 * get removed.
 	 */
 	async stopProcess() {
-		if (!inBenchmark) this.logger.info('\nStopping n8n...');
+		this.logger.info('\nStopping n8n...');
 
 		try {
 			// Stop with trying to activate workflows that could not be activated
@@ -287,7 +287,7 @@ export class Start extends BaseCommand {
 		await this.activeWorkflowRunner.init();
 
 		const editorUrl = Container.get(UrlService).baseUrl;
-		this.log(`\nEditor is now accessible via:\n${editorUrl}`);
+		this.logger.info(`\nEditor is now accessible via:\n${editorUrl}`);
 
 		if (inBenchmark) return;
 
diff --git a/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
index fcadaf911bfa3..bd7d13483916a 100644
--- a/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
+++ b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
@@ -4,7 +4,7 @@ import Container from 'typedi';
 import { Cipher, InstanceSettings } from 'n8n-core';
 import { jsonParse } from 'n8n-workflow';
 import type { MigrationContext, ReversibleMigration } from '@db/types';
-import { inTest, inBenchmark } from '@/constants';
+import { inTest } from '@/constants';
 
 /**
  * Move SSH key pair from file system to database, to enable SSH connections
@@ -32,8 +32,7 @@ export class MoveSshKeysToDatabase1711390882123 implements ReversibleMigration {
 				readFile(this.publicKeyPath, { encoding: 'utf8' }),
 			]);
 		} catch {
-			if (!inTest && !inBenchmark)
-				logger.info(`[${migrationName}] No SSH keys in filesystem, skipping`);
+			if (!inTest) logger.info(`[${migrationName}] No SSH keys in filesystem, skipping`);
 			return;
 		}
 
diff --git a/packages/cli/src/databases/utils/migrationHelpers.ts b/packages/cli/src/databases/utils/migrationHelpers.ts
index 4e40b2755def5..f87076394a8aa 100644
--- a/packages/cli/src/databases/utils/migrationHelpers.ts
+++ b/packages/cli/src/databases/utils/migrationHelpers.ts
@@ -5,7 +5,7 @@ import type { ObjectLiteral } from '@n8n/typeorm';
 import type { QueryRunner } from '@n8n/typeorm/query-runner/QueryRunner';
 import { ApplicationError, jsonParse } from 'n8n-workflow';
 import config from '@/config';
-import { inBenchmark, inTest } from '@/constants';
+import { inTest } from '@/constants';
 import type { BaseMigration, Migration, MigrationContext, MigrationFn } from '@db/types';
 import { createSchemaBuilder } from '@db/dsl';
 import { NodeTypes } from '@/NodeTypes';
@@ -44,7 +44,7 @@ function loadSurveyFromDisk(): string | null {
 let runningMigrations = false;
 
 function logMigrationStart(migrationName: string): void {
-	if (inTest || inBenchmark) return;
+	if (inTest) return;
 
 	const logger = Container.get(Logger);
 	if (!runningMigrations) {
@@ -56,7 +56,7 @@ function logMigrationStart(migrationName: string): void {
 }
 
 function logMigrationEnd(migrationName: string): void {
-	if (inTest || inBenchmark) return;
+	if (inTest) return;
 
 	const logger = Container.get(Logger);
 	logger.info(`Finished migration ${migrationName}`);
diff --git a/packages/core/src/InstanceSettings.ts b/packages/core/src/InstanceSettings.ts
index 194da5cbc5005..0c902c385f8ab 100644
--- a/packages/core/src/InstanceSettings.ts
+++ b/packages/core/src/InstanceSettings.ts
@@ -2,7 +2,7 @@ import path from 'path';
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
 import { createHash, randomBytes } from 'crypto';
 import { Service } from 'typedi';
-import { ApplicationError, jsonParse } from 'n8n-workflow';
+import { ApplicationError, jsonParse, LoggerProxy as Logger } from 'n8n-workflow';
 
 interface ReadOnlySettings {
 	encryptionKey: string;
@@ -72,7 +72,7 @@ export class InstanceSettings {
 			});
 
 			// @TODO: Remove first two logs in benchmark run
-			if (!inTest) console.info(`User settings loaded from: ${this.settingsFile}`);
+			if (!inTest) Logger.info(`User settings loaded from: ${this.settingsFile}`);
 
 			const { encryptionKey, tunnelSubdomain } = settings;
 

From 526bb833a0c5b63587d54a400cc027d34cb9f162 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Sun, 28 Apr 2024 18:35:26 +0200
Subject: [PATCH 77/86] Document

---
 packages/cli/src/benchmark/benchmark.md | 52 +++++++++++++++++++++----
 packages/cli/src/benchmark/lib/log.ts   | 30 +++++++-------
 2 files changed, 57 insertions(+), 25 deletions(-)

diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index 9c00e4c843c9f..4b7efca39d138 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -1,21 +1,57 @@
 # Benchmark
 
-To run all benchmark suites locally in sqlite:
+This package contains benchmarks to measure the execution time of n8n backend operations.
+
+Benchmarks are organized into **suites** for the scenario to benchmark and **tasks** for operations in that scenario, with hooks for per-task setup and teardown, runnning in both sqlite and Postgres. This lib is implemented on top of [`tinybench`](https://github.com/tinylibs/tinybench). Execution in CI is delegated to [Codspeed](https://codspeed.io/) to keep measurements consistent and to monitor improvements and regressions.
+
+## Running benchmarks
+
+To start a benchmarking run:
 
 ```sh
+pnpm build:benchmark
 pnpm benchmark
 ```
 
-## Creating a benchmark suite
+On every run, benchmarking will set up and tear down a temporary Postgres database, so ensure you have a Postgres server running locally and allow n8n to connect to it via [environment variables](https://docs.n8n.io/hosting/configuration/environment-variables/database/#postgresql).
+
+The default benchmarking configuration can be adjusted via [environment variables](https://docs.n8n.io/hosting/configuration/environment-variables/benchmarking).
+
+## Creating benchmarks
+
+To create benchmarks:
+
+1. Create a file at `suites/{suite-id}-{suite-title}.ts` or `suites/{theme}/{suite-id}-{suite-title}.ts`.
+2. Include a `suite()` call for the scenario to benchmark.
+3. Include one or more `task()` calls for operations in that scenario. `task()` must contain only the specific operation whose execution time to measure. Move any per-task setup and teardown to `beforeEachTask()` and `afterEachTask()` in the suite.
+4. Include workflows at `suites/workflows/{suite-id}-{ordinal-number}`. During setup, all workflows at this dir are loaded to the temp DBs and activated in memory. If the workflow is triggered by webhook, set the webhook path to `/{suite-id}-{ordinal-number}`.
+5. Run `pnpm build:benchmark` to add the suite and its tasks to the index in this document.
+
+## Reading benchmarks
+
+In a benchmarking run, a task is repeatedly executed for a duration and for a number of iterations - the run will continue until the number of iterations is reached, even if this exceeds the duration.
+
+```
+BENCHMARK suites/001-production-webhook-with-authless-webhook-node.suite.ts [sqlite]
+
+  • using "Respond immediately" mode
+    · Ran 27 iterations in 509.992 ms at a rate of 52.941 op/s
+    · p75 20.251 ms ··· p99 64.570 ms ··· p999 64.570 ms
+    · min 8.363 ms ···· max 64.570 ms ··· mean 18.888 ms
+    · MoE ±4.1% ··· std err 02.037 ms ··· std dev 10.586 ms
+```
+
+`p{n}` is the percentile, i.e. the percentage of data points in a distribution that are less than or equal to a value. For example, `p75` being 20.251 ms means that 75% of the 27 iterations for the task `using "Respond immediately" mode` took 20.251 ms or less. `p75` is the execution time that the majority of users experience, `p99` captures worst-case scenarios for all but 1% of users, and `p999` includes performance at extreme cases for the slowest 0.1% of users.
+
+`min` is the shortest execution time recorded across all iterations of the task, `max` is the longest, and `mean` is the average.
+
+`MoE` (margin of error) reflects how much the sample mean is expected to differ from the true population mean. For example, a margin of error of ±4.189% in the task `using "Respond immediately" mode` suggests that, if the benchmarking run were repeated multiple times, the sample mean would fall within 4.189% of the true population mean in 95% of those runs, assuming a standard confidence level. This range indicates the variability we might see due to the randomness of selecting a sample.
 
-To create a benchmark suite:
+`std err` (standard error) reflects how closely a sample mean is expected to approximate the true population mean. A smaller standard error indicates that the sample mean is likely to be a more accurate estimate of the population mean because the variation among sample means is less. For example, in the task `using "Respond immediately" mode`, the standard error is 2.037 ms, which suggests that the sample mean is expected to differ from the true population mean by 2.037 ms on average.
 
-- Add a file to `./suites` following the pattern: `{id}-{description}`
-- Add workflows to `./suites/workflows`. These will all be loaded to the temp DB during setup. If a workflow is triggered by webhook, set the filename as its path for clarity.
-- Use `suite()` for the scenario to benchmark and `task()` for operations in that scenario. Ensure `task()` contains only the specific operation whose execution time will be measured. Move any setup and teardown to `beforeEachTask()` and `afterEachTask()` in the suite.
-- Run `build:benchmark` to add it to the list below.
+`std dev` (standard deviation) is the amount of dispersion across samples. When low, it indicates that the samples tend to be close to the mean; when high, it indicates that the samples are spread out over a wider range. For example, in the task `using "Respond immediately" mode`, the standard deviation is `10.586 ms`, which suggests that the execution times varied significantly across iterations.
 
-## Benchmark suites list
+## Index of benchmarking suites
 
 > **Note**: All workflows with default settings unless otherwise specified.
 
diff --git a/packages/cli/src/benchmark/lib/log.ts b/packages/cli/src/benchmark/lib/log.ts
index ec0639e89b794..6c5f5ea04470b 100644
--- a/packages/cli/src/benchmark/lib/log.ts
+++ b/packages/cli/src/benchmark/lib/log.ts
@@ -31,7 +31,11 @@ const indentation = {
 function truncate(n: number, decimalPlaces = 3) {
 	const nStr = n.toString();
 
-	return nStr.slice(0, nStr.indexOf('.') + decimalPlaces + 1);
+	const truncated = nStr.slice(0, nStr.indexOf('.') + decimalPlaces + 1);
+
+	if (truncated.length === 5) return '0' + truncated;
+
+	return truncated;
 }
 
 const toDirsAndFileName = (key: string) => {
@@ -65,7 +69,7 @@ export function logResults(suites: Suites, results: Bench['results']) {
 
 			assert(result !== undefined);
 
-			const { totalTime, samples, sd, hz, moe, sem, variance } = result;
+			const { totalTime, samples, sd, hz, moe, sem } = result;
 
 			const zerothLine = [
 				indentation.second + pico.dim('·'),
@@ -73,7 +77,7 @@ export function logResults(suites: Suites, results: Bench['results']) {
 				pico.magenta(samples.length),
 				pico.dim('iterations in'),
 				pico.magenta(truncate(totalTime) + ' ms'),
-				pico.dim('at'),
+				pico.dim('at a rate of'),
 				pico.magenta(truncate(hz) + ' op/s'),
 			].join(' ');
 
@@ -113,25 +117,17 @@ export function logResults(suites: Suites, results: Bench['results']) {
 
 			const thirdLine = [
 				indentation.second + pico.dim('·'),
-				pico.dim('margin of error'),
-				pico.magenta('±' + truncate(moe) + '%'),
+				pico.dim('MoE'),
+				pico.magenta('±' + truncate(moe, 1) + '%'),
 				columnDivider,
-				pico.dim('std error'),
+				pico.dim('std err'),
 				pico.magenta(truncate(sem) + ' ms'),
-			].join(' ');
-
-			console.log(thirdLine);
-
-			const fourthLine = [
-				indentation.second + pico.dim('·'),
-				pico.dim('std deviation'),
-				pico.magenta(truncate(sd) + ' ms'),
 				columnDivider,
-				pico.dim('variance'),
-				pico.magenta(truncate(variance) + ' ms²'),
+				pico.dim('std dev'),
+				pico.magenta(truncate(sd) + ' ms'),
 			].join(' ');
 
-			console.log(fourthLine + '\n');
+			console.log(thirdLine + '\n');
 		}
 	}
 }

From 57bcc8799793831ba004e7c70b6e2050a2f26ffe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 30 Apr 2024 18:50:57 +0200
Subject: [PATCH 78/86] perf(core): Support Postgres in benchmarks (#9246)

---
 .github/workflows/benchmark.yml               | 34 ++++++----
 packages/cli/package.json                     |  3 +-
 packages/cli/src/benchmark/lib/api.ts         |  2 +-
 packages/cli/src/benchmark/lib/hooks/seed.ts  |  4 +-
 .../benchmark/lib/hooks/setup-and-teardown.ts | 15 ++++-
 packages/cli/src/benchmark/lib/index.ts       |  2 +-
 packages/cli/src/benchmark/lib/log.ts         |  4 +-
 packages/cli/src/benchmark/lib/postgres.ts    | 64 +++++++++++++++++++
 packages/cli/src/benchmark/lib/types.ts       |  1 -
 packages/cli/src/benchmark/main.ts            | 36 ++++-------
 packages/cli/src/commands/start.ts            |  2 +
 11 files changed, 123 insertions(+), 44 deletions(-)
 create mode 100644 packages/cli/src/benchmark/lib/postgres.ts

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index de1fdbe4a0162..2e8652a466309 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -1,4 +1,4 @@
-name: Benchmarks
+name: Benchmark
 
 on:
   pull_request:
@@ -9,27 +9,33 @@ on:
   workflow_dispatch:
 
 jobs:
-  benchmarks:
+  benchmark:
+    name: Benchmark
     runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      DB_POSTGRESDB_PASSWORD: password
     steps:
       - uses: actions/checkout@v4.1.1
+
+      - name: Start Postgres
+        uses: isbang/compose-action@v2.0.0
         with:
-          repository: n8n-io/n8n
-          ref: ${{ inputs.ref }}
+          compose-file: ./.github/docker-compose.yml
+          services: postgres
 
       - run: corepack enable
-      - name: Use Node.js ${{ inputs.nodeVersion }}
-        uses: actions/setup-node@v4.0.1
+
+      - uses: actions/setup-node@v4.0.1
         with:
-          node-version: ${{ inputs.nodeVersion }}
+          node-version: 18.x
           cache: pnpm
 
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
+      - run: pnpm install --frozen-lockfile
 
       - name: Build
         if: ${{ inputs.cacheKey == '' }}
-        run: pnpm build
+        run: pnpm build:backend
 
       - name: Restore cached build artifacts
         if: ${{ inputs.cacheKey != '' }}
@@ -38,7 +44,13 @@ jobs:
           path: ./packages/**/dist
           key: ${{ inputs.cacheKey }}
 
+      - run: pnpm build:benchmark
+        working-directory: packages/cli
+
       - name: Benchmark
         uses: CodSpeedHQ/action@v2
         with:
-          run: pnpm benchmark
+          working-directory: packages/cli
+          run: |
+            pnpm benchmark:sqlite
+            pnpm benchmark:postgres
diff --git a/packages/cli/package.json b/packages/cli/package.json
index adc78f88aed55..4288b2302444e 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -20,7 +20,8 @@
     "bin": "n8n"
   },
   "scripts": {
-    "benchmark": "pnpm build:benchmark && NODE_ENV=benchmark N8N_LOG_LEVEL=silent node dist/benchmark/main.js",
+    "benchmark:sqlite": "NODE_ENV=benchmark N8N_LOG_LEVEL=silent DB_TYPE=sqlite node dist/benchmark/main.js",
+    "benchmark:postgres": "NODE_ENV=benchmark N8N_LOG_LEVEL=silent DB_TYPE=postgresdb node dist/benchmark/main.js",
     "clean": "rimraf dist .turbo",
     "typecheck": "tsc",
     "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/build.mjs",
diff --git a/packages/cli/src/benchmark/lib/api.ts b/packages/cli/src/benchmark/lib/api.ts
index e59c8a7db481a..3a23463370c88 100644
--- a/packages/cli/src/benchmark/lib/api.ts
+++ b/packages/cli/src/benchmark/lib/api.ts
@@ -61,7 +61,7 @@ export function suite(suiteName: string, suiteFn: () => void) {
 
 	if (suites[key]) throw new DuplicateSuiteError(key);
 
-	suites[key] = { name: suiteName, hooks: {}, tasks: [], db: 'sqlite' };
+	suites[key] = { name: suiteName, hooks: {}, tasks: [] };
 
 	suiteFn();
 }
diff --git a/packages/cli/src/benchmark/lib/hooks/seed.ts b/packages/cli/src/benchmark/lib/hooks/seed.ts
index 11eb0e8850d68..ac58623ee25c6 100644
--- a/packages/cli/src/benchmark/lib/hooks/seed.ts
+++ b/packages/cli/src/benchmark/lib/hooks/seed.ts
@@ -13,7 +13,7 @@ export async function seedInstanceOwner() {
 
 	const user = await Container.get(UserRepositoryExtension).createInstanceOwner();
 
-	log('Seeded user', user.email);
+	log('Seeded user in DB', user.email);
 }
 
 export async function seedWorkflows() {
@@ -38,7 +38,7 @@ export async function seedWorkflows() {
 
 	const files = _files.map((f) => f.replace(/.*workflows\//, '')).join(' ');
 
-	log('Seeded workflows', files);
+	log('Seeded workflows in DB', files);
 
 	return files;
 }
diff --git a/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts b/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
index d86ea0d6edcc4..90775f844c9f3 100644
--- a/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
+++ b/packages/cli/src/benchmark/lib/hooks/setup-and-teardown.ts
@@ -5,13 +5,22 @@ import { n8nDir } from './n8nDir';
 import { ActiveWorkflowRunner } from '@/ActiveWorkflowRunner';
 import { seedInstanceOwner, seedWorkflows } from './seed';
 import { log } from '../log';
+import { postgresSetup, postgresTeardown } from '../postgres';
+import * as Db from '@/Db';
+import config from '@/config';
 
 let main: Start;
 
+const dbType = config.getEnv('database.type');
+
 export async function setup() {
 	n8nDir();
 
-	main = new Start([], new Config({ root: __dirname })); // @TODO: Silence stdout
+	log('Selected DB type', dbType);
+
+	if (dbType === 'postgresdb') await postgresSetup();
+
+	main = new Start([], new Config({ root: __dirname }));
 
 	await main.init();
 	await main.run();
@@ -26,4 +35,8 @@ export async function setup() {
 
 export async function teardown() {
 	await main.stopProcess();
+
+	await Db.close();
+
+	if (dbType === 'postgresdb') await postgresTeardown();
 }
diff --git a/packages/cli/src/benchmark/lib/index.ts b/packages/cli/src/benchmark/lib/index.ts
index efcf4c96daa5c..5745c94153f7c 100644
--- a/packages/cli/src/benchmark/lib/index.ts
+++ b/packages/cli/src/benchmark/lib/index.ts
@@ -1,6 +1,6 @@
 import 'reflect-metadata';
 
-export { suite, task, beforeEachTask, afterEachTask, collectSuites, registerSuites } from './api';
+export { suite, task, beforeEachTask, afterEachTask, collectSuites } from './api';
 export { agent } from './agent';
 export { setup, teardown } from './hooks/setup-and-teardown';
 export type { Suites } from './types';
diff --git a/packages/cli/src/benchmark/lib/log.ts b/packages/cli/src/benchmark/lib/log.ts
index 6c5f5ea04470b..d5009000de9f5 100644
--- a/packages/cli/src/benchmark/lib/log.ts
+++ b/packages/cli/src/benchmark/lib/log.ts
@@ -1,5 +1,6 @@
 import pico from 'picocolors';
 import { assert } from 'n8n-workflow';
+import config from '@/config';
 import type Bench from 'tinybench';
 import type { Suites } from './types';
 
@@ -47,6 +48,7 @@ const toDirsAndFileName = (key: string) => {
 };
 
 export function logResults(suites: Suites, results: Bench['results']) {
+	const dbType = config.getEnv('database.type') === 'postgresdb' ? 'postgres' : 'sqlite';
 	const columnDivider = pico.dim('·'.repeat(3));
 
 	for (const [key, suite] of Object.entries(suites)) {
@@ -56,7 +58,7 @@ export function logResults(suites: Suites, results: Bench['results']) {
 			'\n',
 			pico.bgWhite(pico.black(' BENCHMARK ')),
 			pico.gray(dirs) + pico.bold(fileName),
-			pico.dim('[' + suite.db + ']'),
+			pico.dim('[' + dbType + ']'),
 			'\n',
 		].join(' ');
 
diff --git a/packages/cli/src/benchmark/lib/postgres.ts b/packages/cli/src/benchmark/lib/postgres.ts
new file mode 100644
index 0000000000000..2a5e9694cdc2c
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/postgres.ts
@@ -0,0 +1,64 @@
+import { DataSource } from '@n8n/typeorm';
+import type { DataSourceOptions } from '@n8n/typeorm';
+import config from '@/config';
+import { log } from './log';
+
+const BENCHMARK_DB_PREFIX = 'n8n_benchmark';
+
+const pgOptions: DataSourceOptions = {
+	type: 'postgres',
+	database: config.getEnv('database.postgresdb.database'),
+	host: config.getEnv('database.postgresdb.host'),
+	port: config.getEnv('database.postgresdb.port'),
+	username: config.getEnv('database.postgresdb.user'),
+	password: config.getEnv('database.postgresdb.password'),
+	schema: config.getEnv('database.postgresdb.schema'),
+};
+
+function tenRandomChars() {
+	const ALPHABET = 'abcdefghijklmnopqrstuvwxyz';
+
+	let result = '';
+
+	for (let i = 0; i < 10; i++) {
+		result += ALPHABET[Math.floor(Math.random() * ALPHABET.length)];
+	}
+
+	return result;
+}
+
+export async function postgresSetup() {
+	const dbName = [BENCHMARK_DB_PREFIX, tenRandomChars(), Date.now()].join('_');
+
+	const bootstrap = await new DataSource(pgOptions).initialize();
+
+	await bootstrap.query(`CREATE DATABASE ${dbName};`);
+	await bootstrap.destroy();
+
+	log('Created temp Postgres DB', dbName);
+
+	config.set('database.postgresdb.database', dbName);
+}
+
+export async function postgresTeardown() {
+	const bootstrap = new DataSource(pgOptions);
+	await bootstrap.initialize();
+
+	const results: Array<{ dbName: string }> = await bootstrap.query(
+		'SELECT datname AS "dbName" FROM pg_database',
+	);
+
+	const dbNames = results
+		.filter(({ dbName }) => dbName.startsWith(BENCHMARK_DB_PREFIX))
+		.map(({ dbName }) => dbName);
+
+	const promises: Array<Promise<void>> = dbNames.map(
+		async (dbName) => await bootstrap.query(`DROP DATABASE ${dbName};`),
+	);
+
+	await Promise.all(promises);
+
+	log('Dropped temp Postgres DB', dbNames.at(0));
+
+	await bootstrap.destroy();
+}
diff --git a/packages/cli/src/benchmark/lib/types.ts b/packages/cli/src/benchmark/lib/types.ts
index b7db1b6132c34..35830b2052ca6 100644
--- a/packages/cli/src/benchmark/lib/types.ts
+++ b/packages/cli/src/benchmark/lib/types.ts
@@ -1,7 +1,6 @@
 export type Suites = {
 	[key: string]: {
 		name: string;
-		db: 'sqlite' | 'postgres';
 		hooks: {
 			beforeEachTask?: Callback;
 			afterEachTask?: Callback;
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 543a06da1e787..ecf743d2d7341 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -1,26 +1,20 @@
 import 'reflect-metadata';
+import { collectSuites, log, logResults, setup, teardown, toOneLineJson } from './lib';
+import { registerSuites } from './lib/api';
 import config from '@/config';
-import {
-	collectSuites,
-	log,
-	logResults,
-	registerSuites,
-	setup,
-	teardown,
-	toOneLineJson,
-} from './lib';
 
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
+
 /* eslint-enable import/no-extraneous-dependencies */
 
 async function main() {
-	const suites = await collectSuites(); // @TODO: --filter suites
+	const suites = await collectSuites();
 
 	log('Found suites', Object.keys(suites).join(' '));
 
-	await setup(); // 1. create a benchmark postgres DB
+	await setup();
 
 	const benchConfig = {
 		time: config.getEnv('benchmark.time'),
@@ -30,35 +24,27 @@ async function main() {
 		warmupIterations: config.getEnv('benchmark.warmupIterations'),
 	};
 
-	const _bench = new Bench({
-		// @TODO: Temp values
-		// time: 0,
-		// iterations: 1,
-		...benchConfig,
-	});
+	const _bench = new Bench(benchConfig);
 
 	const bench = process.env.CI === 'true' ? withCodSpeed(_bench) : _bench;
 
 	registerSuites(bench);
 
-	// 2. duplicate all suites
-	// - add a `[postgres]` prefix to each suite name
-	// - add a `beforeEachTask` hook to each new task, containing `config.set('database.type', 'postgresdb')`
-
 	// await bench.warmup(); // @TODO: Restore
 
 	log('Set config', toOneLineJson(benchConfig));
+
 	log('Running iterations, please wait...');
 
 	await bench.run();
 
-	log('Iterations completed');
+	log('Iterations completed ✓');
+
+	await teardown();
 
 	if (process.env.CI !== 'true') logResults(suites, bench.results);
-	// console.table(bench.table());
-	// console.log(bench.results);
 
-	await teardown(); // 3. remove benchmark postgres DB
+	process.exit(0);
 }
 
 void main();
diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts
index 048ab9bf2cc05..0f5b286ec3e52 100644
--- a/packages/cli/src/commands/start.ts
+++ b/packages/cli/src/commands/start.ts
@@ -114,6 +114,8 @@ export class Start extends BaseCommand {
 			await this.exitWithCrash('There was an error shutting down n8n.', error);
 		}
 
+		if (inBenchmark) return;
+
 		await this.exitSuccessFully();
 	}
 

From 4ef5a2acbfb502969f34be47d050fe13874d9ee7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 30 Apr 2024 19:00:55 +0200
Subject: [PATCH 79/86] Improve docs

---
 packages/cli/src/benchmark/benchmark.md | 49 ++++++++++++-------------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index 4b7efca39d138..70a8df782b593 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -1,31 +1,44 @@
 # Benchmark
 
-This package contains benchmarks to measure the execution time of n8n backend operations.
+This package contains benchmarks to measure the execution time of n8n backend operations in sqlite and Postgres.
 
-Benchmarks are organized into **suites** for the scenario to benchmark and **tasks** for operations in that scenario, with hooks for per-task setup and teardown, runnning in both sqlite and Postgres. This lib is implemented on top of [`tinybench`](https://github.com/tinylibs/tinybench). Execution in CI is delegated to [Codspeed](https://codspeed.io/) to keep measurements consistent and to monitor improvements and regressions.
+Benchmarks are organized into **suites** for the scenario to benchmark, **tasks** for operations in that scenario, and **hooks** for setup and teardown, implemented on top of [`tinybench`](https://github.com/tinylibs/tinybench). Execution in CI is delegated to [Codspeed](https://codspeed.io/) to keep measurements consistent and to monitor improvements and regressions.
 
 ## Running benchmarks
 
-To start a benchmarking run:
+To run benchmarks:
 
 ```sh
 pnpm build:benchmark
-pnpm benchmark
+pnpm benchmark:sqlite # or
+pnpm benchmark:postgres
 ```
 
-On every run, benchmarking will set up and tear down a temporary Postgres database, so ensure you have a Postgres server running locally and allow n8n to connect to it via [environment variables](https://docs.n8n.io/hosting/configuration/environment-variables/database/#postgresql).
-
-The default benchmarking configuration can be adjusted via [environment variables](https://docs.n8n.io/hosting/configuration/environment-variables/benchmarking).
+Locally, the benchmarking run can be configured (number of iterations, warmup, etc.) via [environment variables](https://docs.n8n.io/hosting/configuration/environment-variables/benchmarking). In CI, the configuration is set by Codspeed.
 
 ## Creating benchmarks
 
 To create benchmarks:
 
-1. Create a file at `suites/{suite-id}-{suite-title}.ts` or `suites/{theme}/{suite-id}-{suite-title}.ts`.
+1. Create a file at `suites/**/{suite-id}-{suite-title}.ts`.
 2. Include a `suite()` call for the scenario to benchmark.
-3. Include one or more `task()` calls for operations in that scenario. `task()` must contain only the specific operation whose execution time to measure. Move any per-task setup and teardown to `beforeEachTask()` and `afterEachTask()` in the suite.
-4. Include workflows at `suites/workflows/{suite-id}-{ordinal-number}`. During setup, all workflows at this dir are loaded to the temp DBs and activated in memory. If the workflow is triggered by webhook, set the webhook path to `/{suite-id}-{ordinal-number}`.
-5. Run `pnpm build:benchmark` to add the suite and its tasks to the index in this document.
+3. Inside the suite, include one or more `task()` calls for operations in that scenario. `task()` must contain only the specific operation whose execution time to measure. Move any per-task setup and teardown to `beforeEachTask()` and `afterEachTask()` in the suite.
+4. Include workflows at `suites/workflows/{suite-id}-{ordinal-number}`. During setup, workflows at this dir are saved in the temp DB and activated in memory.
+5. Run `pnpm build:benchmark` to add the suite and its tasks to the index below.
+
+## Index of benchmarking suites
+
+> **Note**: All workflows with default settings unless otherwise specified.
+
+<!-- BENCHMARK_SUITES_LIST -->
+
+### 001 - Production workflow with authless webhook node
+
+- [using "Respond immediately" mode](./suites/workflows/001-1.json)
+- [using "When last node finishes" mode](./suites/workflows/001-2.json)
+- [using "Respond to Webhook node" mode](./suites/workflows/001-3.json)
+
+<!-- /BENCHMARK_SUITES_LIST -->
 
 ## Reading benchmarks
 
@@ -50,17 +63,3 @@ BENCHMARK suites/001-production-webhook-with-authless-webhook-node.suite.ts [sql
 `std err` (standard error) reflects how closely a sample mean is expected to approximate the true population mean. A smaller standard error indicates that the sample mean is likely to be a more accurate estimate of the population mean because the variation among sample means is less. For example, in the task `using "Respond immediately" mode`, the standard error is 2.037 ms, which suggests that the sample mean is expected to differ from the true population mean by 2.037 ms on average.
 
 `std dev` (standard deviation) is the amount of dispersion across samples. When low, it indicates that the samples tend to be close to the mean; when high, it indicates that the samples are spread out over a wider range. For example, in the task `using "Respond immediately" mode`, the standard deviation is `10.586 ms`, which suggests that the execution times varied significantly across iterations.
-
-## Index of benchmarking suites
-
-> **Note**: All workflows with default settings unless otherwise specified.
-
-<!-- BENCHMARK_SUITES_LIST -->
-
-### 001 - Production workflow with authless webhook node
-
-- [using "Respond immediately" mode](./suites/workflows/001-1.json)
-- [using "When last node finishes" mode](./suites/workflows/001-2.json)
-- [using "Respond to Webhook node" mode](./suites/workflows/001-3.json)
-
-<!-- /BENCHMARK_SUITES_LIST -->

From 1a9110225515f52f6d00956f53491c90ad6f633b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 30 Apr 2024 19:06:53 +0200
Subject: [PATCH 80/86] Handle connection error gracefully

---
 .../lib/errors/postgres-connection.error.ts          | 12 ++++++++++++
 packages/cli/src/benchmark/lib/postgres.ts           |  9 ++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 packages/cli/src/benchmark/lib/errors/postgres-connection.error.ts

diff --git a/packages/cli/src/benchmark/lib/errors/postgres-connection.error.ts b/packages/cli/src/benchmark/lib/errors/postgres-connection.error.ts
new file mode 100644
index 0000000000000..d280e88bb89e7
--- /dev/null
+++ b/packages/cli/src/benchmark/lib/errors/postgres-connection.error.ts
@@ -0,0 +1,12 @@
+import { ApplicationError } from 'n8n-workflow';
+import type { DataSourceOptions } from '@n8n/typeorm';
+
+export class PostgresConnectionError extends ApplicationError {
+	constructor(error: unknown, pgOptions: DataSourceOptions) {
+		super('Failed to connect to Postgres - check your Postgres configuration', {
+			level: 'warning',
+			cause: error,
+			extra: { postgresConfig: { pgOptions } },
+		});
+	}
+}
diff --git a/packages/cli/src/benchmark/lib/postgres.ts b/packages/cli/src/benchmark/lib/postgres.ts
index 2a5e9694cdc2c..23c13dc6658f0 100644
--- a/packages/cli/src/benchmark/lib/postgres.ts
+++ b/packages/cli/src/benchmark/lib/postgres.ts
@@ -2,6 +2,7 @@ import { DataSource } from '@n8n/typeorm';
 import type { DataSourceOptions } from '@n8n/typeorm';
 import config from '@/config';
 import { log } from './log';
+import { PostgresConnectionError } from './errors/postgres-connection.error';
 
 const BENCHMARK_DB_PREFIX = 'n8n_benchmark';
 
@@ -30,7 +31,13 @@ function tenRandomChars() {
 export async function postgresSetup() {
 	const dbName = [BENCHMARK_DB_PREFIX, tenRandomChars(), Date.now()].join('_');
 
-	const bootstrap = await new DataSource(pgOptions).initialize();
+	let bootstrap: DataSource;
+
+	try {
+		bootstrap = await new DataSource(pgOptions).initialize();
+	} catch (error) {
+		throw new PostgresConnectionError(error, pgOptions);
+	}
 
 	await bootstrap.query(`CREATE DATABASE ${dbName};`);
 	await bootstrap.destroy();

From 00ff0b300ae2362822c947218872f8a50dd204f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 30 Apr 2024 19:08:41 +0200
Subject: [PATCH 81/86] Remove outdated comment

---
 packages/core/src/InstanceSettings.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/core/src/InstanceSettings.ts b/packages/core/src/InstanceSettings.ts
index 0c902c385f8ab..7a3494a76e0e0 100644
--- a/packages/core/src/InstanceSettings.ts
+++ b/packages/core/src/InstanceSettings.ts
@@ -71,7 +71,6 @@ export class InstanceSettings {
 				errorMessage: `Error parsing n8n-config file "${this.settingsFile}". It does not seem to be valid JSON.`,
 			});
 
-			// @TODO: Remove first two logs in benchmark run
 			if (!inTest) Logger.info(`User settings loaded from: ${this.settingsFile}`);
 
 			const { encryptionKey, tunnelSubdomain } = settings;

From eaa87907455201de19b24e03ffa59cfe691b3dbb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 30 Apr 2024 19:09:04 +0200
Subject: [PATCH 82/86] Remove another TODO

---
 packages/cli/src/config/schema.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts
index c82acd90c84cd..b16ca6971071d 100644
--- a/packages/cli/src/config/schema.ts
+++ b/packages/cli/src/config/schema.ts
@@ -1437,7 +1437,6 @@ export const schema = {
 		doc: 'Number of reverse-proxies n8n is running behind',
 	},
 
-	// @TODO: Stay with tinybench's defaults?
 	benchmark: {
 		time: {
 			doc: 'Length of time (ms) during which to repeatedly run a benchmarking task',

From c1d69e8947c3cfdf7ccb6495c98bfd7da61e085e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 30 Apr 2024 19:10:22 +0200
Subject: [PATCH 83/86] Reduce diff

---
 .../migrations/common/1711390882123-MoveSshKeysToDatabase.ts   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
index bd7d13483916a..30d7d87c4ac6a 100644
--- a/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
+++ b/packages/cli/src/databases/migrations/common/1711390882123-MoveSshKeysToDatabase.ts
@@ -4,7 +4,6 @@ import Container from 'typedi';
 import { Cipher, InstanceSettings } from 'n8n-core';
 import { jsonParse } from 'n8n-workflow';
 import type { MigrationContext, ReversibleMigration } from '@db/types';
-import { inTest } from '@/constants';
 
 /**
  * Move SSH key pair from file system to database, to enable SSH connections
@@ -32,7 +31,7 @@ export class MoveSshKeysToDatabase1711390882123 implements ReversibleMigration {
 				readFile(this.publicKeyPath, { encoding: 'utf8' }),
 			]);
 		} catch {
-			if (!inTest) logger.info(`[${migrationName}] No SSH keys in filesystem, skipping`);
+			logger.info(`[${migrationName}] No SSH keys in filesystem, skipping`);
 			return;
 		}
 

From f60caf4483e30bf6d7a93872eeb65475c3bd068b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 30 Apr 2024 19:12:27 +0200
Subject: [PATCH 84/86] Cleanup

---
 packages/cli/src/benchmark/main.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index ecf743d2d7341..89ff21a3c7a97 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -6,7 +6,6 @@ import config from '@/config';
 /* eslint-disable import/no-extraneous-dependencies */
 import Bench from 'tinybench';
 import { withCodSpeed } from '@codspeed/tinybench-plugin';
-
 /* eslint-enable import/no-extraneous-dependencies */
 
 async function main() {

From b40652d7cfccdac8a49d78507892df4cd1bcebc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Tue, 30 Apr 2024 21:59:07 +0200
Subject: [PATCH 85/86] Patch @codspeed/tinybench-plugin@3.1.0

---
 package.json                                   |  3 ++-
 packages/cli/src/benchmark/lib/api.ts          |  6 ++++--
 .../@codspeed__tinybench-plugin@3.1.0.patch    | 18 ++++++++++++++++++
 pnpm-lock.yaml                                 | 14 +++++++++-----
 4 files changed, 33 insertions(+), 8 deletions(-)
 create mode 100644 patches/@codspeed__tinybench-plugin@3.1.0.patch

diff --git a/package.json b/package.json
index f2c8efc682af4..1ed7700bc6bc2 100644
--- a/package.json
+++ b/package.json
@@ -96,7 +96,8 @@
       "pyodide@0.23.4": "patches/pyodide@0.23.4.patch",
       "@types/express-serve-static-core@4.17.43": "patches/@types__express-serve-static-core@4.17.43.patch",
       "@types/ws@8.5.4": "patches/@types__ws@8.5.4.patch",
-      "vite-plugin-checker@0.6.4": "patches/vite-plugin-checker@0.6.4.patch"
+      "vite-plugin-checker@0.6.4": "patches/vite-plugin-checker@0.6.4.patch",
+      "@codspeed/tinybench-plugin@3.1.0": "patches/@codspeed__tinybench-plugin@3.1.0.patch"
     }
   }
 }
diff --git a/packages/cli/src/benchmark/lib/api.ts b/packages/cli/src/benchmark/lib/api.ts
index 3a23463370c88..b4bbbc83a4551 100644
--- a/packages/cli/src/benchmark/lib/api.ts
+++ b/packages/cli/src/benchmark/lib/api.ts
@@ -24,7 +24,7 @@ export async function collectSuites() {
 }
 
 export function registerSuites(bench: Bench) {
-	for (const { hooks, tasks } of Object.values(suites)) {
+	for (const { name: suiteName, hooks, tasks } of Object.values(suites)) {
 		/**
 		 * In tinybench, `beforeAll` and `afterAll` refer to all _iterations_ of
 		 * a single task, while `beforeEach` and `afterEach` refer to each _iteration_.
@@ -40,7 +40,9 @@ export function registerSuites(bench: Bench) {
 		if (hooks.afterEachTask) options.afterAll = hooks.afterEachTask;
 
 		for (const t of tasks) {
-			bench.add(t.name, t.operation, options);
+			const taskName = process.env.CI === 'true' ? [suiteName, t.name].join('::') : t.name;
+
+			bench.add(taskName, t.operation, options);
 		}
 	}
 }
diff --git a/patches/@codspeed__tinybench-plugin@3.1.0.patch b/patches/@codspeed__tinybench-plugin@3.1.0.patch
new file mode 100644
index 0000000000000..ce4f38306e2a3
--- /dev/null
+++ b/patches/@codspeed__tinybench-plugin@3.1.0.patch
@@ -0,0 +1,18 @@
+diff --git a/dist/index.cjs.js b/dist/index.cjs.js
+index a75964e40eaeff15df33fabd3c0bef20e9b1e5dd..42f2bec4950f13675ddccba675b2c207b2789d42 100644
+--- a/dist/index.cjs.js
++++ b/dist/index.cjs.js
+@@ -80,7 +80,12 @@ function withCodSpeed(bench) {
+   const rawAdd = bench.add;
+   bench.add = (name, fn, opts) => {
+     const callingFile = getCallingFile();
+-    const uri = `${callingFile}::${name}`;
++    let uri = `${callingFile}::${name}`;
++    if (name.includes('::')) {
++      // ignore calling file misdirecting to benchmark/lib/api.js
++      const [suiteName, taskName] = name.split('::');
++      uri = `${suiteName} - ${taskName}`
++		}
+     const options = Object.assign({}, opts ?? {}, { uri });
+     return rawAdd.bind(bench)(name, fn, options);
+   };
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 3b3f95ac100e0..a9bae4f7c81fa 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -19,6 +19,9 @@ overrides:
   typescript: ^5.4.2
 
 patchedDependencies:
+  '@codspeed/tinybench-plugin@3.1.0':
+    hash: dtpxhamve6ldymzame6xpj4atq
+    path: patches/@codspeed__tinybench-plugin@3.1.0.patch
   '@sentry/cli@2.17.0':
     hash: nchnoezkq6p37qaiku3vrpwraq
     path: patches/@sentry__cli@2.17.0.patch
@@ -759,7 +762,7 @@ importers:
     devDependencies:
       '@codspeed/tinybench-plugin':
         specifier: ^3.1.0
-        version: 3.1.0(tinybench@2.6.0)
+        version: 3.1.0(patch_hash=dtpxhamve6ldymzame6xpj4atq)(tinybench@2.6.0)
       '@redocly/cli':
         specifier: ^1.6.0
         version: 1.6.0
@@ -4726,7 +4729,7 @@ packages:
       - debug
     dev: true
 
-  /@codspeed/tinybench-plugin@3.1.0(tinybench@2.6.0):
+  /@codspeed/tinybench-plugin@3.1.0(patch_hash=dtpxhamve6ldymzame6xpj4atq)(tinybench@2.6.0):
     resolution: {integrity: sha512-yl0WzzUGIXkZzWaw7+2U+xGkuIal1Rs9hS09DtlDZGGAcGRoMMU5d2vyCS8nBrna4hrPQZ5Sx/hIKerO+lqWaw==}
     peerDependencies:
       tinybench: ^2.3.0
@@ -4737,6 +4740,7 @@ packages:
     transitivePeerDependencies:
       - debug
     dev: true
+    patched: true
 
   /@colors/colors@1.5.0:
     resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
@@ -9227,7 +9231,7 @@ packages:
       ts-dedent: 2.2.0
       type-fest: 2.19.0
       vue: 3.4.21(typescript@5.4.2)
-      vue-component-type-helpers: 2.0.14
+      vue-component-type-helpers: 2.0.15
     transitivePeerDependencies:
       - encoding
       - supports-color
@@ -25853,8 +25857,8 @@ packages:
     resolution: {integrity: sha512-0vOfAtI67UjeO1G6UiX5Kd76CqaQ67wrRZiOe7UAb9Jm6GzlUr/fC7CV90XfwapJRjpCMaZFhv1V0ajWRmE9Dg==}
     dev: true
 
-  /vue-component-type-helpers@2.0.14:
-    resolution: {integrity: sha512-DInfgOyXlMyliyqAAD9frK28tTfch0+tMi4qoWJcZlRxUf+NFAtraJBnAsKLep+FOyLMiajkhfyEb3xLK08i7w==}
+  /vue-component-type-helpers@2.0.15:
+    resolution: {integrity: sha512-jR/Hw52gzNQxMovJBsOQ/F9E1UQ8K1Np0CVG3RnueLkaCKqWuyL9XHl/5tUBAGJx+bk5xZ+co7vK23+Pzt75Lg==}
     dev: true
 
   /vue-demi@0.14.5(vue@3.4.21):

From 05ed5486adfb4236c8b278e1de8086cc6a369eaf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= <ivov.src@gmail.com>
Date: Thu, 2 May 2024 10:09:25 +0200
Subject: [PATCH 86/86] Cleanup

---
 packages/cli/src/benchmark/benchmark.md           | 12 ++++++------
 packages/cli/src/benchmark/lib/constants.ts       |  3 +--
 packages/cli/src/benchmark/lib/hooks/n8nDir.ts    |  2 +-
 packages/cli/src/benchmark/main.ts                |  2 +-
 packages/cli/src/benchmark/scripts/list-suites.ts | 11 +++++------
 packages/cli/src/config/schema.ts                 |  4 ++--
 6 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/packages/cli/src/benchmark/benchmark.md b/packages/cli/src/benchmark/benchmark.md
index 70a8df782b593..1df01b8cbe22a 100644
--- a/packages/cli/src/benchmark/benchmark.md
+++ b/packages/cli/src/benchmark/benchmark.md
@@ -14,21 +14,21 @@ pnpm benchmark:sqlite # or
 pnpm benchmark:postgres
 ```
 
-Locally, the benchmarking run can be configured (number of iterations, warmup, etc.) via [environment variables](https://docs.n8n.io/hosting/configuration/environment-variables/benchmarking). In CI, the configuration is set by Codspeed.
+Locally, the benchmarking run can be configured via [environment variables](https://docs.n8n.io/hosting/configuration/environment-variables/benchmarking). In CI, the configuration is set by Codspeed.
 
 ## Creating benchmarks
 
 To create benchmarks:
 
-1. Create a file at `suites/**/{suite-id}-{suite-title}.ts`.
+1. Create a file at `suites/**/{suiteId}-{suiteTitle}.ts`.
 2. Include a `suite()` call for the scenario to benchmark.
 3. Inside the suite, include one or more `task()` calls for operations in that scenario. `task()` must contain only the specific operation whose execution time to measure. Move any per-task setup and teardown to `beforeEachTask()` and `afterEachTask()` in the suite.
-4. Include workflows at `suites/workflows/{suite-id}-{ordinal-number}`. During setup, workflows at this dir are saved in the temp DB and activated in memory.
+4. Include workflows at `suites/workflows/{suiteId}-{ordinalNumber}`. During setup, workflows at this dir are saved in the temp DB and activated in memory.
 5. Run `pnpm build:benchmark` to add the suite and its tasks to the index below.
 
 ## Index of benchmarking suites
 
-> **Note**: All workflows with default settings unless otherwise specified.
+> **Note**: All workflows with default settings unless otherwise specified, e.g. `EXECUTIONS_MODE` is `regular` unless `queue` is specified.
 
 <!-- BENCHMARK_SUITES_LIST -->
 
@@ -58,8 +58,8 @@ BENCHMARK suites/001-production-webhook-with-authless-webhook-node.suite.ts [sql
 
 `min` is the shortest execution time recorded across all iterations of the task, `max` is the longest, and `mean` is the average.
 
-`MoE` (margin of error) reflects how much the sample mean is expected to differ from the true population mean. For example, a margin of error of ±4.189% in the task `using "Respond immediately" mode` suggests that, if the benchmarking run were repeated multiple times, the sample mean would fall within 4.189% of the true population mean in 95% of those runs, assuming a standard confidence level. This range indicates the variability we might see due to the randomness of selecting a sample.
+`MoE` (margin of error) reflects how much the sample mean is expected to differ from the true population mean. For example, a margin of error of ±4.1% in the task `using "Respond immediately" mode` suggests that, if the benchmarking run were repeated multiple times, the sample mean would fall within 4.1% of the true population mean in 95% of those runs, assuming a standard confidence level. This range indicates the variability we might see due to the randomness of selecting a sample.
 
 `std err` (standard error) reflects how closely a sample mean is expected to approximate the true population mean. A smaller standard error indicates that the sample mean is likely to be a more accurate estimate of the population mean because the variation among sample means is less. For example, in the task `using "Respond immediately" mode`, the standard error is 2.037 ms, which suggests that the sample mean is expected to differ from the true population mean by 2.037 ms on average.
 
-`std dev` (standard deviation) is the amount of dispersion across samples. When low, it indicates that the samples tend to be close to the mean; when high, it indicates that the samples are spread out over a wider range. For example, in the task `using "Respond immediately" mode`, the standard deviation is `10.586 ms`, which suggests that the execution times varied significantly across iterations.
+`std dev` (standard deviation) is the amount of dispersion across samples. When low, it indicates that the samples tend to be close to the mean; when high, it indicates that the samples are spread out over a wider range. For example, in the task `using "Respond immediately" mode`, the standard deviation is 10.586 ms, which suggests that the execution times varied significantly across iterations.
diff --git a/packages/cli/src/benchmark/lib/constants.ts b/packages/cli/src/benchmark/lib/constants.ts
index ff1a276db194c..5cb0e8bdb442a 100644
--- a/packages/cli/src/benchmark/lib/constants.ts
+++ b/packages/cli/src/benchmark/lib/constants.ts
@@ -5,6 +5,5 @@ export const INSTANCE_ONWER = {
 	PASSWORD: 'password',
 	FIRST_NAME: 'Instance',
 	LAST_NAME: 'Owner',
-	API_KEY:
-		'n8n_api_96a4804143bdeb044802273c93423c33e1582c89a764c645fd6304fd2df19f3b2c73f4b972e28194',
+	API_KEY: 'n8n_api_123',
 };
diff --git a/packages/cli/src/benchmark/lib/hooks/n8nDir.ts b/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
index 40f4b9215117f..21f0acf93392d 100644
--- a/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
+++ b/packages/cli/src/benchmark/lib/hooks/n8nDir.ts
@@ -21,7 +21,7 @@ export function n8nDir() {
 
 	writeFileSync(
 		path.join(tempN8nDir, 'config'),
-		JSON.stringify({ encryptionKey: 'temp_encryption_key', instanceId: 'temp-123' }),
+		JSON.stringify({ encryptionKey: 'temp-encryption-key', instanceId: 'temp-123' }),
 		'utf-8',
 	);
 
diff --git a/packages/cli/src/benchmark/main.ts b/packages/cli/src/benchmark/main.ts
index 89ff21a3c7a97..93c21fa065f9a 100644
--- a/packages/cli/src/benchmark/main.ts
+++ b/packages/cli/src/benchmark/main.ts
@@ -29,7 +29,7 @@ async function main() {
 
 	registerSuites(bench);
 
-	// await bench.warmup(); // @TODO: Restore
+	await bench.warmup();
 
 	log('Set config', toOneLineJson(benchConfig));
 
diff --git a/packages/cli/src/benchmark/scripts/list-suites.ts b/packages/cli/src/benchmark/scripts/list-suites.ts
index dfc77d98ecce2..672f5fefde6d5 100644
--- a/packages/cli/src/benchmark/scripts/list-suites.ts
+++ b/packages/cli/src/benchmark/scripts/list-suites.ts
@@ -5,10 +5,8 @@ import { collectSuites } from '../lib';
 import type { Suites } from '../lib';
 
 async function exists(filePath: string) {
-	const fullPath = path.resolve('src', 'benchmark', filePath);
-
 	try {
-		await fs.access(fullPath);
+		await fs.access(filePath);
 		return true;
 	} catch {
 		return false;
@@ -25,10 +23,11 @@ async function toSuitesList(suites: Suites) {
 
 		for (let i = 0; i < suite.tasks.length; i++) {
 			const suiteName = suite.tasks[i].name.replace(suite.name, '').trim();
-			const workflowPath = `./suites/workflows/${suiteId}-${i + 1}.json`;
+			const relativeWorkflowPath = `./suites/workflows/${suiteId}-${i + 1}.json`;
+			const absoluteWorkflowPath = path.resolve('src', 'benchmark', relativeWorkflowPath);
 
-			list += (await exists(workflowPath))
-				? `- [${suiteName}](${workflowPath})\n`
+			list += (await exists(absoluteWorkflowPath))
+				? `- [${suiteName}](${relativeWorkflowPath})\n`
 				: `- ${suiteName}\n`;
 		}
 	}
diff --git a/packages/cli/src/config/schema.ts b/packages/cli/src/config/schema.ts
index b16ca6971071d..59e0e593977f9 100644
--- a/packages/cli/src/config/schema.ts
+++ b/packages/cli/src/config/schema.ts
@@ -1457,13 +1457,13 @@ export const schema = {
 			env: 'N8N_BENCHMARK_STOP_ON_ERROR',
 		},
 		warmupTime: {
-			doc: 'Length of time (ms) during which to repeatedly run a benchmarking task for warmup',
+			doc: 'Length of time (ms) during which to repeatedly run each benchmarking task for warmup',
 			format: Number,
 			default: 100,
 			env: 'N8N_BENCHMARK_WARMUP_TIME',
 		},
 		warmupIterations: {
-			doc: 'Number of times to run a benchmarking task for warmup, even if `N8N_BENCHMARK_WARMUP_TIME` is exceeded',
+			doc: 'Number of times to run each benchmarking task for warmup, even if `N8N_BENCHMARK_WARMUP_TIME` is exceeded',
 			format: Number,
 			default: 5,
 			env: 'N8N_BENCHMARK_WARMUP_ITERATIONS',