From 158e7817f482c49bd57dd4bf07ddb186c44e23ad Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 16:37:55 -0700 Subject: [PATCH 01/35] Rearchitect exporter to background exporting tasks. --- ecosystem.config.js | 11 +- package-lock.json | 1222 ++++++++++++++++++++++++-- package.json | 7 +- src/core/index.ts | 4 +- src/core/queues.ts | 29 + src/core/types.ts | 5 + src/scripts/export/handlers/bank.ts | 117 +-- src/scripts/export/handlers/index.ts | 2 +- src/scripts/export/handlers/wasm.ts | 410 ++++----- src/scripts/export/index.ts | 226 ----- src/scripts/export/process.ts | 144 +++ src/scripts/export/trace.ts | 491 +++++++++++ src/scripts/export/types.ts | 55 +- src/scripts/export/utils.ts | 36 + src/scripts/export/worker.ts | 337 ------- src/server/routes/indexer/bull.ts | 14 + src/server/routes/indexer/index.ts | 5 + 17 files changed, 2138 insertions(+), 977 deletions(-) create mode 100644 src/core/queues.ts delete mode 100644 src/scripts/export/index.ts create mode 100644 src/scripts/export/process.ts create mode 100644 src/scripts/export/trace.ts delete mode 100644 src/scripts/export/worker.ts create mode 100644 src/server/routes/indexer/bull.ts diff --git a/ecosystem.config.js b/ecosystem.config.js index 1bf881ed..e0ddb70b 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -1,8 +1,15 @@ module.exports = { apps: [ { - name: 'exporter', - script: 'dist/scripts/export/index.js', + name: 'export-tracer', + script: 'dist/scripts/export/trace.js', + wait_ready: true, + listen_timeout: 30000, + kill_timeout: 30000, + }, + { + name: 'export-processor', + script: 'dist/scripts/export/process.js', wait_ready: true, listen_timeout: 30000, kill_timeout: 30000, diff --git a/package-lock.json b/package-lock.json index c7c88ab2..33fee0aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "AGPL-3.0-only", "dependencies": { + "@bull-board/koa": "^5.8.4", "@cosmjs/amino": "^0.29.5", "@cosmjs/cosmwasm-stargate": "^0.30.1", "@cosmjs/crypto": "^0.29.5", @@ -22,6 +23,7 @@ "@types/koa__router": "^12.0.0", "async-await-retry": "^2.0.1", "axios": "^1.3.6", + "bullmq": "^4.12.0", "commander": "^9.4.1", "cosmjs-types": "^0.8.0", "crypto-js": "^4.1.1", @@ -705,6 +707,54 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@bull-board/api": { + "version": "5.8.4", + "resolved": "https://registry.npmjs.org/@bull-board/api/-/api-5.8.4.tgz", + "integrity": "sha512-uuB2ziEpMc6FrkqqRAqyqSh+yKx1ukTvxlgDenOimuLAiHlcPfhNLgYuYv3qzqWVaZdvUCLhiNqAvOdURoKT5A==", + "dependencies": { + "redis-info": "^3.0.8" + }, + "peerDependencies": { + "@bull-board/ui": "5.8.4" + } + }, + "node_modules/@bull-board/koa": { + "version": "5.8.4", + "resolved": "https://registry.npmjs.org/@bull-board/koa/-/koa-5.8.4.tgz", + "integrity": "sha512-dahAUAEum3tTW7zehGdBibuiQMDXJ46YquG7wP4eQztzw4yfKC7kJKCn5pwM2iBwVvkJnuDRYrYyTbZj2TDMCA==", + "dependencies": { + "@bull-board/api": "5.8.4", + "@bull-board/ui": "5.8.4", + "ejs": "^3.1.7", + "koa": "^2.13.1", + "koa-mount": "^4.0.0", + "koa-router": "^10.0.0", + "koa-static": "^5.0.0", + "koa-views": "^7.0.1" + } + }, + "node_modules/@bull-board/koa/node_modules/ejs": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz", + "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@bull-board/ui": { + "version": "5.8.4", + "resolved": "https://registry.npmjs.org/@bull-board/ui/-/ui-5.8.4.tgz", + "integrity": "sha512-OBCwelyO5aptZUanZOSiHCK46Y3dNsCXMj4XXF8rNKx0sg/BMZMZzINjlpsCkgNNM5Lxl51OM1eYLjzZREvaAQ==", + "dependencies": { + "@bull-board/api": "5.8.4" + } + }, "node_modules/@confio/ics23": { "version": "0.6.8", "resolved": "https://registry.npmjs.org/@confio/ics23/-/ics23-0.6.8.tgz", @@ -1303,6 +1353,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1820,6 +1875,78 @@ "node": ">= 0.8" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz", + "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", + "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", + "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", + "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", + "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", + "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nicolo-ribaudo/semver-v6": { "version": "6.3.3", "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/semver-v6/-/semver-v6-6.3.3.tgz", @@ -2934,8 +3061,7 @@ "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", - "dev": true + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, "node_modules/abort-controller": { "version": "3.0.0", @@ -3091,6 +3217,11 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==" + }, "node_modules/anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -3188,8 +3319,7 @@ "node_modules/async": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==", - "dev": true + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, "node_modules/async-await-retry": { "version": "2.0.1", @@ -3392,8 +3522,7 @@ "node_modules/bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", - "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", - "dev": true + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, "node_modules/bn.js": { "version": "5.2.1", @@ -3504,6 +3633,64 @@ "node": ">=4" } }, + "node_modules/bullmq": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-4.12.0.tgz", + "integrity": "sha512-eAN7WnlR6MszikwQXE16oGMUWngfbYG0SqkxiwUmVm4muR3KYg+etIRZE3vRKNXWKw0WxJZTA+3oBfDMcCBh+w==", + "dependencies": { + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.2", + "lodash": "^4.17.21", + "msgpackr": "^1.6.2", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/bullmq/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/bullmq/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/bullmq/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -3719,6 +3906,14 @@ "node": ">=12" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -3832,16 +4027,40 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/condense-newlines": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/condense-newlines/-/condense-newlines-0.2.1.tgz", + "integrity": "sha512-P7X+QL9Hb9B/c8HI5BFFKmjgBu2XpQuF98WZ9XkO+dBGgk5XgwiQz7o1SmpglNWId3581UcS0SFAWfoIhMHPfg==", + "dependencies": { + "extend-shallow": "^2.0.1", + "is-whitespace": "^0.3.0", + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/config-chain": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", - "dev": true, "dependencies": { "ini": "^1.3.4", "proto-list": "~1.2.1" } }, + "node_modules/consolidate": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/consolidate/-/consolidate-0.16.0.tgz", + "integrity": "sha512-Nhl1wzCslqXYTJVDyJCu3ODohy9OfBMB5uD2BiBTzd7w+QY0lBzafkR8y8755yMYHAaMD4NuzbAw03/xzfw+eQ==", + "deprecated": "Please upgrade to consolidate v1.0.0+ as it has been modernized with several long-awaited fixes implemented. Maintenance is supported by Forward Email at https://forwardemail.net ; follow/watch https://github.com/ladjs/consolidate for updates and release changelog", + "dependencies": { + "bluebird": "^3.7.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -3923,6 +4142,17 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/croner": { "version": "4.1.97", "resolved": "https://registry.npmjs.org/croner/-/croner-4.1.97.tgz", @@ -4085,6 +4315,14 @@ "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==" }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4179,7 +4417,6 @@ "version": "0.15.3", "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", "integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==", - "dev": true, "dependencies": { "commander": "^2.19.0", "lru-cache": "^4.1.5", @@ -4193,14 +4430,12 @@ "node_modules/editorconfig/node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", - "dev": true + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" }, "node_modules/editorconfig/node_modules/lru-cache": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "dev": true, "dependencies": { "pseudomap": "^1.0.2", "yallist": "^2.1.2" @@ -4210,7 +4445,6 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true, "bin": { "semver": "bin/semver" } @@ -4218,8 +4452,7 @@ "node_modules/editorconfig/node_modules/yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", - "dev": true + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" }, "node_modules/ee-first": { "version": "1.1.1", @@ -5009,6 +5242,17 @@ "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==", "dev": true }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -5117,6 +5361,33 @@ "node": ">= 6" } }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -5337,6 +5608,17 @@ "node": ">=8.0.0" } }, + "node_modules/get-paths": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/get-paths/-/get-paths-0.0.7.tgz", + "integrity": "sha512-0wdJt7C1XKQxuCgouqd+ZvLJ56FQixKoki9MrFaO4EriqzXOiH9gbukaDE1ou08S8Ns3/yDzoBAISNPqj6e6tA==", + "dependencies": { + "pify": "^4.0.1" + }, + "engines": { + "node": ">=6.4" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -5789,8 +6071,7 @@ "node_modules/ini": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "node_modules/internal-slot": { "version": "1.0.3", @@ -5806,6 +6087,29 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ip": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.8.tgz", @@ -5867,6 +6171,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -5906,6 +6215,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -6092,6 +6409,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-whitespace": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-whitespace/-/is-whitespace-0.3.0.tgz", + "integrity": "sha512-RydPhl4S6JwAyj0JJjshWJEFG6hNye3pZFBRZaTUfZFwGHxzppNaNOVgQuS/E/SlhrApuMXrpnK1EEIXfdo3Dg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", @@ -6187,6 +6512,23 @@ "node": ">=8" } }, + "node_modules/jake": { + "version": "10.8.7", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz", + "integrity": "sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==", + "dependencies": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.4", + "minimatch": "^3.1.2" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/jest": { "version": "29.6.1", "resolved": "https://registry.npmjs.org/jest/-/jest-29.6.1.tgz", @@ -6766,7 +7108,6 @@ "version": "1.14.8", "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.14.8.tgz", "integrity": "sha512-4S7HFeI9YfRvRgKnEweohs0tgJj28InHVIj4Nl8Htf96Y6pHg3+tJrmo4ucAM9f7l4SHbFI3IvFAZ2a1eQPbyg==", - "dev": true, "dependencies": { "config-chain": "^1.1.13", "editorconfig": "^0.15.3", @@ -6786,7 +7127,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -6795,7 +7135,6 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", - "dev": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -6814,7 +7153,6 @@ "version": "5.1.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -6968,6 +7306,17 @@ "node": ">= 0.6" } }, + "node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -7040,6 +7389,89 @@ "node": ">= 10" } }, + "node_modules/koa-mount": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/koa-mount/-/koa-mount-4.0.0.tgz", + "integrity": "sha512-rm71jaA/P+6HeCpoRhmCv8KVBIi0tfGuO/dMKicbQnQW/YJntJ6MnnspkodoA4QstMVEZArsCphmd0bJEtoMjQ==", + "dependencies": { + "debug": "^4.0.1", + "koa-compose": "^4.1.0" + }, + "engines": { + "node": ">= 7.6.0" + } + }, + "node_modules/koa-router": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/koa-router/-/koa-router-10.1.1.tgz", + "integrity": "sha512-z/OzxVjf5NyuNO3t9nJpx7e1oR3FSBAauiwXtMQu4ppcnuNZzTaQ4p21P8A6r2Es8uJJM339oc4oVW+qX7SqnQ==", + "dependencies": { + "debug": "^4.1.1", + "http-errors": "^1.7.3", + "koa-compose": "^4.1.0", + "methods": "^1.1.2", + "path-to-regexp": "^6.1.0" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/koa-send": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/koa-send/-/koa-send-5.0.1.tgz", + "integrity": "sha512-tmcyQ/wXXuxpDxyNXv5yNNkdAMdFRqwtegBXUaowiQzUKqJehttS0x2j0eOZDQAyloAth5w6wwBImnFzkUz3pQ==", + "dependencies": { + "debug": "^4.1.1", + "http-errors": "^1.7.3", + "resolve-path": "^1.4.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/koa-static": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/koa-static/-/koa-static-5.0.0.tgz", + "integrity": "sha512-UqyYyH5YEXaJrf9S8E23GoJFQZXkBVJ9zYYMPGz919MSX1KuvAcycIuS0ci150HCoPf4XQVhQ84Qf8xRPWxFaQ==", + "dependencies": { + "debug": "^3.1.0", + "koa-send": "^5.0.0" + }, + "engines": { + "node": ">= 7.6.0" + } + }, + "node_modules/koa-static/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/koa-views": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/koa-views/-/koa-views-7.0.2.tgz", + "integrity": "sha512-dvx3mdVeSVuIPEaKAoGbxLcenudvhl821xxyuRbcoA+bOJ2dvN8wlGjkLu0ZFMlkCscXZV6lzxy28rafeazI/w==", + "deprecated": "This package is deprecated, please use the new fork @ladjs/koa-views. Maintenance is supported by Forward Email at https://forwardemail.net ; follow/watch https://github.com/ladjs/koa-views for updates and release changelog", + "dependencies": { + "consolidate": "^0.16.0", + "debug": "^4.1.0", + "get-paths": "0.0.7", + "koa-send": "^5.0.0", + "mz": "^2.4.0", + "pretty": "^2.0.0", + "resolve-path": "^1.4.0" + }, + "peerDependencies": { + "@types/koa": "^2.13.1" + }, + "peerDependenciesMeta": { + "@types/koa": { + "optional": true + } + } + }, "node_modules/lazy": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/lazy/-/lazy-1.0.11.tgz", @@ -7123,11 +7555,21 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "node_modules/lodash.groupby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", @@ -7181,6 +7623,14 @@ "es5-ext": "~0.10.2" } }, + "node_modules/luxon": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.3.tgz", + "integrity": "sha512-tFWBiv3h7z+T/tDaoxA8rqTxy1CHV6gHS//QdaH4pulbq/JuBSGgQspQQqcgnwdAx6pNI7cmvz5Sv/addzHmUg==", + "engines": { + "node": ">=12" + } + }, "node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -7406,6 +7856,35 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/msgpackr": { + "version": "1.9.9", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.9.tgz", + "integrity": "sha512-sbn6mioS2w0lq1O6PpGtsv6Gy8roWM+o3o4Sqjd6DudrL/nOugY+KyJUimoWzHnf9OkO0T6broHFnYE/R05t9A==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz", + "integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.0.7" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -7425,6 +7904,16 @@ "url": "https://github.com/sponsors/raouldeheer" } }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7486,6 +7975,11 @@ "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", "dev": true }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==" + }, "node_modules/node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -7505,6 +7999,17 @@ } } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "optional": true, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -7618,7 +8123,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz", "integrity": "sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g==", - "dev": true, "dependencies": { "abbrev": "^1.0.0" }, @@ -7669,6 +8173,14 @@ "integrity": "sha512-K7J4xq5xAD5jHsGM5ReWXRTFa3JRGofHiMcVgQ8PRwgWxzjHpMWCIzsmyf60+mh8KLsqYPcjUMa0AC4hd6lPyQ==", "dev": true }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-inspect": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", @@ -8059,6 +8571,14 @@ "node": ">=10" } }, + "node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "engines": { + "node": ">=6" + } + }, "node_modules/pirates": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz", @@ -8356,6 +8876,19 @@ "node": ">=6.0.0" } }, + "node_modules/pretty": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pretty/-/pretty-2.0.0.tgz", + "integrity": "sha512-G9xUchgTEiNpormdYBl+Pha50gOUovT18IvAe7EYMZ1/f9W/WWMPRn+xI68yXNMUk3QXHDwo/1wV/4NejVNe1w==", + "dependencies": { + "condense-newlines": "^0.2.1", + "extend-shallow": "^2.0.1", + "js-beautify": "^1.6.12" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/pretty-format": { "version": "29.6.1", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.6.1.tgz", @@ -8407,8 +8940,7 @@ "node_modules/proto-list": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", - "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", - "dev": true + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==" }, "node_modules/protobufjs": { "version": "6.11.3", @@ -8477,8 +9009,7 @@ "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==", - "dev": true + "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==" }, "node_modules/pstree.remy": { "version": "1.1.8", @@ -8660,6 +9191,33 @@ "resolved": "https://registry.npmjs.org/readonly-date/-/readonly-date-1.0.0.tgz", "integrity": "sha512-tMKIV7hlk0h4mO3JTmmVuIlJVXjKk3Sep9Bf5OH0O+758ruuVkUy2J9SttDLm91IEX/WHlXPSpxMGjPj4beMIQ==" }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-info": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redis-info/-/redis-info-3.1.0.tgz", + "integrity": "sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==", + "dependencies": { + "lodash": "^4.17.11" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -8764,6 +9322,50 @@ "node": ">=4" } }, + "node_modules/resolve-path": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/resolve-path/-/resolve-path-1.4.0.tgz", + "integrity": "sha512-i1xevIst/Qa+nA9olDxLWnLk8YZbi8R/7JPbCMcgyWaFR6bKWaexgJgEB5oc2PKMjYdrHynyz0NY+if+H98t1w==", + "dependencies": { + "http-errors": "~1.6.2", + "path-is-absolute": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/resolve-path/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/resolve-path/node_modules/http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/resolve-path/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + }, + "node_modules/resolve-path/node_modules/setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + }, "node_modules/resolve.exports": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", @@ -9160,8 +9762,7 @@ "node_modules/sigmund": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", - "integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==", - "dev": true + "integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==" }, "node_modules/signal-exit": { "version": "3.0.7", @@ -9309,6 +9910,11 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -9541,6 +10147,25 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/timers-ext": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", @@ -9950,6 +10575,18 @@ "punycode": "^2.1.0" } }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -10804,6 +11441,47 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "@bull-board/api": { + "version": "5.8.4", + "resolved": "https://registry.npmjs.org/@bull-board/api/-/api-5.8.4.tgz", + "integrity": "sha512-uuB2ziEpMc6FrkqqRAqyqSh+yKx1ukTvxlgDenOimuLAiHlcPfhNLgYuYv3qzqWVaZdvUCLhiNqAvOdURoKT5A==", + "requires": { + "redis-info": "^3.0.8" + } + }, + "@bull-board/koa": { + "version": "5.8.4", + "resolved": "https://registry.npmjs.org/@bull-board/koa/-/koa-5.8.4.tgz", + "integrity": "sha512-dahAUAEum3tTW7zehGdBibuiQMDXJ46YquG7wP4eQztzw4yfKC7kJKCn5pwM2iBwVvkJnuDRYrYyTbZj2TDMCA==", + "requires": { + "@bull-board/api": "5.8.4", + "@bull-board/ui": "5.8.4", + "ejs": "^3.1.7", + "koa": "^2.13.1", + "koa-mount": "^4.0.0", + "koa-router": "^10.0.0", + "koa-static": "^5.0.0", + "koa-views": "^7.0.1" + }, + "dependencies": { + "ejs": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz", + "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==", + "requires": { + "jake": "^10.8.5" + } + } + } + }, + "@bull-board/ui": { + "version": "5.8.4", + "resolved": "https://registry.npmjs.org/@bull-board/ui/-/ui-5.8.4.tgz", + "integrity": "sha512-OBCwelyO5aptZUanZOSiHCK46Y3dNsCXMj4XXF8rNKx0sg/BMZMZzINjlpsCkgNNM5Lxl51OM1eYLjzZREvaAQ==", + "requires": { + "@bull-board/api": "5.8.4" + } + }, "@confio/ics23": { "version": "0.6.8", "resolved": "https://registry.npmjs.org/@confio/ics23/-/ics23-0.6.8.tgz", @@ -11353,6 +12031,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -11774,6 +12457,42 @@ } } }, + "@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz", + "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", + "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", + "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", + "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", + "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", + "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", + "optional": true + }, "@nicolo-ribaudo/semver-v6": { "version": "6.3.3", "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/semver-v6/-/semver-v6-6.3.3.tgz", @@ -12703,8 +13422,7 @@ "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", - "dev": true + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, "abort-controller": { "version": "3.0.0", @@ -12814,6 +13532,11 @@ "color-convert": "^2.0.1" } }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==" + }, "anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -12892,8 +13615,7 @@ "async": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", - "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==", - "dev": true + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" }, "async-await-retry": { "version": "2.0.1", @@ -13039,8 +13761,7 @@ "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", - "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", - "dev": true + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, "bn.js": { "version": "5.2.1", @@ -13122,6 +13843,57 @@ "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" }, + "bullmq": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-4.12.0.tgz", + "integrity": "sha512-eAN7WnlR6MszikwQXE16oGMUWngfbYG0SqkxiwUmVm4muR3KYg+etIRZE3vRKNXWKw0WxJZTA+3oBfDMcCBh+w==", + "requires": { + "cron-parser": "^4.6.0", + "glob": "^8.0.3", + "ioredis": "^5.3.2", + "lodash": "^4.17.21", + "msgpackr": "^1.6.2", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + } + }, + "minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "requires": { + "brace-expansion": "^2.0.1" + } + }, + "tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + } + } + }, "bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -13268,6 +14040,11 @@ "wrap-ansi": "^7.0.0" } }, + "cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==" + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -13354,16 +14131,33 @@ } } }, + "condense-newlines": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/condense-newlines/-/condense-newlines-0.2.1.tgz", + "integrity": "sha512-P7X+QL9Hb9B/c8HI5BFFKmjgBu2XpQuF98WZ9XkO+dBGgk5XgwiQz7o1SmpglNWId3581UcS0SFAWfoIhMHPfg==", + "requires": { + "extend-shallow": "^2.0.1", + "is-whitespace": "^0.3.0", + "kind-of": "^3.0.2" + } + }, "config-chain": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", - "dev": true, "requires": { "ini": "^1.3.4", "proto-list": "~1.2.1" } }, + "consolidate": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/consolidate/-/consolidate-0.16.0.tgz", + "integrity": "sha512-Nhl1wzCslqXYTJVDyJCu3ODohy9OfBMB5uD2BiBTzd7w+QY0lBzafkR8y8755yMYHAaMD4NuzbAw03/xzfw+eQ==", + "requires": { + "bluebird": "^3.7.2" + } + }, "content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -13433,6 +14227,14 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "requires": { + "luxon": "^3.2.1" + } + }, "croner": { "version": "4.1.97", "resolved": "https://registry.npmjs.org/croner/-/croner-4.1.97.tgz", @@ -13559,6 +14361,11 @@ "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==" }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + }, "depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -13631,7 +14438,6 @@ "version": "0.15.3", "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", "integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==", - "dev": true, "requires": { "commander": "^2.19.0", "lru-cache": "^4.1.5", @@ -13642,14 +14448,12 @@ "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", - "dev": true + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" }, "lru-cache": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", - "dev": true, "requires": { "pseudomap": "^1.0.2", "yallist": "^2.1.2" @@ -13658,14 +14462,12 @@ "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", - "dev": true + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" }, "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", - "dev": true + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" } } }, @@ -14288,6 +15090,14 @@ } } }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "requires": { + "is-extendable": "^0.1.0" + } + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -14386,6 +15196,32 @@ "integrity": "sha512-hjPFI8oE/2iQPVe4gbrJ73Pp+Xfub2+WI2LlXDbsaJBwT5wuMh35WNWVYYTpnz895shtwfyutMFLFywpQAFdLg==", "dev": true }, + "filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "requires": { + "minimatch": "^5.0.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "requires": { + "brace-expansion": "^2.0.1" + } + } + } + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -14537,6 +15373,14 @@ "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "dev": true }, + "get-paths": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/get-paths/-/get-paths-0.0.7.tgz", + "integrity": "sha512-0wdJt7C1XKQxuCgouqd+ZvLJ56FQixKoki9MrFaO4EriqzXOiH9gbukaDE1ou08S8Ns3/yDzoBAISNPqj6e6tA==", + "requires": { + "pify": "^4.0.1" + } + }, "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -14870,8 +15714,7 @@ "ini": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "internal-slot": { "version": "1.0.3", @@ -14884,6 +15727,22 @@ "side-channel": "^1.0.4" } }, + "ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, "ip": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.8.tgz", @@ -14929,6 +15788,11 @@ "has-tostringtag": "^1.0.0" } }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -14953,6 +15817,11 @@ "has-tostringtag": "^1.0.0" } }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==" + }, "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -15073,6 +15942,11 @@ "call-bind": "^1.0.2" } }, + "is-whitespace": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-whitespace/-/is-whitespace-0.3.0.tgz", + "integrity": "sha512-RydPhl4S6JwAyj0JJjshWJEFG6hNye3pZFBRZaTUfZFwGHxzppNaNOVgQuS/E/SlhrApuMXrpnK1EEIXfdo3Dg==" + }, "isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", @@ -15150,6 +16024,17 @@ "istanbul-lib-report": "^3.0.0" } }, + "jake": { + "version": "10.8.7", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.8.7.tgz", + "integrity": "sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==", + "requires": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.4", + "minimatch": "^3.1.2" + } + }, "jest": { "version": "29.6.1", "resolved": "https://registry.npmjs.org/jest/-/jest-29.6.1.tgz", @@ -15601,7 +16486,6 @@ "version": "1.14.8", "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.14.8.tgz", "integrity": "sha512-4S7HFeI9YfRvRgKnEweohs0tgJj28InHVIj4Nl8Htf96Y6pHg3+tJrmo4ucAM9f7l4SHbFI3IvFAZ2a1eQPbyg==", - "dev": true, "requires": { "config-chain": "^1.1.13", "editorconfig": "^0.15.3", @@ -15613,7 +16497,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "requires": { "balanced-match": "^1.0.0" } @@ -15622,7 +16505,6 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -15635,7 +16517,6 @@ "version": "5.1.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, "requires": { "brace-expansion": "^2.0.1" } @@ -15760,6 +16641,14 @@ "tsscmp": "1.0.6" } }, + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "requires": { + "is-buffer": "^1.1.5" + } + }, "kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -15823,6 +16712,70 @@ "koa-compose": "^4.1.0" } }, + "koa-mount": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/koa-mount/-/koa-mount-4.0.0.tgz", + "integrity": "sha512-rm71jaA/P+6HeCpoRhmCv8KVBIi0tfGuO/dMKicbQnQW/YJntJ6MnnspkodoA4QstMVEZArsCphmd0bJEtoMjQ==", + "requires": { + "debug": "^4.0.1", + "koa-compose": "^4.1.0" + } + }, + "koa-router": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/koa-router/-/koa-router-10.1.1.tgz", + "integrity": "sha512-z/OzxVjf5NyuNO3t9nJpx7e1oR3FSBAauiwXtMQu4ppcnuNZzTaQ4p21P8A6r2Es8uJJM339oc4oVW+qX7SqnQ==", + "requires": { + "debug": "^4.1.1", + "http-errors": "^1.7.3", + "koa-compose": "^4.1.0", + "methods": "^1.1.2", + "path-to-regexp": "^6.1.0" + } + }, + "koa-send": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/koa-send/-/koa-send-5.0.1.tgz", + "integrity": "sha512-tmcyQ/wXXuxpDxyNXv5yNNkdAMdFRqwtegBXUaowiQzUKqJehttS0x2j0eOZDQAyloAth5w6wwBImnFzkUz3pQ==", + "requires": { + "debug": "^4.1.1", + "http-errors": "^1.7.3", + "resolve-path": "^1.4.0" + } + }, + "koa-static": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/koa-static/-/koa-static-5.0.0.tgz", + "integrity": "sha512-UqyYyH5YEXaJrf9S8E23GoJFQZXkBVJ9zYYMPGz919MSX1KuvAcycIuS0ci150HCoPf4XQVhQ84Qf8xRPWxFaQ==", + "requires": { + "debug": "^3.1.0", + "koa-send": "^5.0.0" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, + "koa-views": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/koa-views/-/koa-views-7.0.2.tgz", + "integrity": "sha512-dvx3mdVeSVuIPEaKAoGbxLcenudvhl821xxyuRbcoA+bOJ2dvN8wlGjkLu0ZFMlkCscXZV6lzxy28rafeazI/w==", + "requires": { + "consolidate": "^0.16.0", + "debug": "^4.1.0", + "get-paths": "0.0.7", + "koa-send": "^5.0.0", + "mz": "^2.4.0", + "pretty": "^2.0.0", + "resolve-path": "^1.4.0" + } + }, "lazy": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/lazy/-/lazy-1.0.11.tgz", @@ -15891,11 +16844,21 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "lodash.groupby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==" }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", @@ -15943,6 +16906,11 @@ "es5-ext": "~0.10.2" } }, + "luxon": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.3.tgz", + "integrity": "sha512-tFWBiv3h7z+T/tDaoxA8rqTxy1CHV6gHS//QdaH4pulbq/JuBSGgQspQQqcgnwdAx6pNI7cmvz5Sv/addzHmUg==" + }, "make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -16116,6 +17084,29 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "msgpackr": { + "version": "1.9.9", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.9.tgz", + "integrity": "sha512-sbn6mioS2w0lq1O6PpGtsv6Gy8roWM+o3o4Sqjd6DudrL/nOugY+KyJUimoWzHnf9OkO0T6broHFnYE/R05t9A==", + "requires": { + "msgpackr-extract": "^3.0.2" + } + }, + "msgpackr-extract": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz", + "integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==", + "optional": true, + "requires": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2", + "node-gyp-build-optional-packages": "5.0.7" + } + }, "mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -16128,6 +17119,16 @@ "integrity": "sha512-+MrqnJRtxdF+xngFfUUkIMQrUUL0KsxbADUkn23Z/4ibGg192Q+z+CQyiYwvWTsYjJygmMR8+w3ZDa98Zh6ESg==", "dev": true }, + "mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "requires": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -16179,6 +17180,11 @@ "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", "dev": true }, + "node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==" + }, "node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -16187,6 +17193,12 @@ "whatwg-url": "^5.0.0" } }, + "node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "optional": true + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -16279,7 +17291,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz", "integrity": "sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g==", - "dev": true, "requires": { "abbrev": "^1.0.0" } @@ -16317,6 +17328,11 @@ } } }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" + }, "object-inspect": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", @@ -16601,6 +17617,11 @@ "safe-buffer": "^5.2.1" } }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + }, "pirates": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz", @@ -16830,6 +17851,16 @@ "fast-diff": "^1.1.2" } }, + "pretty": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pretty/-/pretty-2.0.0.tgz", + "integrity": "sha512-G9xUchgTEiNpormdYBl+Pha50gOUovT18IvAe7EYMZ1/f9W/WWMPRn+xI68yXNMUk3QXHDwo/1wV/4NejVNe1w==", + "requires": { + "condense-newlines": "^0.2.1", + "extend-shallow": "^2.0.1", + "js-beautify": "^1.6.12" + } + }, "pretty-format": { "version": "29.6.1", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.6.1.tgz", @@ -16871,8 +17902,7 @@ "proto-list": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", - "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", - "dev": true + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==" }, "protobufjs": { "version": "6.11.3", @@ -16935,8 +17965,7 @@ "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==", - "dev": true + "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==" }, "pstree.remy": { "version": "1.1.8", @@ -17066,6 +18095,27 @@ "resolved": "https://registry.npmjs.org/readonly-date/-/readonly-date-1.0.0.tgz", "integrity": "sha512-tMKIV7hlk0h4mO3JTmmVuIlJVXjKk3Sep9Bf5OH0O+758ruuVkUy2J9SttDLm91IEX/WHlXPSpxMGjPj4beMIQ==" }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==" + }, + "redis-info": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redis-info/-/redis-info-3.1.0.tgz", + "integrity": "sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==", + "requires": { + "lodash": "^4.17.11" + } + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "requires": { + "redis-errors": "^1.0.0" + } + }, "reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -17139,6 +18189,43 @@ "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, + "resolve-path": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/resolve-path/-/resolve-path-1.4.0.tgz", + "integrity": "sha512-i1xevIst/Qa+nA9olDxLWnLk8YZbi8R/7JPbCMcgyWaFR6bKWaexgJgEB5oc2PKMjYdrHynyz0NY+if+H98t1w==", + "requires": { + "http-errors": "~1.6.2", + "path-is-absolute": "1.0.1" + }, + "dependencies": { + "depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" + }, + "http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + } + }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + }, + "setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" + } + } + }, "resolve.exports": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz", @@ -17394,8 +18481,7 @@ "sigmund": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", - "integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==", - "dev": true + "integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==" }, "signal-exit": { "version": "3.0.7", @@ -17517,6 +18603,11 @@ } } }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -17674,6 +18765,22 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, + "thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "requires": { + "any-promise": "^1.0.0" + } + }, + "thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "requires": { + "thenify": ">= 3.1.0 < 4" + } + }, "timers-ext": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", @@ -17946,6 +19053,11 @@ "punycode": "^2.1.0" } }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index 88647197..52637475 100644 --- a/package.json +++ b/package.json @@ -19,9 +19,10 @@ "db:migrate:data": "MIGRATING_DB=true sequelize-cli db:migrate --env data", "db:migrate:accounts": "MIGRATING_DB=true sequelize-cli db:migrate --env accounts", "clear-cache": "node dist/scripts/clearCache.js", - "export": "node dist/scripts/export/index.js", + "export:trace": "node dist/scripts/export/trace.js", + "export:process": "node dist/scripts/export/process.js", "export:dev": "docker-compose -f compose.dev.yml up --exit-code-from export", - "export:prod": "npm install && npm run build && pm2 delete all && npm run db:migrate:data && pm2 start ecosystem.config.js --only exporter,webhooks && pm2 save", + "export:prod": "npm install && npm run build && pm2 delete all && npm run db:migrate:data && pm2 start ecosystem.config.js --only export-tracer,export-processor,webhooks && pm2 save", "webhooks": "node dist/scripts/webhooks.js", "accountWebhooks": "node dist/scripts/accountWebhooks.js", "transform": "node dist/scripts/transform.js", @@ -76,6 +77,7 @@ "typescript": "^4.9.3" }, "dependencies": { + "@bull-board/koa": "^5.8.4", "@cosmjs/amino": "^0.29.5", "@cosmjs/cosmwasm-stargate": "^0.30.1", "@cosmjs/crypto": "^0.29.5", @@ -89,6 +91,7 @@ "@types/koa__router": "^12.0.0", "async-await-retry": "^2.0.1", "axios": "^1.3.6", + "bullmq": "^4.12.0", "commander": "^9.4.1", "cosmjs-types": "^0.8.0", "crypto-js": "^4.1.1", diff --git a/src/core/index.ts b/src/core/index.ts index 366370fb..d7037b0b 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -1,5 +1,7 @@ +export * from './utils' + export * from './compute' export * from './config' export * from './env' +export * from './queues' export * from './types' -export * from './utils' diff --git a/src/core/queues.ts b/src/core/queues.ts new file mode 100644 index 00000000..906d0924 --- /dev/null +++ b/src/core/queues.ts @@ -0,0 +1,29 @@ +import { ConnectionOptions, Processor, Queue, Worker } from 'bullmq' + +import { loadConfig } from './config' + +export const EXPORT_QUEUE_NAME = 'export' + +const getBullConnection = (): ConnectionOptions | undefined => { + const { redis } = loadConfig() + return ( + redis && { + host: redis.host, + port: redis.port, + password: redis.password, + } + ) +} + +export const getBullQueue = (name: string) => + new Queue(name, { + connection: getBullConnection(), + }) + +export const WorkerQueue = ( + name: string, + processor: Processor +) => + new Worker(name, processor, { + connection: getBullConnection(), + }) diff --git a/src/core/types.ts b/src/core/types.ts index 221dfa2a..5f18d962 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -37,6 +37,11 @@ export type Config = { data: DB accounts: DB } + redis?: { + host?: string + port?: number + password: string + } meilisearch?: { host: string apiKey?: string diff --git a/src/scripts/export/handlers/bank.ts b/src/scripts/export/handlers/bank.ts index f67df315..a7a71006 100644 --- a/src/scripts/export/handlers/bank.ts +++ b/src/scripts/export/handlers/bank.ts @@ -13,45 +13,12 @@ import { import { Handler, HandlerMaker } from '../types' const STORE_NAME = 'bank' -const MAX_BATCH_SIZE = 5000 -export const bank: HandlerMaker = async ({ - config, - dontUpdateComputations, - getBlockTimeUnixMs, +export const bank: HandlerMaker = async ({ + config: { bech32Prefix }, + updateComputations, }) => { - const pending: ParsedBankStateEvent[] = [] - - const flush = async () => { - if (pending.length === 0) { - return - } - - // For state events with the same blockHeight, address, and denom, only keep - // the last event. This is because the indexer guarantees that events are - // emitted in order, and the last event is the most up-to-date. Multiple - // events may occur if the value is updated multiple times across different - // messages. The indexer can only maintain uniqueness within a message and - // its submessages, but different messages in the same block can write to - // the same key, and the indexer emits all the messages. - const uniqueIndexerEvents = pending.reduce((acc, event) => { - const key = event.blockHeight + event.address + event.denom - acc[key] = event - return acc - }, {} as Record) - const eventsToExport = Object.values(uniqueIndexerEvents) - - // Clear queue. - pending.length = 0 - - // Export events. - await exporter(eventsToExport) - } - - let lastBlockHeightSeen = 0 - let debouncedFlush: NodeJS.Timeout | undefined - - const handle: Handler['handle'] = async (trace) => { + const match: Handler['match'] = (trace) => { // BalancesPrefix = 0x02 // bank keys are formatted as: // BalancesPrefix || len(addressBytes) || addressBytes || denomBytes @@ -66,28 +33,18 @@ export const bank: HandlerMaker = async ({ let address let denom try { - address = toBech32(config.bech32Prefix, keyData.slice(2, 2 + length)) + address = toBech32(bech32Prefix, keyData.slice(2, 2 + length)) denom = fromUtf8(keyData.slice(2 + length)) } catch { // Ignore decoding errors. return } - // If we reached the first event of the next block, flush the previous - // events to the DB. This ensures we batch all events from the same block - // together. - if (trace.metadata.blockHeight > lastBlockHeightSeen) { - await flush() - } - // Get code ID and block timestamp from chain. const blockHeight = BigInt(trace.metadata.blockHeight).toString() - const blockTimeUnixMsNum = await getBlockTimeUnixMs( - trace.metadata.blockHeight, - trace - ) - const blockTimeUnixMs = BigInt(blockTimeUnixMsNum).toString() - const blockTimestamp = new Date(blockTimeUnixMsNum) + + const blockTimeUnixMs = BigInt(trace.blockTimeUnixMs).toString() + const blockTimestamp = new Date(trace.blockTimeUnixMs) // Mimics behavior of `UnmarshalBalanceCompat` in `x/bank/keeper/view.go` to // decode balance. @@ -145,68 +102,46 @@ export const bank: HandlerMaker = async ({ return } - pending.push({ + return { + id: [blockHeight, address, denom].join(':'), address, blockHeight, blockTimeUnixMs, blockTimestamp, denom, balance, - }) - lastBlockHeightSeen = trace.metadata.blockHeight - - // Debounce flush in 200ms. - if (debouncedFlush !== undefined) { - clearTimeout(debouncedFlush) - } - - // If batch size reached, flush immediately. - if (pending.length >= MAX_BATCH_SIZE) { - debouncedFlush = undefined - await flush() - return - } else { - debouncedFlush = setTimeout(flush, 200) } - - return } - const exporter = async (parsedEvents: ParsedBankStateEvent[]) => { - const start = Date.now() - + const process: Handler['process'] = async (events) => { const exportEvents = async () => // Unique index on [blockHeight, address, denom] ensures that we don't // insert duplicate events. If we encounter a duplicate, we update the // `balance` field in case event processing for a block was batched // separately. - parsedEvents.length > 0 - ? await BankStateEvent.bulkCreate(parsedEvents, { + events.length > 0 + ? await BankStateEvent.bulkCreate(events, { updateOnDuplicate: ['balance'], }) : [] // Retry 3 times with exponential backoff starting at 100ms delay. - const events = (await retry(exportEvents, [], { + const exportedEvents = (await retry(exportEvents, [], { retriesMax: 3, exponential: true, interval: 100, })) as BankStateEvent[] - let computationsUpdated = 0 - let computationsDestroyed = 0 - if (!dontUpdateComputations) { - const computationUpdates = - await updateComputationValidityDependentOnChanges(events) - computationsUpdated = computationUpdates.updated - computationsDestroyed = computationUpdates.destroyed + if (updateComputations) { + await updateComputationValidityDependentOnChanges(exportedEvents) } // Store last block height exported, and update latest block // height/time if the last export is newer. - const lastBlockHeightExported = events[events.length - 1].blockHeight + const lastBlockHeightExported = + exportedEvents[exportedEvents.length - 1].blockHeight const lastBlockTimeUnixMsExported = - events[events.length - 1].blockTimeUnixMs + exportedEvents[exportedEvents.length - 1].blockTimeUnixMs await State.update( { lastBankBlockHeightExported: Sequelize.fn( @@ -232,21 +167,11 @@ export const bank: HandlerMaker = async ({ }, } ) - - const end = Date.now() - const duration = end - start - - // Log. - console.log( - `[bank] Exported: ${events.length.toLocaleString()}. Block: ${BigInt( - lastBlockHeightExported - ).toLocaleString()}. Computations updated/destroyed: ${computationsUpdated.toLocaleString()}/${computationsDestroyed.toLocaleString()}. Duration: ${duration.toLocaleString()}ms.` - ) } return { storeName: STORE_NAME, - handle, - flush, + match, + process, } } diff --git a/src/scripts/export/handlers/index.ts b/src/scripts/export/handlers/index.ts index a7dd5f41..c5227ee4 100644 --- a/src/scripts/export/handlers/index.ts +++ b/src/scripts/export/handlers/index.ts @@ -2,7 +2,7 @@ import { HandlerMaker } from '../types' import { bank } from './bank' import { wasm } from './wasm' -export const handlerMakers: Record = { +export const handlerMakers: Record> = { bank, wasm, } diff --git a/src/scripts/export/handlers/wasm.ts b/src/scripts/export/handlers/wasm.ts index 6e7e57a2..be516d71 100644 --- a/src/scripts/export/handlers/wasm.ts +++ b/src/scripts/export/handlers/wasm.ts @@ -20,51 +20,89 @@ import { updateIndexesForContracts } from '@/ms' import { Handler, HandlerMaker } from '../types' const STORE_NAME = 'wasm' -const MAX_BATCH_SIZE = 5000 const CONTRACT_BYTE_LENGTH = 32 -export const wasm: HandlerMaker = async ({ - config, - dontUpdateComputations, - dontSendWebhooks, +type WasmExportData = + | { + type: 'state' + data: Omit + } + | { + type: 'contract' + data: { + address: string + codeId: number + blockHeight: string + blockTimeUnixMs: string + } + } + +export const wasm: HandlerMaker = async ({ + config: { bech32Prefix }, + updateComputations, + sendWebhooks, cosmWasmClient, - getBlockTimeUnixMs, }) => { const chainId = await cosmWasmClient.getChainId() - const pending: ParsedWasmStateEvent[] = [] - const flush = async () => { - if (pending.length === 0) { - return + // Get code ID for contract, cached in memory. + const codeIdCache = new LRUCache({ + max: 1000, + }) + const getCodeId = async (contractAddress: string): Promise => { + if (codeIdCache.has(contractAddress)) { + return codeIdCache.get(contractAddress) ?? 0 } - // For state events with the same blockHeight, contractAddress, and key, - // only keep the last event. This is because the indexer guarantees that - // events are emitted in order, and the last event is the most up-to-date. - // Multiple events may occur if the value is updated multiple times across - // different messages. The indexer can only maintain uniqueness within a - // message and its submessages, but different messages in the same block can - // write to the same key, and the indexer emits all the messages. For tx - // events, all events should be unique, so just use a unique key which - // should get all of them. - const uniqueIndexerEvents = pending.reduce((acc, event) => { - const key = event.blockHeight + event.contractAddress + event.key - acc[key] = event - return acc - }, {} as Record) - const eventsToExport = Object.values(uniqueIndexerEvents) - - // Clear queue. - pending.length = 0 - - // Export events. - await exporter(eventsToExport) - } + const loadIntoCache = async () => { + let codeId = 0 + try { + const contract = await cosmWasmClient.getContract(contractAddress) + codeId = contract.codeId + } catch (err) { + // If contract not found, ignore, leaving as 0. Otherwise, throw err. + if ( + !(err instanceof Error) || + !err.message.includes('not found: invalid request') + ) { + throw err + } + } - let lastBlockHeightSeen = 0 - let debouncedFlush: NodeJS.Timeout | undefined + codeIdCache.set(contractAddress, codeId) + } - const handle: Handler['handle'] = async (trace) => { + try { + // Retry 3 times with exponential backoff starting at 100ms delay. + await retry(loadIntoCache, [], { + retriesMax: 3, + exponential: true, + interval: 100, + }) + } catch (err) { + console.error( + '-------\nFailed to get code ID:\n', + err instanceof Error ? err.message : err, + '\nContract: ' + contractAddress + '\n-------' + ) + Sentry.captureException(err, { + tags: { + type: 'failed-get-code-id', + script: 'export', + handler: 'wasm', + chainId, + contractAddress, + }, + }) + + // Set to 0 on failure so we can continue. + codeIdCache.set(contractAddress, 0) + } + + return codeIdCache.get(contractAddress) ?? 0 + } + + const match: Handler['match'] = (trace) => { // ContractStorePrefix = 0x03 // wasm keys are formatted as: // ContractStorePrefix || contractAddressBytes || keyBytes @@ -85,26 +123,16 @@ export const wasm: HandlerMaker = async ({ } const contractAddress = toBech32( - config.bech32Prefix, + bech32Prefix, keyData.slice(1, CONTRACT_BYTE_LENGTH + 1) ) - const key = keyData.slice(CONTRACT_BYTE_LENGTH + 1) - - // If we reached the first event of the next block, flush the previous - // events to the DB. This ensures we batch all events from the same block - // together. - if (trace.metadata.blockHeight > lastBlockHeightSeen) { - await flush() - } + // Convert key to comma-separated list of bytes. See explanation in `Event` + // model for more information. + const key = keyData.slice(CONTRACT_BYTE_LENGTH + 1).join(',') // Get code ID and block timestamp from chain. const blockHeight = BigInt(trace.metadata.blockHeight).toString() - const blockTimeUnixMsNum = await getBlockTimeUnixMs( - trace.metadata.blockHeight, - trace - ) - const blockTimeUnixMs = BigInt(blockTimeUnixMsNum).toString() - const blockTimestamp = new Date(blockTimeUnixMsNum) + const blockTimeUnixMs = BigInt(trace.blockTimeUnixMs).toString() // If contract key, save contract info. if (trace.operation === 'write' && keyData[0] === 0x02) { @@ -123,39 +151,18 @@ export const wasm: HandlerMaker = async ({ return } - const blockHeightFromContractInfo = - contractInfo.created?.blockHeight.toInt() - const codeId = contractInfo.codeId.toInt() - const [contract, created] = await Contract.findOrCreate({ - where: { - address: contractAddress, - }, - defaults: { + + return { + id: ['contract', blockHeight, contractAddress].join(':'), + type: 'contract', + data: { address: contractAddress, codeId, - instantiatedAtBlockHeight: - blockHeightFromContractInfo ?? Number(blockHeight), - // If block height is from contract info, we don't have block time, so - // just set to 0. - instantiatedAtBlockTimeUnixMs: - blockHeightFromContractInfo === undefined ? 0 : blockTimeUnixMs, - // If block height is from contract info, we don't have block time, so - // just set to 0. - instantiatedAtBlockTimestamp: - blockHeightFromContractInfo === undefined - ? new Date(0) - : blockTimestamp, + blockHeight, + blockTimeUnixMs, }, - }) - // Update code ID if it's changed. - if (!created && contract.codeId !== codeId) { - await contract.update({ - codeId, - }) } - - return } // Otherwise, save state event. @@ -178,60 +185,69 @@ export const wasm: HandlerMaker = async ({ } } - const event: ParsedWasmStateEvent = { + return { + id: ['state', blockHeight, contractAddress, key].join(':'), type: 'state', - // Initialize the code ID to 0 since we don't know it yet. It will be - // retrieved below. - codeId: 0, - contractAddress, - blockHeight, - blockTimeUnixMs, - blockTimestamp, - // Convert key to comma-separated list of bytes. See explanation in - // `Event` model for more information. - key: key.join(','), - value, - valueJson, - delete: trace.operation === 'delete', + data: { + type: 'state', + // Initialize the code ID to 0 since we don't know it yet. It will be + // retrieved later. + codeId: 0, + contractAddress, + blockHeight, + blockTimeUnixMs, + key, + value, + valueJson, + delete: trace.operation === 'delete', + }, } + } - pending.push(event) - lastBlockHeightSeen = trace.metadata.blockHeight - - // Debounce flush in 200ms. - if (debouncedFlush !== undefined) { - clearTimeout(debouncedFlush) + const process: Handler['process'] = async (events) => { + // Export contracts. + const contractEvents = events.flatMap((event) => + event.type === 'contract' ? event.data : [] + ) + if (contractEvents.length > 0) { + await Contract.bulkCreate( + contractEvents.map( + ({ address, codeId, blockHeight, blockTimeUnixMs }) => ({ + address, + codeId, + instantiatedAtBlockHeight: blockHeight, + instantiatedAtBlockTimeUnixMs: blockTimeUnixMs, + instantiatedAtBlockTimestamp: new Date(Number(blockTimeUnixMs)), + }) + ), + { + updateOnDuplicate: ['codeId'], + } + ) } - // If batch size reached, flush immediately. - if (pending.length >= MAX_BATCH_SIZE) { - debouncedFlush = undefined - await flush() + // Export state. + let stateEvents = events.flatMap((event) => + event.type === 'state' ? event.data : [] + ) + if (!stateEvents.length) { return - } else { - debouncedFlush = setTimeout(flush, 200) } - return - } - - const exporter = async (parsedEvents: ParsedWasmStateEvent[]) => { - const start = Date.now() - const state = await State.getSingleton() if (!state) { throw new Error('State not found while exporting.') } const uniqueContracts = [ - ...new Set(parsedEvents.map((event) => event.contractAddress)), + ...new Set(stateEvents.map((stateEvent) => stateEvent.contractAddress)), ] const exportContractsAndEvents = async () => { // Ensure contract exists before creating events. `address` is unique. await Contract.bulkCreate( uniqueContracts.map((address) => { - const event = parsedEvents.find( + const event = stateEvents.find( (event) => event.contractAddress === address ) // Should never happen since `uniqueContracts` is derived from @@ -242,22 +258,23 @@ export const wasm: HandlerMaker = async ({ return { address, - // Initialize the code ID to 0 since we don't know it yet. It will - // be retrieved below. + // Initialize the code ID to 0 since we don't know it here. It will + // be retrieved below if it doesn't already exist in the database. codeId: 0, - // Set the contract instantiation block to the first event found - // in the list of parsed events. Events are sorted in ascending - // order by creation block. These won't get updated if the - // contract already exists, so it's safe to always attempt - // creation with the first event's block. Only `codeId` gets - // updated below when a duplicate is found. + // Set the contract instantiation block to the first event found in + // the list of parsed events. Events are sorted in ascending order + // by creation block. These won't get updated if the contract + // already exists, so it's safe to always attempt creation with the + // first event's block. instantiatedAtBlockHeight: event.blockHeight, instantiatedAtBlockTimeUnixMs: event.blockTimeUnixMs, - instantiatedAtBlockTimestamp: event.blockTimestamp, + instantiatedAtBlockTimestamp: new Date( + Number(event.blockTimeUnixMs) + ), } }), - // Do nothing if contract already exists. { + // Do nothing if contract already exists. ignoreDuplicates: true, } ) @@ -300,14 +317,17 @@ export const wasm: HandlerMaker = async ({ // Unique index on [blockHeight, contractAddress, key] ensures that we // don't insert duplicate events. If we encounter a duplicate, we update - // the `value`, `valueJson`, and `delete` fields in case event - // processing for a block was batched separately. - const events = - parsedEvents.length > 0 - ? await WasmStateEvent.bulkCreate(parsedEvents, { - updateOnDuplicate: ['value', 'valueJson', 'delete'], - }) - : [] + // the `value`, `valueJson`, and `delete` fields in case event processing + // for a block was batched separately. + const events = await WasmStateEvent.bulkCreate( + stateEvents.map((e) => ({ + ...e, + blockTimestamp: new Date(Number(e.blockTimeUnixMs)), + })), + { + updateOnDuplicate: ['value', 'valueJson', 'delete'], + } + ) return { contracts, @@ -316,18 +336,22 @@ export const wasm: HandlerMaker = async ({ } // Retry 3 times with exponential backoff starting at 100ms delay. - let { contracts, events } = (await retry(exportContractsAndEvents, [], { - retriesMax: 3, - exponential: true, - interval: 100, - })) as { + let { contracts, events: exportedEvents } = (await retry( + exportContractsAndEvents, + [], + { + retriesMax: 3, + exponential: true, + interval: 100, + } + )) as { contracts: Contract[] events: WasmStateEvent[] } // Add contract to events. await Promise.all( - events.map(async (event) => { + exportedEvents.map(async (event) => { let contract = contracts.find( (contract) => contract.address === event.contractAddress ) @@ -350,24 +374,26 @@ export const wasm: HandlerMaker = async ({ ) // Add code ID to parsed events. - parsedEvents.forEach((parsedEvent) => { + stateEvents.forEach((stateEvent) => { const contract = contracts.find( - (contract) => contract.address === parsedEvent.contractAddress + (contract) => contract.address === stateEvent.contractAddress ) if (contract) { - parsedEvent.codeId = contract.codeId + stateEvent.codeId = contract.codeId } }) // Remove events that don't have a contract or code ID. - events = events.filter((event) => event.contract !== undefined) - parsedEvents = parsedEvents.filter((event) => event.codeId > 0) + exportedEvents = exportedEvents.filter( + (event) => event.contract !== undefined + ) + stateEvents = stateEvents.filter((stateEvent) => stateEvent.codeId > 0) // Transform events as needed. // Retry 3 times with exponential backoff starting at 100ms delay. const transformations = (await retry( WasmStateEventTransformation.transformParsedStateEvents, - [parsedEvents], + [stateEvents], { retriesMax: 3, exponential: true, @@ -375,30 +401,25 @@ export const wasm: HandlerMaker = async ({ } )) as WasmStateEventTransformation[] - let computationsUpdated = 0 - let computationsDestroyed = 0 - if (!dontUpdateComputations) { - const computationUpdates = - await updateComputationValidityDependentOnChanges([ - ...events, - ...transformations, - ]) - computationsUpdated = computationUpdates.updated - computationsDestroyed = computationUpdates.destroyed + if (updateComputations) { + await updateComputationValidityDependentOnChanges([ + ...exportedEvents, + ...transformations, + ]) } // Queue webhooks as needed. - const webhooksQueued = - dontSendWebhooks || events.length === 0 - ? 0 - : (await PendingWebhook.queueWebhooks(state, events)) + - (await AccountWebhook.queueWebhooks(events)) + if (sendWebhooks && exportedEvents.length > 0) { + await PendingWebhook.queueWebhooks(state, exportedEvents) + await AccountWebhook.queueWebhooks(exportedEvents) + } // Store last block height exported, and update latest block // height/time if the last export is newer. - const lastBlockHeightExported = events[events.length - 1].blockHeight + const lastBlockHeightExported = + exportedEvents[exportedEvents.length - 1].blockHeight const lastBlockTimeUnixMsExported = - events[events.length - 1].blockTimeUnixMs + exportedEvents[exportedEvents.length - 1].blockTimeUnixMs await State.update( { lastWasmBlockHeightExported: Sequelize.fn( @@ -430,78 +451,11 @@ export const wasm: HandlerMaker = async ({ await updateIndexesForContracts({ contracts, }) - - const end = Date.now() - const duration = end - start - - // Log. - console.log( - `[wasm] Exported: ${events.length.toLocaleString()}. Block: ${BigInt( - lastBlockHeightExported - ).toLocaleString()}. Transformed: ${transformations.length.toLocaleString()}. Webhooks: ${webhooksQueued.toLocaleString()}. Computations updated/destroyed: ${computationsUpdated.toLocaleString()}/${computationsDestroyed.toLocaleString()}. Duration: ${duration.toLocaleString()}ms.` - ) - } - - // Get code ID for contract, cached in memory. - const codeIdCache = new LRUCache({ - max: 1000, - }) - const getCodeId = async (contractAddress: string): Promise => { - if (codeIdCache.has(contractAddress)) { - return codeIdCache.get(contractAddress) ?? 0 - } - - const loadIntoCache = async () => { - let codeId = 0 - try { - const contract = await cosmWasmClient.getContract(contractAddress) - codeId = contract.codeId - } catch (err) { - // If contract not found, ignore, leaving as 0. Otherwise, throw err. - if ( - !(err instanceof Error) || - !err.message.includes('not found: invalid request') - ) { - throw err - } - } - - codeIdCache.set(contractAddress, codeId) - } - - try { - // Retry 3 times with exponential backoff starting at 100ms delay. - await retry(loadIntoCache, [], { - retriesMax: 3, - exponential: true, - interval: 100, - }) - } catch (err) { - console.error( - '-------\nFailed to get code ID:\n', - err instanceof Error ? err.message : err, - '\nContract: ' + contractAddress + '\n-------' - ) - Sentry.captureException(err, { - tags: { - type: 'failed-get-code-id', - script: 'export', - handler: 'wasm', - chainId, - contractAddress, - }, - }) - - // Set to 0 on failure so we can continue. - codeIdCache.set(contractAddress, 0) - } - - return codeIdCache.get(contractAddress) ?? 0 } return { storeName: STORE_NAME, - handle, - flush, + match, + process, } } diff --git a/src/scripts/export/index.ts b/src/scripts/export/index.ts deleted file mode 100644 index db1caa14..00000000 --- a/src/scripts/export/index.ts +++ /dev/null @@ -1,226 +0,0 @@ -import * as fs from 'fs' -import path from 'path' -import { Worker } from 'worker_threads' - -import * as Sentry from '@sentry/node' -import { Command } from 'commander' - -import { loadConfig, objectMatchesStructure } from '@/core' -import { setupMeilisearch } from '@/ms' - -import { - FromWorkerMessage, - ToWorkerMessage, - TracedEvent, - WorkerInitData, -} from './types' -import { setUpFifoJsonTracer } from './utils' - -const MAX_QUEUE_SIZE = 5000 - -// Parse arguments. -const program = new Command() -program.option( - '-c, --config ', - 'path to config file, falling back to config.json' -) -program.option( - // Adds inverted `update` boolean to the options object. - '--no-update', - "don't update computation validity based on new events or transformations" -) -program.option( - // Adds inverted `webhooks` boolean to the options object. - '--no-webhooks', - "don't send webhooks" -) -program.option( - // Adds inverted `ws` boolean to the options object. - '--no-ws', - "don't connect to websocket" -) -program.parse() -const { config: _config, update, webhooks, ws } = program.opts() - -// Load config with config option. -const config = loadConfig(_config) - -if (!config.home) { - throw new Error('Config missing home directory.') -} - -// Add Sentry error reporting. -if (config.sentryDsn) { - Sentry.init({ - dsn: config.sentryDsn, - }) -} - -const traceFile = path.join(config.home, 'trace.pipe') - -const main = async () => { - // Setup meilisearch. - await setupMeilisearch() - - // Ensure trace and update files exist. - if (!fs.existsSync(traceFile)) { - throw new Error( - `Trace file not found: ${traceFile}. Create it with "mkfifo ${traceFile}".` - ) - } - - // Verify trace and update files are FIFOs. - const stat = fs.statSync(traceFile) - if (!stat.isFIFO()) { - throw new Error(`Trace file is not a FIFO: ${traceFile}.`) - } - - // Read from trace file. - await trace() -} - -const trace = async () => { - const workerData: WorkerInitData = { - config, - update: !!update, - webhooks: !!webhooks, - websocket: !!ws, - } - - const worker = new Worker(path.join(__dirname, 'worker.js'), { - workerData, - }) - - // Add worker exit handler. - worker.on('exit', (code) => { - if (code !== 0) { - console.error(`Worker stopped with exit code ${code}`) - } - - // Exit with worker's exit code. - process.exit(code) - }) - - let queued = 0 - let paused = false - const queue: TracedEvent[] = [] - // Listen for worker processing queue. - worker.on('message', async (data: FromWorkerMessage) => { - if (data.type === 'ready') { - console.log(`\n[${new Date().toISOString()}] Exporting from trace...`) - - // Tell pm2 we're ready right before we start reading. - if (process.send) { - process.send('ready') - } - - // Process queue. - const processQueue = () => { - if (paused) { - return - } - - while (queue.length > 0) { - const event = queue.shift() - if (!event) { - continue - } - - // Process event. - worker.postMessage({ - type: 'trace', - event, - } as ToWorkerMessage) - queued += 1 - - // If queue fills up, pause sending to worker. Process queue will be - // called again when the queue drains. - if (queued >= MAX_QUEUE_SIZE) { - paused = true - - // Resume once queue drains. - const interval = setInterval(() => { - if (queued < MAX_QUEUE_SIZE / 5) { - paused = false - clearInterval(interval) - processQueue() - } - }, 50) - break - } - } - } - - const { promise: tracer, close: closeTracer } = setUpFifoJsonTracer({ - file: traceFile, - onData: (data) => { - const tracedEvent = data as TracedEvent - // Ensure this is a traced write or delete event. - if ( - !objectMatchesStructure(tracedEvent, { - operation: {}, - key: {}, - value: {}, - metadata: { - blockHeight: {}, - }, - }) - ) { - return - } - - // Only handle writes and deletes. - if ( - tracedEvent.operation !== 'write' && - tracedEvent.operation !== 'delete' - ) { - return - } - - queue.push(tracedEvent) - processQueue() - }, - }) - - // Add shutdown signal handler. - process.on('SIGINT', () => { - // Tell tracer to close. The rest of the data in the buffer will finish - // processing. - closeTracer() - console.log('Shutting down after handlers finish...') - }) - - // Wait for tracer to close. - await tracer - - // Wait for queue to finish processing. - await new Promise((resolve) => { - console.log( - `Waiting for ${queue.length.toLocaleString()} events to finish processing...` - ) - - const interval = setInterval(() => { - if (queued === 0 && queue.length === 0) { - clearInterval(interval) - resolve() - } - }, 50) - }) - - // Give a little time for the worker to queue. - await new Promise((resolve) => setTimeout(resolve, 5000)) - - // Tell worker to shutdown. - worker.postMessage({ - type: 'shutdown', - } as ToWorkerMessage) - } else if (data.type === 'processed') { - queued -= data.count - } - }) -} - -main().catch((err) => { - console.error(err) - process.exit(1) -}) diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts new file mode 100644 index 00000000..08dd7d9d --- /dev/null +++ b/src/scripts/export/process.ts @@ -0,0 +1,144 @@ +import * as Sentry from '@sentry/node' +import retry from 'async-await-retry' +import { Worker } from 'bullmq' +import { Command } from 'commander' + +import { DbType, EXPORT_QUEUE_NAME, loadConfig } from '@/core' +import { State, loadDb } from '@/db' + +import { handlerMakers } from './handlers' +import { ExportQueueData } from './types' +import { getCosmWasmClient } from './utils' + +// Parse arguments. +const program = new Command() +program.option( + '-c, --config ', + 'path to config file, falling back to config.json' +) +program.option( + // Adds inverted `update` boolean to the options object. + '--no-update', + "don't update computation validity based on new events or transformations" +) +program.option( + // Adds inverted `webhooks` boolean to the options object. + '--no-webhooks', + "don't send webhooks" +) +program.parse() +const { config: _config, update, webhooks } = program.opts() + +// Load config with config option. +const config = loadConfig(_config) + +// Add Sentry error reporting. +if (config.sentryDsn) { + Sentry.init({ + dsn: config.sentryDsn, + }) +} + +const main = async () => { + // Load DB on start. + const dataSequelize = await loadDb({ + type: DbType.Data, + }) + const accountsSequelize = await loadDb({ + type: DbType.Accounts, + }) + + // Initialize state. + await State.createSingletonIfMissing() + + const cosmWasmClient = await getCosmWasmClient(config.rpc) + + // Setup handlers. + const handlers = await Promise.all( + Object.entries(handlerMakers).map(async ([name, handlerMaker]) => ({ + name, + handler: await handlerMaker({ + config, + updateComputations: !!update, + sendWebhooks: !!webhooks, + cosmWasmClient, + }), + })) + ) + + // Create queue worker. + const worker = new Worker<{ data: ExportQueueData[] }>( + EXPORT_QUEUE_NAME, + async (job) => { + const { data } = job.data + + // Group data by handler. + const groupedData = data.reduce( + (acc, { handler, data }) => ({ + ...acc, + [handler]: (acc[handler] || []).concat(data), + }), + {} as Record + ) + + // Process data. + for (const { name, handler } of handlers) { + const events = groupedData[name] + if (!events?.length) { + continue + } + + try { + // Retry 3 times with exponential backoff starting at 100ms delay. + await retry(handler.process, [events], { + retriesMax: 3, + exponential: true, + interval: 100, + }) + } catch (err) { + console.error( + '-------\nFailed to process:\n', + err instanceof Error ? err.message : err, + '\n-------' + ) + Sentry.captureException(err, { + tags: { + type: 'failed-flush', + script: 'export', + }, + extra: { + handler: name, + }, + }) + + throw err + } + } + } + ) + + // Add shutdown signal handler. + process.on('SIGINT', () => { + if (worker.closing) { + console.log('Already shutting down.') + } else { + console.log('Shutting down after worker jobs finish...') + // Exit once worker closes. + worker.close().then(async () => { + await dataSequelize.close() + await accountsSequelize.close() + process.exit(0) + }) + } + }) + + // Tell pm2 we're ready. + if (process.send) { + process.send('ready') + } +} + +main().catch((err) => { + console.error('Processor errored', err) + process.exit(1) +}) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts new file mode 100644 index 00000000..9806f26f --- /dev/null +++ b/src/scripts/export/trace.ts @@ -0,0 +1,491 @@ +import * as fs from 'fs' +import path from 'path' + +import * as Sentry from '@sentry/node' +import retry from 'async-await-retry' +import { Command } from 'commander' +import { LRUCache } from 'lru-cache' +import waitPort from 'wait-port' + +import { + EXPORT_QUEUE_NAME, + getBullQueue, + loadConfig, + objectMatchesStructure, +} from '@/core' +import { State } from '@/db' +import { setupMeilisearch } from '@/ms' + +import { handlerMakers } from './handlers' +import { ExportQueueData, TracedEvent, TracedEventWithBlockTime } from './types' +import { + getCosmWasmClient, + setUpFifoJsonTracer, + setUpWebSocketNewBlockListener, +} from './utils' + +const MAX_QUEUE_SIZE = 5000 +const MAX_BATCH_SIZE = 5000 + +// Parse arguments. +const program = new Command() +program.option( + '-c, --config ', + 'path to config file, falling back to config.json' +) +program.option( + // Adds inverted `ws` boolean to the options object. + '--no-ws', + "don't connect to websocket" +) +program.parse() +const { config: _config, ws: webSocketEnabled } = program.opts() + +// Load config with config option. +const config = loadConfig(_config) + +if (!config.home) { + throw new Error('Config missing home directory.') +} + +// Add Sentry error reporting. +if (config.sentryDsn) { + Sentry.init({ + dsn: config.sentryDsn, + }) +} + +const traceFile = path.join(config.home, 'trace.pipe') + +const main = async () => { + // Setup meilisearch. + await setupMeilisearch() + + // Ensure trace and update files exist. + if (!fs.existsSync(traceFile)) { + throw new Error( + `Trace file not found: ${traceFile}. Create it with "mkfifo ${traceFile}".` + ) + } + + // Verify trace and update files are FIFOs. + const stat = fs.statSync(traceFile) + if (!stat.isFIFO()) { + throw new Error(`Trace file is not a FIFO: ${traceFile}.`) + } + + // Initialize state. + await State.createSingletonIfMissing() + + // Read from trace file. + await trace() +} + +const trace = async () => { + const exportQueue = getBullQueue<{ data: ExportQueueData[] }>( + EXPORT_QUEUE_NAME + ) + + // Create CosmWasm client that batches requests. + const cosmWasmClient = await getCosmWasmClient(config.rpc) + + // Helper function that gets block time for height, cached in memory, which is + // filled in by the NewBlock WebSocket listener. + const blockHeightToTimeCache = new LRUCache({ + max: 100, + }) + const getBlockTimeUnixMs = async (trace: TracedEvent): Promise => { + const { blockHeight } = trace.metadata + + if (blockHeightToTimeCache.has(blockHeight)) { + return blockHeightToTimeCache.get(blockHeight) ?? 0 + } + + // This may fail if the RPC does not have the block info at this height + // anymore (i.e. if it's too old and the RPC pruned it) + const loadIntoCache = async () => { + const { + header: { time }, + } = await cosmWasmClient.getBlock(blockHeight) + blockHeightToTimeCache.set(blockHeight, Date.parse(time)) + } + + try { + // Retry 3 times with exponential backoff starting at 150ms delay. + await retry(loadIntoCache, [], { + retriesMax: 3, + exponential: true, + interval: 150, + }) + } catch (err) { + console.error( + '-------\nFailed to get block:\n', + err instanceof Error ? err.message : err, + '\nBlock height: ' + + BigInt(blockHeight).toLocaleString() + + '\nData: ' + + JSON.stringify(trace, null, 2) + + '\n-------' + ) + + // Only log to Sentry if not block height unavailable error. + if ( + !(err instanceof Error) || + !err.message.includes('must be less than or equal to the current') + ) { + Sentry.captureException(err, { + tags: { + type: 'failed-get-block', + script: 'export', + chainId: (await State.getSingleton())?.chainId ?? 'unknown', + }, + extra: { + trace, + blockHeight, + }, + }) + } + + // Set to 0 on failure so we can continue. + blockHeightToTimeCache.set(blockHeight, 0) + } + + return blockHeightToTimeCache.get(blockHeight) ?? 0 + } + + // Set up handlers. + const handlers = await Promise.all( + Object.entries(handlerMakers).map(async ([name, handlerMaker]) => ({ + name, + handler: await handlerMaker({ + config, + cosmWasmClient, + // These are only relevant when processing, not tracing. + updateComputations: false, + sendWebhooks: false, + }), + })) + ) + + console.log(`\n[${new Date().toISOString()}] Exporting from trace...`) + + let webSocketReady = false + const traceQueue: TracedEvent[] = [] + let traceExportPaused = false + let traceExporter = Promise.resolve() + let exporting = 0 + + // Batch events and group by block height. + let exportBatch: { + handler: string + data: any + trace: TracedEvent + }[] = [] + let exportTraceBatchDebounce: NodeJS.Timeout | null = null + const exportTraceBatch = () => { + if (exportTraceBatchDebounce !== null) { + clearTimeout(exportTraceBatchDebounce) + exportTraceBatchDebounce = null + } + + if (exportBatch.length) { + exportQueue.add( + BigInt( + exportBatch[exportBatch.length - 1].trace.metadata.blockHeight + ).toString(), + { + data: exportBatch.map( + ({ handler, data }): ExportQueueData => ({ + handler, + data, + }) + ), + } + ) + exportBatch = [] + } + } + + const exportTrace = async () => { + const trace = traceQueue.shift() + try { + if (!trace) { + return + } + + // Fetch block time. + const blockTimeUnixMs = await getBlockTimeUnixMs(trace) + const eventWithBlockTime: TracedEventWithBlockTime = { + ...trace, + blockTimeUnixMs, + } + + // Match traces with handlers and get queue data. + const matchedData = handlers + .filter( + ({ handler }) => + // Filter by store if present. Osmosis, for example, does not emit + // store_name in metadata, so try all handlers. + !trace.metadata.store_name || + handler.storeName === trace.metadata.store_name + ) + .flatMap(({ name, handler }) => { + const data = handler.match(eventWithBlockTime) + return data + ? { + handler: name, + data, + trace, + } + : [] + }) + + // If this trace is a newer block height than the last trace, export the + // previous batch before batching this one. + if ( + exportBatch.length > 0 && + trace.metadata.blockHeight > + exportBatch[exportBatch.length - 1].trace.metadata.blockHeight + ) { + exportTraceBatch() + } + + exportBatch.push(...matchedData) + + // If batch size reached, immediately export. + if (exportBatch.length >= MAX_BATCH_SIZE) { + exportTraceBatch() + } else if (matchedData.length) { + // Otherwise, if queued new data, debounce export. + if (exportTraceBatchDebounce !== null) { + clearTimeout(exportTraceBatchDebounce) + } + exportTraceBatchDebounce = setTimeout(exportTraceBatch, 200) + } + } catch (err) { + console.error( + '-------\nFailed to export trace:\n', + err instanceof Error ? err.message : err, + '\nBlock height: ' + + BigInt(trace?.metadata.blockHeight ?? '-1').toLocaleString() + + '\nData: ' + + JSON.stringify(trace, null, 2) + + '\n-------' + ) + + Sentry.captureException(err, { + tags: { + type: 'failed-export-trace', + script: 'export', + chainId: (await State.getSingleton())?.chainId ?? 'unknown', + }, + extra: { + trace, + }, + }) + } finally { + exporting-- + } + } + + // Process traced events queue by exporting to a job queue until a certain + // concurrency is reached, then pause until the queue is drained. + const processTraceQueue = () => { + if ( + traceExportPaused || + // If WebSocket is enabled, pause trace queue until WebSocket is ready. + (webSocketEnabled && !webSocketReady) + ) { + return + } + + // Export traces until the trace queue is empty or the trace exporter queue + // is full. + for (let i = 0; i < traceQueue.length; i++) { + exporting++ + traceExporter = traceExporter.then(exportTrace) + + // If trace exporter queue fills up, pause until it drains. + if (exporting >= MAX_QUEUE_SIZE) { + traceExportPaused = true + + // Resume once queue drains. + const interval = setInterval(() => { + if (exporting < MAX_QUEUE_SIZE / 5) { + traceExportPaused = false + clearInterval(interval) + processTraceQueue() + } + }, 100) + + break + } + } + } + + // Tell pm2 we're ready right before we start reading. + if (process.send) { + process.send('ready') + } + + const { promise: tracer, close: closeTracer } = setUpFifoJsonTracer({ + file: traceFile, + onData: (data) => { + const tracedEvent = data as TracedEvent + // Ensure this is a traced write or delete event. + if ( + !objectMatchesStructure(tracedEvent, { + operation: {}, + key: {}, + value: {}, + metadata: { + blockHeight: {}, + }, + }) + ) { + return + } + + // Only handle writes and deletes. + if ( + tracedEvent.operation !== 'write' && + tracedEvent.operation !== 'delete' + ) { + return + } + + traceQueue.push(tracedEvent) + processTraceQueue() + }, + }) + + // If WebSocket enabled, connect to it before queueing. + if (webSocketEnabled) { + // Connect to local RPC WebSocket once ready. We need to read from the trace + // as the server is starting but not start processing the queue until the + // WebSocket block listener has connected. This is because the trace blocks + // the server from starting, but we can only listen for new blocks once the + // WebSocket is connected at some point after the server has started. We + // have to read from the trace to allow the server to start up. + waitPort({ + host: 'localhost', + port: 26657, + output: 'silent', + }).then(async ({ open }) => { + if (open) { + const setUpWebSocket = () => { + // Get new-block WebSocket. + const webSocket = setUpWebSocketNewBlockListener({ + rpc: 'http://127.0.0.1:26657', + onNewBlock: async (block) => { + const { chain_id, height, time } = (block as any).header + const latestBlockHeight = Number(height) + const latestBlockTimeUnixMs = Date.parse(time) + + // Cache block time for block height in cache used by state. + blockHeightToTimeCache.set( + latestBlockHeight, + latestBlockTimeUnixMs + ) + + // Update state singleton with latest information. + await State.update( + { + chainId: chain_id, + latestBlockHeight, + latestBlockTimeUnixMs, + }, + { + where: { + singleton: true, + }, + } + ) + }, + onConnect: () => { + console.log('WebSocket connected.') + webSocketReady = true + processTraceQueue() + }, + onError: async (error) => { + // If fails to connect, retry after three seconds. + if (error.message.includes('ECONNREFUSED')) { + console.error( + 'Failed to connect to WebSocket. Retrying in 3 seconds...', + error + ) + webSocket.terminate() + + setTimeout(setUpWebSocket, 3000) + } else { + console.error('WebSocket error', error) + + Sentry.captureException(error, { + tags: { + type: 'websocket-error', + script: 'export', + chainId: (await State.getSingleton())?.chainId ?? 'unknown', + }, + }) + } + }, + }) + } + + setUpWebSocket() + } else { + console.error( + 'Failed to connect to local RPC WebSocket. Queries may be slower as block times will be fetched from a remote RPC.' + ) + + Sentry.captureMessage( + 'Failed to connect to local RPC WebSocket (not open).', + { + tags: { + type: 'failed-websocket-connection', + script: 'export', + chainId: (await State.getSingleton())?.chainId ?? 'unknown', + }, + } + ) + + webSocketReady = true + processTraceQueue() + } + }) + } + + // Add shutdown signal handler. + process.on('SIGINT', () => { + // Tell tracer to close. The rest of the data in the buffer will finish + // processing. + closeTracer() + console.log('Shutting down after handlers finish...') + }) + + // Wait for tracer to close. Happens on FIFO closure or if `closeTracer` is + // manually called, such as in the SIGINT handler above. + await tracer + + // Wait for queue to finish processing. + await new Promise((resolve) => { + console.log( + `Shutting down after the queue drains ${traceQueue.length.toLocaleString()} events and ${exporting.toLocaleString()} others finish processing...` + ) + + const interval = setInterval(() => { + if (exporting === 0 && traceQueue.length === 0) { + clearInterval(interval) + resolve() + } + }, 50) + }) + + // Give a little time for the worker to queue. + await new Promise((resolve) => setTimeout(resolve, 5000)) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/src/scripts/export/types.ts b/src/scripts/export/types.ts index 70b0777c..28337bb2 100644 --- a/src/scripts/export/types.ts +++ b/src/scripts/export/types.ts @@ -2,27 +2,33 @@ import { CosmWasmClient } from '@cosmjs/cosmwasm-stargate' import { Config } from '@/core' -export type Handler = { +export type Handler = { // What store name to filter by for events to handle. storeName: string - // The function that will be called for each trace in the trace file. - handle: (trace: TracedEvent) => Promise - // The function that will be called after reading the entire trace file. - flush: () => Promise + // The function that will be called for each trace which determines if it will + // be queued for export. If returns an object, it will be queued. If returns + // undefined, it will not be queued. + match: (trace: TracedEventWithBlockTime) => + | (Data & { + // ID that uniquely represents this object. Likely a combination of + // block height and some key or keys. + id: string + }) + | undefined + // The function that will be called with queued objects. + process: (data: Data[]) => Promise } export type HandlerMakerOptions = { config: Config - dontUpdateComputations: boolean - dontSendWebhooks: boolean + updateComputations: boolean + sendWebhooks: boolean cosmWasmClient: CosmWasmClient - getBlockTimeUnixMs: ( - blockHeight: number, - trace: TracedEvent - ) => Promise } -export type HandlerMaker = (options: HandlerMakerOptions) => Promise +export type HandlerMaker = ( + options: HandlerMakerOptions +) => Promise> export type TracedEvent = { operation: 'read' | 'write' | 'delete' @@ -35,6 +41,10 @@ export type TracedEvent = { } } +export type TracedEventWithBlockTime = TracedEvent & { + blockTimeUnixMs: number +} + export type WorkerInitData = { config: Config update: boolean @@ -42,20 +52,7 @@ export type WorkerInitData = { websocket: boolean } -export type ToWorkerMessage = - | { - type: 'trace' - event: TracedEvent - } - | { - type: 'shutdown' - } - -export type FromWorkerMessage = - | { - type: 'ready' - } - | { - type: 'processed' - count: number - } +export type ExportQueueData = { + handler: string + data: unknown +} diff --git a/src/scripts/export/utils.ts b/src/scripts/export/utils.ts index 101ed876..f1aaecd4 100644 --- a/src/scripts/export/utils.ts +++ b/src/scripts/export/utils.ts @@ -1,5 +1,11 @@ import * as fs from 'fs' +import { CosmWasmClient } from '@cosmjs/cosmwasm-stargate' +import { + HttpBatchClient, + Tendermint34Client, + Tendermint37Client, +} from '@cosmjs/tendermint-rpc' import { WebSocket } from 'ws' import { objectMatchesStructure } from '@/core' @@ -218,3 +224,33 @@ export const setUpWebSocketNewBlockListener = ({ return webSocket } + +// Create CosmWasm client that batches requests. +export const getCosmWasmClient = async ( + rpc: string +): Promise => { + const httpClient = new HttpBatchClient(rpc) + const tmClient = await ( + ( + await connectTendermintClient(rpc) + ).constructor as typeof Tendermint34Client | typeof Tendermint37Client + ).create(httpClient) + // @ts-ignore + return new CosmWasmClient(tmClient) +} + +// Connect the correct tendermint client based on the node's version. +export const connectTendermintClient = async (endpoint: string) => { + // Tendermint/CometBFT 0.34/0.37 auto-detection. Starting with 0.37 we seem to + // get reliable versions again 🎉 Using 0.34 as the fallback. + let tmClient + const tm37Client = await Tendermint37Client.connect(endpoint) + const version = (await tm37Client.status()).nodeInfo.version + if (version.startsWith('0.37.')) { + tmClient = tm37Client + } else { + tm37Client.disconnect() + tmClient = await Tendermint34Client.connect(endpoint) + } + return tmClient +} diff --git a/src/scripts/export/worker.ts b/src/scripts/export/worker.ts deleted file mode 100644 index d63fa1dc..00000000 --- a/src/scripts/export/worker.ts +++ /dev/null @@ -1,337 +0,0 @@ -import { parentPort, workerData } from 'worker_threads' - -import { CosmWasmClient } from '@cosmjs/cosmwasm-stargate' -import { HttpBatchClient, Tendermint34Client } from '@cosmjs/tendermint-rpc' -import * as Sentry from '@sentry/node' -import retry from 'async-await-retry' -import { LRUCache } from 'lru-cache' -import waitPort from 'wait-port' - -import { DbType } from '@/core' -import { State, loadDb } from '@/db' - -import { handlerMakers } from './handlers' -import { - FromWorkerMessage, - ToWorkerMessage, - TracedEvent, - WorkerInitData, -} from './types' -import { setUpWebSocketNewBlockListener } from './utils' - -const main = async () => { - if (!parentPort) { - throw new Error('Must be run as a Worker') - } - - const { config, update, webhooks, websocket } = workerData as WorkerInitData - - // Add Sentry error reporting. - if (config.sentryDsn) { - Sentry.init({ - dsn: config.sentryDsn, - }) - } - - // Load DB on start. - await loadDb({ - type: DbType.Data, - }) - await loadDb({ - type: DbType.Accounts, - }) - - // Initialize state. - await State.createSingletonIfMissing() - - // Create CosmWasm client that batches requests. - const httpClient = new HttpBatchClient(config.rpc) - const tmClient = await Tendermint34Client.create(httpClient) - // @ts-ignore - const cosmWasmClient = new CosmWasmClient(tmClient) - - // Helper function that gets block time for height, cached in memory, which is - // filled in by the NewBlock WebSocket listener. - const blockHeightToTimeCache = new LRUCache({ - max: 100, - }) - const getBlockTimeUnixMs = async ( - blockHeight: number, - trace: TracedEvent - ): Promise => { - if (blockHeightToTimeCache.has(blockHeight)) { - return blockHeightToTimeCache.get(blockHeight) ?? 0 - } - - // This may fail if the RPC does not have the block info at this height - // anymore (i.e. if it's too old and the RPC pruned it) - const loadIntoCache = async () => { - const { - header: { time }, - } = await cosmWasmClient.getBlock(blockHeight) - blockHeightToTimeCache.set(blockHeight, Date.parse(time)) - } - - try { - // Retry 3 times with exponential backoff starting at 150ms delay. - await retry(loadIntoCache, [], { - retriesMax: 3, - exponential: true, - interval: 150, - }) - } catch (err) { - console.error( - '-------\nFailed to get block:\n', - err instanceof Error ? err.message : err, - '\nBlock height: ' + - BigInt(blockHeight).toLocaleString() + - '\nData: ' + - JSON.stringify(trace, null, 2) + - '\n-------' - ) - - // Only log to Sentry if not block height unavailable error. - if ( - !(err instanceof Error) || - !err.message.includes('must be less than or equal to the current') - ) { - Sentry.captureException(err, { - tags: { - type: 'failed-get-block', - script: 'export', - }, - extra: { - chainId: (await State.getSingleton())?.chainId ?? 'unknown', - trace, - blockHeight, - }, - }) - } - - // Set to 0 on failure so we can continue. - blockHeightToTimeCache.set(blockHeight, 0) - } - - return blockHeightToTimeCache.get(blockHeight) ?? 0 - } - - // Setup handlers. - const handlers = await Promise.all( - Object.entries(handlerMakers).map(async ([name, handlerMaker]) => ({ - name, - handler: await handlerMaker({ - config, - dontUpdateComputations: !update, - dontSendWebhooks: !webhooks, - cosmWasmClient, - getBlockTimeUnixMs, - }), - })) - ) - - // Flush all handlers. - const flushAll = async () => { - for (const { name, handler } of handlers) { - try { - // Retry 3 times with exponential backoff starting at 100ms delay. - await retry(handler.flush, [], { - retriesMax: 3, - exponential: true, - interval: 100, - }) - } catch (err) { - console.error( - '-------\nFailed to flush:\n', - err instanceof Error ? err.message : err, - '\n-------' - ) - Sentry.captureException(err, { - tags: { - type: 'failed-flush', - script: 'export', - }, - extra: { - handler: name, - }, - }) - throw err - } - } - } - - let webSocketConnected = false - let queueHandler: Promise = websocket - ? // Wait for WebSocket to be ready. - new Promise((resolve) => { - // We need to read from the trace as the server is starting but not - // start processing the queue until the WebSocket block listener has - // connected. This is because the trace blocks the server from starting, - // but we can only listen for new blocks once the WebSocket is connected - // at some point after the server has started. We have to read from the - // trace to allow the server to start up. - - // Wait for WebSocket to be ready. - const interval = setInterval(() => { - if (webSocketConnected) { - clearInterval(interval) - resolve() - } - }, 1000) - }) - : // Don't wait for websocket. - Promise.resolve() - - if (websocket) { - // Connect to local RPC WebSocket once ready. Don't await since we need to - // start reading from the trace FIFO before the RPC starts. - waitPort({ - host: 'localhost', - port: 26657, - output: 'silent', - }).then(({ open }) => { - if (open) { - const setUpWebSocket = () => { - // Get new-block WebSocket. - const webSocket = setUpWebSocketNewBlockListener({ - rpc: 'http://127.0.0.1:26657', - onNewBlock: async (block) => { - const { chain_id, height, time } = (block as any).header - const latestBlockHeight = Number(height) - const latestBlockTimeUnixMs = Date.parse(time) - - // Cache block time for block height in cache used by state. - blockHeightToTimeCache.set( - latestBlockHeight, - latestBlockTimeUnixMs - ) - - // Update state singleton with latest information. - await State.update( - { - chainId: chain_id, - latestBlockHeight, - latestBlockTimeUnixMs, - }, - { - where: { - singleton: true, - }, - } - ) - }, - onConnect: () => { - webSocketConnected = true - console.log('WebSocket connected.') - }, - onError: (error) => { - // If fails to connect, retry after three seconds. - if (error.message.includes('ECONNREFUSED')) { - console.error('Failed to connect to WebSocket.', error) - webSocket.terminate() - - setTimeout(setUpWebSocket, 3000) - } else { - console.error('WebSocket error', error) - } - }, - }) - } - - setUpWebSocket() - } else { - console.error( - 'Failed to connect to local RPC WebSocket. Queries may be slower as block times will be fetched from a remote RPC.' - ) - } - }) - } - - let processed = 0 - // Update parent on processed count. - setInterval(() => { - parentPort!.postMessage({ - type: 'processed', - count: processed, - } as FromWorkerMessage) - - // Reset processed count. - processed = 0 - }, 100).unref() - - parentPort.on('message', (message: ToWorkerMessage) => { - if (message.type === 'trace') { - const tracedEvent = message.event - - // Handle event after previous event is handled. - queueHandler = queueHandler.then(async () => { - // Try to handle with each module. - for (const { - name, - handler: { storeName, handle }, - } of handlers) { - // Filter by handler store if present. Otherwise just try to handle. - // Osmosis, for example, does not emit store_name in metadata. - if ( - tracedEvent.metadata.store_name && - storeName !== tracedEvent.metadata.store_name - ) { - continue - } - - try { - // Retry 3 times with exponential backoff starting at 100ms delay. - await retry(handle, [tracedEvent], { - retriesMax: 3, - exponential: true, - interval: 100, - }) - } catch (err) { - console.error( - '-------\nFailed to handle:\n', - err instanceof Error ? err.message : err, - '\nHandler: ' + - name + - '\nData: ' + - JSON.stringify(tracedEvent, null, 2) + - '\n-------' - ) - Sentry.captureException(err, { - tags: { - type: 'failed-handle', - script: 'export', - }, - extra: { - handler: name, - tracedEvent, - }, - }) - } - } - - // Increment processed count. - processed++ - }) - } else if (message.type === 'shutdown') { - ;(async () => { - // Wait for queue to finish. - await queueHandler - - // Flush all handlers. - await flushAll() - - // Exit worker process. - process.exit(0) - })() - } - }) - - // Tell parent we're ready to process traces. - parentPort.postMessage({ - type: 'ready', - }) -} - -main().catch((err) => { - console.error('Worker errored', err) - process.exit(1) -}) diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts new file mode 100644 index 00000000..f06e9b3a --- /dev/null +++ b/src/server/routes/indexer/bull.ts @@ -0,0 +1,14 @@ +import { createBullBoard } from '@bull-board/api' +import { BullMQAdapter } from '@bull-board/api/bullMQAdapter' +import { KoaAdapter } from '@bull-board/koa' + +import { EXPORT_QUEUE_NAME, getBullQueue } from '@/core' + +const serverAdapter = new KoaAdapter().setBasePath('/jobs') + +createBullBoard({ + queues: [new BullMQAdapter(getBullQueue(EXPORT_QUEUE_NAME))], + serverAdapter, +}) + +export const bullBoardJobsMiddleware = serverAdapter.registerPlugin() diff --git a/src/server/routes/indexer/index.ts b/src/server/routes/indexer/index.ts index 9f002021..2db6ce22 100644 --- a/src/server/routes/indexer/index.ts +++ b/src/server/routes/indexer/index.ts @@ -1,5 +1,6 @@ import Router from '@koa/router' +import { bullBoardJobsMiddleware } from './bull' import { computer } from './computer' import { getStatus } from './getStatus' import { up } from './up' @@ -12,5 +13,9 @@ indexerRouter.get('/status', getStatus) // Check if indexer is caught up. indexerRouter.get('/up', up) +// Bull board (background worker dashboard) +// Route: /jobs (defined in ./bull.ts) +indexerRouter.use(bullBoardJobsMiddleware) + // Formula computer. This must be the last route since it's a catch-all. indexerRouter.get('/(.+)', computer) From 5c2267279b0da61873bd9477e9ca64093f1938f6 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 16:42:21 -0700 Subject: [PATCH 02/35] Added password auth to bull dashboard. --- package-lock.json | 69 ++++++++++++++++++++++++++++++ package.json | 2 + src/core/types.ts | 2 + src/server/routes/indexer/index.ts | 13 +++++- 4 files changed, 85 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 33fee0aa..b9088946 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "crypto-js": "^4.1.1", "jsonwebtoken": "^9.0.0", "koa": "^2.13.4", + "koa-basic-auth": "^4.0.0", "koa-body": "^6.0.1", "lodash.groupby": "^4.6.0", "lodash.isequal": "^4.5.0", @@ -52,6 +53,7 @@ "@types/jest": "^29.4.0", "@types/jsonwebtoken": "^9.0.1", "@types/koa": "^2.13.5", + "@types/koa-basic-auth": "^2.0.4", "@types/lodash.groupby": "^4.6.7", "@types/lodash.isequal": "^4.5.6", "@types/node": "^18.11.10", @@ -2712,6 +2714,15 @@ "@types/koa": "*" } }, + "node_modules/@types/koa-basic-auth": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/koa-basic-auth/-/koa-basic-auth-2.0.4.tgz", + "integrity": "sha512-PJKvoF5OMGlEEzUnctZDGRQVqV12xB0V4KplDJvHQDX9egh9ADFa456zGXRNnhNr43t3Fe4/VzD6ziM61uM5RQ==", + "dev": true, + "dependencies": { + "@types/koa": "*" + } + }, "node_modules/@types/koa-compose": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/@types/koa-compose/-/koa-compose-3.2.5.tgz", @@ -3493,6 +3504,22 @@ } ] }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/basic-auth/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/bech32": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/bech32/-/bech32-1.1.4.tgz", @@ -7359,6 +7386,15 @@ "node": "^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4" } }, + "node_modules/koa-basic-auth": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/koa-basic-auth/-/koa-basic-auth-4.0.0.tgz", + "integrity": "sha512-eV1sGVAizDuFWNpY43VF3Z1ND4PotQZB/igxHNrcJXzXw+Flmj8Uv+4hP9LyNXyvqLJz/X5bmXeMu84AAGD9Jw==", + "dependencies": { + "basic-auth": "^2.0.0", + "tsscmp": "^1.0.6" + } + }, "node_modules/koa-body": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/koa-body/-/koa-body-6.0.1.tgz", @@ -13163,6 +13199,15 @@ "@types/koa": "*" } }, + "@types/koa-basic-auth": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/koa-basic-auth/-/koa-basic-auth-2.0.4.tgz", + "integrity": "sha512-PJKvoF5OMGlEEzUnctZDGRQVqV12xB0V4KplDJvHQDX9egh9ADFa456zGXRNnhNr43t3Fe4/VzD6ziM61uM5RQ==", + "dev": true, + "requires": { + "@types/koa": "*" + } + }, "@types/koa-compose": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/@types/koa-compose/-/koa-compose-3.2.5.tgz", @@ -13741,6 +13786,21 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, + "basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "requires": { + "safe-buffer": "5.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, "bech32": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/bech32/-/bech32-1.1.4.tgz", @@ -16685,6 +16745,15 @@ "vary": "^1.1.2" } }, + "koa-basic-auth": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/koa-basic-auth/-/koa-basic-auth-4.0.0.tgz", + "integrity": "sha512-eV1sGVAizDuFWNpY43VF3Z1ND4PotQZB/igxHNrcJXzXw+Flmj8Uv+4hP9LyNXyvqLJz/X5bmXeMu84AAGD9Jw==", + "requires": { + "basic-auth": "^2.0.0", + "tsscmp": "^1.0.6" + } + }, "koa-body": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/koa-body/-/koa-body-6.0.1.tgz", diff --git a/package.json b/package.json index 52637475..f09d7c1a 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "@types/jest": "^29.4.0", "@types/jsonwebtoken": "^9.0.1", "@types/koa": "^2.13.5", + "@types/koa-basic-auth": "^2.0.4", "@types/lodash.groupby": "^4.6.7", "@types/lodash.isequal": "^4.5.6", "@types/node": "^18.11.10", @@ -97,6 +98,7 @@ "crypto-js": "^4.1.1", "jsonwebtoken": "^9.0.0", "koa": "^2.13.4", + "koa-basic-auth": "^4.0.0", "koa-body": "^6.0.1", "lodash.groupby": "^4.6.0", "lodash.isequal": "^4.5.0", diff --git a/src/core/types.ts b/src/core/types.ts index 5f18d962..529dd554 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -81,6 +81,8 @@ export type Config = { soketi?: PusherOptions // Accounts server JWT secret. accountsJwtSecret?: string + // Indexer exporter dashboard password. + exporterDashboardPassword?: string // Other config options. [key: string]: any diff --git a/src/server/routes/indexer/index.ts b/src/server/routes/indexer/index.ts index 2db6ce22..2b397793 100644 --- a/src/server/routes/indexer/index.ts +++ b/src/server/routes/indexer/index.ts @@ -1,4 +1,7 @@ import Router from '@koa/router' +import auth from 'koa-basic-auth' + +import { loadConfig } from '@/core' import { bullBoardJobsMiddleware } from './bull' import { computer } from './computer' @@ -7,6 +10,8 @@ import { up } from './up' export const indexerRouter = new Router() +const { exporterDashboardPassword = 'exporter' } = loadConfig() + // Status. indexerRouter.get('/status', getStatus) @@ -15,7 +20,13 @@ indexerRouter.get('/up', up) // Bull board (background worker dashboard) // Route: /jobs (defined in ./bull.ts) -indexerRouter.use(bullBoardJobsMiddleware) +indexerRouter.use( + auth({ + name: 'exporter', + pass: exporterDashboardPassword, + }), + bullBoardJobsMiddleware +) // Formula computer. This must be the last route since it's a catch-all. indexerRouter.get('/(.+)', computer) From 9f2474ca956d101083be38a1e159da4bf70736da Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 16:43:39 -0700 Subject: [PATCH 03/35] Fixed missing types. --- package-lock.json | 13 +++++++++++++ package.json | 1 + 2 files changed, 14 insertions(+) diff --git a/package-lock.json b/package-lock.json index b9088946..2147f81f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -58,6 +58,7 @@ "@types/lodash.isequal": "^4.5.6", "@types/node": "^18.11.10", "@types/prettier": "^2.7.1", + "@types/redis-info": "^3.0.1", "@types/supertest": "^2.0.12", "@types/validator": "^13.7.10", "@types/ws": "^8.5.5", @@ -2813,6 +2814,12 @@ "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" }, + "node_modules/@types/redis-info": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/redis-info/-/redis-info-3.0.1.tgz", + "integrity": "sha512-P/zVo/P5XEnqbztGhuO9gTzyVKdr2royHT5/HxGdN4JYAs7PMWtOF5BQJktn1hXsPNcsNnW4NBSMruPOa1zuhQ==", + "dev": true + }, "node_modules/@types/semver": { "version": "7.3.13", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz", @@ -13297,6 +13304,12 @@ "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==" }, + "@types/redis-info": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/redis-info/-/redis-info-3.0.1.tgz", + "integrity": "sha512-P/zVo/P5XEnqbztGhuO9gTzyVKdr2royHT5/HxGdN4JYAs7PMWtOF5BQJktn1hXsPNcsNnW4NBSMruPOa1zuhQ==", + "dev": true + }, "@types/semver": { "version": "7.3.13", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz", diff --git a/package.json b/package.json index f09d7c1a..a1331a42 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@types/lodash.isequal": "^4.5.6", "@types/node": "^18.11.10", "@types/prettier": "^2.7.1", + "@types/redis-info": "^3.0.1", "@types/supertest": "^2.0.12", "@types/validator": "^13.7.10", "@types/ws": "^8.5.5", From 5c50b5687a947783a9285f868a28cfe4b8e8f2f5 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 16:48:07 -0700 Subject: [PATCH 04/35] Pass config from server. --- src/server/routes/index.ts | 11 ++++++-- src/server/routes/indexer/index.ts | 40 ++++++++++++++++-------------- src/server/serve.ts | 1 + src/server/test/account/app.ts | 3 +++ src/server/test/indexer/app.ts | 3 +++ 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src/server/routes/index.ts b/src/server/routes/index.ts index 0c78033d..da0eb8d4 100644 --- a/src/server/routes/index.ts +++ b/src/server/routes/index.ts @@ -1,15 +1,21 @@ import Router from '@koa/router' import Koa from 'koa' +import { Config } from '@/core/types' + import { accountRouter } from './account' -import { indexerRouter } from './indexer' +import { makeIndexerRouter } from './indexer' export type SetupRouterOptions = { + config: Config // Whether to run the account server. If false, runs indexer server. accounts: boolean } -export const setupRouter = (app: Koa, { accounts }: SetupRouterOptions) => { +export const setupRouter = ( + app: Koa, + { config, accounts }: SetupRouterOptions +) => { const router = new Router() // Ping. @@ -23,6 +29,7 @@ export const setupRouter = (app: Koa, { accounts }: SetupRouterOptions) => { router.use(accountRouter.routes(), accountRouter.allowedMethods()) } else { // Indexer API. + const indexerRouter = makeIndexerRouter(config) router.use(indexerRouter.routes(), indexerRouter.allowedMethods()) } diff --git a/src/server/routes/indexer/index.ts b/src/server/routes/indexer/index.ts index 2b397793..d6d750f7 100644 --- a/src/server/routes/indexer/index.ts +++ b/src/server/routes/indexer/index.ts @@ -1,32 +1,36 @@ import Router from '@koa/router' import auth from 'koa-basic-auth' -import { loadConfig } from '@/core' +import { Config } from '@/core' import { bullBoardJobsMiddleware } from './bull' import { computer } from './computer' import { getStatus } from './getStatus' import { up } from './up' -export const indexerRouter = new Router() +export const makeIndexerRouter = ({ + exporterDashboardPassword = 'exporter', +}: Config) => { + const indexerRouter = new Router() -const { exporterDashboardPassword = 'exporter' } = loadConfig() + // Status. + indexerRouter.get('/status', getStatus) -// Status. -indexerRouter.get('/status', getStatus) + // Check if indexer is caught up. + indexerRouter.get('/up', up) -// Check if indexer is caught up. -indexerRouter.get('/up', up) + // Bull board (background worker dashboard) + // Route: /jobs (defined in ./bull.ts) + indexerRouter.use( + auth({ + name: 'exporter', + pass: exporterDashboardPassword, + }), + bullBoardJobsMiddleware + ) -// Bull board (background worker dashboard) -// Route: /jobs (defined in ./bull.ts) -indexerRouter.use( - auth({ - name: 'exporter', - pass: exporterDashboardPassword, - }), - bullBoardJobsMiddleware -) + // Formula computer. This must be the last route since it's a catch-all. + indexerRouter.get('/(.+)', computer) -// Formula computer. This must be the last route since it's a catch-all. -indexerRouter.get('/(.+)', computer) + return indexerRouter +} diff --git a/src/server/serve.ts b/src/server/serve.ts index 02b7edea..2998863f 100644 --- a/src/server/serve.ts +++ b/src/server/serve.ts @@ -67,6 +67,7 @@ app.use(async (ctx, next) => { // Add routes. setupRouter(app, { + config, accounts, }) diff --git a/src/server/test/account/app.ts b/src/server/test/account/app.ts index 03dfdd8c..ebd46036 100644 --- a/src/server/test/account/app.ts +++ b/src/server/test/account/app.ts @@ -1,9 +1,12 @@ import Koa from 'koa' +import { loadConfig } from '@/core' + import { setupRouter } from '../../routes' export const app = new Koa() setupRouter(app, { + config: loadConfig(), accounts: true, }) diff --git a/src/server/test/indexer/app.ts b/src/server/test/indexer/app.ts index bcb1aa16..ef4b5a06 100644 --- a/src/server/test/indexer/app.ts +++ b/src/server/test/indexer/app.ts @@ -1,9 +1,12 @@ import Koa from 'koa' +import { loadConfig } from '@/core' + import { setupRouter } from '../../routes' export const app = new Koa() setupRouter(app, { + config: loadConfig(), accounts: false, }) From fb4cd49b6ad44b59cf2d1501cd72ee2be2795326 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 16:50:08 -0700 Subject: [PATCH 05/35] Load bull board in function so config loads. --- src/server/routes/indexer/bull.ts | 14 ++++++++------ src/server/routes/indexer/index.ts | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index f06e9b3a..a8c24bf4 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -4,11 +4,13 @@ import { KoaAdapter } from '@bull-board/koa' import { EXPORT_QUEUE_NAME, getBullQueue } from '@/core' -const serverAdapter = new KoaAdapter().setBasePath('/jobs') +export const makeBullBoardJobsMiddleware = () => { + const serverAdapter = new KoaAdapter().setBasePath('/jobs') -createBullBoard({ - queues: [new BullMQAdapter(getBullQueue(EXPORT_QUEUE_NAME))], - serverAdapter, -}) + createBullBoard({ + queues: [new BullMQAdapter(getBullQueue(EXPORT_QUEUE_NAME))], + serverAdapter, + }) -export const bullBoardJobsMiddleware = serverAdapter.registerPlugin() + return serverAdapter.registerPlugin() +} diff --git a/src/server/routes/indexer/index.ts b/src/server/routes/indexer/index.ts index d6d750f7..a6e4fa6c 100644 --- a/src/server/routes/indexer/index.ts +++ b/src/server/routes/indexer/index.ts @@ -3,7 +3,7 @@ import auth from 'koa-basic-auth' import { Config } from '@/core' -import { bullBoardJobsMiddleware } from './bull' +import { makeBullBoardJobsMiddleware } from './bull' import { computer } from './computer' import { getStatus } from './getStatus' import { up } from './up' @@ -26,7 +26,7 @@ export const makeIndexerRouter = ({ name: 'exporter', pass: exporterDashboardPassword, }), - bullBoardJobsMiddleware + makeBullBoardJobsMiddleware() ) // Formula computer. This must be the last route since it's a catch-all. From 024242f12b91986f61ca7270fdf2ce401cc846b5 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 16:51:28 -0700 Subject: [PATCH 06/35] Fixed bull connection getter. --- src/core/queues.ts | 2 +- src/scripts/export/process.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/queues.ts b/src/core/queues.ts index 906d0924..5fbd4e1a 100644 --- a/src/core/queues.ts +++ b/src/core/queues.ts @@ -20,7 +20,7 @@ export const getBullQueue = (name: string) => connection: getBullConnection(), }) -export const WorkerQueue = ( +export const getBullWorker = ( name: string, processor: Processor ) => diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index 08dd7d9d..ba251fce 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -3,7 +3,7 @@ import retry from 'async-await-retry' import { Worker } from 'bullmq' import { Command } from 'commander' -import { DbType, EXPORT_QUEUE_NAME, loadConfig } from '@/core' +import { DbType, EXPORT_QUEUE_NAME, getBullWorker, loadConfig } from '@/core' import { State, loadDb } from '@/db' import { handlerMakers } from './handlers' @@ -67,7 +67,7 @@ const main = async () => { ) // Create queue worker. - const worker = new Worker<{ data: ExportQueueData[] }>( + const worker = getBullWorker<{ data: ExportQueueData[] }>( EXPORT_QUEUE_NAME, async (job) => { const { data } = job.data From 1bc2e3c38fc9b5abb2434de798d52659f17a4edb Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 16:52:54 -0700 Subject: [PATCH 07/35] Fixed bull board route. --- src/server/routes/indexer/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/routes/indexer/index.ts b/src/server/routes/indexer/index.ts index a6e4fa6c..5f762ce3 100644 --- a/src/server/routes/indexer/index.ts +++ b/src/server/routes/indexer/index.ts @@ -20,8 +20,8 @@ export const makeIndexerRouter = ({ indexerRouter.get('/up', up) // Bull board (background worker dashboard) - // Route: /jobs (defined in ./bull.ts) indexerRouter.use( + '/jobs', auth({ name: 'exporter', pass: exporterDashboardPassword, From 63668fb5cda23e2bfa660d4525627ebc8bc0a7f4 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:00:13 -0700 Subject: [PATCH 08/35] Attempt fix bull board router. --- src/server/routes/indexer/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/routes/indexer/index.ts b/src/server/routes/indexer/index.ts index 5f762ce3..5e4f62f7 100644 --- a/src/server/routes/indexer/index.ts +++ b/src/server/routes/indexer/index.ts @@ -20,14 +20,15 @@ export const makeIndexerRouter = ({ indexerRouter.get('/up', up) // Bull board (background worker dashboard) - indexerRouter.use( - '/jobs', + const router = new Router() + router.use( auth({ name: 'exporter', pass: exporterDashboardPassword, }), makeBullBoardJobsMiddleware() ) + indexerRouter.use('/jobs', router.routes(), router.allowedMethods()) // Formula computer. This must be the last route since it's a catch-all. indexerRouter.get('/(.+)', computer) From 4371ca561aeee226d59c0f8f507adff6e4efc80f Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:01:32 -0700 Subject: [PATCH 09/35] Attempt fix bull board router 2. --- src/server/routes/indexer/bull.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index a8c24bf4..634d3b7a 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -5,7 +5,7 @@ import { KoaAdapter } from '@bull-board/koa' import { EXPORT_QUEUE_NAME, getBullQueue } from '@/core' export const makeBullBoardJobsMiddleware = () => { - const serverAdapter = new KoaAdapter().setBasePath('/jobs') + const serverAdapter = new KoaAdapter() createBullBoard({ queues: [new BullMQAdapter(getBullQueue(EXPORT_QUEUE_NAME))], From 9e4821b739f3b12c2bfef0c7b83e4fc39396d3f8 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:06:11 -0700 Subject: [PATCH 10/35] Attempt fix bull board router 3. --- src/server/routes/indexer/bull.ts | 2 +- src/server/routes/indexer/index.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index 634d3b7a..a8c24bf4 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -5,7 +5,7 @@ import { KoaAdapter } from '@bull-board/koa' import { EXPORT_QUEUE_NAME, getBullQueue } from '@/core' export const makeBullBoardJobsMiddleware = () => { - const serverAdapter = new KoaAdapter() + const serverAdapter = new KoaAdapter().setBasePath('/jobs') createBullBoard({ queues: [new BullMQAdapter(getBullQueue(EXPORT_QUEUE_NAME))], diff --git a/src/server/routes/indexer/index.ts b/src/server/routes/indexer/index.ts index 5e4f62f7..f107a8b2 100644 --- a/src/server/routes/indexer/index.ts +++ b/src/server/routes/indexer/index.ts @@ -20,15 +20,14 @@ export const makeIndexerRouter = ({ indexerRouter.get('/up', up) // Bull board (background worker dashboard) - const router = new Router() - router.use( + indexerRouter.all( + '/jobs', auth({ name: 'exporter', pass: exporterDashboardPassword, }), makeBullBoardJobsMiddleware() ) - indexerRouter.use('/jobs', router.routes(), router.allowedMethods()) // Formula computer. This must be the last route since it's a catch-all. indexerRouter.get('/(.+)', computer) From f7414dd2d0697efe6dae89f8ee5504191d20431a Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:13:04 -0700 Subject: [PATCH 11/35] Attempt fix bull board router 4. --- package-lock.json | 20 ++++++++++++++++++++ package.json | 2 ++ src/server/routes/indexer/bull.ts | 2 +- src/server/routes/indexer/index.ts | 11 +++++++---- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2147f81f..6ca624ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,6 +31,7 @@ "koa": "^2.13.4", "koa-basic-auth": "^4.0.0", "koa-body": "^6.0.1", + "koa-mount": "^4.0.0", "lodash.groupby": "^4.6.0", "lodash.isequal": "^4.5.0", "lru-cache": "^10.0.0", @@ -54,6 +55,7 @@ "@types/jsonwebtoken": "^9.0.1", "@types/koa": "^2.13.5", "@types/koa-basic-auth": "^2.0.4", + "@types/koa-mount": "^4.0.3", "@types/lodash.groupby": "^4.6.7", "@types/lodash.isequal": "^4.5.6", "@types/node": "^18.11.10", @@ -2732,6 +2734,15 @@ "@types/koa": "*" } }, + "node_modules/@types/koa-mount": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/koa-mount/-/koa-mount-4.0.3.tgz", + "integrity": "sha512-WXhyitlW5B6zW31cKZO+RBl38afLmO9847M8PaKmcnO5tqGJy/+XcH5N/69Nsp+vKvFXhDOY9GM5uF34HrWgrw==", + "dev": true, + "dependencies": { + "@types/koa": "*" + } + }, "node_modules/@types/lodash": { "version": "4.14.191", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz", @@ -13223,6 +13234,15 @@ "@types/koa": "*" } }, + "@types/koa-mount": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/koa-mount/-/koa-mount-4.0.3.tgz", + "integrity": "sha512-WXhyitlW5B6zW31cKZO+RBl38afLmO9847M8PaKmcnO5tqGJy/+XcH5N/69Nsp+vKvFXhDOY9GM5uF34HrWgrw==", + "dev": true, + "requires": { + "@types/koa": "*" + } + }, "@types/lodash": { "version": "4.14.191", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.191.tgz", diff --git a/package.json b/package.json index a1331a42..ee385e7e 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "@types/jsonwebtoken": "^9.0.1", "@types/koa": "^2.13.5", "@types/koa-basic-auth": "^2.0.4", + "@types/koa-mount": "^4.0.3", "@types/lodash.groupby": "^4.6.7", "@types/lodash.isequal": "^4.5.6", "@types/node": "^18.11.10", @@ -101,6 +102,7 @@ "koa": "^2.13.4", "koa-basic-auth": "^4.0.0", "koa-body": "^6.0.1", + "koa-mount": "^4.0.0", "lodash.groupby": "^4.6.0", "lodash.isequal": "^4.5.0", "lru-cache": "^10.0.0", diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index a8c24bf4..634d3b7a 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -5,7 +5,7 @@ import { KoaAdapter } from '@bull-board/koa' import { EXPORT_QUEUE_NAME, getBullQueue } from '@/core' export const makeBullBoardJobsMiddleware = () => { - const serverAdapter = new KoaAdapter().setBasePath('/jobs') + const serverAdapter = new KoaAdapter() createBullBoard({ queues: [new BullMQAdapter(getBullQueue(EXPORT_QUEUE_NAME))], diff --git a/src/server/routes/indexer/index.ts b/src/server/routes/indexer/index.ts index f107a8b2..644f7651 100644 --- a/src/server/routes/indexer/index.ts +++ b/src/server/routes/indexer/index.ts @@ -1,5 +1,7 @@ import Router from '@koa/router' +import Koa from 'koa' import auth from 'koa-basic-auth' +import mount from 'koa-mount' import { Config } from '@/core' @@ -20,14 +22,15 @@ export const makeIndexerRouter = ({ indexerRouter.get('/up', up) // Bull board (background worker dashboard) - indexerRouter.all( - '/jobs', + const jobsApp = new Koa() + jobsApp.use( auth({ name: 'exporter', pass: exporterDashboardPassword, - }), - makeBullBoardJobsMiddleware() + }) ) + jobsApp.use(makeBullBoardJobsMiddleware()) + indexerRouter.use(mount('/jobs', jobsApp)) // Formula computer. This must be the last route since it's a catch-all. indexerRouter.get('/(.+)', computer) From 1a1f4180355ae51e6f55dd56646f7b3e7434f36c Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:15:49 -0700 Subject: [PATCH 12/35] Attempt fix bull board router 5. --- src/server/routes/indexer/bull.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index 634d3b7a..b781b7fa 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -12,5 +12,5 @@ export const makeBullBoardJobsMiddleware = () => { serverAdapter, }) - return serverAdapter.registerPlugin() + return serverAdapter.registerPlugin({ mount: '/jobs' }) } From 69c00d0ce2e23b04d6ce6e27641222c31af6bbb7 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:21:48 -0700 Subject: [PATCH 13/35] Attempt fix bull board router 6. --- src/server/routes/index.ts | 18 ++++++++++++--- src/server/routes/indexer/bull.ts | 2 +- src/server/routes/indexer/index.ts | 37 ++++++------------------------ 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/src/server/routes/index.ts b/src/server/routes/index.ts index da0eb8d4..9174cfbd 100644 --- a/src/server/routes/index.ts +++ b/src/server/routes/index.ts @@ -1,10 +1,13 @@ import Router from '@koa/router' import Koa from 'koa' +import auth from 'koa-basic-auth' +import mount from 'koa-mount' import { Config } from '@/core/types' import { accountRouter } from './account' -import { makeIndexerRouter } from './indexer' +import { indexerRouter } from './indexer' +import { makeBullBoardJobsMiddleware } from './indexer/bull' export type SetupRouterOptions = { config: Config @@ -14,7 +17,7 @@ export type SetupRouterOptions = { export const setupRouter = ( app: Koa, - { config, accounts }: SetupRouterOptions + { config: { exporterDashboardPassword }, accounts }: SetupRouterOptions ) => { const router = new Router() @@ -28,8 +31,17 @@ export const setupRouter = ( // Account API. router.use(accountRouter.routes(), accountRouter.allowedMethods()) } else { + const bullApp = new Koa() + bullApp.use( + auth({ + name: 'exporter', + pass: exporterDashboardPassword || 'exporter', + }) + ) + bullApp.use(makeBullBoardJobsMiddleware()) + app.use(mount('/bull', bullApp)) + // Indexer API. - const indexerRouter = makeIndexerRouter(config) router.use(indexerRouter.routes(), indexerRouter.allowedMethods()) } diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index b781b7fa..634d3b7a 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -12,5 +12,5 @@ export const makeBullBoardJobsMiddleware = () => { serverAdapter, }) - return serverAdapter.registerPlugin({ mount: '/jobs' }) + return serverAdapter.registerPlugin() } diff --git a/src/server/routes/indexer/index.ts b/src/server/routes/indexer/index.ts index 644f7651..9f002021 100644 --- a/src/server/routes/indexer/index.ts +++ b/src/server/routes/indexer/index.ts @@ -1,39 +1,16 @@ import Router from '@koa/router' -import Koa from 'koa' -import auth from 'koa-basic-auth' -import mount from 'koa-mount' -import { Config } from '@/core' - -import { makeBullBoardJobsMiddleware } from './bull' import { computer } from './computer' import { getStatus } from './getStatus' import { up } from './up' -export const makeIndexerRouter = ({ - exporterDashboardPassword = 'exporter', -}: Config) => { - const indexerRouter = new Router() - - // Status. - indexerRouter.get('/status', getStatus) - - // Check if indexer is caught up. - indexerRouter.get('/up', up) +export const indexerRouter = new Router() - // Bull board (background worker dashboard) - const jobsApp = new Koa() - jobsApp.use( - auth({ - name: 'exporter', - pass: exporterDashboardPassword, - }) - ) - jobsApp.use(makeBullBoardJobsMiddleware()) - indexerRouter.use(mount('/jobs', jobsApp)) +// Status. +indexerRouter.get('/status', getStatus) - // Formula computer. This must be the last route since it's a catch-all. - indexerRouter.get('/(.+)', computer) +// Check if indexer is caught up. +indexerRouter.get('/up', up) - return indexerRouter -} +// Formula computer. This must be the last route since it's a catch-all. +indexerRouter.get('/(.+)', computer) From 7566f051a2393ec2a03a56f6eb18f92d57726491 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:23:26 -0700 Subject: [PATCH 14/35] Installed missing package. --- package-lock.json | 47 +++++++++++++++++++++++------------------------ package.json | 1 + 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6ca624ae..9a154961 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "commander": "^9.4.1", "cosmjs-types": "^0.8.0", "crypto-js": "^4.1.1", + "ejs": "^3.1.9", "jsonwebtoken": "^9.0.0", "koa": "^2.13.4", "koa-basic-auth": "^4.0.0", @@ -738,20 +739,6 @@ "koa-views": "^7.0.1" } }, - "node_modules/@bull-board/koa/node_modules/ejs": { - "version": "3.1.9", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz", - "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==", - "dependencies": { - "jake": "^10.8.5" - }, - "bin": { - "ejs": "bin/cli.js" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/@bull-board/ui": { "version": "5.8.4", "resolved": "https://registry.npmjs.org/@bull-board/ui/-/ui-5.8.4.tgz", @@ -4504,6 +4491,20 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/ejs": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz", + "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.454", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.454.tgz", @@ -11516,16 +11517,6 @@ "koa-router": "^10.0.0", "koa-static": "^5.0.0", "koa-views": "^7.0.1" - }, - "dependencies": { - "ejs": { - "version": "3.1.9", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz", - "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==", - "requires": { - "jake": "^10.8.5" - } - } } }, "@bull-board/ui": { @@ -14569,6 +14560,14 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "ejs": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz", + "integrity": "sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==", + "requires": { + "jake": "^10.8.5" + } + }, "electron-to-chromium": { "version": "1.4.454", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.454.tgz", diff --git a/package.json b/package.json index ee385e7e..b49d8b7b 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "commander": "^9.4.1", "cosmjs-types": "^0.8.0", "crypto-js": "^4.1.1", + "ejs": "^3.1.9", "jsonwebtoken": "^9.0.0", "koa": "^2.13.4", "koa-basic-auth": "^4.0.0", From bd9dc48d239881bd155fa216458fdc914079729e Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:27:36 -0700 Subject: [PATCH 15/35] Attempt fix bull board router 7. --- src/server/routes/indexer/bull.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index 634d3b7a..d4564cf9 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -5,12 +5,15 @@ import { KoaAdapter } from '@bull-board/koa' import { EXPORT_QUEUE_NAME, getBullQueue } from '@/core' export const makeBullBoardJobsMiddleware = () => { - const serverAdapter = new KoaAdapter() + const serverAdapter = new KoaAdapter().setBasePath('/bull') createBullBoard({ queues: [new BullMQAdapter(getBullQueue(EXPORT_QUEUE_NAME))], serverAdapter, }) - return serverAdapter.registerPlugin() + return serverAdapter.registerPlugin({ + // Mount on root since we wrap this in our own app with auth. + mount: '/', + }) } From a5c2429d1b5a211a67133052d06999ebe67adbb1 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:28:45 -0700 Subject: [PATCH 16/35] Change to /jobs. --- src/server/routes/index.ts | 4 ++-- src/server/routes/indexer/bull.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/routes/index.ts b/src/server/routes/index.ts index 9174cfbd..9322783c 100644 --- a/src/server/routes/index.ts +++ b/src/server/routes/index.ts @@ -38,8 +38,8 @@ export const setupRouter = ( pass: exporterDashboardPassword || 'exporter', }) ) - bullApp.use(makeBullBoardJobsMiddleware()) - app.use(mount('/bull', bullApp)) + bullApp.use(makeBullBoardJobsMiddleware('/jobs')) + app.use(mount('/jobs', bullApp)) // Indexer API. router.use(indexerRouter.routes(), indexerRouter.allowedMethods()) diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index d4564cf9..80bfb6e4 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -4,8 +4,8 @@ import { KoaAdapter } from '@bull-board/koa' import { EXPORT_QUEUE_NAME, getBullQueue } from '@/core' -export const makeBullBoardJobsMiddleware = () => { - const serverAdapter = new KoaAdapter().setBasePath('/bull') +export const makeBullBoardJobsMiddleware = (basePath: string) => { + const serverAdapter = new KoaAdapter().setBasePath(basePath) createBullBoard({ queues: [new BullMQAdapter(getBullQueue(EXPORT_QUEUE_NAME))], From 0336b239a79b0e4518bb06cc2be9159abf344ef3 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:35:43 -0700 Subject: [PATCH 17/35] Reordered core exports. --- src/core/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/index.ts b/src/core/index.ts index d7037b0b..6696f42e 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -1,7 +1,7 @@ -export * from './utils' - export * from './compute' export * from './config' export * from './env' export * from './queues' export * from './types' +// Must be last for circular imports. +export * from './utils' From 7afac4040fc8594f4975bf30778ed7770390bcae Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:38:33 -0700 Subject: [PATCH 18/35] Load DB and close at the end. --- src/scripts/export/trace.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 9806f26f..fd63db0a 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -8,12 +8,13 @@ import { LRUCache } from 'lru-cache' import waitPort from 'wait-port' import { + DbType, EXPORT_QUEUE_NAME, getBullQueue, loadConfig, objectMatchesStructure, } from '@/core' -import { State } from '@/db' +import { State, loadDb } from '@/db' import { setupMeilisearch } from '@/ms' import { handlerMakers } from './handlers' @@ -74,14 +75,18 @@ const main = async () => { throw new Error(`Trace file is not a FIFO: ${traceFile}.`) } - // Initialize state. - await State.createSingletonIfMissing() - // Read from trace file. await trace() } const trace = async () => { + const dataSequelize = await loadDb({ + type: DbType.Data, + }) + + // Initialize state. + await State.createSingletonIfMissing() + const exportQueue = getBullQueue<{ data: ExportQueueData[] }>( EXPORT_QUEUE_NAME ) @@ -481,8 +486,11 @@ const trace = async () => { }, 50) }) - // Give a little time for the worker to queue. - await new Promise((resolve) => setTimeout(resolve, 5000)) + // Close database connection. + await dataSequelize.close() + + // Close queue. + await exportQueue.close() } main().catch((err) => { From edcac5cd1f18683c985038a573669c281a16d593 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:41:01 -0700 Subject: [PATCH 19/35] Improved log. --- src/scripts/export/trace.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index fd63db0a..3d61d14d 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -475,7 +475,7 @@ const trace = async () => { // Wait for queue to finish processing. await new Promise((resolve) => { console.log( - `Shutting down after the queue drains ${traceQueue.length.toLocaleString()} events and ${exporting.toLocaleString()} others finish processing...` + `Shutting down after the queue drains ${traceQueue.length.toLocaleString()} traces and ${exporting.toLocaleString()} finish exporting...` ) const interval = setInterval(() => { From d7f545a8e3cb5f826d89b7d410f20d1b2f7af78b Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:41:26 -0700 Subject: [PATCH 20/35] Exit process manually at end. --- src/scripts/export/trace.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 3d61d14d..1ccaf3f9 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -491,6 +491,8 @@ const trace = async () => { // Close queue. await exportQueue.close() + + process.exit(0) } main().catch((err) => { From 28a3e2ed597787d539fbe96187ba97b35d8e7210 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 17:52:18 -0700 Subject: [PATCH 21/35] Validate ID uniqueness and remove once queued. --- src/scripts/export/process.ts | 6 +---- src/scripts/export/trace.ts | 50 ++++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index ba251fce..61877e26 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -96,11 +96,7 @@ const main = async () => { interval: 100, }) } catch (err) { - console.error( - '-------\nFailed to process:\n', - err instanceof Error ? err.message : err, - '\n-------' - ) + console.error('-------\nFailed to process:\n', err, '\n-------') Sentry.captureException(err, { tags: { type: 'failed-flush', diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 1ccaf3f9..7d16bc96 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -194,19 +194,45 @@ const trace = async () => { } if (exportBatch.length) { - exportQueue.add( - BigInt( - exportBatch[exportBatch.length - 1].trace.metadata.blockHeight - ).toString(), - { - data: exportBatch.map( - ({ handler, data }): ExportQueueData => ({ - handler, - data, - }) - ), - } + // For state events with the same ID, only keep the last event. This is + // because the indexer guarantees that events are emitted in order, and + // the last event is the most up-to-date. Multiple events may occur if a + // state key is updated multiple times across different messages within + // the same block. + const uniqueBatchData = Object.values( + exportBatch.reduce( + (acc, data, index) => ({ + ...acc, + [data.handler + ':' + data.data.id]: { + ...data, + index, + }, + }), + {} as Record + ) ) + // Ensure order is preserved. + uniqueBatchData.sort((a, b) => a.index - b.index) + + if (uniqueBatchData.length) { + exportQueue.add( + BigInt( + uniqueBatchData[uniqueBatchData.length - 1].trace.metadata + .blockHeight + ).toString(), + { + data: uniqueBatchData.map(({ handler, data }): ExportQueueData => { + // Remove ID since it's no longer relevant. + delete data.id + + return { + handler, + data, + } + }), + } + ) + } exportBatch = [] } } From 8fbc3bd28ed72326c8ae60bc837f55b596a4f49e Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 18:01:12 -0700 Subject: [PATCH 22/35] Auto remove jobs after 1 or 3 days depending on their status. --- src/core/queues.ts | 8 ++++++++ src/scripts/export/trace.ts | 37 ++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/core/queues.ts b/src/core/queues.ts index 5fbd4e1a..5b1fbf91 100644 --- a/src/core/queues.ts +++ b/src/core/queues.ts @@ -26,4 +26,12 @@ export const getBullWorker = ( ) => new Worker(name, processor, { connection: getBullConnection(), + removeOnComplete: { + // Keep last 1 day of successful jobs. + age: 1 * 24 * 60 * 60, + }, + removeOnFail: { + // Keep last 3 days of failed jobs. + age: 3 * 24 * 60 * 60, + }, }) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 7d16bc96..05f461f8 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -214,26 +214,29 @@ const trace = async () => { // Ensure order is preserved. uniqueBatchData.sort((a, b) => a.index - b.index) + const blockHeight = BigInt( + exportBatch[exportBatch.length - 1].trace.metadata.blockHeight + ) if (uniqueBatchData.length) { - exportQueue.add( - BigInt( - uniqueBatchData[uniqueBatchData.length - 1].trace.metadata - .blockHeight - ).toString(), - { - data: uniqueBatchData.map(({ handler, data }): ExportQueueData => { - // Remove ID since it's no longer relevant. - delete data.id - - return { - handler, - data, - } - }), - } - ) + exportQueue.add(blockHeight.toString(), { + data: uniqueBatchData.map(({ handler, data }): ExportQueueData => { + // Remove ID since it's no longer relevant. + delete data.id + + return { + handler, + data, + } + }), + }) } exportBatch = [] + + console.log( + `\n[${new Date().toISOString()}] Exported ${ + exportBatch.length + } events for block ${blockHeight.toLocaleString()}...` + ) } } From 4116aa702e847beb63fe6d3548bc911854cae243 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 18:09:42 -0700 Subject: [PATCH 23/35] Fix fifo read stream not resolving once closed. --- src/scripts/export/utils.ts | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/scripts/export/utils.ts b/src/scripts/export/utils.ts index f1aaecd4..09bf4a9f 100644 --- a/src/scripts/export/utils.ts +++ b/src/scripts/export/utils.ts @@ -136,26 +136,51 @@ export const setUpFifoJsonTracer = ({ fifoRs.on('data', dataListener) // Wait for FIFO to error or end. - const promise = new Promise((resolve, reject) => { + const promise = new Promise((_resolve, reject) => { + let done = false + const resolve = () => { + if (!done) { + done = true + _resolve() + } + } + const resolveDelayed = () => setTimeout(resolve, 5000) + fifoRs.on('error', (error) => { fifoRs.off('end', resolve) + fifoRs.off('close', resolveDelayed) // Reject once the FIFO ends. fifoRs.on('end', () => { - reject(error) + if (!done) { + done = true + reject(error) + } + }) + // If closed and promise not done after 5 seconds, reject. + fifoRs.on('close', () => { + setTimeout(() => { + if (!done) { + done = true + reject(error) + } + }, 5000) }) // Close the FIFO if it is not already closed, so it ends. if (!fifoRs.closed) { - fifoRs.close() + fifoRs.destroy() } }) // Once data ends, resolve. fifoRs.on('end', resolve) + + // If closed and promise not done after 5 seconds, resolve. + fifoRs.on('close', resolveDelayed) }) return { promise, - close: () => fifoRs.close(), + close: () => fifoRs.destroy(), } } From 0eb92706f542605984c0b16916618130f8e9fdef Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 18:10:33 -0700 Subject: [PATCH 24/35] Fixed log. --- src/scripts/export/trace.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 05f461f8..fd6569d2 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -230,13 +230,14 @@ const trace = async () => { }), }) } - exportBatch = [] console.log( `\n[${new Date().toISOString()}] Exported ${ exportBatch.length - } events for block ${blockHeight.toLocaleString()}...` + } events for block ${blockHeight.toLocaleString()}.` ) + + exportBatch = [] } } From 6664169772da6bd4fc779686d252889aa0343e96 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 18:14:16 -0700 Subject: [PATCH 25/35] Improved log. --- src/scripts/export/trace.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index fd6569d2..8d3a5deb 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -232,9 +232,7 @@ const trace = async () => { } console.log( - `\n[${new Date().toISOString()}] Exported ${ - exportBatch.length - } events for block ${blockHeight.toLocaleString()}.` + `\n[${new Date().toISOString()}] Exported ${exportBatch.length.toLocaleString()} events for block ${blockHeight.toLocaleString()}.` ) exportBatch = [] From 2da858bb7c025c9e540c54292fc9a4e207d480dc Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 18:20:41 -0700 Subject: [PATCH 26/35] Added block log. --- src/scripts/export/trace.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 8d3a5deb..6daa5442 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -421,6 +421,10 @@ const trace = async () => { latestBlockTimeUnixMs ) + console.log( + `Got new block height ${latestBlockHeight.toLocaleString()} time: ${latestBlockTimeUnixMs.toLocaleString()}.` + ) + // Update state singleton with latest information. await State.update( { From 038941eac704656c23b43c25e0877c7410e9261e Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 18:28:34 -0700 Subject: [PATCH 27/35] Added buffer to wait for block height if WebSocket connected but not yet loaded block height. --- src/scripts/export/trace.ts | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 6daa5442..b7e2de45 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -102,6 +102,26 @@ const trace = async () => { const getBlockTimeUnixMs = async (trace: TracedEvent): Promise => { const { blockHeight } = trace.metadata + // If not in cache but WebSocket is connected, wait for up to 1 second for + // it to be added to the cache. We might be just a moment ahead of the new + // block event. + if (!blockHeightToTimeCache.has(blockHeight) && webSocketConnected) { + await new Promise((resolve) => { + const interval = setInterval(() => { + if (blockHeightToTimeCache.has(blockHeight)) { + clearInterval(interval) + clearTimeout(timeout) + resolve() + } + }, 50) + + const timeout = setTimeout(() => { + clearInterval(interval) + resolve() + }, 1000) + }) + } + if (blockHeightToTimeCache.has(blockHeight)) { return blockHeightToTimeCache.get(blockHeight) ?? 0 } @@ -175,6 +195,7 @@ const trace = async () => { console.log(`\n[${new Date().toISOString()}] Exporting from trace...`) let webSocketReady = false + let webSocketConnected = false const traceQueue: TracedEvent[] = [] let traceExportPaused = false let traceExporter = Promise.resolve() @@ -421,10 +442,6 @@ const trace = async () => { latestBlockTimeUnixMs ) - console.log( - `Got new block height ${latestBlockHeight.toLocaleString()} time: ${latestBlockTimeUnixMs.toLocaleString()}.` - ) - // Update state singleton with latest information. await State.update( { @@ -442,6 +459,7 @@ const trace = async () => { onConnect: () => { console.log('WebSocket connected.') webSocketReady = true + webSocketConnected = true processTraceQueue() }, onError: async (error) => { From ee14bb9c14abcc07ae3b37ccf67e45e434aefdcd Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 18:33:33 -0700 Subject: [PATCH 28/35] Added cache to prevent waiting for the same block more than once. --- src/scripts/export/trace.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index b7e2de45..128f142d 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -99,13 +99,23 @@ const trace = async () => { const blockHeightToTimeCache = new LRUCache({ max: 100, }) + // Store whether or not we've already tried to buffer and wait for WebSocket + // to load a block. If so, don't wait again. Use cache to prevent memory + // buildup. + const waitedForBlockCache = new LRUCache({ + max: 100, + }) const getBlockTimeUnixMs = async (trace: TracedEvent): Promise => { const { blockHeight } = trace.metadata // If not in cache but WebSocket is connected, wait for up to 1 second for // it to be added to the cache. We might be just a moment ahead of the new - // block event. - if (!blockHeightToTimeCache.has(blockHeight) && webSocketConnected) { + // block event. If we've already waited for it before, don't wait again. + if ( + !blockHeightToTimeCache.has(blockHeight) && + webSocketConnected && + !waitedForBlockCache.get(blockHeight) + ) { await new Promise((resolve) => { const interval = setInterval(() => { if (blockHeightToTimeCache.has(blockHeight)) { @@ -120,6 +130,8 @@ const trace = async () => { resolve() }, 1000) }) + + waitedForBlockCache.set(blockHeight, true) } if (blockHeightToTimeCache.has(blockHeight)) { From 44beb419b2c9bc789d902c52a97ab8301e83708f Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sat, 30 Sep 2023 18:36:47 -0700 Subject: [PATCH 29/35] Allow capturing any block height error because we should not surpass block heights anymore. --- src/scripts/export/trace.ts | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 128f142d..2905867c 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -165,23 +165,17 @@ const trace = async () => { '\n-------' ) - // Only log to Sentry if not block height unavailable error. - if ( - !(err instanceof Error) || - !err.message.includes('must be less than or equal to the current') - ) { - Sentry.captureException(err, { - tags: { - type: 'failed-get-block', - script: 'export', - chainId: (await State.getSingleton())?.chainId ?? 'unknown', - }, - extra: { - trace, - blockHeight, - }, - }) - } + Sentry.captureException(err, { + tags: { + type: 'failed-get-block', + script: 'export', + chainId: (await State.getSingleton())?.chainId ?? 'unknown', + }, + extra: { + trace, + blockHeight, + }, + }) // Set to 0 on failure so we can continue. blockHeightToTimeCache.set(blockHeight, 0) From c778cb19761d50efc5a0b2ecfeeb89f9a4483666 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sun, 1 Oct 2023 00:09:39 -0700 Subject: [PATCH 30/35] Capture failed workers errors, keep all failed jobs, and auto retry failed jobs 3 times. --- src/core/queues.ts | 14 +++++++++----- src/scripts/export/process.ts | 12 ++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/core/queues.ts b/src/core/queues.ts index 5b1fbf91..b229515a 100644 --- a/src/core/queues.ts +++ b/src/core/queues.ts @@ -18,6 +18,13 @@ const getBullConnection = (): ConnectionOptions | undefined => { export const getBullQueue = (name: string) => new Queue(name, { connection: getBullConnection(), + defaultJobOptions: { + attempts: 3, + backoff: { + type: 'exponential', + delay: 300, + }, + }, }) export const getBullWorker = ( @@ -27,11 +34,8 @@ export const getBullWorker = ( new Worker(name, processor, { connection: getBullConnection(), removeOnComplete: { - // Keep last 1 day of successful jobs. - age: 1 * 24 * 60 * 60, - }, - removeOnFail: { - // Keep last 3 days of failed jobs. + // Keep last 3 days of successful jobs. age: 3 * 24 * 60 * 60, }, + // Keep all failed jobs. }) diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index 61877e26..8f2cd63c 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -113,6 +113,18 @@ const main = async () => { } ) + worker.on('error', async (err) => { + console.error('Worker errored', err) + + Sentry.captureException(err, { + tags: { + type: 'export-worker-error', + script: 'export:process', + chainId: (await State.getSingleton())?.chainId ?? 'unknown', + }, + }) + }) + // Add shutdown signal handler. process.on('SIGINT', () => { if (worker.closing) { From e1c549529c1aad74f6d37d419ea8277c84352f80 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sun, 1 Oct 2023 00:12:27 -0700 Subject: [PATCH 31/35] Capture error on export queue. --- src/scripts/export/trace.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 2905867c..bacc96b9 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -90,6 +90,17 @@ const trace = async () => { const exportQueue = getBullQueue<{ data: ExportQueueData[] }>( EXPORT_QUEUE_NAME ) + exportQueue.on('error', async (err) => { + console.error('Queue errored', err) + + Sentry.captureException(err, { + tags: { + type: 'export-queue-error', + script: 'export:trace', + chainId: (await State.getSingleton())?.chainId ?? 'unknown', + }, + }) + }) // Create CosmWasm client that batches requests. const cosmWasmClient = await getCosmWasmClient(config.rpc) From d238d69e815c18e71b8a5f2776a747c540d70c49 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sun, 1 Oct 2023 01:02:35 -0700 Subject: [PATCH 32/35] Converted pending webhooks model into worker job queue. --- ecosystem.config.js | 5 - package.json | 3 +- src/core/queues.ts | 7 +- src/core/types.ts | 6 + src/db/connection.ts | 2 - .../20231001075953-drop-pending-webhooks.ts | 11 ++ src/db/models/index.ts | 1 - src/scripts/export/handlers/wasm.ts | 4 +- src/scripts/export/process.ts | 100 +++-------- src/scripts/export/trace.ts | 6 +- src/scripts/export/types.ts | 17 +- src/scripts/export/webhooks.ts | 164 ++++++++++++++++++ src/scripts/export/workers/export.ts | 53 ++++++ src/scripts/export/workers/index.ts | 8 + src/scripts/export/workers/webhooks.ts | 43 +++++ src/scripts/webhooks.ts | 94 ---------- src/server/routes/indexer/bull.ts | 4 +- 17 files changed, 338 insertions(+), 190 deletions(-) create mode 100644 src/db/migrations/20231001075953-drop-pending-webhooks.ts create mode 100644 src/scripts/export/webhooks.ts create mode 100644 src/scripts/export/workers/export.ts create mode 100644 src/scripts/export/workers/index.ts create mode 100644 src/scripts/export/workers/webhooks.ts delete mode 100644 src/scripts/webhooks.ts diff --git a/ecosystem.config.js b/ecosystem.config.js index e0ddb70b..566baa45 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -14,11 +14,6 @@ module.exports = { listen_timeout: 30000, kill_timeout: 30000, }, - { - name: 'webhooks', - script: 'dist/scripts/webhooks.js', - kill_timeout: 30000, - }, { name: 'account-webhooks', script: 'dist/scripts/accountWebhooks.js', diff --git a/package.json b/package.json index b49d8b7b..535425a7 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,7 @@ "export:trace": "node dist/scripts/export/trace.js", "export:process": "node dist/scripts/export/process.js", "export:dev": "docker-compose -f compose.dev.yml up --exit-code-from export", - "export:prod": "npm install && npm run build && pm2 delete all && npm run db:migrate:data && pm2 start ecosystem.config.js --only export-tracer,export-processor,webhooks && pm2 save", - "webhooks": "node dist/scripts/webhooks.js", + "export:prod": "npm install && npm run build && pm2 delete all && npm run db:migrate:data && pm2 start ecosystem.config.js --only export-tracer,export-processor && pm2 save", "accountWebhooks": "node dist/scripts/accountWebhooks.js", "transform": "node dist/scripts/transform.js", "pre-compute": "node dist/scripts/preCompute.js", diff --git a/src/core/queues.ts b/src/core/queues.ts index b229515a..af0af669 100644 --- a/src/core/queues.ts +++ b/src/core/queues.ts @@ -1,8 +1,7 @@ import { ConnectionOptions, Processor, Queue, Worker } from 'bullmq' import { loadConfig } from './config' - -export const EXPORT_QUEUE_NAME = 'export' +import { QueueName } from './types' const getBullConnection = (): ConnectionOptions | undefined => { const { redis } = loadConfig() @@ -15,7 +14,7 @@ const getBullConnection = (): ConnectionOptions | undefined => { ) } -export const getBullQueue = (name: string) => +export const getBullQueue = (name: QueueName) => new Queue(name, { connection: getBullConnection(), defaultJobOptions: { @@ -28,7 +27,7 @@ export const getBullQueue = (name: string) => }) export const getBullWorker = ( - name: string, + name: QueueName, processor: Processor ) => new Worker(name, processor, { diff --git a/src/core/types.ts b/src/core/types.ts index 529dd554..cfebb924 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -552,6 +552,7 @@ export type ProcessedWebhook = Omit, 'filter'> & { } export type PendingWebhook = { + wasmEventId: number endpoint: WebhookEndpoint value: any } @@ -560,3 +561,8 @@ export enum DbType { Accounts = 'accounts', Data = 'data', } + +export enum QueueName { + Export = 'export', + Webhooks = 'webhooks', +} diff --git a/src/db/connection.ts b/src/db/connection.ts index f705d21e..2ff3e0d1 100644 --- a/src/db/connection.ts +++ b/src/db/connection.ts @@ -16,7 +16,6 @@ import { Computation, ComputationDependency, Contract, - PendingWebhook, StakingSlashEvent, State, Validator, @@ -41,7 +40,6 @@ const getModelsForType = (type: DbType): SequelizeOptions['models'] => Computation, ComputationDependency, Contract, - PendingWebhook, StakingSlashEvent, State, Validator, diff --git a/src/db/migrations/20231001075953-drop-pending-webhooks.ts b/src/db/migrations/20231001075953-drop-pending-webhooks.ts new file mode 100644 index 00000000..cfb81dce --- /dev/null +++ b/src/db/migrations/20231001075953-drop-pending-webhooks.ts @@ -0,0 +1,11 @@ +import { QueryInterface } from 'sequelize' + +module.exports = { + async up(queryInterface: QueryInterface) { + await queryInterface.dropTable('PendingWebhooks') + }, + + async down() { + throw new Error('Not implemented.') + }, +} diff --git a/src/db/models/index.ts b/src/db/models/index.ts index 51e31b59..1d3a17b0 100644 --- a/src/db/models/index.ts +++ b/src/db/models/index.ts @@ -10,7 +10,6 @@ export * from './BankStateEvent' export * from './Computation' export * from './ComputationDependency' export * from './Contract' -export * from './PendingWebhook' export * from './StakingSlashEvent' export * from './State' export * from './Validator' diff --git a/src/scripts/export/handlers/wasm.ts b/src/scripts/export/handlers/wasm.ts index be516d71..fc8ef8f1 100644 --- a/src/scripts/export/handlers/wasm.ts +++ b/src/scripts/export/handlers/wasm.ts @@ -9,7 +9,6 @@ import { ParsedWasmStateEvent } from '@/core' import { AccountWebhook, Contract, - PendingWebhook, State, WasmStateEvent, WasmStateEventTransformation, @@ -18,6 +17,7 @@ import { import { updateIndexesForContracts } from '@/ms' import { Handler, HandlerMaker } from '../types' +import { queueWebhooks } from '../webhooks' const STORE_NAME = 'wasm' const CONTRACT_BYTE_LENGTH = 32 @@ -410,7 +410,7 @@ export const wasm: HandlerMaker = async ({ // Queue webhooks as needed. if (sendWebhooks && exportedEvents.length > 0) { - await PendingWebhook.queueWebhooks(state, exportedEvents) + await queueWebhooks(state, exportedEvents) await AccountWebhook.queueWebhooks(exportedEvents) } diff --git a/src/scripts/export/process.ts b/src/scripts/export/process.ts index 8f2cd63c..5bfee0dc 100644 --- a/src/scripts/export/process.ts +++ b/src/scripts/export/process.ts @@ -1,14 +1,10 @@ import * as Sentry from '@sentry/node' -import retry from 'async-await-retry' -import { Worker } from 'bullmq' import { Command } from 'commander' -import { DbType, EXPORT_QUEUE_NAME, getBullWorker, loadConfig } from '@/core' +import { DbType, getBullWorker, loadConfig } from '@/core' import { State, loadDb } from '@/db' -import { handlerMakers } from './handlers' -import { ExportQueueData } from './types' -import { getCosmWasmClient } from './utils' +import { workerMakers } from './workers' // Parse arguments. const program = new Command() @@ -51,88 +47,44 @@ const main = async () => { // Initialize state. await State.createSingletonIfMissing() - const cosmWasmClient = await getCosmWasmClient(config.rpc) - - // Setup handlers. - const handlers = await Promise.all( - Object.entries(handlerMakers).map(async ([name, handlerMaker]) => ({ - name, - handler: await handlerMaker({ + // Create queue workers. + const madeWorkers = await Promise.all( + workerMakers.map((makeWorker) => + makeWorker({ config, updateComputations: !!update, sendWebhooks: !!webhooks, - cosmWasmClient, - }), - })) + }) + ) ) - // Create queue worker. - const worker = getBullWorker<{ data: ExportQueueData[] }>( - EXPORT_QUEUE_NAME, - async (job) => { - const { data } = job.data - - // Group data by handler. - const groupedData = data.reduce( - (acc, { handler, data }) => ({ - ...acc, - [handler]: (acc[handler] || []).concat(data), - }), - {} as Record - ) - - // Process data. - for (const { name, handler } of handlers) { - const events = groupedData[name] - if (!events?.length) { - continue - } - - try { - // Retry 3 times with exponential backoff starting at 100ms delay. - await retry(handler.process, [events], { - retriesMax: 3, - exponential: true, - interval: 100, - }) - } catch (err) { - console.error('-------\nFailed to process:\n', err, '\n-------') - Sentry.captureException(err, { - tags: { - type: 'failed-flush', - script: 'export', - }, - extra: { - handler: name, - }, - }) + const workers = madeWorkers.map(({ queueName, processor }) => { + const worker = getBullWorker(queueName, processor) - throw err - } - } - } - ) - - worker.on('error', async (err) => { - console.error('Worker errored', err) + worker.on('error', async (err) => { + console.error('Worker errored', err) - Sentry.captureException(err, { - tags: { - type: 'export-worker-error', - script: 'export:process', - chainId: (await State.getSingleton())?.chainId ?? 'unknown', - }, + Sentry.captureException(err, { + tags: { + type: 'worker-error', + script: 'export:process', + chainId: (await State.getSingleton())?.chainId ?? 'unknown', + queueName, + }, + }) }) + + return worker }) // Add shutdown signal handler. process.on('SIGINT', () => { - if (worker.closing) { + if (workers.every((w) => w.closing)) { console.log('Already shutting down.') } else { - console.log('Shutting down after worker jobs finish...') - // Exit once worker closes. - worker.close().then(async () => { + console.log('Shutting down after current worker jobs complete...') + // Exit once all workers close. + Promise.all(workers.map((worker) => worker.close())).then(async () => { await dataSequelize.close() await accountsSequelize.close() process.exit(0) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index bacc96b9..6362a3b5 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -9,7 +9,7 @@ import waitPort from 'wait-port' import { DbType, - EXPORT_QUEUE_NAME, + QueueName, getBullQueue, loadConfig, objectMatchesStructure, @@ -88,10 +88,10 @@ const trace = async () => { await State.createSingletonIfMissing() const exportQueue = getBullQueue<{ data: ExportQueueData[] }>( - EXPORT_QUEUE_NAME + QueueName.Export ) exportQueue.on('error', async (err) => { - console.error('Queue errored', err) + console.error('Export queue errored', err) Sentry.captureException(err, { tags: { diff --git a/src/scripts/export/types.ts b/src/scripts/export/types.ts index 28337bb2..9acb28ac 100644 --- a/src/scripts/export/types.ts +++ b/src/scripts/export/types.ts @@ -1,6 +1,7 @@ import { CosmWasmClient } from '@cosmjs/cosmwasm-stargate' +import { Processor } from 'bullmq' -import { Config } from '@/core' +import { Config, QueueName } from '@/core/types' export type Handler = { // What store name to filter by for events to handle. @@ -56,3 +57,17 @@ export type ExportQueueData = { handler: string data: unknown } + +export type ExportWorker = { + queueName: QueueName + processor: Processor +} + +export type ExportWorkerMakerOptions = Omit< + HandlerMakerOptions, + 'cosmWasmClient' +> + +export type ExportWorkerMaker = ( + options: ExportWorkerMakerOptions +) => Promise> diff --git a/src/scripts/export/webhooks.ts b/src/scripts/export/webhooks.ts new file mode 100644 index 00000000..daea2393 --- /dev/null +++ b/src/scripts/export/webhooks.ts @@ -0,0 +1,164 @@ +import { randomUUID } from 'crypto' + +import * as Sentry from '@sentry/node' + +import { + ContractEnv, + PendingWebhook, + QueueName, + getBullQueue, + getEnv, + loadConfig, +} from '@/core' +import { getProcessedWebhooks } from '@/data/webhooks' +import { State, WasmStateEvent } from '@/db' + +export const queueWebhooks = async ( + state: State, + wasmEvents: WasmStateEvent[] +): Promise => { + const webhooks = getProcessedWebhooks(loadConfig(), state) + if (webhooks.length === 0) { + return + } + + const pendingWebhooks = ( + await Promise.all( + wasmEvents.flatMap((wasmEvent) => { + const webhooksForEvent = webhooks.filter((webhook) => + webhook.filter(wasmEvent) + ) + + return webhooksForEvent.map( + async (webhook): Promise => { + const env: ContractEnv = { + ...getEnv({ + chainId: state.chainId, + block: wasmEvent.block, + cache: { + contracts: { + [wasmEvent.contract.address]: wasmEvent.contract, + }, + }, + }), + contractAddress: wasmEvent.contractAddress, + } + + // Wrap in try/catch in case a webhook errors. Don't want to prevent + // other webhooks from sending. + let value + try { + value = await webhook.getValue( + wasmEvent, + async () => { + // Find most recent event for this contract and key before + // this block. + + // Check events in case the most recent event is in the + // current group of events. + const previousEvent = wasmEvents + .filter( + (e) => + e.contractAddress === wasmEvent.contractAddress && + e.key === e.key && + e.blockHeight < wasmEvent.blockHeight + ) + .slice(-1)[0] + + if (previousEvent) { + return previousEvent.delete ? null : previousEvent.valueJson + } + + // Fallback to database. + const lastEvent = await wasmEvent.getPreviousEvent() + return !lastEvent || lastEvent.delete + ? null + : lastEvent.valueJson + }, + env + ) + } catch (error) { + console.error( + `Error getting webhook value for event ${wasmEvent.blockHeight}/${wasmEvent.contractAddress}/${wasmEvent.key}: ${error}` + ) + Sentry.captureException(error, { + tags: { + type: 'queue-webhook-value', + script: 'export:trace', + chainId: state.chainId, + }, + extra: { + wasmEvent, + }, + }) + } + + // Wrap in try/catch in case a webhook errors. Don't want to prevent + // other webhooks from sending. + let endpoint + try { + endpoint = + typeof webhook.endpoint === 'function' + ? await webhook.endpoint(wasmEvent, env) + : webhook.endpoint + } catch (error) { + console.error( + `Error getting webhook endpoint for event ${wasmEvent.blockHeight}/${wasmEvent.contractAddress}/${wasmEvent.key}: ${error}` + ) + Sentry.captureException(error, { + tags: { + type: 'queue-webhook-endpoint', + script: 'export:trace', + chainId: state.chainId, + }, + extra: { + wasmEvent, + }, + }) + } + + // If value or endpoint is undefined, one either errored or the + // function returned undefined. In either case, don't send a + // webhook. + if (value === undefined || endpoint === undefined) { + return + } + + return { + wasmEventId: wasmEvent.id, + endpoint, + value, + } + } + ) + }) + ) + ).filter((w): w is PendingWebhook => w !== undefined) + + if (pendingWebhooks.length) { + const webhookQueue = getBullQueue(QueueName.Webhooks) + webhookQueue.on('error', async (err) => { + console.error('Webhook queue errored', err) + + Sentry.captureException(err, { + tags: { + type: 'webhook-queue-error', + script: 'export:trace', + chainId: state.chainId, + }, + extra: { + pendingWebhooks, + }, + }) + }) + + webhookQueue.addBulk( + pendingWebhooks.map((data) => ({ + name: randomUUID(), + data, + })) + ) + + await webhookQueue.close() + } +} diff --git a/src/scripts/export/workers/export.ts b/src/scripts/export/workers/export.ts new file mode 100644 index 00000000..6965b8f0 --- /dev/null +++ b/src/scripts/export/workers/export.ts @@ -0,0 +1,53 @@ +import retry from 'async-await-retry' + +import { QueueName } from '@/core/types' + +import { handlerMakers } from '../handlers' +import { ExportQueueData, ExportWorkerMaker } from '../types' +import { getCosmWasmClient } from '../utils' + +export const makeExportWorker: ExportWorkerMaker<{ + data: ExportQueueData[] +}> = async (options) => { + const cosmWasmClient = await getCosmWasmClient(options.config.rpc) + + // Set up handlers. + const handlers = await Promise.all( + Object.entries(handlerMakers).map(async ([name, handlerMaker]) => ({ + name, + handler: await handlerMaker({ + ...options, + cosmWasmClient, + }), + })) + ) + + return { + queueName: QueueName.Export, + processor: async ({ data: { data } }) => { + // Group data by handler. + const groupedData = data.reduce( + (acc, { handler, data }) => ({ + ...acc, + [handler]: (acc[handler] || []).concat(data), + }), + {} as Record + ) + + // Process data. + for (const { name, handler } of handlers) { + const events = groupedData[name] + if (!events?.length) { + continue + } + + // Retry 3 times with exponential backoff starting at 100ms delay. + await retry(handler.process, [events], { + retriesMax: 3, + exponential: true, + interval: 100, + }) + } + }, + } +} diff --git a/src/scripts/export/workers/index.ts b/src/scripts/export/workers/index.ts new file mode 100644 index 00000000..1bdbfac4 --- /dev/null +++ b/src/scripts/export/workers/index.ts @@ -0,0 +1,8 @@ +import { ExportWorkerMaker } from '../types' +import { makeExportWorker } from './export' +import { makeWebhooksWorker } from './webhooks' + +export const workerMakers: ExportWorkerMaker[] = [ + makeExportWorker, + makeWebhooksWorker, +] diff --git a/src/scripts/export/workers/webhooks.ts b/src/scripts/export/workers/webhooks.ts new file mode 100644 index 00000000..79a8b260 --- /dev/null +++ b/src/scripts/export/workers/webhooks.ts @@ -0,0 +1,43 @@ +import axios from 'axios' +import Pusher from 'pusher' + +import { PendingWebhook, QueueName, WebhookType } from '@/core/types' + +import { ExportWorkerMaker } from '../types' + +export const makeWebhooksWorker: ExportWorkerMaker = async ({ + config: { soketi }, +}) => ({ + queueName: QueueName.Webhooks, + processor: async ({ data: { endpoint, value } }) => { + switch (endpoint.type) { + case WebhookType.Url: { + await axios(endpoint.url, { + method: endpoint.method, + // https://stackoverflow.com/a/74735197 + headers: { + 'Accept-Encoding': 'gzip,deflate,compress', + ...endpoint.headers, + }, + data: value, + }) + + break + } + + case WebhookType.Soketi: { + if (!soketi) { + throw new Error('Soketi config not found') + } + + const pusher = new Pusher(soketi) + await pusher.trigger(endpoint.channel, endpoint.event, value) + + break + } + + default: + throw new Error('Unknown webhook type: ' + (endpoint as any).type) + } + }, +}) diff --git a/src/scripts/webhooks.ts b/src/scripts/webhooks.ts deleted file mode 100644 index 743f181c..00000000 --- a/src/scripts/webhooks.ts +++ /dev/null @@ -1,94 +0,0 @@ -import * as Sentry from '@sentry/node' -import { Command } from 'commander' -import { Op } from 'sequelize' - -import { loadConfig } from '@/core/config' -import { DbType } from '@/core/types' -import { PendingWebhook, loadDb } from '@/db' - -let shuttingDown = false - -// Parse arguments. -const program = new Command() -program.option( - '-c, --config ', - 'path to config file, falling back to config.json' -) -program.option( - '-b, --batch ', - 'webhook batch size', - (value) => parseInt(value, 10), - 50 -) -program.parse() -const { config: _config, batch } = program.opts() - -const config = loadConfig(_config) - -// Add Sentry error reporting. -if (config.sentryDsn) { - Sentry.init({ - dsn: config.sentryDsn, - }) -} - -const main = async () => { - // Connect to both DBs. - await loadDb({ - type: DbType.Data, - }) - await loadDb({ - type: DbType.Accounts, - }) - - console.log(`\n[${new Date().toISOString()}] Firing webhooks...`) - - while (!shuttingDown) { - const pending = await PendingWebhook.findAll({ - where: { - failures: { - // Retry up to 3 times. - [Op.lt]: 3, - }, - }, - limit: batch, - }) - - let succeeded = 0 - if (pending.length > 0) { - const requests = await Promise.allSettled( - pending.map((pendingWebhook) => pendingWebhook.fire()) - ) - - succeeded = requests.filter( - (request) => request.status === 'fulfilled' - ).length - const failed = requests.filter( - (request) => request.status === 'rejected' - ).length - - console.log( - `[webhooks] ${[ - succeeded > 0 && `${succeeded.toLocaleString()} succeeded`, - failed > 0 && `${failed.toLocaleString()} failed`, - ] - .filter(Boolean) - .join(', ')}` - ) - } - - // If no webhooks or all failed, wait between loops to prevent spamming. - if (succeeded === 0) { - await new Promise((resolve) => setTimeout(resolve, 1000)) - } - } - - process.exit(0) -} - -main() - -process.on('SIGINT', () => { - shuttingDown = true - console.log('\nShutting down after current batch finishes...') -}) diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index 80bfb6e4..f5436fbe 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -2,13 +2,13 @@ import { createBullBoard } from '@bull-board/api' import { BullMQAdapter } from '@bull-board/api/bullMQAdapter' import { KoaAdapter } from '@bull-board/koa' -import { EXPORT_QUEUE_NAME, getBullQueue } from '@/core' +import { QueueName, getBullQueue } from '@/core' export const makeBullBoardJobsMiddleware = (basePath: string) => { const serverAdapter = new KoaAdapter().setBasePath(basePath) createBullBoard({ - queues: [new BullMQAdapter(getBullQueue(EXPORT_QUEUE_NAME))], + queues: [new BullMQAdapter(getBullQueue(QueueName.Export))], serverAdapter, }) From e09589acf0a562508d6cd520252644e160b7ad14 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sun, 1 Oct 2023 01:08:15 -0700 Subject: [PATCH 33/35] Added all queues to bull dashboard. --- src/server/routes/indexer/bull.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/routes/indexer/bull.ts b/src/server/routes/indexer/bull.ts index f5436fbe..f90f3431 100644 --- a/src/server/routes/indexer/bull.ts +++ b/src/server/routes/indexer/bull.ts @@ -8,7 +8,9 @@ export const makeBullBoardJobsMiddleware = (basePath: string) => { const serverAdapter = new KoaAdapter().setBasePath(basePath) createBullBoard({ - queues: [new BullMQAdapter(getBullQueue(QueueName.Export))], + queues: Object.values(QueueName).map( + (name) => new BullMQAdapter(getBullQueue(name)) + ), serverAdapter, }) From 344b0603485d3b02368e1bb5f3094c256fcb8060 Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sun, 1 Oct 2023 20:12:48 -0700 Subject: [PATCH 34/35] Wait up to 5 seconds for the block time to load and no need to prevent waiting in the future since it will be set to 0 if not found. Also clean up trace exporter queue promise. --- src/scripts/export/trace.ts | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/scripts/export/trace.ts b/src/scripts/export/trace.ts index 6362a3b5..4728138d 100644 --- a/src/scripts/export/trace.ts +++ b/src/scripts/export/trace.ts @@ -110,39 +110,31 @@ const trace = async () => { const blockHeightToTimeCache = new LRUCache({ max: 100, }) - // Store whether or not we've already tried to buffer and wait for WebSocket - // to load a block. If so, don't wait again. Use cache to prevent memory - // buildup. - const waitedForBlockCache = new LRUCache({ - max: 100, - }) const getBlockTimeUnixMs = async (trace: TracedEvent): Promise => { const { blockHeight } = trace.metadata - // If not in cache but WebSocket is connected, wait for up to 1 second for + // If not in cache but WebSocket is connected, wait for up to 5 seconds for // it to be added to the cache. We might be just a moment ahead of the new - // block event. If we've already waited for it before, don't wait again. - if ( - !blockHeightToTimeCache.has(blockHeight) && - webSocketConnected && - !waitedForBlockCache.get(blockHeight) - ) { - await new Promise((resolve) => { + // block event. + if (!blockHeightToTimeCache.has(blockHeight) && webSocketConnected) { + const time = await new Promise((resolve) => { const interval = setInterval(() => { if (blockHeightToTimeCache.has(blockHeight)) { clearInterval(interval) clearTimeout(timeout) - resolve() + resolve(blockHeightToTimeCache.get(blockHeight)) } }, 50) const timeout = setTimeout(() => { clearInterval(interval) - resolve() - }, 1000) + resolve(undefined) + }, 5000) }) - waitedForBlockCache.set(blockHeight, true) + if (time !== undefined) { + return time + } } if (blockHeightToTimeCache.has(blockHeight)) { @@ -356,6 +348,12 @@ const trace = async () => { }) } finally { exporting-- + + // If no more events being exported, reset trace exporter queue promise + // variable so that memory of the promises in the chain can be cleaned up. + if (exporting === 0) { + traceExporter = Promise.resolve() + } } } From 704ca8478a2a7eb6c3a77a4313d8b0bcf9e5960b Mon Sep 17 00:00:00 2001 From: Noah Saso Date: Sun, 1 Oct 2023 20:20:40 -0700 Subject: [PATCH 35/35] Added redis container to docker compose to fix tests. --- compose.dev.yml | 12 +++++++++++- compose.test.yml | 12 +++++++++++- config-dev.json | 3 +++ config-test.json | 3 +++ config.json.example | 4 ++++ 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index 9cfcc2a3..fa2612e8 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -12,12 +12,14 @@ services: NODE_ENV: development # contains db connection info that matches db service setup below CONFIG_FILE: config-dev.json - # waits for db to start + # waits for db and redis to start depends_on: db_accounts: condition: service_healthy db_data: condition: service_healthy + redis: + condition: service_healthy # colorizes output tty: true @@ -47,5 +49,13 @@ services: timeout: 3s retries: 5 + redis: + image: redis:7-alpine + healthcheck: + test: ["CMD-SHELL", "redis-cli ping"] + interval: 1s + timeout: 3s + retries: 5 + volumes: node_modules: diff --git a/compose.test.yml b/compose.test.yml index 8c3650af..7dc6e5b0 100644 --- a/compose.test.yml +++ b/compose.test.yml @@ -12,10 +12,12 @@ services: NODE_ENV: test # contains db connection info that matches db service setup below CONFIG_FILE: config-test.json - # waits for db to start + # waits for db and redis to start depends_on: db: condition: service_healthy + redis: + condition: service_healthy # colorizes output tty: true @@ -32,5 +34,13 @@ services: timeout: 3s retries: 5 + redis: + image: redis:7-alpine + healthcheck: + test: ["CMD-SHELL", "redis-cli ping"] + interval: 1s + timeout: 3s + retries: 5 + volumes: node_modules: diff --git a/config-dev.json b/config-dev.json index 7c7f004e..3141046b 100644 --- a/config-dev.json +++ b/config-dev.json @@ -2,6 +2,9 @@ "home": "./fake", "rpc": "https://uni-rpc.reece.sh", "bech32Prefix": "juno", + "redis": { + "host": "redis" + }, "db": { "data": { "dialect": "postgres", diff --git a/config-test.json b/config-test.json index 9e3e0b39..a48ac84d 100644 --- a/config-test.json +++ b/config-test.json @@ -1,4 +1,7 @@ { + "redis": { + "host": "redis" + }, "db": { "data": { "dialect": "postgres", diff --git a/config.json.example b/config.json.example index 9f9a7d38..f553e62c 100644 --- a/config.json.example +++ b/config.json.example @@ -2,6 +2,10 @@ "home": "/Users/noah/.juno/indexer", "rpc": "https://uni-rpc.reece.sh", "bech32Prefix": "juno", + "redis": { + "host": "127.0.0.1", + "password": "" + }, "db": { "data": { "dialect": "postgres",