From cf217c55d881765c2963557c9d4c2c50a34a0494 Mon Sep 17 00:00:00 2001 From: Pedro Tamborindeguy Date: Tue, 10 Jan 2023 12:09:50 -0300 Subject: [PATCH 1/3] chore: update snapshots-fetcher with new synchronizer component --- .env | 2 +- package-lock.json | 409 +++++++++++++++++++++++++++++---- package.json | 6 +- src/adapters/deployer/index.ts | 11 +- src/components.ts | 74 ++++-- src/service.ts | 2 +- src/types.ts | 18 +- 7 files changed, 439 insertions(+), 83 deletions(-) diff --git a/.env b/.env index 4e07515..79d8211 100644 --- a/.env +++ b/.env @@ -3,4 +3,4 @@ # You may want to add this .env file to .gitignore to save development # credentials -HTTP_SERVER_PORT=5000 \ No newline at end of file +HTTP_SERVER_PORT=5001 \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 9b88d73..3ac00bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,9 +6,9 @@ "": { "name": "template-server", "dependencies": { - "@dcl/catalyst-storage": "^1.0.0", + "@dcl/catalyst-storage": "^2.0.0", "@dcl/schemas": "^5.22.0", - "@dcl/snapshots-fetcher": "^3.0.2", + "@dcl/snapshots-fetcher": "^5.0.1", "@types/destroy": "^1.0.0", "@well-known-components/env-config-provider": "^1.1.1", "@well-known-components/http-server": "^1.1.1", @@ -640,21 +640,21 @@ "dev": true }, "node_modules/@dcl/catalyst-storage": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@dcl/catalyst-storage/-/catalyst-storage-1.0.0.tgz", - "integrity": "sha512-TtKpP8P/v/kzKiIWR2eB+OlCmTkvXlGL9ZlVBHHSs0XCdrxcc7p0SROFd7qzt7aFQA2hMFjODrhHAxACDD1ijA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@dcl/catalyst-storage/-/catalyst-storage-2.0.2.tgz", + "integrity": "sha512-fL4xNCwqMIGB7nvhV7UTqOpEEF8Unub3nOMvstjjG6mS7tpUVoeIbXFY8to16awdOShcHNHoi5GvI4WaBKgbnw==", "dependencies": { "@types/destroy": "^1.0.0", - "@types/node": "^17.0.39", + "@types/node": "^18.6.3", "@well-known-components/interfaces": "^1.1.1", - "aws-sdk": "^2.1148.0", + "aws-sdk": "^2.1248.0", "destroy": "^1.2.0" } }, "node_modules/@dcl/catalyst-storage/node_modules/@types/node": { - "version": "17.0.39", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.39.tgz", - "integrity": "sha512-JDU3YLlnPK3WDao6/DlXLOgSNpG13ct+CwIO17V8q0/9fWJyeMJJ/VyZ1lv8kDprihvZMydzVwf0tQOqGiY2Nw==" + "version": "18.11.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz", + "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==" }, "node_modules/@dcl/hashing": { "version": "1.1.0", @@ -677,19 +677,39 @@ } }, "node_modules/@dcl/snapshots-fetcher": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@dcl/snapshots-fetcher/-/snapshots-fetcher-3.0.2.tgz", - "integrity": "sha512-W0M8NoGJ2iBuxxu6beF3iZI2dT4gFcSfrJJM45QhMoa5/kTaDgShYIeIBkv2N+9rC6xZw+U57+KMNhFkshNKZw==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@dcl/snapshots-fetcher/-/snapshots-fetcher-5.0.1.tgz", + "integrity": "sha512-afGKsn03JJ1iBDPJv1auRdKsnxmUeA7Z6ontv6yCicTw+eNcaDDz5T+SZjFwz5FH9i8Cr9jfdCqs03/ZGwtEhQ==", "dependencies": { + "@dcl/catalyst-storage": "^2.0.0", "@dcl/hashing": "^1.0.0", + "@dcl/schemas": "^6.2.0", "@well-known-components/interfaces": "^1.1.0", - "@well-known-components/metrics": "^1.1.2", + "@well-known-components/metrics": "^2.0.0", "p-queue": "^6.6.2" }, "engines": { "node": ">16.0.0" } }, + "node_modules/@dcl/snapshots-fetcher/node_modules/@dcl/schemas": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@dcl/schemas/-/schemas-6.3.0.tgz", + "integrity": "sha512-Zhbly35/SZ5f4h0VnCtSjxxIaEbCpRzAo487Pd+PIuzx1vt5C9srJRHbMz0Cns6h3KiF8/E4Zsjl141NYIijzA==", + "dependencies": { + "ajv": "^8.11.0", + "ajv-errors": "^3.0.0", + "ajv-keywords": "^5.1.0" + } + }, + "node_modules/@dcl/snapshots-fetcher/node_modules/@well-known-components/metrics": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@well-known-components/metrics/-/metrics-2.0.1.tgz", + "integrity": "sha512-jgT9TuxVS9GzVMMYWXNJRM2qXvexuf4xfrFNBYa3w+gqEWCK4Id5R4zBee/bm6/F5yIPauAWxbOS6KNNDBIrGw==", + "dependencies": { + "prom-client": "^14.1.0" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1698,10 +1718,21 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "node_modules/available-typed-arrays": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", + "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/aws-sdk": { - "version": "2.1148.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1148.0.tgz", - "integrity": "sha512-FUYAyveKmS5eqIiGQgrGVsLZwwtI+K6S6Gz8oJf56pgypZCo9dV+cXO4aaS+vN0+LSmGh6dSKc6G8h8FYASIJg==", + "version": "2.1291.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1291.0.tgz", + "integrity": "sha512-iM82Md2Wb29MZ72BpNF4YeIHtkJTyw+JMGXJG48Dwois2Rq7rLcriX6NN/4Fx4b5EEtUkwAO/wJc+NTHI7QeJw==", "dependencies": { "buffer": "4.9.2", "events": "1.1.1", @@ -1710,6 +1741,7 @@ "querystring": "0.2.0", "sax": "1.2.1", "url": "0.10.3", + "util": "^0.12.4", "uuid": "8.0.0", "xml2js": "0.4.19" }, @@ -2031,6 +2063,18 @@ "node": ">= 0.8" } }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -2776,6 +2820,14 @@ "node": ">=8" } }, + "node_modules/for-each": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", + "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "dependencies": { + "is-callable": "^1.1.3" + } + }, "node_modules/form-data": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", @@ -2833,8 +2885,7 @@ "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "node_modules/gensync": { "version": "1.0.0-beta.2", @@ -2854,6 +2905,19 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", @@ -2904,6 +2968,17 @@ "node": ">=4" } }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graceful-fs": { "version": "4.2.9", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", @@ -2935,7 +3010,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "dependencies": { "function-bind": "^1.1.1" }, @@ -2952,6 +3026,31 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/html-encoding-sniffer": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", @@ -3221,6 +3320,32 @@ "npm": ">=3.0.0" } }, + "node_modules/is-arguments": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", + "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-core-module": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz", @@ -3251,6 +3376,20 @@ "node": ">=6" } }, + "node_modules/is-generator-function": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", + "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -3286,6 +3425,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-typed-array": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.10.tgz", + "integrity": "sha512-PJqgEHiWZvMpaFZ3uTc8kHPM4+4ADTlDniuQL7cU/UDA0Ql7F70yGfHph3cLNe+c9toaigv+DFzTJKhc2CtO6A==", + "dependencies": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "gopd": "^1.0.1", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -4878,9 +5035,9 @@ } }, "node_modules/prom-client": { - "version": "14.0.1", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.0.1.tgz", - "integrity": "sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.1.1.tgz", + "integrity": "sha512-hFU32q7UZQ59bVJQGUtm3I2PrJ3gWvoCkilX9sF165ks1qflhugVCeK+S1JjJYHvyt3o5kj68+q3bchormjnzw==", "dependencies": { "tdigest": "^0.1.1" }, @@ -5790,6 +5947,18 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -5921,6 +6090,25 @@ "node": ">= 8" } }, + "node_modules/which-typed-array": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.9.tgz", + "integrity": "sha512-w9c4xkx6mPidwp7180ckYWfMmvxpjlZuIudNtDf4N/tTAUB8VJbX25qZoAsrtGuYNnGw3pa0AXgbGKRB8/EceA==", + "dependencies": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "gopd": "^1.0.1", + "has-tostringtag": "^1.0.0", + "is-typed-array": "^1.1.10" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -6525,21 +6713,21 @@ "dev": true }, "@dcl/catalyst-storage": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@dcl/catalyst-storage/-/catalyst-storage-1.0.0.tgz", - "integrity": "sha512-TtKpP8P/v/kzKiIWR2eB+OlCmTkvXlGL9ZlVBHHSs0XCdrxcc7p0SROFd7qzt7aFQA2hMFjODrhHAxACDD1ijA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@dcl/catalyst-storage/-/catalyst-storage-2.0.2.tgz", + "integrity": "sha512-fL4xNCwqMIGB7nvhV7UTqOpEEF8Unub3nOMvstjjG6mS7tpUVoeIbXFY8to16awdOShcHNHoi5GvI4WaBKgbnw==", "requires": { "@types/destroy": "^1.0.0", - "@types/node": "^17.0.39", + "@types/node": "^18.6.3", "@well-known-components/interfaces": "^1.1.1", - "aws-sdk": "^2.1148.0", + "aws-sdk": "^2.1248.0", "destroy": "^1.2.0" }, "dependencies": { "@types/node": { - "version": "17.0.39", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.39.tgz", - "integrity": "sha512-JDU3YLlnPK3WDao6/DlXLOgSNpG13ct+CwIO17V8q0/9fWJyeMJJ/VyZ1lv8kDprihvZMydzVwf0tQOqGiY2Nw==" + "version": "18.11.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz", + "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==" } } }, @@ -6564,14 +6752,36 @@ } }, "@dcl/snapshots-fetcher": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@dcl/snapshots-fetcher/-/snapshots-fetcher-3.0.2.tgz", - "integrity": "sha512-W0M8NoGJ2iBuxxu6beF3iZI2dT4gFcSfrJJM45QhMoa5/kTaDgShYIeIBkv2N+9rC6xZw+U57+KMNhFkshNKZw==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@dcl/snapshots-fetcher/-/snapshots-fetcher-5.0.1.tgz", + "integrity": "sha512-afGKsn03JJ1iBDPJv1auRdKsnxmUeA7Z6ontv6yCicTw+eNcaDDz5T+SZjFwz5FH9i8Cr9jfdCqs03/ZGwtEhQ==", "requires": { + "@dcl/catalyst-storage": "^2.0.0", "@dcl/hashing": "^1.0.0", + "@dcl/schemas": "^6.2.0", "@well-known-components/interfaces": "^1.1.0", - "@well-known-components/metrics": "^1.1.2", + "@well-known-components/metrics": "^2.0.0", "p-queue": "^6.6.2" + }, + "dependencies": { + "@dcl/schemas": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@dcl/schemas/-/schemas-6.3.0.tgz", + "integrity": "sha512-Zhbly35/SZ5f4h0VnCtSjxxIaEbCpRzAo487Pd+PIuzx1vt5C9srJRHbMz0Cns6h3KiF8/E4Zsjl141NYIijzA==", + "requires": { + "ajv": "^8.11.0", + "ajv-errors": "^3.0.0", + "ajv-keywords": "^5.1.0" + } + }, + "@well-known-components/metrics": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@well-known-components/metrics/-/metrics-2.0.1.tgz", + "integrity": "sha512-jgT9TuxVS9GzVMMYWXNJRM2qXvexuf4xfrFNBYa3w+gqEWCK4Id5R4zBee/bm6/F5yIPauAWxbOS6KNNDBIrGw==", + "requires": { + "prom-client": "^14.1.0" + } + } } }, "@istanbuljs/load-nyc-config": { @@ -7424,10 +7634,15 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "available-typed-arrays": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", + "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" + }, "aws-sdk": { - "version": "2.1148.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1148.0.tgz", - "integrity": "sha512-FUYAyveKmS5eqIiGQgrGVsLZwwtI+K6S6Gz8oJf56pgypZCo9dV+cXO4aaS+vN0+LSmGh6dSKc6G8h8FYASIJg==", + "version": "2.1291.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1291.0.tgz", + "integrity": "sha512-iM82Md2Wb29MZ72BpNF4YeIHtkJTyw+JMGXJG48Dwois2Rq7rLcriX6NN/4Fx4b5EEtUkwAO/wJc+NTHI7QeJw==", "requires": { "buffer": "4.9.2", "events": "1.1.1", @@ -7436,6 +7651,7 @@ "querystring": "0.2.0", "sax": "1.2.1", "url": "0.10.3", + "util": "^0.12.4", "uuid": "8.0.0", "xml2js": "0.4.19" }, @@ -7681,6 +7897,15 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -8268,6 +8493,14 @@ "path-exists": "^4.0.0" } }, + "for-each": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", + "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "requires": { + "is-callable": "^1.1.3" + } + }, "form-data": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", @@ -8309,8 +8542,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "gensync": { "version": "1.0.0-beta.2", @@ -8324,6 +8556,16 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true }, + "get-intrinsic": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.3.tgz", + "integrity": "sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, "get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", @@ -8356,6 +8598,14 @@ "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", "dev": true }, + "gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "requires": { + "get-intrinsic": "^1.1.3" + } + }, "graceful-fs": { "version": "4.2.9", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", @@ -8385,7 +8635,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -8396,6 +8645,19 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, + "has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "requires": { + "has-symbols": "^1.0.2" + } + }, "html-encoding-sniffer": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", @@ -8594,6 +8856,20 @@ "uint8arrays": "^2.0.5" } }, + "is-arguments": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", + "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==" + }, "is-core-module": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz", @@ -8615,6 +8891,14 @@ "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", "dev": true }, + "is-generator-function": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", + "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, "is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -8638,6 +8922,18 @@ "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", "dev": true }, + "is-typed-array": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.10.tgz", + "integrity": "sha512-PJqgEHiWZvMpaFZ3uTc8kHPM4+4ADTlDniuQL7cU/UDA0Ql7F70yGfHph3cLNe+c9toaigv+DFzTJKhc2CtO6A==", + "requires": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "gopd": "^1.0.1", + "has-tostringtag": "^1.0.0" + } + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -9881,9 +10177,9 @@ } }, "prom-client": { - "version": "14.0.1", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.0.1.tgz", - "integrity": "sha512-HxTArb6fkOntQHoRGvv4qd/BkorjliiuO2uSWC2KC17MUTKYttWdDoXX/vxOhQdkoECEM9BBH0pj2l8G8kev6w==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.1.1.tgz", + "integrity": "sha512-hFU32q7UZQ59bVJQGUtm3I2PrJ3gWvoCkilX9sF165ks1qflhugVCeK+S1JjJYHvyt3o5kj68+q3bchormjnzw==", "requires": { "tdigest": "^0.1.1" } @@ -10569,6 +10865,18 @@ } } }, + "util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "requires": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -10678,6 +10986,19 @@ "isexe": "^2.0.0" } }, + "which-typed-array": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.9.tgz", + "integrity": "sha512-w9c4xkx6mPidwp7180ckYWfMmvxpjlZuIudNtDf4N/tTAUB8VJbX25qZoAsrtGuYNnGw3pa0AXgbGKRB8/EceA==", + "requires": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "gopd": "^1.0.1", + "has-tostringtag": "^1.0.0", + "is-typed-array": "^1.1.10" + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", diff --git a/package.json b/package.json index ed35b68..3c8d1e8 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,8 @@ }, "dependencies": { "@dcl/schemas": "^5.22.0", - "@dcl/catalyst-storage": "^1.0.0", - "@dcl/snapshots-fetcher": "^3.0.2", + "@dcl/catalyst-storage": "^2.0.0", + "@dcl/snapshots-fetcher": "^5.0.1", "@types/destroy": "^1.0.0", "@well-known-components/env-config-provider": "^1.1.1", "@well-known-components/http-server": "^1.1.1", @@ -27,4 +27,4 @@ "aws-sdk": "^2.1138.0", "destroy": "^1.2.0" } -} +} \ No newline at end of file diff --git a/src/adapters/deployer/index.ts b/src/adapters/deployer/index.ts index ea3ab86..3060b1e 100644 --- a/src/adapters/deployer/index.ts +++ b/src/adapters/deployer/index.ts @@ -2,7 +2,7 @@ import { downloadEntityAndContentFiles } from "@dcl/snapshots-fetcher" import { IDeployerComponent } from "@dcl/snapshots-fetcher/dist/types" import { SNS } from "aws-sdk" import { AppComponents } from "../../types" -import { DeploymentToSqs } from "@dcl/schemas/dist/misc/deployments-to-sqs"; +import { DeploymentToSqs } from "@dcl/schemas/dist/misc/deployments-to-sqs" export function createDeployerComponent( components: Pick @@ -48,10 +48,17 @@ export function createDeployerComponent( SequenceNumber: receipt.SequenceNumber as any, }) } + if (entity.markAsDeployed) { + await entity.markAsDeployed() + } }) } + } else { + if (entity.markAsDeployed) { + await entity.markAsDeployed() + } } }, - async onIdle() {}, + async onIdle() { }, } } diff --git a/src/components.ts b/src/components.ts index 8562dc6..c433ed9 100644 --- a/src/components.ts +++ b/src/components.ts @@ -6,7 +6,7 @@ import { createMetricsComponent } from "@well-known-components/metrics" import { AppComponents, GlobalContext, SnsComponent } from "./types" import { metricDeclarations } from "./metrics" import { createJobQueue } from "@dcl/snapshots-fetcher/dist/job-queue-port" -import { createCatalystDeploymentStream } from "@dcl/snapshots-fetcher" +import { createSynchronizer } from "@dcl/snapshots-fetcher" import { createJobLifecycleManagerComponent } from "@dcl/snapshots-fetcher/dist/job-lifecycle-manager" import { createDeployerComponent } from "./adapters/deployer" import { createAwsS3BasedFileSystemContentStorage, createFolderBasedFileSystemContentStorage, createFsComponent } from "@dcl/catalyst-storage" @@ -43,31 +43,55 @@ export async function initComponents(): Promise { const deployer = createDeployerComponent({ storage, downloadQueue, fetch, logs, metrics, sns }) - const synchronizationJobManager = createJobLifecycleManagerComponent( - { logs }, - { - jobManagerName: "SynchronizationJobManager", - createJob(contentServer) { - return createCatalystDeploymentStream( - { logs, downloadQueue, fetcher: fetch, metrics, deployer, storage }, - { - tmpDownloadFolder: downloadsFolder, - contentServer, - - // time between every poll to /pointer-changes - pointerChangesWaitTime: 5000, + const processedSnapshots = new Set() + const processedSnapshotStorage = { + async processedFrom(snapshotHashes: string[]): Promise> { + return new Set(snapshotHashes.filter(h => processedSnapshots.has(h))) + }, + async saveProcessed(snapshotHash: string): Promise { + processedSnapshots.add(snapshotHash) + } + } - // reconnection time for the whole catalyst - reconnectTime: 1000 /* one second */, - reconnectRetryTimeExponent: 1.2, - maxReconnectionTime: 3_600_000 /* one hour */, + const snapshotStorage = { + async has(snapshotHas: string) { + return false + } + } - // download entities retry - requestMaxRetries: 10, - requestRetryWaitTime: 5000, - } - ) + const synchronizer = await createSynchronizer( + { + logs, + downloadQueue, + fetcher: fetch, + metrics, + deployer, + storage, + processedSnapshotStorage, + snapshotStorage + }, + { + // reconnection options + bootstrapReconnection: { + reconnectTime: 5000 /* five second */, + reconnectRetryTimeExponent: 1.5, + maxReconnectionTime: 3_600_000 /* one hour */ + }, + syncingReconnection: { + reconnectTime: 1000 /* one second */, + reconnectRetryTimeExponent: 1.2, + maxReconnectionTime: 3_600_000 /* one hour */ }, + + // snapshot stream options + tmpDownloadFolder: downloadsFolder, + // download entities retry + requestMaxRetries: 10, + requestRetryWaitTime: 5000, + + // pointer chagnes stream options + // time between every poll to /pointer-changes + pointerChangesWaitTime: 5000 } ) @@ -81,8 +105,10 @@ export async function initComponents(): Promise { storage, fs, downloadQueue, - synchronizationJobManager, + synchronizer, deployer, sns, + processedSnapshotStorage, + snapshotStorage } } diff --git a/src/service.ts b/src/service.ts index d930776..99b4bd1 100644 --- a/src/service.ts +++ b/src/service.ts @@ -21,5 +21,5 @@ export async function main(program: Lifecycle.EntryPointParameters fs: IFileSystemComponent storage: IContentStorageComponent - synchronizationJobManager: JobLifecycleManagerComponent + synchronizer: SynchronizerComponent + processedSnapshotStorage: IProcessedSnapshotStorageComponent + snapshotStorage: ISnapshotStorageComponent deployer: IDeployerComponent sns: SnsComponent } @@ -50,11 +52,11 @@ export type TestComponents = BaseComponents & { export type HandlerContextWithPath< ComponentNames extends keyof AppComponents, Path extends string = any -> = IHttpServerComponent.PathAwareContext< - IHttpServerComponent.DefaultContext<{ - components: Pick - }>, - Path -> + > = IHttpServerComponent.PathAwareContext< + IHttpServerComponent.DefaultContext<{ + components: Pick + }>, + Path + > export type Context = IHttpServerComponent.PathAwareContext From d9036462c97cce9d5e8434b7a1a923cd9772a019 Mon Sep 17 00:00:00 2001 From: Pedro Tamborindeguy Date: Tue, 10 Jan 2023 12:10:26 -0300 Subject: [PATCH 2/3] chore: fix --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 79d8211..4e07515 100644 --- a/.env +++ b/.env @@ -3,4 +3,4 @@ # You may want to add this .env file to .gitignore to save development # credentials -HTTP_SERVER_PORT=5001 \ No newline at end of file +HTTP_SERVER_PORT=5000 \ No newline at end of file From 88eec2d3c01bba8ba10ccbbd9e042cac17e88417 Mon Sep 17 00:00:00 2001 From: Pedro Tamborindeguy Date: Tue, 10 Jan 2023 12:24:40 -0300 Subject: [PATCH 3/3] fix: fix --- src/adapters/deployer/index.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/adapters/deployer/index.ts b/src/adapters/deployer/index.ts index 3060b1e..3646dd5 100644 --- a/src/adapters/deployer/index.ts +++ b/src/adapters/deployer/index.ts @@ -13,6 +13,7 @@ export function createDeployerComponent( return { async deployEntity(entity, servers) { + const markAsDeployed = entity.markAsDeployed ? entity.markAsDeployed : async () => { } if (entity.entityType == "scene" || entity.entityType == "wearable" || entity.entityType == "emote") { const exists = await components.storage.exist(entity.entityId) @@ -48,15 +49,13 @@ export function createDeployerComponent( SequenceNumber: receipt.SequenceNumber as any, }) } - if (entity.markAsDeployed) { - await entity.markAsDeployed() - } + await markAsDeployed() }) + } else { + await markAsDeployed() } } else { - if (entity.markAsDeployed) { - await entity.markAsDeployed() - } + await markAsDeployed() } }, async onIdle() { },