From 21d910b676790aaf47332f448194d55031e8eb27 Mon Sep 17 00:00:00 2001 From: Gunar Gessner Date: Tue, 6 Dec 2022 09:43:47 -0300 Subject: [PATCH] feat(sync): add limit cli options (#1309) * feat(sync): add limit cli options * fix(ci): reduce concurrency to avoid 505 errors --- .github/workflows/sync-staging-db.yml | 2 +- scripts/seed-database/package.json | 4 +- scripts/seed-database/pnpm-lock.yaml | 413 +++++++++++++++++- .../seed-database/upsert-production-flows.js | 40 +- 4 files changed, 439 insertions(+), 20 deletions(-) diff --git a/.github/workflows/sync-staging-db.yml b/.github/workflows/sync-staging-db.yml index 32d18e9f23..4baa0ac016 100644 --- a/.github/workflows/sync-staging-db.yml +++ b/.github/workflows/sync-staging-db.yml @@ -26,7 +26,7 @@ jobs: working-directory: scripts/seed-database - run: pnpm install working-directory: scripts/seed-database - - run: node ./upsert-production-flows.js --overwrite + - run: node ./upsert-production-flows.js --overwrite --limit 1 working-directory: scripts/seed-database env: HASURA_GRAPHQL_URL: https://hasura.editor.planx.dev/v1/graphql diff --git a/scripts/seed-database/package.json b/scripts/seed-database/package.json index ec064bc76f..1e9d394f61 100644 --- a/scripts/seed-database/package.json +++ b/scripts/seed-database/package.json @@ -13,8 +13,8 @@ "dependencies": { "graphql": "^16.5.0", "graphql-request": "^4.3.0", - "lodash": "^4.17.21", - "p-throttle": "^5.0.0" + "meow": "^11.0.0", + "p-limit": "^4.0.0" }, "devDependencies": { "@types/lodash": "^4.14.184" diff --git a/scripts/seed-database/pnpm-lock.yaml b/scripts/seed-database/pnpm-lock.yaml index dff9b90335..4664f85839 100644 --- a/scripts/seed-database/pnpm-lock.yaml +++ b/scripts/seed-database/pnpm-lock.yaml @@ -4,28 +4,103 @@ specifiers: '@types/lodash': ^4.14.184 graphql: ^16.5.0 graphql-request: ^4.3.0 - lodash: ^4.17.21 - p-throttle: ^5.0.0 + meow: ^11.0.0 + p-limit: ^4.0.0 dependencies: graphql: 16.6.0 graphql-request: 4.3.0_graphql@16.6.0 - lodash: 4.17.21 - p-throttle: 5.0.0 + meow: 11.0.0 + p-limit: 4.0.0 devDependencies: '@types/lodash': 4.14.184 packages: + /@babel/code-frame/7.18.6: + resolution: {integrity: sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/highlight': 7.18.6 + dev: false + + /@babel/helper-validator-identifier/7.19.1: + resolution: {integrity: sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/highlight/7.18.6: + resolution: {integrity: sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-validator-identifier': 7.19.1 + chalk: 2.4.2 + js-tokens: 4.0.0 + dev: false + /@types/lodash/4.14.184: resolution: {integrity: sha512-RoZphVtHbxPZizt4IcILciSWiC6dcn+eZ8oX9IWEYfDMcocdd42f7NPI6fQj+6zI8y4E0L7gu2pcZKLGTRaV9Q==} dev: true + /@types/minimist/1.2.2: + resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} + dev: false + + /@types/normalize-package-data/2.4.1: + resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} + dev: false + + /ansi-styles/3.2.1: + resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} + engines: {node: '>=4'} + dependencies: + color-convert: 1.9.3 + dev: false + + /arrify/1.0.1: + resolution: {integrity: sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==} + engines: {node: '>=0.10.0'} + dev: false + /asynckit/0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} dev: false + /camelcase-keys/8.0.2: + resolution: {integrity: sha512-qMKdlOfsjlezMqxkUGGMaWWs17i2HoL15tM+wtx8ld4nLrUwU58TFdvyGOz/piNP842KeO8yXvggVQSdQ828NA==} + engines: {node: '>=14.16'} + dependencies: + camelcase: 7.0.0 + map-obj: 4.3.0 + quick-lru: 6.1.1 + type-fest: 2.19.0 + dev: false + + /camelcase/7.0.0: + resolution: {integrity: sha512-JToIvOmz6nhGsUhAYScbo2d6Py5wojjNfoxoc2mEVLUdJ70gJK2gnd+ABY1Tc3sVMyK7QDPtN0T/XdlCQWITyQ==} + engines: {node: '>=14.16'} + dev: false + + /chalk/2.4.2: + resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} + engines: {node: '>=4'} + dependencies: + ansi-styles: 3.2.1 + escape-string-regexp: 1.0.5 + supports-color: 5.5.0 + dev: false + + /color-convert/1.9.3: + resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} + dependencies: + color-name: 1.1.3 + dev: false + + /color-name/1.1.3: + resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} + dev: false + /combined-stream/1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -41,16 +116,53 @@ packages: - encoding dev: false + /decamelize-keys/1.1.1: + resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} + engines: {node: '>=0.10.0'} + dependencies: + decamelize: 1.2.0 + map-obj: 1.0.1 + dev: false + + /decamelize/1.2.0: + resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} + engines: {node: '>=0.10.0'} + dev: false + + /decamelize/6.0.0: + resolution: {integrity: sha512-Fv96DCsdOgB6mdGl67MT5JaTNKRzrzill5OH5s8bjYJXVlcXyPYGyPsUkWyGV5p1TXI5esYIYMMeDJL0hEIwaA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + /delayed-stream/1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} dev: false + /error-ex/1.3.2: + resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} + dependencies: + is-arrayish: 0.2.1 + dev: false + + /escape-string-regexp/1.0.5: + resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} + engines: {node: '>=0.8.0'} + dev: false + /extract-files/9.0.0: resolution: {integrity: sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ==} engines: {node: ^10.17.0 || ^12.0.0 || >= 13.7.0} dev: false + /find-up/6.3.0: + resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + locate-path: 7.1.1 + path-exists: 5.0.0 + dev: false + /form-data/3.0.1: resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} engines: {node: '>= 6'} @@ -60,6 +172,10 @@ packages: mime-types: 2.1.35 dev: false + /function-bind/1.1.1: + resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} + dev: false + /graphql-request/4.3.0_graphql@16.6.0: resolution: {integrity: sha512-2v6hQViJvSsifK606AliqiNiijb1uwWp6Re7o0RTyH+uRTv/u7Uqm2g4Fjq/LgZIzARB38RZEvVBFOQOVdlBow==} peerDependencies: @@ -78,8 +194,119 @@ packages: engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} dev: false - /lodash/4.17.21: - resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + /hard-rejection/2.1.0: + resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} + engines: {node: '>=6'} + dev: false + + /has-flag/3.0.0: + resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} + engines: {node: '>=4'} + dev: false + + /has/1.0.3: + resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} + engines: {node: '>= 0.4.0'} + dependencies: + function-bind: 1.1.1 + dev: false + + /hosted-git-info/4.1.0: + resolution: {integrity: sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==} + engines: {node: '>=10'} + dependencies: + lru-cache: 6.0.0 + dev: false + + /hosted-git-info/5.2.1: + resolution: {integrity: sha512-xIcQYMnhcx2Nr4JTjsFmwwnr9vldugPy9uVm0o87bjqqWMv9GaqsTeT+i99wTl0mk1uLxJtHxLb8kymqTENQsw==} + engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + dependencies: + lru-cache: 7.14.1 + dev: false + + /indent-string/5.0.0: + resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==} + engines: {node: '>=12'} + dev: false + + /is-arrayish/0.2.1: + resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + dev: false + + /is-core-module/2.11.0: + resolution: {integrity: sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==} + dependencies: + has: 1.0.3 + dev: false + + /is-plain-obj/1.1.0: + resolution: {integrity: sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==} + engines: {node: '>=0.10.0'} + dev: false + + /js-tokens/4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + dev: false + + /json-parse-even-better-errors/2.3.1: + resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + dev: false + + /kind-of/6.0.3: + resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} + engines: {node: '>=0.10.0'} + dev: false + + /lines-and-columns/1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + dev: false + + /locate-path/7.1.1: + resolution: {integrity: sha512-vJXaRMJgRVD3+cUZs3Mncj2mxpt5mP0EmNOsxRSZRMlbqjvxzDEOIUWXGmavo0ZC9+tNZCBLQ66reA11nbpHZg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-locate: 6.0.0 + dev: false + + /lru-cache/6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + dependencies: + yallist: 4.0.0 + dev: false + + /lru-cache/7.14.1: + resolution: {integrity: sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==} + engines: {node: '>=12'} + dev: false + + /map-obj/1.0.1: + resolution: {integrity: sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==} + engines: {node: '>=0.10.0'} + dev: false + + /map-obj/4.3.0: + resolution: {integrity: sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==} + engines: {node: '>=8'} + dev: false + + /meow/11.0.0: + resolution: {integrity: sha512-Cl0yeeIrko6d94KpUo1M+0X1sB14ikoaqlIGuTH1fW4I+E3+YljL54/hb/BWmVfrV9tTV9zU04+xjw08Fh2WkA==} + engines: {node: '>=14.16'} + dependencies: + '@types/minimist': 1.2.2 + camelcase-keys: 8.0.2 + decamelize: 6.0.0 + decamelize-keys: 1.1.1 + hard-rejection: 2.1.0 + minimist-options: 4.1.0 + normalize-package-data: 4.0.1 + read-pkg-up: 9.1.0 + redent: 4.0.0 + trim-newlines: 4.0.2 + type-fest: 3.3.0 + yargs-parser: 21.1.1 dev: false /mime-db/1.52.0: @@ -94,6 +321,20 @@ packages: mime-db: 1.52.0 dev: false + /min-indent/1.0.1: + resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} + engines: {node: '>=4'} + dev: false + + /minimist-options/4.1.0: + resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} + engines: {node: '>= 6'} + dependencies: + arrify: 1.0.1 + is-plain-obj: 1.1.0 + kind-of: 6.0.3 + dev: false + /node-fetch/2.6.7: resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} engines: {node: 4.x || >=6.0.0} @@ -106,15 +347,157 @@ packages: whatwg-url: 5.0.0 dev: false - /p-throttle/5.0.0: - resolution: {integrity: sha512-iXBFjW4kP/5Ivw7uC9EDnj+/xo3pNn4Rws3zgMGPwXnWTv1M3P0LVdZxLrqRUI5JK0Fp3Du0bt6lCaVrI3WF7g==} + /normalize-package-data/3.0.3: + resolution: {integrity: sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==} + engines: {node: '>=10'} + dependencies: + hosted-git-info: 4.1.0 + is-core-module: 2.11.0 + semver: 7.3.8 + validate-npm-package-license: 3.0.4 + dev: false + + /normalize-package-data/4.0.1: + resolution: {integrity: sha512-EBk5QKKuocMJhB3BILuKhmaPjI8vNRSpIfO9woLC6NyHVkKKdVEdAO1mrT0ZfxNR1lKwCcTkuZfmGIFdizZ8Pg==} + engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} + dependencies: + hosted-git-info: 5.2.1 + is-core-module: 2.11.0 + semver: 7.3.8 + validate-npm-package-license: 3.0.4 + dev: false + + /p-limit/4.0.0: + resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + yocto-queue: 1.0.0 + dev: false + + /p-locate/6.0.0: + resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-limit: 4.0.0 + dev: false + + /parse-json/5.2.0: + resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} + engines: {node: '>=8'} + dependencies: + '@babel/code-frame': 7.18.6 + error-ex: 1.3.2 + json-parse-even-better-errors: 2.3.1 + lines-and-columns: 1.2.4 + dev: false + + /path-exists/5.0.0: + resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /quick-lru/6.1.1: + resolution: {integrity: sha512-S27GBT+F0NTRiehtbrgaSE1idUAJ5bX8dPAQTdylEyNlrdcH5X4Lz7Edz3DYzecbsCluD5zO8ZNEe04z3D3u6Q==} + engines: {node: '>=12'} + dev: false + + /read-pkg-up/9.1.0: + resolution: {integrity: sha512-vaMRR1AC1nrd5CQM0PhlRsO5oc2AAigqr7cCrZ/MW/Rsaflz4RlgzkpL4qoU/z1F6wrbd85iFv1OQj/y5RdGvg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + find-up: 6.3.0 + read-pkg: 7.1.0 + type-fest: 2.19.0 + dev: false + + /read-pkg/7.1.0: + resolution: {integrity: sha512-5iOehe+WF75IccPc30bWTbpdDQLOCc3Uu8bi3Dte3Eueij81yx1Mrufk8qBx/YAbR4uL1FdUr+7BKXDwEtisXg==} + engines: {node: '>=12.20'} + dependencies: + '@types/normalize-package-data': 2.4.1 + normalize-package-data: 3.0.3 + parse-json: 5.2.0 + type-fest: 2.19.0 + dev: false + + /redent/4.0.0: + resolution: {integrity: sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==} + engines: {node: '>=12'} + dependencies: + indent-string: 5.0.0 + strip-indent: 4.0.0 + dev: false + + /semver/7.3.8: + resolution: {integrity: sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==} + engines: {node: '>=10'} + hasBin: true + dependencies: + lru-cache: 6.0.0 + dev: false + + /spdx-correct/3.1.1: + resolution: {integrity: sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==} + dependencies: + spdx-expression-parse: 3.0.1 + spdx-license-ids: 3.0.12 + dev: false + + /spdx-exceptions/2.3.0: + resolution: {integrity: sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==} + dev: false + + /spdx-expression-parse/3.0.1: + resolution: {integrity: sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==} + dependencies: + spdx-exceptions: 2.3.0 + spdx-license-ids: 3.0.12 + dev: false + + /spdx-license-ids/3.0.12: + resolution: {integrity: sha512-rr+VVSXtRhO4OHbXUiAF7xW3Bo9DuuF6C5jH+q/x15j2jniycgKbxU09Hr0WqlSLUs4i4ltHGXqTe7VHclYWyA==} + dev: false + + /strip-indent/4.0.0: + resolution: {integrity: sha512-mnVSV2l+Zv6BLpSD/8V87CW/y9EmmbYzGCIavsnsI6/nwn26DwffM/yztm30Z/I2DY9wdS3vXVCMnHDgZaVNoA==} + engines: {node: '>=12'} + dependencies: + min-indent: 1.0.1 + dev: false + + /supports-color/5.5.0: + resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} + engines: {node: '>=4'} + dependencies: + has-flag: 3.0.0 dev: false /tr46/0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} dev: false + /trim-newlines/4.0.2: + resolution: {integrity: sha512-GJtWyq9InR/2HRiLZgpIKv+ufIKrVrvjQWEj7PxAXNc5dwbNJkqhAUoAGgzRmULAnoOM5EIpveYd3J2VeSAIew==} + engines: {node: '>=12'} + dev: false + + /type-fest/2.19.0: + resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} + engines: {node: '>=12.20'} + dev: false + + /type-fest/3.3.0: + resolution: {integrity: sha512-gezeeOIZyQLGW5uuCeEnXF1aXmtt2afKspXz3YqoOcZ3l/YMJq1pujvgT+cz/Nw1O/7q/kSav5fihJHsC/AOUg==} + engines: {node: '>=14.16'} + dev: false + + /validate-npm-package-license/3.0.4: + resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} + dependencies: + spdx-correct: 3.1.1 + spdx-expression-parse: 3.0.1 + dev: false + /webidl-conversions/3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} dev: false @@ -125,3 +508,17 @@ packages: tr46: 0.0.3 webidl-conversions: 3.0.1 dev: false + + /yallist/4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: false + + /yargs-parser/21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + dev: false + + /yocto-queue/1.0.0: + resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} + engines: {node: '>=12.20'} + dev: false diff --git a/scripts/seed-database/upsert-production-flows.js b/scripts/seed-database/upsert-production-flows.js index d55226864e..d7c30058ad 100644 --- a/scripts/seed-database/upsert-production-flows.js +++ b/scripts/seed-database/upsert-production-flows.js @@ -1,7 +1,26 @@ -import _ from "lodash"; import { gql, GraphQLClient } from "graphql-request"; -import pThrottle from 'p-throttle'; -const args = process.argv.slice(2); +import pLimit from 'p-limit'; +import meow from 'meow'; + +const DEFAULT_LIMIT = 10; + +const cli = meow(` + Options + --overwrite, -o Overwrites existing flows and teams + --limit, -l Rate limit for published flows insertion (Default: ${DEFAULT_LIMIT}) +`, { + importMeta: import.meta, + flags: { + overwrite: { + type: 'boolean', + alias: 'o', + }, + limit: { + type: 'number', + alias: 'l' + } + } +}); const PRODUCTION_GRAPHQL_URL = 'https://hasura.editor.planx.uk/v1/graphql'; const LOCAL_GRAPHQL_URL = process.env.HASURA_GRAPHQL_URL; @@ -16,7 +35,8 @@ const LOCAL_GRAPHQL_ADMIN_SECRET = process.env.HASURA_GRAPHQL_ADMIN_SECRET; }, }); - const shouldOverwrite = args?.includes('-o') || args?.includes('--overwrite'); + const shouldOverwrite = cli.flags.overwrite; + const rateLimit = cli.flags.limit || DEFAULT_LIMIT; if (!shouldOverwrite) { const { flows: localFlows } = await localClient.request(gql` @@ -85,11 +105,11 @@ const LOCAL_GRAPHQL_ADMIN_SECRET = process.env.HASURA_GRAPHQL_ADMIN_SECRET; console.log('Fetching published flows...'); // throttling is needed to prevent errors when fetching a large amount of data - const throttledSync = pThrottle({ limit: 10, interval: 5000 })( - syncPublishedFlow(productionClient, localClient) - ); + const limit = pLimit(rateLimit); + const rateLimitedSync = (flowId) => + limit(() => syncPublishedFlow(productionClient, localClient, flowId)); - await Promise.all(productionFlows.map(async flow => throttledSync(flow.id))); + await Promise.all(productionFlows.map(async flow => rateLimitedSync(flow.id))); console.log("Production flows and teams inserted successfully."); } catch (err) { @@ -149,9 +169,11 @@ const insertFlowsMutation = gql` } `; -const syncPublishedFlow = (productionClient, localClient) => async (flowId) => { +const syncPublishedFlow = async (productionClient, localClient, flowId) => { const publishedFlows = await publishedFlowsByFlowIdQuery(flowId, productionClient); + console.log(`Inserting ${publishedFlows.length} published flows for flow id ${flowId}`); + await Promise.all(publishedFlows.map( async publishedFlow => insertPublishedFlowMutation( {