From 483ac3492133a0ccdc0f62a0e46f1535b9db66ff Mon Sep 17 00:00:00 2001 From: Choro Abdymanapov Date: Mon, 22 Mar 2021 11:07:23 +0300 Subject: [PATCH] feat: add filtering by range to download (#24) --- README.md | 4 +- packages/cli/README.md | 2 +- .../cli/src/main/ts/executors/download.ts | 5 +- packages/cli/src/main/ts/interfaces.ts | 1 + packages/cli/src/main/ts/utils/validators.ts | 2 + packages/helper/README.md | 5 +- packages/helper/src/main/ts/helper/helper.ts | 19 +++-- .../helper/src/main/ts/helper/interfaces.ts | 2 +- packages/helper/src/test/ts/helper/helper.ts | 47 ++++++++++++- yarn.lock | 69 ++++++++++--------- 10 files changed, 109 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index de168dc..e9200ef 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ await helper.deletePackagesByIds(['foo', 'bar', 'baz']) ``` ## [@qiwi/nexus-cli](https://github.com/qiwi/nexus/tree/master/packages/cli) -CLI utility for getting and deleting package components. +CLI utility for getting, downloading and deleting package components. ```shell script -> @qiwi/nexus-cli --nexus.username=foo --nexus.password=bar --nexus.url=baz --package.repo=npm --package.name=react --package.group=null --package.range='>16.0.0' +> @qiwi/nexus-cli --auth.username=foo --auth.password=bar --url=baz --data.repo=npm --data.name=react --data.group=null --data.range='>16.0.0' --action=delete ┌─────────┬─────────────┬───────┬─────────┬───────────┬────────────────────────────────────────────────────────────┐ │ (index) │ repository │ group │ name │ version │ id │ diff --git a/packages/cli/README.md b/packages/cli/README.md index 314dd41..9c8382b 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -22,7 +22,7 @@ nexus-cli --auth.username foo --auth.password bar --url baz --data.repo npm --da ### Download For this action utility saves tarballs and prints metadata in given `cwd` ```shell script -nexus-cli --auth.username foo --auth.password bar --url baz --data.repo npm --data.name foo --data.group qiwi --data.range --data.cwd nexus-downloads '<1.0.0' --action download +nexus-cli --auth.username foo --auth.password bar --url baz --data.repo npm --data.name foo --data.group qiwi --data.range '<1.0.0' --data.cwd nexus-downloads --action download ``` Output in `nexus-downloads/nexus-cli-downloads-meta-2021-03-17T06:48:21.347Z.json`: ```json diff --git a/packages/cli/src/main/ts/executors/download.ts b/packages/cli/src/main/ts/executors/download.ts index b1c64f4..2a44e28 100644 --- a/packages/cli/src/main/ts/executors/download.ts +++ b/packages/cli/src/main/ts/executors/download.ts @@ -7,11 +7,12 @@ import { TDownloadConfigData, TDownloadConfigDataStrict, TPackageAccess } from ' import { writeJson } from '../utils' export const performDownload = async (data: TDownloadConfigDataStrict, helper: INexusHelper): Promise => { - const { repo, group, name, cwd, npmBatch } = data + const { repo, group, name, cwd, npmBatch, range } = data const opts = { repository: repo, group, - name + name, + range, } const { downloadsPath, infoOutputPath } = getPathsWithTimestamp(cwd) diff --git a/packages/cli/src/main/ts/interfaces.ts b/packages/cli/src/main/ts/interfaces.ts index f7505df..2a4005e 100644 --- a/packages/cli/src/main/ts/interfaces.ts +++ b/packages/cli/src/main/ts/interfaces.ts @@ -30,6 +30,7 @@ export type TDeleteConfig = IBaseConfig<'delete', TDeleteConfigData> export type TPackageAccess = 'public' | 'restricted' export type TDownloadConfigMetaData = { + range?: string cwd: string npmBatch?: { access: TPackageAccess diff --git a/packages/cli/src/main/ts/utils/validators.ts b/packages/cli/src/main/ts/utils/validators.ts index 27542ca..f97f608 100644 --- a/packages/cli/src/main/ts/utils/validators.ts +++ b/packages/cli/src/main/ts/utils/validators.ts @@ -30,6 +30,8 @@ export const validateDownloadConfig = (config: IBaseConfig): TDownloadConfig => check(config.data.version, 'config.data.version: str?') check(config.data.cwd, 'config.data.cwd: str?') check(config.data.repo, 'config.data.repo: str') + check(config.data.range, 'config.data.range: str?') + if (config.data.npmBatch) { check(config.data.npmBatch.access, 'config.data.npmBatch.access: "public" | "restricted"') } diff --git a/packages/helper/README.md b/packages/helper/README.md index cae4dcd..d322832 100644 --- a/packages/helper/README.md +++ b/packages/helper/README.md @@ -166,8 +166,9 @@ The same as a `getPackageAssets`, but with saving tarballs to given path const data = await helper.downloadPackageAssets( { repository: 'npm', - group: 'qiwi', - name: 'substrate' + group: 'qiwi', // optional + name: 'substrate', // optional + range: '<1.0.0' // optional }, 'pathToSaveTarball' ) diff --git a/packages/helper/src/main/ts/helper/helper.ts b/packages/helper/src/main/ts/helper/helper.ts index 5154bb1..cee0bf9 100644 --- a/packages/helper/src/main/ts/helper/helper.ts +++ b/packages/helper/src/main/ts/helper/helper.ts @@ -100,12 +100,19 @@ export class NexusComponentsHelper implements INexusHelper { } return { - items: await Promise.allSettled(items.map(({ downloadUrl, path }) => { - if (!downloadUrl || !path) { - return Promise.reject(new Error('downloadUrl or path is absent in Nexus API response')) - } - return this.downloadPackageAsset(downloadUrl, path, cwd) - })), + items: await Promise.allSettled( + items + .filter(item => opts.range + ? satisfies(NexusComponentsHelper.extractNameAndVersionFromPath(item.path + '').version, opts.range) + : true + ) + .map(({ downloadUrl, path }) => { + if (!downloadUrl || !path) { + return Promise.reject(new Error('downloadUrl or path is absent in Nexus API response')) + } + return this.downloadPackageAsset(downloadUrl, path, cwd) + }) + ), continuationToken } } diff --git a/packages/helper/src/main/ts/helper/interfaces.ts b/packages/helper/src/main/ts/helper/interfaces.ts index 61236f0..49a604e 100644 --- a/packages/helper/src/main/ts/helper/interfaces.ts +++ b/packages/helper/src/main/ts/helper/interfaces.ts @@ -14,7 +14,7 @@ export type TGetPackageVersionsOpts = { timeout?: number } -export type TGetPackageAssetsOpts = Partial & { repository: string } +export type TGetPackageAssetsOpts = Partial & { repository: string, range?: string } export type TPaginatedSettledResult = { items: PromiseSettledResult[] diff --git a/packages/helper/src/test/ts/helper/helper.ts b/packages/helper/src/test/ts/helper/helper.ts index 37e5b4a..dd42a31 100644 --- a/packages/helper/src/test/ts/helper/helper.ts +++ b/packages/helper/src/test/ts/helper/helper.ts @@ -190,6 +190,51 @@ describe('NexusContentsHelper', () => { expect(mock.isDone()).toEqual(true) }) + it ('downloads assets which satisfy given range', async () => { + const params = { + group: 'types', + repository: 'npm', + name: 'react', + } + const page = [ + { ...asset, downloadUrl: 'http://localhost/repository/npm/@types/react/-/react-15.9.41.tgz', path: '@types/react/-/react-15.9.41.tgz' }, + { ...asset, downloadUrl: 'http://localhost/repository/npm/@types/react/-/react-14.9.41.tgz', path: '@types/react/-/react-14.9.41.tgz' }, + { ...asset, downloadUrl: 'http://localhost/repository/npm/@types/react/-/react-13.9.41.tgz', path: '@types/react/-/react-13.9.41.tgz' }, + { ...asset, downloadUrl: 'http://localhost/repository/npm/@types/react/-/react-12.9.41.tgz', path: '@types/react/-/react-12.9.41.tgz' }, + ] + const assetsMockToBeDownloaded = [ + nock(assetsBasePath) + .get('/repository/npm/@types/react/-/react-15.9.41.tgz') + .once() + .reply(200, 'foo'), + nock(assetsBasePath) + .get('/repository/npm/@types/react/-/react-14.9.41.tgz') + .once() + .reply(200, 'foo') + ] + const assetsMockNotToBeDownloaded = [ + nock(assetsBasePath) + .get('/repository/npm/@types/react/-/react-13.9.41.tgz') + .once() + .reply(200, 'foo'), + nock(assetsBasePath) + .get('/repository/npm/@types/react/-/react-12.9.41.tgz') + .once() + .reply(200, 'foo'), + ] + const getAssetsMock = nock(basePath) + .get(assetsSearchUri) + .query(params) + .once() + .reply(200, { items: page }) + + await helper.downloadPackageAssets({ ...params, range: '>14.0.0' }, tempDirPath) + + expect(getAssetsMock.isDone()).toEqual(true) + expect(assetsMockToBeDownloaded.every(mock => mock.isDone())).toEqual(true) + expect(assetsMockNotToBeDownloaded.every(mock => mock.isDone() === false)).toEqual(true) + }) + it('returns list of settled promises and creates files', async () => { const params = { group: 'types', @@ -217,7 +262,7 @@ describe('NexusContentsHelper', () => { .once() .reply(200, 'foo'), ] - // package asset mockы + // package asset mocks const getAssetsMock = [ nock(basePath) .get(assetsSearchUri) diff --git a/yarn.lock b/yarn.lock index 8be8ed1..bdc8ffb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1660,7 +1660,7 @@ "@qiwi/npm-batch-cli-api@^1.5.1": version "1.5.2" resolved "https://registry.yarnpkg.com/@qiwi/npm-batch-cli-api/-/npm-batch-cli-api-1.5.2.tgz#b629276bb9ea8ee420690e575775634154bab974" - integrity "sha1-tikna7nqjuQgaQ5XV3VjQVS6uXQ= sha512-F/a8JKdiHZhbjXClLTlTvwc8e0OtkqHzM9zf6CvovxTUCDZyYd6K1K5RjBNvnXei7vZ045FU5WF0udJ/csX64A==" + integrity sha512-F/a8JKdiHZhbjXClLTlTvwc8e0OtkqHzM9zf6CvovxTUCDZyYd6K1K5RjBNvnXei7vZ045FU5WF0udJ/csX64A== dependencies: "@qiwi/deep-proxy" "^1.8.1" "@qiwi/npm-batch-client" "1.5.1" @@ -2087,9 +2087,9 @@ ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4: uri-js "^4.2.2" ajv@^7.0.2: - version "7.2.1" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-7.2.1.tgz#a5ac226171912447683524fa2f1248fcf8bac83d" - integrity sha512-+nu0HDv7kNSOua9apAVc979qd932rrZeb3WOvoiD31A/p1mIE5/9bN2027pE2rOPYEdS3UHzsvof4hY+lM9/WQ== + version "7.2.3" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-7.2.3.tgz#ca78d1cf458d7d36d1c3fa0794dd143406db5772" + integrity sha512-idv5WZvKVXDqKralOImQgPM9v6WOdLNa0IY3B3doOjw/YxRGT8I+allIJ6kd7Uaj+SF1xZUSU+nPM5aDNBVtnw== dependencies: fast-deep-equal "^3.1.1" json-schema-traverse "^1.0.0" @@ -2594,9 +2594,9 @@ camelcase@^6.0.0: integrity sha512-WCMml9ivU60+8rEJgELlFp1gxFcEGxwYleE3bziHEDeqsqAWGHdimB7beBFGjLzVNgPGyDsfgXLQEYMpmIFnVQ== caniuse-lite@^1.0.30001181: - version "1.0.30001203" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001203.tgz#a7a34df21a387d9deffcd56c000b8cf5ab540580" - integrity sha512-/I9tvnzU/PHMH7wBPrfDMSuecDeUKerjCPX7D0xBbaJZPxoT9m+yYxt0zCTkcijCkjTdim3H56Zm0i5Adxch4w== + version "1.0.30001204" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001204.tgz#256c85709a348ec4d175e847a3b515c66e79f2aa" + integrity sha512-JUdjWpcxfJ9IPamy2f5JaRDCaqJOxDzOSKtbdx4rH9VivMd1vIzoPumsJa9LoMIi4Fx2BV2KZOxWhNkBjaYivQ== capture-exit@^2.0.0: version "2.0.0" @@ -2657,6 +2657,11 @@ ci-info@^2.0.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== +ci-info@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.1.1.tgz#9a32fcefdf7bcdb6f0a7e1c0f8098ec57897b80a" + integrity sha512-kdRWLBIJwdsYJWYJFtAFFYxybguqeF91qpZaggjG5Nf8QKdizFG2hjqvaTXbxFIcYbSaD74KpAXv6BSm17DHEQ== + cjs-module-lexer@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-0.6.0.tgz#4186fcca0eae175970aee870b9fe2d6cf8d5655f" @@ -2818,9 +2823,9 @@ commander@^6.1.0: integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== commander@^7.1.0: - version "7.1.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-7.1.0.tgz#f2eaecf131f10e36e07d894698226e36ae0eb5ff" - integrity sha512-pRxBna3MJe6HKnBGsDyMv8ETbptw3axEdYHoqNh7gu5oDcew8fs0xnivZGm06Ogk8zGAJ9VX+OPEr2GXEQK4dg== + version "7.2.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" + integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== commondir@^1.0.1: version "1.0.1" @@ -3284,9 +3289,9 @@ ecc-jsbn@~0.1.1: safer-buffer "^2.1.0" electron-to-chromium@^1.3.649: - version "1.3.692" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.692.tgz#4d00479055a7282cdd1b19caec09ed7779529640" - integrity sha512-Ix+zDUAXWZuUzqKdhkgN5dP7ZM+IwMG4yAGFGDLpGJP/3vNEEwuHG1LIhtXUfW0FFV0j38t5PUv2n/3MFSRviQ== + version "1.3.693" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.693.tgz#5089c506a925c31f93fcb173a003a22e341115dd" + integrity sha512-vUdsE8yyeu30RecppQtI+XTz2++LWLVEIYmzeCaCRLSdtKZ2eXqdJcrs85KwLiPOPVc6PELgWyXBsfqIvzGZag== emittery@^0.7.1: version "0.7.2" @@ -3440,9 +3445,9 @@ eslint-config-prettier@^8.1.0: integrity sha512-oKMhGv3ihGbCIimCAjqkdzx2Q+jthoqnXSP+d86M9tptwugycmTFdVR4IpLgq2c4SHifbwO90z2fQ8/Aio73yw== eslint-config-qiwi@^1.10.7: - version "1.10.7" - resolved "https://registry.yarnpkg.com/eslint-config-qiwi/-/eslint-config-qiwi-1.10.7.tgz#12957d5ed09c8a3ec5f89187fbd6a5e1255b7400" - integrity sha512-cKks7wtFwi7SkRhDerGgK5ge7GYSJn5shd2BYXm+Di3iu7eOhWNl+2cdIKPpxhRPKSrPXcnkQQ+fAAmzGTNKLA== + version "1.10.8" + resolved "https://registry.yarnpkg.com/eslint-config-qiwi/-/eslint-config-qiwi-1.10.8.tgz#4761ae0d668c04503bdd1523f0b702627c57fe6c" + integrity sha512-kKqqWdzdUxvLXbSrVVDWk1vRSsMldp4hCs4eVEmtuRjG9JiRb0cdBUg5282b4OWqPeVDtx7dILLqB7h5FkFjNQ== dependencies: "@typescript-eslint/eslint-plugin" "^4.15.1" "@typescript-eslint/parser" "^4.15.1" @@ -3457,7 +3462,7 @@ eslint-config-qiwi@^1.10.7: eslint-plugin-simple-import-sort "^7.0.0" eslint-plugin-sonarjs "^0.6.0" eslint-plugin-standard "^5.0.0" - eslint-plugin-unicorn "^28.0.2" + eslint-plugin-unicorn "^29.0.0" eslint-config-standard@^16.0.2: version "16.0.2" @@ -3573,21 +3578,21 @@ eslint-plugin-standard@^5.0.0: resolved "https://registry.yarnpkg.com/eslint-plugin-standard/-/eslint-plugin-standard-5.0.0.tgz#c43f6925d669f177db46f095ea30be95476b1ee4" integrity sha512-eSIXPc9wBM4BrniMzJRBm2uoVuXz2EPa+NXPk2+itrVt+r5SbKFERx/IgrK/HmfjddyKVz2f+j+7gBRvu19xLg== -eslint-plugin-unicorn@^28.0.2: - version "28.0.2" - resolved "https://registry.yarnpkg.com/eslint-plugin-unicorn/-/eslint-plugin-unicorn-28.0.2.tgz#ab9884ebae04590ecd9c1c294330d889a74b7c37" - integrity sha512-k4AoFP7n8/oq6lBXkdc9Flid6vw2B8j7aXFCxgzJCyKvmaKrCUFb1TFPhG9eSJQFZowqmymMPRtl8oo9NKLUbw== +eslint-plugin-unicorn@^29.0.0: + version "29.0.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-unicorn/-/eslint-plugin-unicorn-29.0.0.tgz#7c97cdb5afe932e9f8dc34108e4a5a2a2fbb1906" + integrity sha512-R9jGLKb2p6LuOixviByGlH2mkfY72EBELXAPeUufveebN0M2Woa7B7dUO3gN2xPn/+eGjrIm4I2u7dDtr9G4iA== dependencies: - ci-info "^2.0.0" + ci-info "^3.1.1" clean-regexp "^1.0.0" - eslint-template-visitor "^2.2.2" + eslint-template-visitor "^2.3.2" eslint-utils "^2.1.0" eslint-visitor-keys "^2.0.0" import-modules "^2.1.0" lodash "^4.17.20" pluralize "^8.0.0" read-pkg-up "^7.0.1" - regexp-tree "^0.1.22" + regexp-tree "^0.1.23" reserved-words "^0.1.2" safe-regex "^2.1.1" semver "^7.3.4" @@ -3616,7 +3621,7 @@ eslint-scope@^5.0.0, eslint-scope@^5.1.1: esrecurse "^4.3.0" estraverse "^4.1.1" -eslint-template-visitor@^2.2.2: +eslint-template-visitor@^2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/eslint-template-visitor/-/eslint-template-visitor-2.3.2.tgz#b52f96ff311e773a345d79053ccc78275bbc463d" integrity sha512-3ydhqFpuV7x1M9EK52BPNj6V0Kwu0KKkcIAfpUhwHbR8ocRln/oUHgfxQupY8O1h4Qv/POHDumb/BwwNfxbtnA== @@ -7238,7 +7243,7 @@ regex-not@^1.0.0, regex-not@^1.0.2: extend-shallow "^3.0.2" safe-regex "^1.1.0" -regexp-tree@^0.1.20, regexp-tree@^0.1.22, regexp-tree@~0.1.1: +regexp-tree@^0.1.20, regexp-tree@^0.1.23, regexp-tree@~0.1.1: version "0.1.23" resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.23.tgz#8a8ce1cc5e971acef62213a7ecdb1f6e18a1f1b2" integrity sha512-+7HWfb4Bvu8Rs2eQTUIpX9I/PlQkYOuTNbRpKLJlQpSgwSkzFYh+pUj0gtvglnOZLKB6YgnIgRuJ2/IlpL48qw== @@ -8352,9 +8357,9 @@ typedoc-default-themes@^0.12.9: integrity sha512-Jd5fYTiqzinZdoIY382W7tQXTwAzWRdg8KbHfaxmb78m1/3jL9riXtk23oBOKwhi8GFVykCOdPzEJKY87/D0LQ== typedoc@^0.20.32: - version "0.20.32" - resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.20.32.tgz#d06fc4cce12dca66797220bbd289bac9f4481b5b" - integrity sha512-GSopd/tiqoKE3fEdvhoaEpR9yrEPsR9tknAjkoeSPL6p1Rq5aVsKxBhhF6cwoDJ7oWjpvnm8vs0rQN0BxEHuWQ== + version "0.20.33" + resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.20.33.tgz#dd9591afc2a4b944bd167eeee3425380daf9047d" + integrity sha512-jzNdHmjZRQKwguhpXjIPuIjz+TpdgG2AVY8ta+qpAukv+3rBhTs4AAVd+mkonrHVYlC0EAbuAJ4urkfnn42Hwg== dependencies: colors "^1.4.0" fs-extra "^9.1.0" @@ -8396,9 +8401,9 @@ typescript@4.2.3, typescript@^4.0.2: integrity sha512-qOcYwxaByStAWrBf4x0fibwZvMRG+r4cQoTjbPtUlrWjBHbmCAww1i448U0GJ+3cNNEtebDteo/cHOR3xJ4wEw== uglify-js@^3.1.4: - version "3.13.1" - resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.13.1.tgz#2749d4b8b5b7d67460b4a418023ff73c3fefa60a" - integrity sha512-EWhx3fHy3M9JbaeTnO+rEqzCe1wtyQClv6q3YWq0voOj4E+bMZBErVS1GAHPDiRGONYq34M1/d8KuQMgvi6Gjw== + version "3.13.2" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.13.2.tgz#fe10319861bccc8682bfe2e8151fbdd8aa921c44" + integrity sha512-SbMu4D2Vo95LMC/MetNaso1194M1htEA+JrqE9Hk+G2DhI+itfS9TRu9ZKeCahLDNa/J3n4MqUJ/fOHMzQpRWw== uid-number@0.0.6: version "0.0.6"