From fa1dc36fdff9151652671b86c091c74039bd6671 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 29 Sep 2023 12:45:19 -0500 Subject: [PATCH 1/9] ci: try windows perf-scale nuts differences --- .github/workflows/perfScaleNut.yml | 13 ++++++++++--- .github/workflows/test.yml | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/perfScaleNut.yml b/.github/workflows/perfScaleNut.yml index 7ac0b31f6f..2a088c518d 100644 --- a/.github/workflows/perfScaleNut.yml +++ b/.github/workflows/perfScaleNut.yml @@ -13,16 +13,23 @@ jobs: perf-scale-nuts: strategy: matrix: - os: [ubuntu-latest, windows-latest] + # os: [ubuntu-latest, windows-latest] + os: [windows-latest] fail-fast: false runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: + node-version: lts/* cache: yarn - - run: npm install -g @salesforce/cli --omit=dev - - run: yarn install + - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd + name: add CLI as global dependency + with: + max_attempts: ${{ inputs.retries }} + command: npm install @salesforce/cli@nightly -g + timeout_minutes: 60 + - uses: salesforcecli/github-workflows/.github/actions/yarnInstallWithRetries@main - run: yarn test:nuts:scale # Run `github-action-benchmark` action diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2b8a51d035..3f2dcb5007 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,8 @@ name: tests on: - push: - branches-ignore: [main] + # push: + # branches-ignore: [main] workflow_dispatch: jobs: From 5bb2b4b88724789679c453a34978d814e77efa04 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 29 Sep 2023 12:48:41 -0500 Subject: [PATCH 2/9] ci: retries, no eda --- .github/workflows/perfScaleNut.yml | 1 - package.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/perfScaleNut.yml b/.github/workflows/perfScaleNut.yml index 2a088c518d..9d540a9a16 100644 --- a/.github/workflows/perfScaleNut.yml +++ b/.github/workflows/perfScaleNut.yml @@ -26,7 +26,6 @@ jobs: - uses: nick-fields/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd name: add CLI as global dependency with: - max_attempts: ${{ inputs.retries }} command: npm install @salesforce/cli@nightly -g timeout_minutes: 60 - uses: salesforcecli/github-workflows/.github/actions/yarnInstallWithRetries@main diff --git a/package.json b/package.json index b715cfb62b..5e62616c11 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "repl": "node --inspect ./scripts/repl.js", "test": "wireit", "test:nuts": "mocha \"test/nuts/local/**/*.nut.ts\" --timeout 500000", - "test:nuts:scale": "mocha \"test/nuts/scale/eda.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClasses.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClassesOneDir.nut.ts\" --timeout 500000", + "test:nuts:scale": "mocha \"test/nuts/scale/lotsOfClasses.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClassesOneDir.nut.ts\" --timeout 500000", "test:nuts:suggest": "mocha \"test/nuts/suggestType/suggestType.nut.ts\" --timeout 10000", "test:only": "wireit", "test:registry": "mocha ./test/registry/registryCompleteness.test.ts --timeout 50000", From f0077c41d03047f77048c01dd5bb6dece4789ba4 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 29 Sep 2023 13:03:04 -0500 Subject: [PATCH 3/9] ci: set session dir --- .github/workflows/perfScaleNut.yml | 3 +-- package.json | 2 +- test/nuts/scale/lotsOfClasses.nut.ts | 1 + test/nuts/scale/lotsOfClassesOneDir.nut.ts | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/perfScaleNut.yml b/.github/workflows/perfScaleNut.yml index 9d540a9a16..41e61c9540 100644 --- a/.github/workflows/perfScaleNut.yml +++ b/.github/workflows/perfScaleNut.yml @@ -13,8 +13,7 @@ jobs: perf-scale-nuts: strategy: matrix: - # os: [ubuntu-latest, windows-latest] - os: [windows-latest] + os: [ubuntu-latest, windows-latest] fail-fast: false runs-on: ${{ matrix.os }} steps: diff --git a/package.json b/package.json index 5e62616c11..b715cfb62b 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "repl": "node --inspect ./scripts/repl.js", "test": "wireit", "test:nuts": "mocha \"test/nuts/local/**/*.nut.ts\" --timeout 500000", - "test:nuts:scale": "mocha \"test/nuts/scale/lotsOfClasses.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClassesOneDir.nut.ts\" --timeout 500000", + "test:nuts:scale": "mocha \"test/nuts/scale/eda.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClasses.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClassesOneDir.nut.ts\" --timeout 500000", "test:nuts:suggest": "mocha \"test/nuts/suggestType/suggestType.nut.ts\" --timeout 10000", "test:only": "wireit", "test:registry": "mocha ./test/registry/registryCompleteness.test.ts --timeout 50000", diff --git a/test/nuts/scale/lotsOfClasses.nut.ts b/test/nuts/scale/lotsOfClasses.nut.ts index 19f02bd8b0..6c91c4d634 100644 --- a/test/nuts/scale/lotsOfClasses.nut.ts +++ b/test/nuts/scale/lotsOfClasses.nut.ts @@ -28,6 +28,7 @@ describe(`handles ${classCount.toLocaleString()} classes (${( before(async () => { session = await TestSession.create({ + sessionDir: 'lotsOfClasses', project: { name: 'lotsOfClasses', }, diff --git a/test/nuts/scale/lotsOfClassesOneDir.nut.ts b/test/nuts/scale/lotsOfClassesOneDir.nut.ts index 2060795d84..67528c7321 100644 --- a/test/nuts/scale/lotsOfClassesOneDir.nut.ts +++ b/test/nuts/scale/lotsOfClassesOneDir.nut.ts @@ -28,8 +28,9 @@ describe(`handles ${classCount.toLocaleString()} classes (${( before(async () => { session = await TestSession.create({ + sessionDir: 'lotsOfClassesOneDir', project: { - name: 'lotsOfClasses', + name: 'lotsOfClassesOneDir', }, devhubAuthStrategy: 'NONE', }); From 1f82c57d59003052ed9b3db0105d769696774a34 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 29 Sep 2023 13:14:14 -0500 Subject: [PATCH 4/9] ci: undo sessionDir sets --- package.json | 4 ++-- test/nuts/scale/lotsOfClasses.nut.ts | 1 - test/nuts/scale/lotsOfClassesOneDir.nut.ts | 1 - yarn.lock | 18 +++++++++--------- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index b715cfb62b..f3d3156a52 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "node": ">=16.0.0" }, "dependencies": { - "@salesforce/core": "^5.2.10", + "@salesforce/core": "^5.3.1", "@salesforce/kit": "^3.0.11", "@salesforce/ts-types": "^2.0.7", "fast-levenshtein": "^3.0.0", @@ -40,7 +40,7 @@ "unzipper": "0.10.14" }, "devDependencies": { - "@salesforce/cli-plugins-testkit": "^4.3.6", + "@salesforce/cli-plugins-testkit": "^4.4.2", "@salesforce/dev-config": "^4.0.1", "@salesforce/dev-scripts": "^5.10.0", "@salesforce/prettier-config": "^0.0.3", diff --git a/test/nuts/scale/lotsOfClasses.nut.ts b/test/nuts/scale/lotsOfClasses.nut.ts index 6c91c4d634..19f02bd8b0 100644 --- a/test/nuts/scale/lotsOfClasses.nut.ts +++ b/test/nuts/scale/lotsOfClasses.nut.ts @@ -28,7 +28,6 @@ describe(`handles ${classCount.toLocaleString()} classes (${( before(async () => { session = await TestSession.create({ - sessionDir: 'lotsOfClasses', project: { name: 'lotsOfClasses', }, diff --git a/test/nuts/scale/lotsOfClassesOneDir.nut.ts b/test/nuts/scale/lotsOfClassesOneDir.nut.ts index 67528c7321..775b10f296 100644 --- a/test/nuts/scale/lotsOfClassesOneDir.nut.ts +++ b/test/nuts/scale/lotsOfClassesOneDir.nut.ts @@ -28,7 +28,6 @@ describe(`handles ${classCount.toLocaleString()} classes (${( before(async () => { session = await TestSession.create({ - sessionDir: 'lotsOfClassesOneDir', project: { name: 'lotsOfClassesOneDir', }, diff --git a/yarn.lock b/yarn.lock index 0c2e636014..4eb27ce549 100644 --- a/yarn.lock +++ b/yarn.lock @@ -535,12 +535,12 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@salesforce/cli-plugins-testkit@^4.3.6": - version "4.3.6" - resolved "https://registry.yarnpkg.com/@salesforce/cli-plugins-testkit/-/cli-plugins-testkit-4.3.6.tgz#7d2258d34b1033af81e9531bd4613d2683327eb1" - integrity sha512-gcwXoMA6PtxzXL/Gd5XxlCpiV6Mbb8MehdAWWlmTb7ExfElHqvSPAwZEbim32ACcfJeqVVK0Y1CbRdbmNMNUYw== +"@salesforce/cli-plugins-testkit@^4.4.2": + version "4.4.2" + resolved "https://registry.yarnpkg.com/@salesforce/cli-plugins-testkit/-/cli-plugins-testkit-4.4.2.tgz#5b3a70906a032698303c220d768dec3608cfff9d" + integrity sha512-1KRhBKAT+3aAChlhvFenmfgsrhphpA9bzvg/J/gAB3O113iwLCuH5ipzo1SU+fR3wWogIPubOWX0U7JNoHDkIw== dependencies: - "@salesforce/core" "^5.2.9" + "@salesforce/core" "^5.2.10" "@salesforce/kit" "^3.0.11" "@salesforce/ts-types" "^2.0.6" "@types/shelljs" "^0.8.12" @@ -550,10 +550,10 @@ strip-ansi "6.0.1" ts-retry-promise "^0.7.1" -"@salesforce/core@^5.2.10", "@salesforce/core@^5.2.7", "@salesforce/core@^5.2.9": - version "5.2.10" - resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-5.2.10.tgz#7f4824ddcef108b6ccec24434260c620de39012e" - integrity sha512-Xj1QRajmHWgl0ahivjKFGKJlGXwe9yFOZ3PwF91qEupGbO74XrCJ8OUM7EVlk53LKy9LlPZQFuy2ATX9MyEDKA== +"@salesforce/core@^5.2.10", "@salesforce/core@^5.2.7", "@salesforce/core@^5.3.1": + version "5.3.1" + resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-5.3.1.tgz#93c49760faa7b503bbfea8376d80a66815065e4a" + integrity sha512-LEQ3Yp7GxexsGXxT85Pp4AgOzzqY9Y3kol8ByVsHpHGvs8XLRbKl44IPxTXBdMB2roFM+Iosd46i0KbTuh2MQg== dependencies: "@salesforce/kit" "^3.0.11" "@salesforce/schemas" "^1.6.0" From cd59159a6148d199ebab5e93e514d1b1ffed410b Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 29 Sep 2023 13:31:31 -0500 Subject: [PATCH 5/9] ci: sleep before cleaning sessionDir --- test/nuts/scale/eda.nut.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/nuts/scale/eda.nut.ts b/test/nuts/scale/eda.nut.ts index 28559a02b1..f5dcdb430c 100644 --- a/test/nuts/scale/eda.nut.ts +++ b/test/nuts/scale/eda.nut.ts @@ -8,6 +8,7 @@ import * as path from 'path'; import { performance } from 'node:perf_hooks'; import { TestSession } from '@salesforce/cli-plugins-testkit'; +import { sleep } from '@salesforce/kit'; import { MetadataResolver } from '../../../src'; import { MetadataConverter } from '../../../src'; import { ComponentSetBuilder } from '../../../src'; @@ -29,6 +30,7 @@ describe('tests using EDA', () => { after(async () => { await recordPerf(testName, performance); + await sleep(10000); await session?.clean(); }); From 8a91199cab4226e79ab79f35c45382069d69e6de Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 29 Sep 2023 13:33:21 -0500 Subject: [PATCH 6/9] ci: fatal log-level --- .github/workflows/perfScaleNut.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/perfScaleNut.yml b/.github/workflows/perfScaleNut.yml index 41e61c9540..d93019b6c0 100644 --- a/.github/workflows/perfScaleNut.yml +++ b/.github/workflows/perfScaleNut.yml @@ -29,6 +29,8 @@ jobs: timeout_minutes: 60 - uses: salesforcecli/github-workflows/.github/actions/yarnInstallWithRetries@main - run: yarn test:nuts:scale + env: + SF_LOG_LEVEL: fatal # Run `github-action-benchmark` action - name: Store benchmark result From 5686ba1f32daa145f7fb4b66ac8b626abd9c5a8a Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 29 Sep 2023 14:09:54 -0500 Subject: [PATCH 7/9] ci: 3 separate run steps --- .github/workflows/perfScaleNut.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/perfScaleNut.yml b/.github/workflows/perfScaleNut.yml index d93019b6c0..738704a628 100644 --- a/.github/workflows/perfScaleNut.yml +++ b/.github/workflows/perfScaleNut.yml @@ -28,9 +28,10 @@ jobs: command: npm install @salesforce/cli@nightly -g timeout_minutes: 60 - uses: salesforcecli/github-workflows/.github/actions/yarnInstallWithRetries@main - - run: yarn test:nuts:scale - env: - SF_LOG_LEVEL: fatal + - run: | + yarn mocha \"test/nuts/scale/eda.nut.ts\ --timeout 500000 + yarn mocha \"test/nuts/scale/lotsOfClasses.nut.ts\ --timeout 500000 + yarn mocha \"test/nuts/scale/lotsOfClassesOneDir.nut.ts\ --timeout 500000 # Run `github-action-benchmark` action - name: Store benchmark result From ffaf2cc71db9ba031a5adfed9f0f386b07df3a12 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 29 Sep 2023 14:11:45 -0500 Subject: [PATCH 8/9] ci: no quotes --- .github/workflows/perfScaleNut.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/perfScaleNut.yml b/.github/workflows/perfScaleNut.yml index 738704a628..bf9d70deb2 100644 --- a/.github/workflows/perfScaleNut.yml +++ b/.github/workflows/perfScaleNut.yml @@ -29,9 +29,9 @@ jobs: timeout_minutes: 60 - uses: salesforcecli/github-workflows/.github/actions/yarnInstallWithRetries@main - run: | - yarn mocha \"test/nuts/scale/eda.nut.ts\ --timeout 500000 - yarn mocha \"test/nuts/scale/lotsOfClasses.nut.ts\ --timeout 500000 - yarn mocha \"test/nuts/scale/lotsOfClassesOneDir.nut.ts\ --timeout 500000 + yarn mocha test/nuts/scale/eda.nut.ts --timeout 500000 + yarn mocha test/nuts/scale/lotsOfClasses.nut.ts --timeout 500000 + yarn mocha test/nuts/scale/lotsOfClassesOneDir.nut.ts --timeout 500000 # Run `github-action-benchmark` action - name: Store benchmark result From 7185c05cbb4161b78e3b11e6c8e4a32082ec0dd4 Mon Sep 17 00:00:00 2001 From: mshanemc Date: Fri, 29 Sep 2023 14:29:00 -0500 Subject: [PATCH 9/9] chore: restore unneeded changes --- .github/workflows/test.yml | 4 ++-- package.json | 1 - test/nuts/scale/eda.nut.ts | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f2dcb5007..2b8a51d035 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,8 +2,8 @@ name: tests on: - # push: - # branches-ignore: [main] + push: + branches-ignore: [main] workflow_dispatch: jobs: diff --git a/package.json b/package.json index f3d3156a52..9cd2033251 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,6 @@ "repl": "node --inspect ./scripts/repl.js", "test": "wireit", "test:nuts": "mocha \"test/nuts/local/**/*.nut.ts\" --timeout 500000", - "test:nuts:scale": "mocha \"test/nuts/scale/eda.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClasses.nut.ts\" --timeout 500000; mocha \"test/nuts/scale/lotsOfClassesOneDir.nut.ts\" --timeout 500000", "test:nuts:suggest": "mocha \"test/nuts/suggestType/suggestType.nut.ts\" --timeout 10000", "test:only": "wireit", "test:registry": "mocha ./test/registry/registryCompleteness.test.ts --timeout 50000", diff --git a/test/nuts/scale/eda.nut.ts b/test/nuts/scale/eda.nut.ts index f5dcdb430c..28559a02b1 100644 --- a/test/nuts/scale/eda.nut.ts +++ b/test/nuts/scale/eda.nut.ts @@ -8,7 +8,6 @@ import * as path from 'path'; import { performance } from 'node:perf_hooks'; import { TestSession } from '@salesforce/cli-plugins-testkit'; -import { sleep } from '@salesforce/kit'; import { MetadataResolver } from '../../../src'; import { MetadataConverter } from '../../../src'; import { ComponentSetBuilder } from '../../../src'; @@ -30,7 +29,6 @@ describe('tests using EDA', () => { after(async () => { await recordPerf(testName, performance); - await sleep(10000); await session?.clean(); });