From 36f3e36a2598d55f09360fb8c84a54705c59ccc1 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Wed, 17 Apr 2024 15:35:39 +0300 Subject: [PATCH 1/6] Increase timeouts for mocha hooks --- test/1-api-sync-mode-sqlite.spec.js | 2 +- test/2-additional-api-sync-mode-sqlite.spec.js | 2 +- test/3-api-filter-sync-mode-sqlite.spec.js | 4 ++-- test/4-sub-account.spec.js | 2 +- test/5-recover-password.spec.js | 2 +- test/6-update-sub-account.spec.js | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/1-api-sync-mode-sqlite.spec.js b/test/1-api-sync-mode-sqlite.spec.js index 555bc7779..44dbf8581 100644 --- a/test/1-api-sync-mode-sqlite.spec.js +++ b/test/1-api-sync-mode-sqlite.spec.js @@ -93,7 +93,7 @@ describe('Sync mode API with SQLite', () => { }) after(async function () { - this.timeout(5000) + this.timeout(20000) await stopEnvironment() await rmDB(dbDirPath) diff --git a/test/2-additional-api-sync-mode-sqlite.spec.js b/test/2-additional-api-sync-mode-sqlite.spec.js index 1b862c12c..163ba67fe 100644 --- a/test/2-additional-api-sync-mode-sqlite.spec.js +++ b/test/2-additional-api-sync-mode-sqlite.spec.js @@ -92,7 +92,7 @@ describe('Additional sync mode API with SQLite', () => { }) after(async function () { - this.timeout(5000) + this.timeout(20000) await stopEnvironment() await rmDB(dbDirPath) diff --git a/test/3-api-filter-sync-mode-sqlite.spec.js b/test/3-api-filter-sync-mode-sqlite.spec.js index e57666328..e6398d24b 100644 --- a/test/3-api-filter-sync-mode-sqlite.spec.js +++ b/test/3-api-filter-sync-mode-sqlite.spec.js @@ -69,7 +69,7 @@ describe('API filter', () => { const auth = { token: '' } before(async function () { - this.timeout(5000) + this.timeout(20000) mockRESTv2Srv = createMockRESTv2SrvWithDate(start, end, 10) @@ -85,7 +85,7 @@ describe('API filter', () => { }) after(async function () { - this.timeout(5000) + this.timeout(20000) await stopEnvironment() await rmDB(dbDirPath) diff --git a/test/4-sub-account.spec.js b/test/4-sub-account.spec.js index b2bb493c2..e32c15cd4 100644 --- a/test/4-sub-account.spec.js +++ b/test/4-sub-account.spec.js @@ -128,7 +128,7 @@ describe('Sub-account', () => { }) after(async function () { - this.timeout(5000) + this.timeout(20000) await stopEnvironment() await rmDB(dbDirPath) diff --git a/test/5-recover-password.spec.js b/test/5-recover-password.spec.js index c8933663d..5a2d50438 100644 --- a/test/5-recover-password.spec.js +++ b/test/5-recover-password.spec.js @@ -127,7 +127,7 @@ describe('Recover password', () => { }) after(async function () { - this.timeout(5000) + this.timeout(20000) await stopEnvironment() await rmDB(dbDirPath) diff --git a/test/6-update-sub-account.spec.js b/test/6-update-sub-account.spec.js index e3ee1440c..c26783441 100644 --- a/test/6-update-sub-account.spec.js +++ b/test/6-update-sub-account.spec.js @@ -144,7 +144,7 @@ describe('Update sub-account', () => { }) after(async function () { - this.timeout(5000) + this.timeout(20000) await stopEnvironment() await rmDB(dbDirPath) From c1388fd08046abc1dc40a5f5d18bfca4033fb65d Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Tue, 30 Apr 2024 09:17:25 +0300 Subject: [PATCH 2/6] Add several retries for the test run before it fails --- .github/workflows/ci.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9b2c6fa2..b9c10c444 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: jobs: linux-test-runner: name: Linux Test Runner - timeout-minutes: 30 + timeout-minutes: 90 runs-on: ubuntu-22.04 steps: - name: Checkout @@ -31,7 +31,14 @@ jobs: - name: Install deps run: npm i - name: Run tests - run: npm test -- --reporter=json --reporter-option output=test-report.json + uses: nick-fields/retry@v3 + continue-on-error: false + with: + timeout_minutes: 20 + retry_wait_seconds: 10 + max_attempts: 3 + retry_on: any + command: npm test -- --reporter=json --reporter-option output=test-report.json - uses: actions/upload-artifact@v4 if: success() || failure() with: From 94d7c501897b97a2d0c2de9021ec6b155a5521e4 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Fri, 3 May 2024 14:36:07 +0300 Subject: [PATCH 3/6] Enhance sub-account ledger balance recalc --- ...ecalc.sub.account.ledgers.balances.hook.js | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js b/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js index 67a8d667c..2ae33ff00 100644 --- a/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js +++ b/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js @@ -82,7 +82,8 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook { ) { return { balance: _nativeBalance, - balanceUsd: _nativeBalanceUsd + balanceUsd: _nativeBalanceUsd, + _isBalanceRecalced: null } } @@ -143,7 +144,8 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook { return { balance, - balanceUsd + balanceUsd, + _isBalanceRecalced: 1 } } @@ -316,11 +318,17 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook { return } + const authIds = [...auth] + .map(([key, auth]) => auth?._id) + .filter((id) => Number.isFinite(id)) + const requiredUserIds = [...new Set(authIds)] + const firstNotRecalcedElem = await this.dao.getElemInCollBy( tableName, { $isNotNull: 'subUserId', - $isNull: '_isBalanceRecalced' + $isNull: '_isBalanceRecalced', + $in: { user_id: requiredUserIds } }, [['mts', 1], ['id', 1]] ) @@ -346,6 +354,7 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook { filter: { $gte: { mts }, $nin: { _id: skipedIds }, + $in: { user_id: requiredUserIds }, $isNotNull: 'subUserId' }, sort: [['mts', 1], ['id', 1]], @@ -377,7 +386,8 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook { const _elem = elem ?? {} const { balance, - balanceUsd + balanceUsd, + _isBalanceRecalced } = await this._getRecalcBalanceAsync( auth, recordsToGetBalances, @@ -387,7 +397,7 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook { recalcElems.push(Object.assign(_elem, { balance, balanceUsd, - _isBalanceRecalced: 1 + _isBalanceRecalced })) } From f605371dc51646028485481296184780dcbf841b Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 6 May 2024 12:37:49 +0300 Subject: [PATCH 4/6] Prevent funding trades sync issue when end less than start --- workers/loc.api/sync/data.inserter/index.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/workers/loc.api/sync/data.inserter/index.js b/workers/loc.api/sync/data.inserter/index.js index b8befa895..a67a04a52 100644 --- a/workers/loc.api/sync/data.inserter/index.js +++ b/workers/loc.api/sync/data.inserter/index.js @@ -552,6 +552,7 @@ class DataInserter extends EventEmitter { let count = 0 let serialRequestsCount = 0 + let serialRequestsCountWithEndLessThanStart = 0 let prevRes = [] while (true) { @@ -617,6 +618,15 @@ class DataInserter extends EventEmitter { isAllData = true } + // Prevent very rare issue: end <= start + if (currIterationArgs.params.start >= currIterationArgs.params.end) { + currIterationArgs.params.end = currIterationArgs.params.start + 1 + serialRequestsCountWithEndLessThanStart += 1 + } + if (serialRequestsCountWithEndLessThanStart > 1) { + isAllData = true + } + const normalizedApiData = normalizeApiData(res, model) /* From 58de9d33d29a68abb533ecfe866b1dec00f378aa Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Wed, 8 May 2024 11:39:16 +0300 Subject: [PATCH 5/6] Update bfx-report-ui sub-module --- bfx-report-ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bfx-report-ui b/bfx-report-ui index 49972f849..b14ccb940 160000 --- a/bfx-report-ui +++ b/bfx-report-ui @@ -1 +1 @@ -Subproject commit 49972f84980a59b9bc9481ca8865959b4ff2abb1 +Subproject commit b14ccb94004977332f8f6838bda0d43c6f0d3914 From 62777f5c08a61233f910683b6d449f6b128865ab Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Wed, 8 May 2024 11:39:57 +0300 Subject: [PATCH 6/6] Bump version up to 4.17.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7076cacef..62b46ba7d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bfx-reports-framework", - "version": "4.17.1", + "version": "4.17.2", "description": "Bitfinex reports framework", "main": "worker.js", "license": "Apache-2.0",