Skip to content

Commit

Permalink
Merge branch 'main' into update-transfer-type-index
Browse files Browse the repository at this point in the history
  • Loading branch information
Romsters authored Mar 11, 2024
2 parents f5906cd + 39336aa commit 0c11081
Show file tree
Hide file tree
Showing 7 changed files with 119 additions and 143 deletions.
8 changes: 5 additions & 3 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion packages/data-fetcher/.eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ module.exports = {
node: true,
jest: true,
},
ignorePatterns: ['.eslintrc.js', 'migrationScripts'],
ignorePatterns: ['.eslintrc.js'],
rules: {
'@typescript-eslint/interface-name-prefix': 'off',
'@typescript-eslint/explicit-function-return-type': 'off',
Expand Down
2 changes: 1 addition & 1 deletion packages/worker/.eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ module.exports = {
node: true,
jest: true,
},
ignorePatterns: ['.eslintrc.js', 'migrationScripts'],
ignorePatterns: ['.eslintrc.js'],
rules: {
'@typescript-eslint/interface-name-prefix': 'off',
'@typescript-eslint/explicit-function-return-type': 'off',
Expand Down
78 changes: 0 additions & 78 deletions packages/worker/migrationScripts/checkAddressBalances.js

This file was deleted.

58 changes: 0 additions & 58 deletions packages/worker/migrationScripts/updateTransactions.js

This file was deleted.

8 changes: 6 additions & 2 deletions packages/worker/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
"migration:generate": "npm run typeorm migration:generate ./src/migrations/$npm_config_name -- -d ./src/typeorm.config.ts",
"migration:create": "npm run typeorm migration:create ./src/migrations/$npm_config_name",
"migration:run": "npm run typeorm migration:run -- -d ./src/typeorm.config.ts",
"migration:revert": "npm run typeorm migration:revert -- -d ./src/typeorm.config.ts"
"migration:revert": "npm run typeorm migration:revert -- -d ./src/typeorm.config.ts",
"migration-script:run": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register ./src/migrationScripts/{migrationFileName}.ts --runInBand"
},
"dependencies": {
"@nestjs/axios": "^3.0.0",
Expand All @@ -50,6 +51,7 @@
"rxjs": "^7.2.0",
"typeorm": "^0.3.15",
"winston": "^3.8.2",
"yargs": "^17.7.2",
"zksync-web3": "0.15.4"
},
"devDependencies": {
Expand All @@ -59,6 +61,7 @@
"@types/express": "^4.17.13",
"@types/jest": "28.1.8",
"@types/supertest": "^2.0.11",
"@types/yargs": "^17.0.32",
"@typescript-eslint/eslint-plugin": "^5.0.0",
"@typescript-eslint/parser": "^5.0.0",
"eslint-config-prettier": "^8.3.0",
Expand Down Expand Up @@ -105,7 +108,8 @@
".module.ts",
"src/logger.ts",
"src/typeorm.config.ts",
"src/migrations"
"src/migrations",
"src/migrationScripts"
],
"reporters": [
"default",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
import { config } from "dotenv";
import { DataSource } from "typeorm";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import { setTimeout } from "timers/promises";
import { typeOrmModuleOptions } from "../typeorm.config";
import logger from "../logger";

config();

const QUERY_MAX_RETRIES = 5;
const QUERY_RETRY_MIN_INTERVAL_MS = 1000;

// eslint-disable-next-line prefer-const
let { fromTransferNumber, toTransferNumber, updateBatchSize, parallelWorkers } = yargs(hideBin(process.argv))
.options({
fromTransferNumber: {
default: 0,
type: "number",
},
toTransferNumber: {
default: 0,
type: "number",
},
updateBatchSize: {
default: 4000,
type: "number",
},
parallelWorkers: {
default: 50,
type: "number",
},
})
.parseSync();

const updateAddressTransfers = async (dataSource: DataSource, from: number, to: number, attempt = 0): Promise<void> => {
try {
await dataSource.query(
`Update "addressTransfers"
Set "type" = "transfers".type::VARCHAR::"addressTransfers_type_enum"
From "transfers"
WHERE "transfers"."number" = "addressTransfers"."transferNumber"
AND "transfers"."number" >= ${from}
AND "transfers"."number" < ${to}
AND "transfers"."type" != 'transfer'`
);
} catch (error) {
if (attempt >= QUERY_MAX_RETRIES) {
logger.error(`Failed to update AddressTransfers from ${from} to ${to} after ${QUERY_MAX_RETRIES} retries.`);
throw error;
}
await setTimeout(QUERY_RETRY_MIN_INTERVAL_MS * Math.pow(2, attempt));
logger.error(`Failed to update AddressTransfers from ${from} to ${to}, retrying...`);
return updateAddressTransfers(dataSource, from, to, attempt + 1);
}
};

const main = async () => {
const typeOrmCliDataSource = new DataSource(typeOrmModuleOptions);
await typeOrmCliDataSource.initialize();

if (!toTransferNumber) {
const lastTransferNumber = await typeOrmCliDataSource.query(
`Select "number" from "transfers" order by "number" DESC limit 1;`
);
toTransferNumber = parseInt(lastTransferNumber[0].number, 10);
}
logger.log(
`Starting migration with params: { fromTransferNumber: ${fromTransferNumber}, toTransferNumber: ${toTransferNumber}, updateBatchSize: ${updateBatchSize}, parallelWorkers: ${parallelWorkers} }`
);

let cursor = fromTransferNumber;
while (cursor <= toTransferNumber) {
const tasks = [];
for (let workerIndex = 0; workerIndex < parallelWorkers; workerIndex++) {
const batchStartNumber = cursor + workerIndex * updateBatchSize;
if (batchStartNumber > toTransferNumber) {
break;
}
let batchEndNumber = batchStartNumber + updateBatchSize;
if (batchEndNumber > toTransferNumber) {
batchEndNumber = toTransferNumber + 1;
}
tasks.push(updateAddressTransfers(typeOrmCliDataSource, batchStartNumber, batchEndNumber));
}
await Promise.all(tasks);

logger.log(
`Updated address transfers from ${cursor} to ${
cursor + parallelWorkers * updateBatchSize
}. Time: ${new Date().toJSON()}.`
);
cursor = cursor + parallelWorkers * updateBatchSize;
}
};

main()
.then(() => {
logger.log(`Migration script 1709722093204-AddAddressTransferType executed successfully.`);
process.exit(0);
})
.catch((error) => {
logger.error(`Migration script 1709722093204-AddAddressTransferType failed.`);
logger.error(error);
process.exit(0);
});

0 comments on commit 0c11081

Please sign in to comment.