From 7c8cab52f2945a2289f50eb350889b00f4570b91 Mon Sep 17 00:00:00 2001 From: Vasyl Ivanchuk Date: Thu, 18 Jan 2024 14:46:31 +0200 Subject: [PATCH] feat: add data fetcher service (#145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What ❔ Add data fetcher service that can be vertically scaled. ## Why ❔ Having data fetching vertically scaled we will be able to keep up with higher TPS. ## Checklist - [X] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [X] Tests for the changes have been added / updated. --- .github/workflows/release.yml | 14 + .github/workflows/validate-pr.yml | 1 + .gitignore | 1 + .vscode/settings.json | 3 + README.md | 21 +- docker-compose-cli.yaml | 35 +- docker-compose.yaml | 28 +- package-lock.json | 203 ++ .../app/src/components/NetworkDeprecated.vue | 3 - packages/data-fetcher/.env.example | 13 + packages/data-fetcher/.eslintignore | 1 + packages/data-fetcher/.eslintrc.js | 25 + packages/data-fetcher/Dockerfile | 43 + packages/data-fetcher/README.md | 52 + packages/data-fetcher/nest-cli.json | 5 + packages/data-fetcher/package.json | 113 + packages/data-fetcher/src/abis/erc721.json | 333 +++ .../src/abis/l2StandardERC20.json | 64 + .../src/abis/transferEventWithNoIndexes.json | 0 .../src/address/address.service.spec.ts | 58 +- .../src/address/address.service.ts | 34 +- .../default.handler.spec.ts | 0 .../default.handler.ts | 0 .../extractContractDeployedHandlers/index.ts | 0 .../interface/contractAddress.interface.ts | 1 + ...extractContractAddressHandler.interface.ts | 0 packages/data-fetcher/src/app.module.ts | 38 + .../src/balance/balance.service.spec.ts | 534 ++++ .../src/balance/balance.service.ts | 122 + packages/data-fetcher/src/balance/index.ts | 1 + .../src/block/block.controller.spec.ts | 89 + .../src/block/block.controller.ts | 37 + .../src/block/block.service.spec.ts | 315 +++ .../data-fetcher/src/block/block.service.ts | 102 + packages/data-fetcher/src/block/index.ts | 2 + .../src/blockchain/blockchain.service.spec.ts | 2384 +++++++++++++++++ .../src/blockchain/blockchain.service.ts | 193 ++ packages/data-fetcher/src/blockchain/index.ts | 1 + .../src/blockchain/retryableContract.spec.ts | 267 ++ .../src/blockchain/retryableContract.ts | 124 + .../responseTransform.interceptor.spec.ts | 69 + .../responseTransform.interceptor.ts | 28 + .../common/pipes/parseLimitedInt.pipe.spec.ts | 84 + .../src/common/pipes/parseLimitedInt.pipe.ts | 34 + packages/data-fetcher/src/config.spec.ts | 30 + packages/data-fetcher/src/config.ts | 35 + packages/data-fetcher/src/constants.ts | 36 + .../src/health/health.controller.spec.ts | 84 + .../src/health/health.controller.ts | 26 + .../data-fetcher/src/health/health.module.ts | 11 + .../src/health/jsonRpcProvider.health.spec.ts | 56 + .../src/health/jsonRpcProvider.health.ts | 22 + packages/data-fetcher/src/log/index.ts | 2 + .../data-fetcher/src/log/log.service.spec.ts | 150 ++ packages/data-fetcher/src/log/log.service.ts | 69 + packages/data-fetcher/src/log/logType.spec.ts | 22 + packages/data-fetcher/src/log/logType.ts | 54 + packages/data-fetcher/src/logger.ts | 27 + packages/data-fetcher/src/main.ts | 23 + packages/data-fetcher/src/metrics/index.ts | 2 + .../src/metrics/metrics.module.ts | 8 + .../src/metrics/metrics.provider.ts | 63 + .../data-fetcher/src/rpcProvider/index.ts | 4 + .../src/rpcProvider/jsonRpcProvider.module.ts | 57 + .../src/rpcProvider/jsonRpcProviderBase.ts | 7 + .../jsonRpcProviderExtended.spec.ts | 119 + .../rpcProvider/jsonRpcProviderExtended.ts | 64 + .../rpcProvider/webSocketProviderExtended.ts | 120 + .../rpcProvider/wrappedWebSocketProvider.ts | 56 + .../src/token/token.service.spec.ts | 468 ++++ .../data-fetcher/src/token/token.service.ts | 121 + .../data-fetcher/src/transaction/index.ts | 1 + .../transaction/transaction.service.spec.ts | 202 ++ .../src/transaction/transaction.service.ts | 90 + .../finalizeDeposit/default.handler.spec.ts | 4 +- .../finalizeDeposit/default.handler.ts | 4 +- .../src/transfer/extractHandlers/index.ts | 0 .../mint/ethMintFromL1.handler.spec.ts | 4 +- .../mint/ethMintFromL1.handler.ts | 4 +- .../contractDeployerTransfer.handler.spec.ts | 4 +- .../contractDeployerTransfer.handler.ts | 4 +- .../transfer/default.handler.spec.ts | 4 +- .../transfer/default.handler.ts | 4 +- .../transfer/erc721Transfer.handle.spec.ts | 4 +- .../transfer/erc721Transfer.handler.ts | 4 +- .../ethWithdrawalToL1.handler.spec.ts | 4 +- .../withdrawal/ethWithdrawalToL1.handler.ts | 4 +- .../default.handler.spec.ts | 4 +- .../withdrawalInitiated/default.handler.ts | 4 +- .../extractTransferHandler.interface.ts | 3 +- .../transfer/interfaces/transfer.interface.ts | 4 +- .../src/transfer/transfer.service.spec.ts | 353 ++- .../src/transfer/transfer.service.ts | 28 +- packages/data-fetcher/src/utils/date.spec.ts | 8 + packages/data-fetcher/src/utils/date.ts | 1 + .../src/utils/isInternalTransaction.spec.ts | 2 +- .../src/utils/isInternalTransaction.ts | 2 +- .../data-fetcher/src/utils/parseLog.spec.ts | 397 +++ packages/data-fetcher/src/utils/parseLog.ts | 63 + packages/data-fetcher/temp-response-new.json | 534 ++++ packages/data-fetcher/temp-response-old.json | 636 +++++ packages/data-fetcher/test/app.e2e-spec.ts | 25 + packages/data-fetcher/test/jest-e2e.json | 9 + .../test/logs/block-with-no-txs-logs.json | 18 + .../address-out-of-range.json | 80 + ...-deposit-from-l1-to-different-address.json | 103 + .../erc20/bridge-deposit-from-l1.json | 148 + .../transactionReceipts/erc20/deploy-l2.json | 107 + .../transactionReceipts/erc20/transfer.json | 62 + .../transactionReceipts/erc20/withdrawal.json | 136 + .../eth/deposit-no-fee.json | 83 + .../eth/deposit-to-different-address.json | 103 + .../eth/deposit-zero-value.json | 74 + .../test/transactionReceipts/eth/deposit.json | 103 + .../eth/transfer-to-zero-address.json | 62 + .../transactionReceipts/eth/transfer.json | 62 + .../eth/withdrawal-to-different-address.json | 125 + .../eth/withdrawal-zero-value.json | 106 + .../transactionReceipts/eth/withdrawal.json | 125 + .../erc20-transfer-to-zero-address.json | 47 + .../fee-with-no-deposits.json | 61 + .../greeter/deploy-to-l2.json | 63 + .../greeter/exec-set-greeting.json | 47 + .../log-parsing-error.json | 48 + .../eth-usdc-erc20-through-paymaster.json | 347 +++ .../eth-usdc-erc20-transfer.json | 287 ++ .../multiTransfer/multi-eth-transfer.json | 152 ++ .../sub-calls-to-other-contracts.json | 91 + .../test/transactionReceipts/nft/approve.json | 63 + .../transactionReceipts/nft/deploy-l2.json | 107 + .../test/transactionReceipts/nft/mint.json | 63 + .../transactionReceipts/nft/transfer.json | 63 + .../no-deposit-after-fee.json | 90 + .../no-matching-handlers.json | 91 + .../paymasters/transfer.json | 107 + .../pre-approved-erc20/deposit.json | 91 + .../pre-approved-erc20/transfer.json | 62 + .../withdrawal-to-diff-address.json | 121 + .../pre-approved-erc20/withdrawal.json | 121 + .../transactionReceipts/tx-with-no-logs.json | 45 + packages/data-fetcher/tsconfig.build.json | 4 + packages/data-fetcher/tsconfig.json | 22 + packages/worker/.env.example | 3 +- packages/worker/README.md | 3 +- packages/worker/package.json | 8 +- packages/worker/src/app.module.ts | 8 +- .../src/balance/balance.service.spec.ts | 622 +---- .../worker/src/balance/balance.service.ts | 124 +- .../worker/src/block/block.processor.spec.ts | 246 +- packages/worker/src/block/block.processor.ts | 62 +- packages/worker/src/block/block.utils.spec.ts | 14 +- packages/worker/src/block/block.utils.ts | 4 +- .../worker/src/block/block.watcher.spec.ts | 14 +- packages/worker/src/block/block.watcher.ts | 25 +- .../src/blockchain/blockchain.service.spec.ts | 1635 +++++++++++ .../src/blockchain/blockchain.service.ts | 56 +- packages/worker/src/config.spec.ts | 8 +- packages/worker/src/config.ts | 14 +- packages/worker/src/constants.ts | 27 - .../dataFetcher/dataFetcher.service.spec.ts | 99 + .../src/dataFetcher/dataFetcher.service.ts | 46 + packages/worker/src/dataFetcher/types.ts | 113 + .../src/entities/addressTransaction.entity.ts | 2 +- .../src/entities/addressTransfer.entity.ts | 7 +- .../worker/src/entities/balance.entity.ts | 6 +- packages/worker/src/entities/block.entity.ts | 18 +- packages/worker/src/entities/log.entity.ts | 2 +- .../worker/src/entities/transaction.entity.ts | 24 +- .../src/entities/transactionReceipt.entity.ts | 14 +- .../worker/src/entities/transfer.entity.ts | 13 +- packages/worker/src/log/index.ts | 2 - packages/worker/src/log/log.processor.spec.ts | 157 -- packages/worker/src/log/log.processor.ts | 66 - .../worker/src/metrics/metrics.provider.ts | 6 - .../src/repositories/block.repository.spec.ts | 5 +- .../src/repositories/block.repository.ts | 3 +- .../worker/src/token/token.service.spec.ts | 2 +- packages/worker/src/token/token.service.ts | 2 +- .../transaction/transaction.processor.spec.ts | 268 +- .../src/transaction/transaction.processor.ts | 114 +- .../transformers/toLower.transformer.spec.ts | 29 - .../src/transformers/toLower.transformer.ts | 13 - .../transferFields.transformer.spec.ts | 46 - .../transferFields.transformer.ts | 22 - packages/worker/src/utils/date.spec.ts | 9 +- packages/worker/src/utils/date.ts | 2 + scripts/setup-hyperchain-config.ts | 13 + 187 files changed, 15363 insertions(+), 1928 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 packages/data-fetcher/.env.example create mode 100644 packages/data-fetcher/.eslintignore create mode 100644 packages/data-fetcher/.eslintrc.js create mode 100644 packages/data-fetcher/Dockerfile create mode 100644 packages/data-fetcher/README.md create mode 100644 packages/data-fetcher/nest-cli.json create mode 100644 packages/data-fetcher/package.json create mode 100644 packages/data-fetcher/src/abis/erc721.json create mode 100644 packages/data-fetcher/src/abis/l2StandardERC20.json rename packages/{worker => data-fetcher}/src/abis/transferEventWithNoIndexes.json (100%) rename packages/{worker => data-fetcher}/src/address/address.service.spec.ts (60%) rename packages/{worker => data-fetcher}/src/address/address.service.ts (59%) rename packages/{worker => data-fetcher}/src/address/extractContractDeployedHandlers/default.handler.spec.ts (100%) rename packages/{worker => data-fetcher}/src/address/extractContractDeployedHandlers/default.handler.ts (100%) rename packages/{worker => data-fetcher}/src/address/extractContractDeployedHandlers/index.ts (100%) rename packages/{worker => data-fetcher}/src/address/interface/contractAddress.interface.ts (87%) rename packages/{worker => data-fetcher}/src/address/interface/extractContractAddressHandler.interface.ts (100%) create mode 100644 packages/data-fetcher/src/app.module.ts create mode 100644 packages/data-fetcher/src/balance/balance.service.spec.ts create mode 100644 packages/data-fetcher/src/balance/balance.service.ts create mode 100644 packages/data-fetcher/src/balance/index.ts create mode 100644 packages/data-fetcher/src/block/block.controller.spec.ts create mode 100644 packages/data-fetcher/src/block/block.controller.ts create mode 100644 packages/data-fetcher/src/block/block.service.spec.ts create mode 100644 packages/data-fetcher/src/block/block.service.ts create mode 100644 packages/data-fetcher/src/block/index.ts create mode 100644 packages/data-fetcher/src/blockchain/blockchain.service.spec.ts create mode 100644 packages/data-fetcher/src/blockchain/blockchain.service.ts create mode 100644 packages/data-fetcher/src/blockchain/index.ts create mode 100644 packages/data-fetcher/src/blockchain/retryableContract.spec.ts create mode 100644 packages/data-fetcher/src/blockchain/retryableContract.ts create mode 100644 packages/data-fetcher/src/common/interceptors/responseTransform.interceptor.spec.ts create mode 100644 packages/data-fetcher/src/common/interceptors/responseTransform.interceptor.ts create mode 100644 packages/data-fetcher/src/common/pipes/parseLimitedInt.pipe.spec.ts create mode 100644 packages/data-fetcher/src/common/pipes/parseLimitedInt.pipe.ts create mode 100644 packages/data-fetcher/src/config.spec.ts create mode 100644 packages/data-fetcher/src/config.ts create mode 100644 packages/data-fetcher/src/constants.ts create mode 100644 packages/data-fetcher/src/health/health.controller.spec.ts create mode 100644 packages/data-fetcher/src/health/health.controller.ts create mode 100644 packages/data-fetcher/src/health/health.module.ts create mode 100644 packages/data-fetcher/src/health/jsonRpcProvider.health.spec.ts create mode 100644 packages/data-fetcher/src/health/jsonRpcProvider.health.ts create mode 100644 packages/data-fetcher/src/log/index.ts create mode 100644 packages/data-fetcher/src/log/log.service.spec.ts create mode 100644 packages/data-fetcher/src/log/log.service.ts create mode 100644 packages/data-fetcher/src/log/logType.spec.ts create mode 100644 packages/data-fetcher/src/log/logType.ts create mode 100644 packages/data-fetcher/src/logger.ts create mode 100644 packages/data-fetcher/src/main.ts create mode 100644 packages/data-fetcher/src/metrics/index.ts create mode 100644 packages/data-fetcher/src/metrics/metrics.module.ts create mode 100644 packages/data-fetcher/src/metrics/metrics.provider.ts create mode 100644 packages/data-fetcher/src/rpcProvider/index.ts create mode 100644 packages/data-fetcher/src/rpcProvider/jsonRpcProvider.module.ts create mode 100644 packages/data-fetcher/src/rpcProvider/jsonRpcProviderBase.ts create mode 100644 packages/data-fetcher/src/rpcProvider/jsonRpcProviderExtended.spec.ts create mode 100644 packages/data-fetcher/src/rpcProvider/jsonRpcProviderExtended.ts create mode 100644 packages/data-fetcher/src/rpcProvider/webSocketProviderExtended.ts create mode 100644 packages/data-fetcher/src/rpcProvider/wrappedWebSocketProvider.ts create mode 100644 packages/data-fetcher/src/token/token.service.spec.ts create mode 100644 packages/data-fetcher/src/token/token.service.ts create mode 100644 packages/data-fetcher/src/transaction/index.ts create mode 100644 packages/data-fetcher/src/transaction/transaction.service.spec.ts create mode 100644 packages/data-fetcher/src/transaction/transaction.service.ts rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/finalizeDeposit/default.handler.spec.ts (97%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/finalizeDeposit/default.handler.ts (91%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/index.ts (100%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/mint/ethMintFromL1.handler.spec.ts (97%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/mint/ethMintFromL1.handler.ts (91%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/transfer/contractDeployerTransfer.handler.spec.ts (98%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/transfer/contractDeployerTransfer.handler.ts (92%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/transfer/default.handler.spec.ts (98%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/transfer/default.handler.ts (93%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/transfer/erc721Transfer.handle.spec.ts (97%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/transfer/erc721Transfer.handler.ts (92%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/withdrawal/ethWithdrawalToL1.handler.spec.ts (97%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/withdrawal/ethWithdrawalToL1.handler.ts (91%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/withdrawalInitiated/default.handler.spec.ts (97%) rename packages/{worker => data-fetcher}/src/transfer/extractHandlers/withdrawalInitiated/default.handler.ts (91%) rename packages/{worker => data-fetcher}/src/transfer/interfaces/extractTransferHandler.interface.ts (77%) rename packages/{worker => data-fetcher}/src/transfer/interfaces/transfer.interface.ts (78%) rename packages/{worker => data-fetcher}/src/transfer/transfer.service.spec.ts (85%) rename packages/{worker => data-fetcher}/src/transfer/transfer.service.ts (88%) create mode 100644 packages/data-fetcher/src/utils/date.spec.ts create mode 100644 packages/data-fetcher/src/utils/date.ts rename packages/{worker => data-fetcher}/src/utils/isInternalTransaction.spec.ts (97%) rename packages/{worker => data-fetcher}/src/utils/isInternalTransaction.ts (90%) create mode 100644 packages/data-fetcher/src/utils/parseLog.spec.ts create mode 100644 packages/data-fetcher/src/utils/parseLog.ts create mode 100644 packages/data-fetcher/temp-response-new.json create mode 100644 packages/data-fetcher/temp-response-old.json create mode 100644 packages/data-fetcher/test/app.e2e-spec.ts create mode 100644 packages/data-fetcher/test/jest-e2e.json create mode 100644 packages/data-fetcher/test/logs/block-with-no-txs-logs.json create mode 100644 packages/data-fetcher/test/transactionReceipts/address-out-of-range.json create mode 100644 packages/data-fetcher/test/transactionReceipts/erc20/bridge-deposit-from-l1-to-different-address.json create mode 100644 packages/data-fetcher/test/transactionReceipts/erc20/bridge-deposit-from-l1.json create mode 100644 packages/data-fetcher/test/transactionReceipts/erc20/deploy-l2.json create mode 100644 packages/data-fetcher/test/transactionReceipts/erc20/transfer.json create mode 100644 packages/data-fetcher/test/transactionReceipts/erc20/withdrawal.json create mode 100644 packages/data-fetcher/test/transactionReceipts/eth/deposit-no-fee.json create mode 100644 packages/data-fetcher/test/transactionReceipts/eth/deposit-to-different-address.json create mode 100644 packages/data-fetcher/test/transactionReceipts/eth/deposit-zero-value.json create mode 100644 packages/data-fetcher/test/transactionReceipts/eth/deposit.json create mode 100644 packages/data-fetcher/test/transactionReceipts/eth/transfer-to-zero-address.json create mode 100644 packages/data-fetcher/test/transactionReceipts/eth/transfer.json create mode 100644 packages/data-fetcher/test/transactionReceipts/eth/withdrawal-to-different-address.json create mode 100644 packages/data-fetcher/test/transactionReceipts/eth/withdrawal-zero-value.json create mode 100644 packages/data-fetcher/test/transactionReceipts/eth/withdrawal.json create mode 100644 packages/data-fetcher/test/transactionReceipts/failedTx/erc20-transfer-to-zero-address.json create mode 100644 packages/data-fetcher/test/transactionReceipts/fee-with-no-deposits.json create mode 100644 packages/data-fetcher/test/transactionReceipts/greeter/deploy-to-l2.json create mode 100644 packages/data-fetcher/test/transactionReceipts/greeter/exec-set-greeting.json create mode 100644 packages/data-fetcher/test/transactionReceipts/log-parsing-error.json create mode 100644 packages/data-fetcher/test/transactionReceipts/multiTransfer/eth-usdc-erc20-through-paymaster.json create mode 100644 packages/data-fetcher/test/transactionReceipts/multiTransfer/eth-usdc-erc20-transfer.json create mode 100644 packages/data-fetcher/test/transactionReceipts/multiTransfer/multi-eth-transfer.json create mode 100644 packages/data-fetcher/test/transactionReceipts/nestedContractsCalls/sub-calls-to-other-contracts.json create mode 100644 packages/data-fetcher/test/transactionReceipts/nft/approve.json create mode 100644 packages/data-fetcher/test/transactionReceipts/nft/deploy-l2.json create mode 100644 packages/data-fetcher/test/transactionReceipts/nft/mint.json create mode 100644 packages/data-fetcher/test/transactionReceipts/nft/transfer.json create mode 100644 packages/data-fetcher/test/transactionReceipts/no-deposit-after-fee.json create mode 100644 packages/data-fetcher/test/transactionReceipts/no-matching-handlers.json create mode 100644 packages/data-fetcher/test/transactionReceipts/paymasters/transfer.json create mode 100644 packages/data-fetcher/test/transactionReceipts/pre-approved-erc20/deposit.json create mode 100644 packages/data-fetcher/test/transactionReceipts/pre-approved-erc20/transfer.json create mode 100644 packages/data-fetcher/test/transactionReceipts/pre-approved-erc20/withdrawal-to-diff-address.json create mode 100644 packages/data-fetcher/test/transactionReceipts/pre-approved-erc20/withdrawal.json create mode 100644 packages/data-fetcher/test/transactionReceipts/tx-with-no-logs.json create mode 100644 packages/data-fetcher/tsconfig.build.json create mode 100644 packages/data-fetcher/tsconfig.json create mode 100644 packages/worker/src/blockchain/blockchain.service.spec.ts create mode 100644 packages/worker/src/dataFetcher/dataFetcher.service.spec.ts create mode 100644 packages/worker/src/dataFetcher/dataFetcher.service.ts create mode 100644 packages/worker/src/dataFetcher/types.ts delete mode 100644 packages/worker/src/log/index.ts delete mode 100644 packages/worker/src/log/log.processor.spec.ts delete mode 100644 packages/worker/src/log/log.processor.ts delete mode 100644 packages/worker/src/transformers/toLower.transformer.spec.ts delete mode 100644 packages/worker/src/transformers/toLower.transformer.ts delete mode 100644 packages/worker/src/transformers/transferFields.transformer.spec.ts delete mode 100644 packages/worker/src/transformers/transferFields.transformer.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 74b9091810..8415c96558 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,6 +111,20 @@ jobs: file: packages/worker/Dockerfile no-cache: true + - name: Build and push Docker image for Data Fetcher + uses: docker/build-push-action@v4 + with: + push: true + tags: | + "matterlabs/block-explorer-data-fetcher:latest" + "matterlabs/block-explorer-data-fetcher:v${{ needs.createReleaseVersion.outputs.releaseVersion }}" + "matterlabs/block-explorer-data-fetcher:${{ steps.setVersionForFlux.outputs.imageTag }}" + "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/block-explorer-data-fetcher:latest" + "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/block-explorer-data-fetcher:v${{ needs.createReleaseVersion.outputs.releaseVersion }}" + "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/block-explorer-data-fetcher:${{ steps.setVersionForFlux.outputs.imageTag }}" + file: packages/data-fetcher/Dockerfile + no-cache: true + - name: Build and push Docker image for App uses: docker/build-push-action@v4 with: diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml index 539a8e0679..a837a4a49e 100644 --- a/.github/workflows/validate-pr.yml +++ b/.github/workflows/validate-pr.yml @@ -66,6 +66,7 @@ jobs: packages/app/junit.xml packages/api/junit.xml packages/worker/junit.xml + packages/data-fetcher/junit.xml check_run_annotations: all tests, skipped tests report_individual_runs: "true" check_name: Unit Test Results diff --git a/.gitignore b/.gitignore index ecbf0116b8..50a0a84644 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ tests/e2e/reports/ # Logs logs !/packages/worker/test/logs/ +!/packages/data-fetcher/test/logs/ *.log npm-debug.log* yarn-debug.log* diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..55712c19f1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "typescript.tsdk": "node_modules/typescript/lib" +} \ No newline at end of file diff --git a/README.md b/README.md index e9276bde56..3cdd7e3dd5 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,9 @@

Online blockchain browser for viewing and analyzing zkSync Era blockchain.

## 📌 Overview -This repository is a monorepo consisting of 3 packages: -- [Worker](./packages/worker) - an indexer service for [zkSync Era](https://zksync.io) blockchain data. The purpose of the service is to read the data from the blockchain in real time, transform it and fill in it's database with the data in a way that makes it easy to be queried by the [API](./packages/api) service. +This repository is a monorepo consisting of 4 packages: +- [Worker](./packages/worker) - an indexer service for [zkSync Era](https://zksync.io) blockchain data. The purpose of the service is to read blockchain data in real time, transform it and fill in it's database with the data in a way that makes it easy to be queried by the [API](./packages/api) service. +- [Data Fetcher](./packages/data-fetcher) - a service that exposes and implements an HTTP endpoint to retrieve aggregated data for a certain block / range of blocks from the blockchain. This endpoint is called by the [Worker](./packages/worker) service. - [API](./packages/api) - a service providing Web API for retrieving structured [zkSync Era](https://zksync.io) blockchain data collected by [Worker](./packages/worker). It connects to the Worker's database to be able to query the collected data. - [App](./packages/app) - a front-end app providing an easy-to-use interface for users to view and inspect transactions, blocks, contracts and more. It makes requests to the [API](./packages/api) to get the data and presents it in a way that's easy to read and understand. @@ -20,10 +21,14 @@ flowchart subgraph explorer[Block explorer] Database[("Block explorer DB
(PostgreSQL)")] Worker(Worker service) + Data-Fetcher(Data Fetcher service) API(API service) App(App) - + + Worker-."Request aggregated data (HTTP)".->Data-Fetcher + Data-Fetcher-."Request data (HTTP)".->Blockchain Worker-.Save processed data.->Database + API-.Query data.->Database App-."Request data (HTTP)".->API App-."Request data (HTTP)".->Blockchain @@ -32,7 +37,7 @@ flowchart Worker-."Request data (HTTP)".->Blockchain ``` -[Worker](./packages/worker) service is responsible for getting data from blockchain using [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html), processing it and saving into the database. [API](./packages/api) service is connected to the same database where it gets the data from to handle API requests. It performs only read requests to the database. The front-end [App](./packages/app) makes HTTP calls to the Block Explorer [API](./packages/api) to get blockchain data and to the [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html) for reading contracts, performing transactions etc. +[Worker](./packages/worker) service retrieves aggregated data from the [Data Fetcher](./packages/data-fetcher) via HTTP and also directly from the blockchain using [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html), processes it and saves into the database. [API](./packages/api) service is connected to the same database where it gets the data from to handle API requests. It performs only read requests to the database. The front-end [App](./packages/app) makes HTTP calls to the Block Explorer [API](./packages/api) to get blockchain data and to the [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html) for reading contracts, performing transactions etc. ## 🚀 Features @@ -56,12 +61,12 @@ npm install ## ⚙️ Setting up env variables ### Manually set up env variables -Make sure you have set up all the necessary env variables. Follow [Setting up env variables for Worker](./packages/worker#setting-up-env-variables) and [Setting up env variables for API](./packages/api#setting-up-env-variables) for instructions. For the [App](./packages/app) package you might want to edit environment config, see [Environment configs](./packages/app#environment-configs). +Make sure you have set up all the necessary env variables. Follow setting up env variables instructions for [Worker](./packages/worker#setting-up-env-variables), [Data Fetcher](./packages/data-fetcher#setting-up-env-variables) and [API](./packages/api#setting-up-env-variables). For the [App](./packages/app) package you might want to edit environment config, see [Environment configs](./packages/app#environment-configs). ### Build env variables based on your [zksync-era](https://github.com/matter-labs/zksync-era) local repo setup Make sure you have [zksync-era](https://github.com/matter-labs/zksync-era) repo set up locally. You must have your environment variables files present in the [zksync-era](https://github.com/matter-labs/zksync-era) repo at `/etc/env/*.env` for the build envs script to work. -The following script sets `.env` files for [Worker](./packages/worker) and [API](./packages/api) packages as well as environment configuration file for [App](./packages/app) package based on your local [zksync-era](https://github.com/matter-labs/zksync-era) repo setup. +The following script sets `.env` files for [Worker](./packages/worker), [Data Fetcher](./packages/data-fetcher) and [API](./packages/api) packages as well as environment configuration file for [App](./packages/app) package based on your local [zksync-era](https://github.com/matter-labs/zksync-era) repo setup. ```bash npm run hyperchain:configure ``` @@ -75,7 +80,7 @@ To create a database run the following command: npm run db:create ``` -To run all the packages (`Worker`, `API` and front-end `App`) in `development` mode run the following command from the root directory. +To run all the packages (`Worker`, `Data Fetcher`, `API` and front-end `App`) in `development` mode run the following command from the root directory. ```bash npm run dev ``` @@ -100,7 +105,7 @@ To get block-explorer connected to your ZK Stack Hyperchain you need to set up a ## 🔍 Verify Block Explorer is up and running -To verify front-end `App` is running open http://localhost:3010 in your browser. `API` should be available at http://localhost:3020. `Worker` - http://localhost:3001. +To verify front-end `App` is running open http://localhost:3010 in your browser. `API` should be available at http://localhost:3020, `Worker` at http://localhost:3001 and `Data Fetcher` at http://localhost:3040. ## 🕵️‍♂️ Testing Run unit tests for all packages: diff --git a/docker-compose-cli.yaml b/docker-compose-cli.yaml index 1343eb8fa2..c01ce5d059 100644 --- a/docker-compose-cli.yaml +++ b/docker-compose-cli.yaml @@ -1,10 +1,10 @@ -version: '3.2' +version: "3.2" +name: "zkcli-block-explorer" services: app: - build: - context: . - dockerfile: ./packages/app/Dockerfile + platform: linux/amd64 + image: "matterlabs/block-explorer-app:${VERSION}" ports: - '3010:3010' depends_on: @@ -12,9 +12,8 @@ services: restart: unless-stopped worker: - build: - context: . - dockerfile: ./packages/worker/Dockerfile + platform: linux/amd64 + image: "matterlabs/block-explorer-worker:${VERSION}" environment: - PORT=3001 - LOG_LEVEL=verbose @@ -23,14 +22,26 @@ services: - DATABASE_USER=postgres - DATABASE_PASSWORD=postgres - DATABASE_NAME=block-explorer - - BLOCKCHAIN_RPC_URL=http://host.docker.internal:3050 + - BLOCKCHAIN_RPC_URL=http://host.docker.internal:${RPC_PORT} + - DATA_FETCHER_URL=http://data-fetcher:3040 - BATCHES_PROCESSING_POLLING_INTERVAL=1000 restart: unless-stopped + data-fetcher: + platform: linux/amd64 + image: "matterlabs/block-explorer-data-fetcher:${VERSION}" + environment: + - PORT=3040 + - LOG_LEVEL=verbose + - NODE_ENV=development + - BLOCKCHAIN_RPC_URL=http://host.docker.internal:${RPC_PORT} + ports: + - '3040:3040' + restart: unless-stopped + api: - build: - context: . - dockerfile: ./packages/api/Dockerfile + platform: linux/amd64 + image: "matterlabs/block-explorer-api:${VERSION}" environment: - PORT=3020 - METRICS_PORT=3005 @@ -60,4 +71,4 @@ services: - POSTGRES_DB=block-explorer volumes: - postgres: + postgres: \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index cd747b2e49..1020cc35bc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -30,6 +30,7 @@ services: - DATABASE_PASSWORD=postgres - DATABASE_NAME=block-explorer - BLOCKCHAIN_RPC_URL=http://zksync:3050 + - DATA_FETCHER_URL=http://data-fetcher:3040 - BATCHES_PROCESSING_POLLING_INTERVAL=1000 ports: - '3001:3001' @@ -43,6 +44,29 @@ services: condition: service_healthy restart: unless-stopped + data-fetcher: + build: + context: . + dockerfile: ./packages/data-fetcher/Dockerfile + target: development-stage + command: npm run --prefix packages/data-fetcher dev:debug + environment: + - PORT=3040 + - LOG_LEVEL=verbose + - NODE_ENV=development + - BLOCKCHAIN_RPC_URL=http://zksync:3050 + ports: + - '3040:3040' + - '9231:9229' + - '9232:9230' + volumes: + - ./packages/data-fetcher:/usr/src/app/packages/data-fetcher + - /usr/src/app/packages/data-fetcher/node_modules + depends_on: + zksync: + condition: service_healthy + restart: unless-stopped + api: build: context: . @@ -58,8 +82,8 @@ services: ports: - '3020:3020' - '3005:3005' - - '9231:9229' - - '9232:9230' + - '9233:9229' + - '9234:9230' volumes: - ./packages/api:/usr/src/app/packages/api - /usr/src/app/packages/api/node_modules diff --git a/package-lock.json b/package-lock.json index b5e9876c27..28760ca985 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23466,6 +23466,10 @@ "node": ">=8" } }, + "node_modules/data-fetcher": { + "resolved": "packages/data-fetcher", + "link": true + }, "node_modules/data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -55159,6 +55163,205 @@ "typedarray-to-buffer": "^3.1.5" } }, + "packages/data-fetcher": { + "version": "0.0.0", + "license": "MIT", + "dependencies": { + "@nestjs/common": "^9.0.0", + "@nestjs/config": "^2.2.0", + "@nestjs/core": "^9.0.0", + "@nestjs/platform-express": "^9.0.0", + "@nestjs/terminus": "^9.1.2", + "@willsoto/nestjs-prometheus": "^4.7.0", + "ethers": "^5.7.1", + "nest-winston": "^1.7.0", + "prom-client": "^14.1.0", + "reflect-metadata": "^0.1.13", + "rimraf": "^3.0.2", + "rxjs": "^7.2.0", + "winston": "^3.8.2", + "zksync-web3": "0.15.4" + }, + "devDependencies": { + "@nestjs/cli": "^9.0.0", + "@nestjs/schematics": "^9.0.0", + "@nestjs/testing": "^9.0.0", + "@types/express": "^4.17.13", + "@types/jest": "28.1.8", + "@types/supertest": "^2.0.11", + "@typescript-eslint/eslint-plugin": "^5.0.0", + "@typescript-eslint/parser": "^5.0.0", + "eslint-config-prettier": "^8.3.0", + "eslint-plugin-prettier": "^4.0.0", + "jest": "29.2.1", + "jest-junit": "^14.0.1", + "jest-mock-extended": "^3.0.1", + "lint-staged": "^13.0.3", + "source-map-support": "^0.5.20", + "supertest": "^6.1.3", + "ts-jest": "29.0.3", + "ts-loader": "^9.2.3", + "ts-node": "^10.0.0", + "tsconfig-paths": "4.1.0" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=9.0.0" + } + }, + "packages/data-fetcher/node_modules/@ethersproject/networks": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/@ethersproject/networks/-/networks-5.7.1.tgz", + "integrity": "sha512-n/MufjFYv3yFcUyfhnXotyDlNdFb7onmkSy8aQERi2PjNcnWQ66xXxa3XlS8nCcA8aJKJjIIMNJTC7tu80GwpQ==", + "funding": [ + { + "type": "individual", + "url": "https://gitcoin.co/grants/13/ethersjs-complete-simple-and-tiny-2" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "dependencies": { + "@ethersproject/logger": "^5.7.0" + } + }, + "packages/data-fetcher/node_modules/@ethersproject/providers": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/@ethersproject/providers/-/providers-5.7.2.tgz", + "integrity": "sha512-g34EWZ1WWAVgr4aptGlVBF8mhl3VWjv+8hoAnzStu8Ah22VHBsuGzP17eb6xDVRzw895G4W7vvx60lFFur/1Rg==", + "funding": [ + { + "type": "individual", + "url": "https://gitcoin.co/grants/13/ethersjs-complete-simple-and-tiny-2" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "dependencies": { + "@ethersproject/abstract-provider": "^5.7.0", + "@ethersproject/abstract-signer": "^5.7.0", + "@ethersproject/address": "^5.7.0", + "@ethersproject/base64": "^5.7.0", + "@ethersproject/basex": "^5.7.0", + "@ethersproject/bignumber": "^5.7.0", + "@ethersproject/bytes": "^5.7.0", + "@ethersproject/constants": "^5.7.0", + "@ethersproject/hash": "^5.7.0", + "@ethersproject/logger": "^5.7.0", + "@ethersproject/networks": "^5.7.0", + "@ethersproject/properties": "^5.7.0", + "@ethersproject/random": "^5.7.0", + "@ethersproject/rlp": "^5.7.0", + "@ethersproject/sha2": "^5.7.0", + "@ethersproject/strings": "^5.7.0", + "@ethersproject/transactions": "^5.7.0", + "@ethersproject/web": "^5.7.0", + "bech32": "1.1.4", + "ws": "7.4.6" + } + }, + "packages/data-fetcher/node_modules/@ethersproject/web": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/@ethersproject/web/-/web-5.7.1.tgz", + "integrity": "sha512-Gueu8lSvyjBWL4cYsWsjh6MtMwM0+H4HvqFPZfB6dV8ctbP9zFAO73VG1cMWae0FLPCtz0peKPpZY8/ugJJX2w==", + "funding": [ + { + "type": "individual", + "url": "https://gitcoin.co/grants/13/ethersjs-complete-simple-and-tiny-2" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "dependencies": { + "@ethersproject/base64": "^5.7.0", + "@ethersproject/bytes": "^5.7.0", + "@ethersproject/logger": "^5.7.0", + "@ethersproject/properties": "^5.7.0", + "@ethersproject/strings": "^5.7.0" + } + }, + "packages/data-fetcher/node_modules/ethers": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/ethers/-/ethers-5.7.2.tgz", + "integrity": "sha512-wswUsmWo1aOK8rR7DIKiWSw9DbLWe6x98Jrn8wcTflTVvaXhAMaB5zGAXy0GYQEQp9iO1iSHWVyARQm11zUtyg==", + "funding": [ + { + "type": "individual", + "url": "https://gitcoin.co/grants/13/ethersjs-complete-simple-and-tiny-2" + }, + { + "type": "individual", + "url": "https://www.buymeacoffee.com/ricmoo" + } + ], + "dependencies": { + "@ethersproject/abi": "5.7.0", + "@ethersproject/abstract-provider": "5.7.0", + "@ethersproject/abstract-signer": "5.7.0", + "@ethersproject/address": "5.7.0", + "@ethersproject/base64": "5.7.0", + "@ethersproject/basex": "5.7.0", + "@ethersproject/bignumber": "5.7.0", + "@ethersproject/bytes": "5.7.0", + "@ethersproject/constants": "5.7.0", + "@ethersproject/contracts": "5.7.0", + "@ethersproject/hash": "5.7.0", + "@ethersproject/hdnode": "5.7.0", + "@ethersproject/json-wallets": "5.7.0", + "@ethersproject/keccak256": "5.7.0", + "@ethersproject/logger": "5.7.0", + "@ethersproject/networks": "5.7.1", + "@ethersproject/pbkdf2": "5.7.0", + "@ethersproject/properties": "5.7.0", + "@ethersproject/providers": "5.7.2", + "@ethersproject/random": "5.7.0", + "@ethersproject/rlp": "5.7.0", + "@ethersproject/sha2": "5.7.0", + "@ethersproject/signing-key": "5.7.0", + "@ethersproject/solidity": "5.7.0", + "@ethersproject/strings": "5.7.0", + "@ethersproject/transactions": "5.7.0", + "@ethersproject/units": "5.7.0", + "@ethersproject/wallet": "5.7.0", + "@ethersproject/web": "5.7.1", + "@ethersproject/wordlists": "5.7.0" + } + }, + "packages/data-fetcher/node_modules/ws": { + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", + "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "packages/data-fetcher/node_modules/zksync-web3": { + "version": "0.15.4", + "resolved": "https://registry.npmjs.org/zksync-web3/-/zksync-web3-0.15.4.tgz", + "integrity": "sha512-6CEpRBbF4nGwRYSF3KvPGqg2aNJFYTl8AR+cejBnC2Uyu1v3NYSkmkXXVuMGupJ7HIQR1aTqFEDsUFPyO/bL0Q==", + "deprecated": "This package has been deprecated in favor of zksync-ethers@5.0.0", + "peerDependencies": { + "ethers": "^5.7.0" + } + }, "packages/worker": { "version": "0.0.0", "license": "MIT", diff --git a/packages/app/src/components/NetworkDeprecated.vue b/packages/app/src/components/NetworkDeprecated.vue index 8b9e8a7ea3..de32dc159d 100644 --- a/packages/app/src/components/NetworkDeprecated.vue +++ b/packages/app/src/components/NetworkDeprecated.vue @@ -11,7 +11,6 @@