test: rebase integration tests to main

.dockerignore

@@ -1 +1,2 @@
-**/dist/
+**/dist/
+**.env

.github/workflows/app-deploy-feature-branch.yml

@@ -70,7 +70,7 @@ jobs:
     uses: ./.github/workflows/app-e2e.yml
     secrets: inherit
     permissions:
-      contents: read
+      contents: write
     with:
       targetUrl: ${{ needs.build.outputs.dappUrl }}
       testnet_network_value_for_e2e: "/?network=sepolia"

.github/workflows/app-deploy-preview.yml

@@ -67,7 +67,7 @@ jobs:
     uses: ./.github/workflows/app-e2e.yml
     secrets: inherit
     permissions:
-      contents: read
+      contents: write
     with:
       targetUrl: ${{ needs.deploy.outputs.dappUrl }}
       testnet_network_value_for_e2e: "/?network=sepolia"

.github/workflows/app-e2e.yml

@@ -79,8 +79,6 @@ jobs:
         uses: actions/cache@v3
         env:
           cache-name: cache-node-modules
-          # Workaround for bug https://github.com/typicode/husky/issues/991
-          HUSKY: 0
         with:
           path: node_modules
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
@@ -116,27 +114,21 @@ jobs:
                 E2ENETWORK='${{ inputs.default_network_value_for_e2e }}' npx cucumber-js --tags "${{ matrix.tags }} ${{ inputs.environmentTags }} and not @testnet"
             fi
 
-      - name: Reset tags quotes
+      - name: Save artifacts to Git
         if: always()
-        run: |
-          echo "MATRIX_TAG_WITHOUT_QUOTES=$(echo ${{ matrix.tags }} | sed -e 's/@//g' )" >> $GITHUB_ENV
+        uses: actions/upload-artifact@v3
+        with:
+          name: allure-results
+          path: packages/app/allure-results
 
-      - name: Create launch ID
+      - name: Upload test results to Allure reporter
         if: always()
         env:
-          ALLURE_LAUNCH_NAME: "#${{ github.run_number }} ${{ env.MATRIX_TAG_WITHOUT_QUOTES }}"
-          ALLURE_LAUNCH_TAGS: "${{ env.ALLURE_BASIC_TAGS }}, ${{ env.MATRIX_TAG_WITHOUT_QUOTES }}"
           ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
         run: |
-          echo "ALLURE_LAUNCH_ID=$(./allurectl launch create --launch-name '${{ env.ALLURE_LAUNCH_NAME }}' --no-header --format ID | tail -n1)" >> $GITHUB_ENV
-
-      - name: Upload tests to the Allure proj
-        if: always() && inputs.publish_to_allure == true
-        env:
-          ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
-        run: |
-          ./allurectl upload allure-results --launch-id ${{ env.ALLURE_LAUNCH_ID }}
-          ./allurectl launch close ${{ env.ALLURE_LAUNCH_ID }}
+          ./allurectl upload allure-results
+          echo "*Public report link: https://raw.githack.com/matter-labs/block-explorer/gh-pages/${{ github.run_number }}/index.html"
+          echo "*Public report link will be available when the 'Allure Report' job will be succesfully executed."
 
       - if: failure()
         name: Save artifacts
@@ -146,18 +138,53 @@ jobs:
           path: packages/app/tests/e2e/artifacts/*
 
   publish:
-    name: Publish Allure link to GIT
+    name: Allure Report
     runs-on: ubuntu-latest
     permissions:
-      contents: read
+      contents: write
     needs: e2e
     if: always()
     steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: allure-results
+          path: packages/app/allure-results
+
+      - name: Get Allure history
+        uses: actions/checkout@v3
+        if: always()
+        continue-on-error: true
+        with:
+          ref: gh-pages
+          path: gh-pages
+
+      - name: Allure Report action from marketplace
+        uses: simple-elf/[email protected]
+        if: always()
+        id: allure-report
+        with:
+          allure_results: packages/app/allure-results
+          gh_pages: gh-pages
+          allure_report: allure-report
+          allure_history: allure-history
+          keep_reports: 10
+
+      - name: Deploy report to Github Pages
+        if: always()
+        uses: peaceiris/actions-gh-pages@v2
+        env:
+          PERSONAL_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PUBLISH_BRANCH: gh-pages
+          PUBLISH_DIR: allure-history
+
       - name: Prepare a link
         run: |
           echo "BASE64_SEARCH_REQUEST=$(echo '${{ env.ALLURE_SEARCH_REQUEST }}' | base64)" >> $GITHUB_ENV
 
       - name: Publish Allure link to GIT Summary
         run: |
-          LINK="${{ vars.ALLURE_ENDPOINT }}project/${{ vars.ALLURE_PROJECT_ID }}/launches?search=${{ env.BASE64_SEARCH_REQUEST }}"
-          echo "Allure [e2e tests]($LINK) :rocket: in git run #${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY
+          LINK1="${{ vars.ALLURE_ENDPOINT }}project/${{ vars.ALLURE_PROJECT_ID }}/launches?search=${{ env.BASE64_SEARCH_REQUEST }}"
+          LINK2="https://raw.githack.com/matter-labs/block-explorer/gh-pages/${{ github.run_number }}/index.html"
+          echo "Allure [Private]($LINK1) and [Public]($LINK2) links:rocket: in git run #${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY

.github/workflows/release.yml

@@ -111,6 +111,20 @@ jobs:
           file: packages/worker/Dockerfile
           no-cache: true
 
+      - name: Build and push Docker image for Data Fetcher
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          tags: |
+              "matterlabs/block-explorer-data-fetcher:latest"
+              "matterlabs/block-explorer-data-fetcher:v${{ needs.createReleaseVersion.outputs.releaseVersion }}"
+              "matterlabs/block-explorer-data-fetcher:${{ steps.setVersionForFlux.outputs.imageTag }}"
+              "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/block-explorer-data-fetcher:latest"
+              "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/block-explorer-data-fetcher:v${{ needs.createReleaseVersion.outputs.releaseVersion }}"
+              "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/block-explorer-data-fetcher:${{ steps.setVersionForFlux.outputs.imageTag }}"
+          file: packages/data-fetcher/Dockerfile
+          no-cache: true
+
       - name: Build and push Docker image for App
         uses: docker/build-push-action@v4
         with:

.github/workflows/validate-pr.yml

@@ -66,6 +66,7 @@ jobs:
             packages/app/junit.xml
             packages/api/junit.xml
             packages/worker/junit.xml
+            packages/data-fetcher/junit.xml
           check_run_annotations: all tests, skipped tests
           report_individual_runs: "true"
           check_name: Unit Test Results

.gitignore

@@ -27,6 +27,7 @@ tests/e2e/reports/
 # Logs
 logs
 !/packages/worker/test/logs/
+!/packages/data-fetcher/test/logs/
 *.log
 npm-debug.log*
 yarn-debug.log*

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "typescript.tsdk": "node_modules/typescript/lib"
+}

README.md

@@ -3,8 +3,9 @@
 <p align="center">Online blockchain browser for viewing and analyzing <a href="https://zksync.io">zkSync Era</a> blockchain.</p>
 
 ## 📌 Overview
-This repository is a monorepo consisting of 3 packages:
-- [Worker](./packages/worker) - an indexer service for [zkSync Era](https://zksync.io) blockchain data. The purpose of the service is to read the data from the blockchain in real time, transform it and fill in it's database with the data in a way that makes it easy to be queried by the [API](./packages/api) service.
+This repository is a monorepo consisting of 4 packages:
+- [Worker](./packages/worker) - an indexer service for [zkSync Era](https://zksync.io) blockchain data. The purpose of the service is to read blockchain data in real time, transform it and fill in it's database with the data in a way that makes it easy to be queried by the [API](./packages/api) service.
+- [Data Fetcher](./packages/data-fetcher) - a service that exposes and implements an HTTP endpoint to retrieve aggregated data for a certain block / range of blocks from the blockchain. This endpoint is called by the [Worker](./packages/worker) service.
 - [API](./packages/api) - a service providing Web API for retrieving structured [zkSync Era](https://zksync.io) blockchain data collected by [Worker](./packages/worker). It connects to the Worker's database to be able to query the collected data.
 - [App](./packages/app) - a front-end app providing an easy-to-use interface for users to view and inspect transactions, blocks, contracts and more. It makes requests to the [API](./packages/api) to get the data and presents it in a way that's easy to read and understand.
 
@@ -22,10 +23,14 @@ flowchart
   subgraph explorer[Block explorer]
     Database[("Block explorer DB<br/>(PostgreSQL)")]
     Worker(Worker service)
+    Data-Fetcher(Data Fetcher service)
     API(API service)
     App(App)
-
+
+    Worker-."Request aggregated data (HTTP)".->Data-Fetcher
+    Data-Fetcher-."Request data (HTTP)".->Blockchain
     Worker-.Save processed data.->Database
+
     API-.Query data.->Database
     App-."Request data (HTTP)".->API
     App-."Request data (HTTP)".->Blockchain
@@ -34,7 +39,7 @@ flowchart
   Worker-."Request data (HTTP)".->Blockchain
 ```
 
-[Worker](./packages/worker) service is responsible for getting data from blockchain using [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html), processing it and saving into the database. [API](./packages/api) service is connected to the same database where it gets the data from to handle API requests. It performs only read requests to the database. The front-end [App](./packages/app) makes HTTP calls to the Block Explorer [API](./packages/api) to get blockchain data and to the [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html) for reading contracts, performing transactions etc.
+[Worker](./packages/worker) service retrieves aggregated data from the [Data Fetcher](./packages/data-fetcher) via HTTP and also directly from the blockchain using [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html), processes it and saves into the database. [API](./packages/api) service is connected to the same database where it gets the data from to handle API requests. It performs only read requests to the database. The front-end [App](./packages/app) makes HTTP calls to the Block Explorer [API](./packages/api) to get blockchain data and to the [zkSync Era JSON-RPC API](https://era.zksync.io/docs/api/api.html) for reading contracts, performing transactions etc.
 
 ## 🚀 Features
 
@@ -58,12 +63,12 @@ npm install
 ## ⚙️ Setting up env variables
 
 ### Manually set up env variables
-Make sure you have set up all the necessary env variables. Follow [Setting up env variables for Worker](./packages/worker#setting-up-env-variables) and [Setting up env variables for API](./packages/api#setting-up-env-variables) for instructions. For the [App](./packages/app) package you might want to edit environment config, see [Environment configs](./packages/app#environment-configs).
+Make sure you have set up all the necessary env variables. Follow setting up env variables instructions for [Worker](./packages/worker#setting-up-env-variables), [Data Fetcher](./packages/data-fetcher#setting-up-env-variables) and [API](./packages/api#setting-up-env-variables). For the [App](./packages/app) package you might want to edit environment config, see [Environment configs](./packages/app#environment-configs).
 
 ### Build env variables based on your [zksync-era](https://github.com/matter-labs/zksync-era) local repo setup
 Make sure you have [zksync-era](https://github.com/matter-labs/zksync-era) repo set up locally. You must have your environment variables files present in the [zksync-era](https://github.com/matter-labs/zksync-era) repo at `/etc/env/*.env` for the build envs script to work.
 
-The following script sets `.env` files for [Worker](./packages/worker) and [API](./packages/api) packages as well as environment configuration file for [App](./packages/app) package based on your local [zksync-era](https://github.com/matter-labs/zksync-era) repo setup.
+The following script sets `.env` files for [Worker](./packages/worker), [Data Fetcher](./packages/data-fetcher) and [API](./packages/api) packages as well as environment configuration file for [App](./packages/app) package based on your local [zksync-era](https://github.com/matter-labs/zksync-era) repo setup.
 ```bash
 npm run hyperchain:configure
 ```
@@ -77,7 +82,7 @@ To create a database run the following command:
 npm run db:create
 ```
 
-To run all the packages (`Worker`, `API` and front-end `App`) in `development` mode run the following command from the root directory.
+To run all the packages (`Worker`, `Data Fetcher`, `API` and front-end `App`) in `development` mode run the following command from the root directory.
 ```bash
 npm run dev
 ```
@@ -102,7 +107,7 @@ To get block-explorer connected to your ZK Stack Hyperchain you need to set up a
 
 ## 🔍 Verify Block Explorer is up and running
 
-To verify front-end `App` is running open http://localhost:3010 in your browser. `API` should be available at http://localhost:3020. `Worker` - http://localhost:3001.
+To verify front-end `App` is running open http://localhost:3010 in your browser. `API` should be available at http://localhost:3020, `Worker` at http://localhost:3001 and `Data Fetcher` at http://localhost:3040.
 
 ## 🕵️‍♂️ Testing
 Run unit tests for all packages:

docker-compose-cli.yaml

@@ -1,20 +1,16 @@
-version: '3.2'
-
 services:
   app:
-    build:
-      context: .
-      dockerfile: ./packages/app/Dockerfile
+    platform: linux/amd64
+    image: "matterlabs/block-explorer-app:${VERSION}"
     ports:
       - '3010:3010'
     depends_on:
       - api
     restart: unless-stopped
 
   worker:
-    build:
-      context: .
-      dockerfile: ./packages/worker/Dockerfile
+    platform: linux/amd64
+    image: "matterlabs/block-explorer-worker:${VERSION}"
     environment:
       - PORT=3001
       - LOG_LEVEL=verbose
@@ -23,14 +19,30 @@ services:
       - DATABASE_USER=postgres
       - DATABASE_PASSWORD=postgres
       - DATABASE_NAME=block-explorer
-      - BLOCKCHAIN_RPC_URL=http://host.docker.internal:3050
+      - BLOCKCHAIN_RPC_URL=http://host.docker.internal:${RPC_PORT}
+      - DATA_FETCHER_URL=http://data-fetcher:3040
       - BATCHES_PROCESSING_POLLING_INTERVAL=1000
     restart: unless-stopped
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  data-fetcher:
+    platform: linux/amd64
+    image: "matterlabs/block-explorer-data-fetcher:${VERSION}"
+    environment:
+      - PORT=3040
+      - LOG_LEVEL=verbose
+      - NODE_ENV=development
+      - BLOCKCHAIN_RPC_URL=http://host.docker.internal:${RPC_PORT}
+    ports:
+      - '3040:3040'
+    restart: unless-stopped
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
   api:
-    build:
-      context: .
-      dockerfile: ./packages/api/Dockerfile
+    platform: linux/amd64
+    image: "matterlabs/block-explorer-api:${VERSION}"
     environment:
       - PORT=3020
       - METRICS_PORT=3005

docker-compose.yaml

@@ -5,13 +5,8 @@ services:
     build:
       context: .
       dockerfile: ./packages/app/Dockerfile
-      target: development-stage
-    command: npm run --prefix packages/app dev -- --host
     ports:
       - '3010:3010'
-    volumes:
-      - ./packages/app:/usr/src/app/packages/app
-      - /usr/src/app/packages/app/node_modules
     depends_on:
       - api
     restart: unless-stopped
@@ -20,25 +15,33 @@ services:
     build:
       context: .
       dockerfile: ./packages/worker/Dockerfile
-      target: development-stage
-    command: npm run --prefix packages/worker dev:debug
     environment:
       - PORT=3001
       - LOG_LEVEL=verbose
-      - NODE_ENV=development
       - DATABASE_HOST=postgres
       - DATABASE_USER=postgres
       - DATABASE_PASSWORD=postgres
       - DATABASE_NAME=block-explorer
       - BLOCKCHAIN_RPC_URL=http://zksync:3050
+      - DATA_FETCHER_URL=http://data-fetcher:3040
       - BATCHES_PROCESSING_POLLING_INTERVAL=1000
     ports:
       - '3001:3001'
-      - '9229:9229'
-      - '9230:9230'
-    volumes:
-      - ./packages/worker:/usr/src/app/packages/worker
-      - /usr/src/app/packages/worker/node_modules
+    depends_on:
+      zksync:
+        condition: service_healthy
+    restart: unless-stopped
+
+  data-fetcher:
+    build:
+      context: .
+      dockerfile: ./packages/data-fetcher/Dockerfile
+    environment:
+      - PORT=3040
+      - LOG_LEVEL=verbose
+      - BLOCKCHAIN_RPC_URL=http://zksync:3050
+    ports:
+      - '3040:3040'
     depends_on:
       zksync:
         condition: service_healthy
@@ -48,22 +51,14 @@ services:
     build:
       context: .
       dockerfile: ./packages/api/Dockerfile
-      target: development-stage
-    command: npm run --prefix packages/api dev:debug
     environment:
       - PORT=3020
       - METRICS_PORT=3005
       - LOG_LEVEL=verbose
-      - NODE_ENV=development
       - DATABASE_URL=postgres://postgres:postgres@postgres:5432/block-explorer
     ports:
       - '3020:3020'
       - '3005:3005'
-      - '9231:9229'
-      - '9232:9230'
-    volumes:
-      - ./packages/api:/usr/src/app/packages/api
-      - /usr/src/app/packages/api/node_modules
     depends_on:
       - worker
     restart: unless-stopped
@@ -120,11 +115,11 @@ services:
       test: "curl -H \"Content-Type: application/json\" -X POST --data '{\"jsonrpc\":\"2.0\",\"method\":\"web3_clientVersion\",\"params\":[],\"id\":67}' 127.0.0.1:3050 || exit 1"
       interval: 5s
       timeout: 5s
-      retries: 120
+      retries: 300
     restart: unless-stopped
 
 volumes:
   geth:
   postgres:
   zksync-config:
-  zksync-data:
+  zksync-data: