feat: add missing blocks metric (#371)

# What ❔

Add missing blocks metric.

## Why ❔

To detect when there are missing blocks in the DB. Issue with missing
blocks is described
[here](#370).

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [X] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [X] Tests for the changes have been added / updated.

packages/app/src/components/TheWelcome.vue

@@ -64,7 +64,7 @@ import ToolingIcon from "@/components/icons/IconTooling.vue";
     <a target="_blank" href="https://chat.vuejs.org">Vue Land</a>, our official Discord server, or
     <a target="_blank" href="https://stackoverflow.com/questions/tagged/vue.js">StackOverflow</a>. You should also
     subscribe to <a target="_blank" href="https://news.vuejs.org">our mailing list</a> and follow the official
-    <a target="_blank" href="https://twitter.com/vuejs">@vuejs</a>
+    <a target="_blank" href="https://x.com/vuejs">@vuejs</a>
     twitter account for latest news in the Vue world.
   </WelcomeItem>
 

packages/app/src/components/header/TheHeader.vue

@@ -204,7 +204,7 @@ const toolsLinks = reactive(links);
 
 const socials = [
   { url: "https://join.zksync.dev/", component: DiscordIcon },
-  { url: "https://twitter.com/zksync", component: TwitterIcon },
+  { url: "https://x.com/zksync", component: TwitterIcon },
 ];
 
 const hasContent = computed(() => {

packages/app/src/views/MaintenanceView.vue

@@ -8,13 +8,13 @@
           {{ currentNetwork.l2NetworkName }}
         </template>
         <template #twitter>
-          <a href="https://twitter.com/ZKsyncDevs" target="_blank">{{ t("maintenance.twitterLink") }}</a>
+          <a href="https://x.com/ZKsyncDevs" target="_blank">{{ t("maintenance.twitterLink") }}</a>
         </template>
         <template #uptime>
           <a href="https://uptime.com/statuspage/era" target="_blank">{{ t("maintenance.uptimeLink") }}</a>
         </template>
       </i18n-t>
-      <Button class="twitter-button" tag="a" href="https://twitter.com/ZKsyncDevs" target="_blank">
+      <Button class="twitter-button" tag="a" href="https://x.com/ZKsyncDevs" target="_blank">
         {{ t("maintenance.twitterButton") }}
       </Button>
       <a class="uptime-link" href="https://uptime.com/statuspage/era" target="_blank">

packages/app/tests/components/TheHeader.spec.ts

@@ -74,7 +74,7 @@ describe("TheHeader:", () => {
     });
     const routerArray = wrapper.findAll(".socials-container > a");
     expect(routerArray[0].attributes("href")).toBe("https://join.zksync.dev/");
-    expect(routerArray[1].attributes("href")).toBe("https://twitter.com/zksync");
+    expect(routerArray[1].attributes("href")).toBe("https://x.com/zksync");
   });
   it("renders network switch", () => {
     const wrapper = mount(TheHeader, {

packages/app/tests/e2e/features/artifacts/artifactsSet1.feature

@@ -39,9 +39,9 @@ Feature: Main Page
       | Value   | url                        |
       # discord renamed to "join"
       | join    | https://join.zksync.dev/   |
-      | twitter | https://twitter.com/zksync |
+      | x.com | https://x.com/zksync |
 
-  @id254:I
+  @id254:
   Scenario Outline: Check dropdown "<Dropdown>" for "<Value>" and verify
     Given Set the "<Value>" value for "<Dropdown>" switcher
     Then Check the "<Value>" value is actual for "<Dropdown>" switcher

packages/app/tests/e2e/features/redirection/redirectionSet1.feature

@@ -21,13 +21,13 @@ Feature: Redirection
   @id231
   Scenario Outline: Verify redirection for "<Icon>" social network icon  on Header
     When I click by element with partial href "<Icon>"
-    Then New page have "<url>" address
+    Then New page address matches the "<regexp>"
 
     Examples:
-      | Icon    | url                                |
+      | Icon    | regexp                                |
       # discord renamed to "join"
-      | join    | https://join.zksync.dev/           |
-      | twitter | https://x.com/zksync               |
+      | join    | ^https://join.zksync.dev/$           |
+      | x.com | ^https://x.com/zksync(\\?.*)?$               |
 
   @id251
   Scenario: Verify redirection for Documentation link

packages/app/tests/e2e/src/steps/blockexplorer.steps.ts

@@ -88,6 +88,16 @@ Then("New page have {string} address", async function (this: ICustomWorld, url:
   await expect(result).toBe(url);
 });
 
+Then("New page address matches the {string}", async function (this: ICustomWorld, regexp: string) {
+  mainPage = new MainPage(this);
+  helper = new Helper(this);
+  await this.page?.waitForTimeout(config.increasedTimeout.timeout);
+  const pages: any = this.context?.pages();
+
+  result = await pages[1].url();
+  await expect(RegExp(regexp).test(result)).toBe(true);
+});
+
 Given("I go to page {string}", async function (this: ICustomWorld, route: string) {
   await this.page?.goto(config.BASE_URL + route + config.DAPP_NETWORK);
   await this.page?.waitForLoadState();

packages/app/tests/views/MaintenanceView.spec.ts

@@ -46,9 +46,9 @@ describe("MaintenanceView:", () => {
       },
     });
 
-    expect(wrapper.findAll(`.description a`)[0].attributes("href")).toBe("https://twitter.com/ZKsyncDevs");
+    expect(wrapper.findAll(`.description a`)[0].attributes("href")).toBe("https://x.com/ZKsyncDevs");
     expect(wrapper.findAll(`.description a`)[1].attributes("href")).toBe("https://uptime.com/statuspage/era");
-    expect(wrapper.find(`.twitter-button`).attributes("href")).toBe("https://twitter.com/ZKsyncDevs");
+    expect(wrapper.find(`.twitter-button`).attributes("href")).toBe("https://x.com/ZKsyncDevs");
     expect(wrapper.find(`.uptime-link`).attributes("href")).toBe("https://uptime.com/statuspage/era");
   });
 });

packages/worker/.env.example

@@ -30,6 +30,9 @@ RPC_BATCH_STALL_TIME_MS=0
 COLLECT_DB_CONNECTION_POOL_METRICS_INTERVAL=10000
 COLLECT_BLOCKS_TO_PROCESS_METRIC_INTERVAL=10000
 
+DISABLE_MISSING_BLOCKS_METRIC=false
+CHECK_MISSING_BLOCKS_METRIC_INTERVAL=86400000
+
 DISABLE_BATCHES_PROCESSING=false
 DISABLE_COUNTERS_PROCESSING=false
 DISABLE_OLD_BALANCES_CLEANER=false

packages/worker/src/block/block.watcher.spec.ts

@@ -5,6 +5,7 @@ import { ConfigService } from "@nestjs/config";
 import { DataFetcherService } from "../dataFetcher/dataFetcher.service";
 import { BlockWatcher } from "./block.watcher";
 import { BlockchainService } from "../blockchain";
+import { BlockRepository } from "../repositories";
 
 jest.useFakeTimers();
 
@@ -18,14 +19,20 @@ describe("BlockWatcher", () => {
   let dataFetcherServiceMock: DataFetcherService;
   let blockchainBlocksMetricMock: jest.Mock;
   let blocksToProcessMetricMock: jest.Mock;
+  let missingBlocksMetricMock: jest.Mock;
   let getBlockInfoDurationMetricStartMock: jest.Mock;
   let getBlockInfoDurationMetricStopMock: jest.Mock;
   let configServiceMock: ConfigService;
+  let blockRepositoryMock: BlockRepository;
 
   const getBlockWatcher = async () => {
     const app = await Test.createTestingModule({
       providers: [
         BlockWatcher,
+        {
+          provide: BlockRepository,
+          useValue: blockRepositoryMock,
+        },
         {
           provide: BlockchainService,
           useValue: blockchainServiceMock,
@@ -46,6 +53,12 @@ describe("BlockWatcher", () => {
             set: blocksToProcessMetricMock,
           },
         },
+        {
+          provide: "PROM_METRIC_MISSING_BLOCKS",
+          useValue: {
+            set: missingBlocksMetricMock,
+          },
+        },
         {
           provide: "PROM_METRIC_GET_BLOCK_INFO_DURATION_SECONDS",
           useValue: {
@@ -67,8 +80,12 @@ describe("BlockWatcher", () => {
   beforeEach(async () => {
     blockchainBlocksMetricMock = jest.fn();
     blocksToProcessMetricMock = jest.fn();
+    missingBlocksMetricMock = jest.fn();
     getBlockInfoDurationMetricStopMock = jest.fn();
     getBlockInfoDurationMetricStartMock = jest.fn().mockReturnValue(getBlockInfoDurationMetricStopMock);
+    blockRepositoryMock = mock<BlockRepository>({
+      getMissingBlocksCount: jest.fn().mockResolvedValue(50),
+    });
     configServiceMock = mock<ConfigService>({
       get: jest.fn().mockImplementation((key: string) => {
         if (key === "blocks.blocksProcessingBatchSize") {
@@ -77,6 +94,10 @@ describe("BlockWatcher", () => {
           return 0;
         } else if (key === "metrics.collectBlocksToProcessMetricInterval") {
           return 10000;
+        } else if (key === "metrics.missingBlocks.disabled") {
+          return true;
+        } else if (key === "metrics.missingBlocks.interval") {
+          return 20000;
         }
         return null;
       }),
@@ -446,6 +467,39 @@ describe("BlockWatcher", () => {
         expect(blocksToProcessMetricMock).toBeCalledWith(0);
       });
     });
+
+    describe("when missing blocks metric is disabled", () => {
+      beforeEach(async () => {
+        (configServiceMock.get as jest.Mock).mockImplementation((key: string) => {
+          if (key === "metrics.missingBlocks.disabled") return true;
+          if (key === "metrics.missingBlocks.interval") return 1000;
+        });
+        blockWatcher = await getBlockWatcher();
+      });
+
+      it("does not set the metric", async () => {
+        await blockWatcher.onModuleInit();
+        expect(blockRepositoryMock.getMissingBlocksCount).toHaveBeenCalledTimes(0);
+        expect(missingBlocksMetricMock).toBeCalledTimes(0);
+      });
+    });
+
+    describe("when missing blocks metric is enabled", () => {
+      beforeEach(async () => {
+        (configServiceMock.get as jest.Mock).mockImplementation((key: string) => {
+          if (key === "metrics.missingBlocks.disabled") return false;
+          if (key === "metrics.missingBlocks.interval") return 1000;
+        });
+        blockWatcher = await getBlockWatcher();
+      });
+
+      it("sets the metric to the proper value", async () => {
+        await blockWatcher.onModuleInit();
+        expect(blockRepositoryMock.getMissingBlocksCount).toHaveBeenCalledTimes(1);
+        expect(missingBlocksMetricMock).toBeCalledTimes(1);
+        expect(missingBlocksMetricMock).toBeCalledWith(50);
+      });
+    });
   });
 
   describe("onModuleDestroy", () => {
@@ -454,5 +508,39 @@ describe("BlockWatcher", () => {
       blockWatcher.onModuleDestroy();
       expect(global.clearInterval).toBeCalledWith(timer);
     });
+
+    describe("when missing blocks metric is disabled", () => {
+      beforeEach(async () => {
+        (configServiceMock.get as jest.Mock).mockImplementation((key: string) => {
+          if (key === "metrics.missingBlocks.disabled") return true;
+          if (key === "metrics.missingBlocks.interval") return 1000;
+        });
+        blockWatcher = await getBlockWatcher();
+      });
+
+      it("does not clear the interval for the metric", async () => {
+        await blockWatcher.onModuleInit();
+        blockWatcher.onModuleDestroy();
+        // first call is for blocks to process metric
+        expect(global.clearInterval).toBeCalledTimes(1);
+      });
+    });
+
+    describe("when missing blocks metric is enabled", () => {
+      beforeEach(async () => {
+        (configServiceMock.get as jest.Mock).mockImplementation((key: string) => {
+          if (key === "metrics.missingBlocks.disabled") return false;
+          if (key === "metrics.missingBlocks.interval") return 1000;
+        });
+        blockWatcher = await getBlockWatcher();
+      });
+
+      it("clears the interval for the metric", async () => {
+        await blockWatcher.onModuleInit();
+        blockWatcher.onModuleDestroy();
+        // first call is for blocks to process metric
+        expect(global.clearInterval).toBeCalledTimes(2);
+      });
+    });
   });
 });

packages/worker/src/block/block.watcher.ts

@@ -5,9 +5,11 @@ import { Gauge, Histogram } from "prom-client";
 import { DataFetcherService } from "../dataFetcher/dataFetcher.service";
 import { BlockData } from "../dataFetcher/types";
 import { BlockchainService } from "../blockchain/blockchain.service";
+import { BlockRepository } from "../repositories";
 import {
   BLOCKCHAIN_BLOCKS_METRIC_NAME,
   BLOCKS_TO_PROCESS_METRIC_NAME,
+  MISSING_BLOCKS_METRIC_NAME,
   GET_BLOCK_INFO_DURATION_METRIC_NAME,
   ProcessingActionMetricLabel,
 } from "../metrics";
@@ -20,6 +22,9 @@ export class BlockWatcher implements OnModuleInit, OnModuleDestroy {
   private readonly batchSize: number;
   private readonly fromBlock: number;
   private readonly toBlock: number;
+  private readonly missingBlocksMetricEnabled: boolean;
+  private readonly missingBlocksMetricInterval: number;
+  private missingBlocksMetricTimer: NodeJS.Timer = null;
   private collectBlocksToProcessMetricInterval: number;
   private collectBlocksToProcessMetricTimer: NodeJS.Timer = null;
 
@@ -28,12 +33,15 @@ export class BlockWatcher implements OnModuleInit, OnModuleDestroy {
   }
 
   public constructor(
+    private readonly blockRepository: BlockRepository,
     private readonly blockchainService: BlockchainService,
     private readonly dataFetchService: DataFetcherService,
     @InjectMetric(BLOCKCHAIN_BLOCKS_METRIC_NAME)
     private readonly blockchainBlocksMetric: Gauge,
     @InjectMetric(BLOCKS_TO_PROCESS_METRIC_NAME)
     private readonly blocksToProcessMetric: Gauge,
+    @InjectMetric(MISSING_BLOCKS_METRIC_NAME)
+    private readonly missingBlocksMetric: Gauge,
     @InjectMetric(GET_BLOCK_INFO_DURATION_METRIC_NAME)
     private readonly getBlockInfoDurationMetric: Histogram<ProcessingActionMetricLabel>,
     configService: ConfigService
@@ -45,6 +53,8 @@ export class BlockWatcher implements OnModuleInit, OnModuleDestroy {
     this.collectBlocksToProcessMetricInterval = configService.get<number>(
       "metrics.collectBlocksToProcessMetricInterval"
     );
+    this.missingBlocksMetricEnabled = !configService.get<boolean>("metrics.missingBlocks.disabled");
+    this.missingBlocksMetricInterval = configService.get<number>("metrics.missingBlocks.interval");
   }
 
   public async getNextBlocksToProcess(lastDbBlockNumber: number = null): Promise<BlockData[]> {
@@ -79,6 +89,11 @@ export class BlockWatcher implements OnModuleInit, OnModuleDestroy {
     }
   }
 
+  private async updateMissingBlocksMetric(): Promise<void> {
+    const missingBlocksCount = await this.blockRepository.getMissingBlocksCount();
+    this.missingBlocksMetric.set(missingBlocksCount);
+  }
+
   private getBlockInfoListFromBlockchain(startBlockNumber: number, endBlockNumber: number): Promise<BlockData[]> {
     const getBlockInfoTasks = [];
     for (let blockNumber = startBlockNumber; blockNumber <= endBlockNumber; blockNumber++) {
@@ -112,9 +127,18 @@ export class BlockWatcher implements OnModuleInit, OnModuleDestroy {
     this.collectBlocksToProcessMetricTimer = setInterval(() => {
       this.setBlocksToProcessMetric();
     }, this.collectBlocksToProcessMetricInterval);
+
+    if (this.missingBlocksMetricEnabled) {
+      this.missingBlocksMetricTimer = setInterval(() => {
+        this.updateMissingBlocksMetric();
+      }, this.missingBlocksMetricInterval);
+    }
   }
 
   public onModuleDestroy() {
     clearInterval(this.collectBlocksToProcessMetricTimer as unknown as number);
+    if (this.missingBlocksMetricEnabled) {
+      clearInterval(this.missingBlocksMetricTimer as unknown as number);
+    }
   }
 }

packages/worker/src/config.spec.ts

@@ -56,6 +56,10 @@ describe("config", () => {
       metrics: {
         collectDbConnectionPoolMetricsInterval: 10000,
         collectBlocksToProcessMetricInterval: 10000,
+        missingBlocks: {
+          disabled: false,
+          interval: 86_400_000,
+        },
       },
     };
   });
@@ -114,6 +118,10 @@ describe("config", () => {
       metrics: {
         collectDbConnectionPoolMetricsInterval: 10000,
         collectBlocksToProcessMetricInterval: 10000,
+        missingBlocks: {
+          disabled: false,
+          interval: 86_400_000,
+        },
       },
     });
   });

packages/worker/src/config.ts

@@ -31,6 +31,8 @@ export default () => {
     TO_BLOCK,
     COINGECKO_IS_PRO_PLAN,
     COINGECKO_API_KEY,
+    DISABLE_MISSING_BLOCKS_METRIC,
+    CHECK_MISSING_BLOCKS_METRIC_INTERVAL,
   } = process.env;
 
   return {
@@ -90,6 +92,10 @@ export default () => {
     metrics: {
       collectDbConnectionPoolMetricsInterval: parseInt(COLLECT_DB_CONNECTION_POOL_METRICS_INTERVAL, 10) || 10000,
       collectBlocksToProcessMetricInterval: parseInt(COLLECT_BLOCKS_TO_PROCESS_METRIC_INTERVAL, 10) || 10000,
+      missingBlocks: {
+        disabled: DISABLE_MISSING_BLOCKS_METRIC === "true",
+        interval: parseInt(CHECK_MISSING_BLOCKS_METRIC_INTERVAL, 10) || 86_400_000, // 1 day
+      },
     },
   };
 };

packages/worker/src/metrics/metrics.provider.ts

@@ -23,6 +23,7 @@ export type BlockchainRpcCallMetricLabel = "function";
 
 export const BLOCKCHAIN_BLOCKS_METRIC_NAME = "blockchain_blocks";
 export const BLOCKS_TO_PROCESS_METRIC_NAME = "blocks_to_process";
+export const MISSING_BLOCKS_METRIC_NAME = "missing_blocks";
 export const BLOCKS_REVERT_DURATION_METRIC_NAME = "blocks_revert_duration_seconds";
 export const BLOCKS_REVERT_DETECT_METRIC_NAME = "blocks_revert_detect";
 
@@ -96,6 +97,10 @@ export const metricProviders: Provider<any>[] = [
     name: BLOCKS_TO_PROCESS_METRIC_NAME,
     help: "total number of remaining blocks to process.",
   }),
+  makeGaugeProvider({
+    name: MISSING_BLOCKS_METRIC_NAME,
+    help: "total number of missing blocks should be 0. A value > 0 indicates an issue that should be investigated.",
+  }),
   makeHistogramProvider({
     name: BLOCKS_REVERT_DURATION_METRIC_NAME,
     help: "revert duration in seconds.",