feat: replace mithril cdn config with list of snapshot mirrors

CHANGELOG.md

@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Custom network support
 - Option to disable token registry lookups
 
+### Changed
+
+- Mithril: Snapshot mirrors are now configured via config option `mithril.snapshotMirrors`, `mithril.snapshotCDN` config option and `BLOCKFROST_MITHRIL_SNAPSHOT_CDN` env config variable were removed
+
 ### Fixed
 
 - Issue with attempting to release an already-released PostgreSQL client during error handling

README.md

@@ -67,7 +67,9 @@ If you are using an authenticated db connection that requires a password, you'd
   mithril:
     enabled: true # ENV var BLOCKFROST_MITHRIL_ENABLED=true
     aggregator: "https://aggregator.pre-release-preview.api.mithril.network/aggregator" # ENV var BLOCKFROST_MITHRIL_AGGREGATOR
-    snapshotCDN: "https://example.com/" # ENV var BLOCKFROST_MITHRIL_SNAPSHOT_CDN
+    snapshotMirrors:
+    - originalUrl: "https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network"
+      mirrorUrl: "https://dummy-mithril-snapshot-cdn.com"
 ```
 
 <details>
@@ -129,14 +131,17 @@ To enable this experimental feature add following lines to your config:
 mithril:
   enabled: true # ENV var BLOCKFROST_MITHRIL_ENABLED=true
   aggregator: 'https://aggregator.pre-release-preview.api.mithril.network/aggregator' # ENV var BLOCKFROST_MITHRIL_AGGREGATOR
-  snapshotCDN: 'https://example.com/' # Optional, ENV var BLOCKFROST_MITHRIL_SNAPSHOT_CDN
-```
+  snapshotMirrors: # Optional
+    - originalUrl: "https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network"
+    mirrorUrl: "https://dummy-mithril-snapshot-cdn.com"
 
 Then you can simply query Mithril API using Blockfrost Backend:
 
 ```
+
 curl localhost:3000/mithril/artifact/snapshots
-```
+
+````
 
 If you set `mithril.snapshotCDN` option, then the response of `/artifact/snapshots` and `/artifact/snapshot/{digest}` endpoints is enhanced with additional link to the list of snapshot locations.
 
@@ -151,7 +156,7 @@ docker run --rm \
   -e BLOCKFROST_CONFIG_SERVER_LISTEN_ADDRESS=0.0.0.0 \
   -v $PWD/config:/app/config \
   blockfrost/backend-ryo:latest
-```
+````
 
 You can also generate a Docker image using Nix instead of the `Dockerfile` running
 
@@ -232,6 +237,7 @@ A minimal usage example is:
 Check the [nixos-module.nix file](./nixos-module.nix) to check options and the default values.
 
 ## Custom Networks
+
 blockfrost-ryo can be configured to run with any genesis parameters. Setting network to `custom`, and using `genesisDataFolder` value in the yaml configuration or environment variable `BLOCKFROST_CONFIG_GENESIS_DATA_FOLDER` you can specify your genesis details.
 
 ## Developing

config/test.yaml

@@ -14,4 +14,6 @@ tokenRegistryUrl: "http://localhost:3100"
 mithril:
   enabled: true
   aggregator: "https://aggregator.release-mainnet.api.mithril.network/aggregator"
-  snapshotCDN: "https://dummy-mithril-snapshot-cdn.com"
+  snapshotMirrors:
+    - originalUrl: "https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network"
+      mirrorUrl: "https://dummy-mithril-snapshot-cdn.com"

src/config.ts

@@ -1,5 +1,5 @@
 import config from 'config';
-import { ByronEraParameters, CARDANO_NETWORKS, Network } from './types/common.js';
+import { ByronEraParameters, CARDANO_NETWORKS, Network, SnapshotMirror } from './types/common.js';
 import { readFileSync } from 'node:fs';
 import * as ResponseTypes from './types/responses/ledger.js';
 import { fileURLToPath } from 'url';
@@ -90,10 +90,9 @@ export const loadConfig = () => {
       ? config.get<string>('mithril.aggregator')
       : undefined;
 
-  const mithrilSnapshotCDN =
-    process.env.BLOCKFROST_MITHRIL_SNAPSHOT_CDN ?? config.has('mithril.snapshotCDN')
-      ? config.get<string>('mithril.snapshotCDN')
-      : undefined;
+  const mithrilSnapshotMirrors = config.has('mithril.snapshotMirrors')
+    ? config.get<SnapshotMirror[]>('mithril.snapshotMirrors')
+    : undefined;
 
   const mithrilAllowedEndpoints = config.has('mithril.mithrilAllowedEndpoints')
     ? config.get<string[]>('mithril.mithrilAllowedEndpoints')
@@ -150,7 +149,7 @@ export const loadConfig = () => {
     mithril: {
       enabled: mithrilEnabled,
       aggregator: mithrilAggregator as string,
-      snapshotCDN: mithrilSnapshotCDN,
+      snapshotMirrors: mithrilSnapshotMirrors,
       allowedEndpoints: mithrilAllowedEndpoints,
     },
   };

src/proxies/mithril.ts

@@ -15,7 +15,7 @@ export const registerMithrilProxy = (app: FastifyInstance) => {
 
   console.log(`Mithril proxy enabled. Aggregator: ${config.mithril.aggregator}.`);
 
-  const snapshotCDN = config.mithril.snapshotCDN;
+  const snapshotMirrors = config.mithril.snapshotMirrors;
 
   app.register(fastifyHttpProxy, {
     upstream: config.mithril.aggregator,
@@ -77,7 +77,7 @@ export const registerMithrilProxy = (app: FastifyInstance) => {
         const isSnapshotEndpoint = matchUrlToEndpoint(url, ['/artifact/snapshot/:digest']);
         const isSnapshotsEndpoint = matchUrlToEndpoint(url, ['/artifact/snapshots']);
 
-        if (snapshotCDN && (isSnapshotEndpoint || isSnapshotsEndpoint)) {
+        if (snapshotMirrors && (isSnapshotEndpoint || isSnapshotsEndpoint)) {
           // Custom snapshot CDN was set
           // Modify response of /artifact/snapshots and /artifact/snapshot/{digest} to append CDN link to list of snapshot locations
           const body = await convertStreamToString(response);
@@ -87,8 +87,10 @@ export const registerMithrilProxy = (app: FastifyInstance) => {
             const jsonBody = JSON.parse(body);
 
             alteredJSONBody = isSnapshotsEndpoint
-              ? jsonBody.map((snapshot: unknown) => appendLocationToSnapshot(snapshot, snapshotCDN))
-              : appendLocationToSnapshot(jsonBody, snapshotCDN);
+              ? jsonBody.map((snapshot: unknown) =>
+                  appendLocationToSnapshot(snapshot, snapshotMirrors),
+                )
+              : appendLocationToSnapshot(jsonBody, snapshotMirrors);
 
             // When replying with a body of a different length it is necessary to remove the content-length header.
             reply.removeHeader('content-length');

src/types/common.ts

@@ -35,3 +35,8 @@ type OnchainMetadataItem = components['schemas']['asset']['onchain_metadata'];
 export type OnchainMetadata = {
   version?: number;
 } & Record<string, Record<string, OnchainMetadataItem>>;
+
+export interface SnapshotMirror {
+  originalUrl: string;
+  mirrorUrl: string;
+}

src/utils/mithril.ts

@@ -1,10 +1,10 @@
-export const appendLocationToSnapshot = (snapshot: unknown, baseSnapshotURL: string) => {
+import { SnapshotMirror } from '../types/common.js';
+
+export const appendLocationToSnapshot = (snapshot: unknown, snapshotMirrors: SnapshotMirror[]) => {
   if (
     typeof snapshot !== 'object' ||
     snapshot === null ||
-    !('digest' in snapshot) ||
     !('locations' in snapshot) ||
-    typeof snapshot.digest !== 'string' ||
     !Array.isArray(snapshot.locations)
   ) {
     console.error('Could not append URL to snapshot locations. Invalid data format.', snapshot);
@@ -13,8 +13,28 @@ export const appendLocationToSnapshot = (snapshot: unknown, baseSnapshotURL: str
 
   const snapshotCopy = { ...snapshot };
 
-  const additionalSnapshotUrl = new URL(snapshot.digest, baseSnapshotURL);
+  if (!Array.isArray(snapshotCopy.locations)) {
+    throw new TypeError('Invalid snapshot format');
+  }
+
+  const mirroredUrls: string[] = [];
+
+  // Create mirror url for every snapshot location matching snapshotMirror configuration
+  for (const snapshotLocation of snapshotCopy.locations) {
+    if (typeof snapshotLocation !== 'string') {
+      continue;
+    }
+
+    for (const snapshotMirror of snapshotMirrors) {
+      const mirroredUrl = snapshotLocation.replace(
+        snapshotMirror.originalUrl,
+        snapshotMirror.mirrorUrl,
+      );
+
+      mirroredUrls.push(mirroredUrl);
+    }
+  }
+  snapshotCopy.locations.push(...mirroredUrls);
 
-  (snapshotCopy.locations as unknown[]).push(additionalSnapshotUrl);
   return snapshotCopy;
 };

test/unit/tests/proxies/mithril.test.ts

@@ -154,7 +154,7 @@ describe('mithril proxy text', () => {
         created_at: '2024-05-22T15:17:47.601798793Z',
         locations: [
           'https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network/preview-e575-i11509.3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba.tar.zst',
-          'https://dummy-mithril-snapshot-cdn.com/3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba',
+          'https://dummy-mithril-snapshot-cdn.com/preview-e575-i11509.3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba.tar.zst',
         ],
         compression_algorithm: 'zstandard',
         cardano_node_version: '8.9.0',
@@ -171,7 +171,7 @@ describe('mithril proxy text', () => {
         created_at: '2024-05-22T14:02:55.976983297Z',
         locations: [
           'https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network/preview-e575-i11508.dc55f5508a3beedf990a362037ddc21a8d39e3ed81ab81eb5fa62c0a2835c0f6.tar.zst',
-          'https://dummy-mithril-snapshot-cdn.com/dc55f5508a3beedf990a362037ddc21a8d39e3ed81ab81eb5fa62c0a2835c0f6',
+          'https://dummy-mithril-snapshot-cdn.com/preview-e575-i11508.dc55f5508a3beedf990a362037ddc21a8d39e3ed81ab81eb5fa62c0a2835c0f6.tar.zst',
         ],
         compression_algorithm: 'zstandard',
         cardano_node_version: '8.9.0',
@@ -225,20 +225,20 @@ describe('mithril proxy text', () => {
       created_at: '2024-05-22T15:17:47.601798793Z',
       locations: [
         'https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network/preview-e575-i11509.3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba.tar.zst',
-        'https://dummy-mithril-snapshot-cdn.com/3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba',
+        'https://dummy-mithril-snapshot-cdn.com/preview-e575-i11509.3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba.tar.zst',
       ],
       compression_algorithm: 'zstandard',
       cardano_node_version: '8.9.0',
     });
   });
 
-  test('DOES NOT modifies response /artifact/snapshot/:digest when mithril.snapshotCDN is not set', async () => {
+  test('DOES NOT modifies response /artifact/snapshot/:digest when mithril.snapshotMirrors is not set', async () => {
     vi.spyOn(config, 'getConfig').mockReturnValue({
       ...config.mainConfig,
       network: 'mainnet',
       mithril: {
         ...config.mainConfig.mithril,
-        snapshotCDN: undefined,
+        snapshotMirrors: undefined,
       },
     });
 

test/unit/tests/utils/mithril.ts

@@ -0,0 +1,50 @@
+import { describe, expect, test } from 'vitest';
+import { appendLocationToSnapshot } from '../../../../src/utils/mithril.js';
+
+describe('mithril utils', () => {
+  test('appendLocationToSnapshot', () => {
+    expect(
+      appendLocationToSnapshot(
+        {
+          digest: '726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78',
+          beacon: {
+            network: 'mainnet',
+            epoch: 522,
+            immutable_file_number: 6492,
+          },
+          certificate_hash: '95306acc7c57b20f1fb451c4abf4acdb09345edf866cb28823dc1acf3f032349',
+          size: 51_840_671_826,
+          created_at: '2024-11-17T16:45:55.499648095Z',
+          locations: [
+            'https://storage.googleapis.com/cdn.aggregator.release-mainnet.api.mithril.network/mainnet-e522-i6492.726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78.tar.zst',
+          ],
+          compression_algorithm: 'zstandard',
+          cardano_node_version: '10.1.2',
+        },
+        [
+          {
+            originalUrl:
+              'https://storage.googleapis.com/cdn.aggregator.release-mainnet.api.mithril.network',
+            mirrorUrl: 'https://mirror.com',
+          },
+        ],
+      ),
+    ).toStrictEqual({
+      digest: '726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78',
+      beacon: {
+        network: 'mainnet',
+        epoch: 522,
+        immutable_file_number: 6492,
+      },
+      certificate_hash: '95306acc7c57b20f1fb451c4abf4acdb09345edf866cb28823dc1acf3f032349',
+      size: 51_840_671_826,
+      created_at: '2024-11-17T16:45:55.499648095Z',
+      locations: [
+        'https://storage.googleapis.com/cdn.aggregator.release-mainnet.api.mithril.network/mainnet-e522-i6492.726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78.tar.zst',
+        'https://mirror.com/mainnet-e522-i6492.726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78.tar.zst',
+      ],
+      compression_algorithm: 'zstandard',
+      cardano_node_version: '10.1.2',
+    });
+  });
+});