From 24ae6776a8d6c472d669d5da45b618abe6d198da Mon Sep 17 00:00:00 2001 From: slowbackspace Date: Mon, 18 Nov 2024 00:39:39 +0100 Subject: [PATCH] feat: replace mithril cdn config with list of snapshot mirrors --- CHANGELOG.md | 4 ++ README.md | 16 +++++--- config/test.yaml | 4 +- src/config.ts | 11 +++--- src/proxies/mithril.ts | 10 +++-- src/types/common.ts | 5 +++ src/utils/mithril.ts | 30 ++++++++++++--- test/unit/tests/proxies/mithril.test.ts | 10 ++--- test/unit/tests/utils/mithril.ts | 50 +++++++++++++++++++++++++ 9 files changed, 114 insertions(+), 26 deletions(-) create mode 100644 test/unit/tests/utils/mithril.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 468144df..035aa137 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Custom network support - Option to disable token registry lookups +### Changed + +- Mithril: Snapshot mirrors are now configured via config option `mithril.snapshotMirrors`, `mithril.snapshotCDN` config option and `BLOCKFROST_MITHRIL_SNAPSHOT_CDN` env config variable were removed + ### Fixed - Issue with attempting to release an already-released PostgreSQL client during error handling diff --git a/README.md b/README.md index 1160aa7c..2ff293f0 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,9 @@ If you are using an authenticated db connection that requires a password, you'd mithril: enabled: true # ENV var BLOCKFROST_MITHRIL_ENABLED=true aggregator: "https://aggregator.pre-release-preview.api.mithril.network/aggregator" # ENV var BLOCKFROST_MITHRIL_AGGREGATOR - snapshotCDN: "https://example.com/" # ENV var BLOCKFROST_MITHRIL_SNAPSHOT_CDN + snapshotMirrors: + - originalUrl: "https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network" + mirrorUrl: "https://dummy-mithril-snapshot-cdn.com" ```
@@ -129,14 +131,17 @@ To enable this experimental feature add following lines to your config: mithril: enabled: true # ENV var BLOCKFROST_MITHRIL_ENABLED=true aggregator: 'https://aggregator.pre-release-preview.api.mithril.network/aggregator' # ENV var BLOCKFROST_MITHRIL_AGGREGATOR - snapshotCDN: 'https://example.com/' # Optional, ENV var BLOCKFROST_MITHRIL_SNAPSHOT_CDN -``` + snapshotMirrors: # Optional + - originalUrl: "https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network" + mirrorUrl: "https://dummy-mithril-snapshot-cdn.com" Then you can simply query Mithril API using Blockfrost Backend: ``` + curl localhost:3000/mithril/artifact/snapshots -``` + +```` If you set `mithril.snapshotCDN` option, then the response of `/artifact/snapshots` and `/artifact/snapshot/{digest}` endpoints is enhanced with additional link to the list of snapshot locations. @@ -151,7 +156,7 @@ docker run --rm \ -e BLOCKFROST_CONFIG_SERVER_LISTEN_ADDRESS=0.0.0.0 \ -v $PWD/config:/app/config \ blockfrost/backend-ryo:latest -``` +```` You can also generate a Docker image using Nix instead of the `Dockerfile` running @@ -232,6 +237,7 @@ A minimal usage example is: Check the [nixos-module.nix file](./nixos-module.nix) to check options and the default values. ## Custom Networks + blockfrost-ryo can be configured to run with any genesis parameters. Setting network to `custom`, and using `genesisDataFolder` value in the yaml configuration or environment variable `BLOCKFROST_CONFIG_GENESIS_DATA_FOLDER` you can specify your genesis details. ## Developing diff --git a/config/test.yaml b/config/test.yaml index bcc469c2..98c82e25 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -14,4 +14,6 @@ tokenRegistryUrl: "http://localhost:3100" mithril: enabled: true aggregator: "https://aggregator.release-mainnet.api.mithril.network/aggregator" - snapshotCDN: "https://dummy-mithril-snapshot-cdn.com" + snapshotMirrors: + - originalUrl: "https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network" + mirrorUrl: "https://dummy-mithril-snapshot-cdn.com" diff --git a/src/config.ts b/src/config.ts index 4496f967..3fcc3240 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,5 @@ import config from 'config'; -import { ByronEraParameters, CARDANO_NETWORKS, Network } from './types/common.js'; +import { ByronEraParameters, CARDANO_NETWORKS, Network, SnapshotMirror } from './types/common.js'; import { readFileSync } from 'node:fs'; import * as ResponseTypes from './types/responses/ledger.js'; import { fileURLToPath } from 'url'; @@ -90,10 +90,9 @@ export const loadConfig = () => { ? config.get('mithril.aggregator') : undefined; - const mithrilSnapshotCDN = - process.env.BLOCKFROST_MITHRIL_SNAPSHOT_CDN ?? config.has('mithril.snapshotCDN') - ? config.get('mithril.snapshotCDN') - : undefined; + const mithrilSnapshotMirrors = config.has('mithril.snapshotMirrors') + ? config.get('mithril.snapshotMirrors') + : undefined; const mithrilAllowedEndpoints = config.has('mithril.mithrilAllowedEndpoints') ? config.get('mithril.mithrilAllowedEndpoints') @@ -150,7 +149,7 @@ export const loadConfig = () => { mithril: { enabled: mithrilEnabled, aggregator: mithrilAggregator as string, - snapshotCDN: mithrilSnapshotCDN, + snapshotMirrors: mithrilSnapshotMirrors, allowedEndpoints: mithrilAllowedEndpoints, }, }; diff --git a/src/proxies/mithril.ts b/src/proxies/mithril.ts index b7166a00..cb747404 100644 --- a/src/proxies/mithril.ts +++ b/src/proxies/mithril.ts @@ -15,7 +15,7 @@ export const registerMithrilProxy = (app: FastifyInstance) => { console.log(`Mithril proxy enabled. Aggregator: ${config.mithril.aggregator}.`); - const snapshotCDN = config.mithril.snapshotCDN; + const snapshotMirrors = config.mithril.snapshotMirrors; app.register(fastifyHttpProxy, { upstream: config.mithril.aggregator, @@ -77,7 +77,7 @@ export const registerMithrilProxy = (app: FastifyInstance) => { const isSnapshotEndpoint = matchUrlToEndpoint(url, ['/artifact/snapshot/:digest']); const isSnapshotsEndpoint = matchUrlToEndpoint(url, ['/artifact/snapshots']); - if (snapshotCDN && (isSnapshotEndpoint || isSnapshotsEndpoint)) { + if (snapshotMirrors && (isSnapshotEndpoint || isSnapshotsEndpoint)) { // Custom snapshot CDN was set // Modify response of /artifact/snapshots and /artifact/snapshot/{digest} to append CDN link to list of snapshot locations const body = await convertStreamToString(response); @@ -87,8 +87,10 @@ export const registerMithrilProxy = (app: FastifyInstance) => { const jsonBody = JSON.parse(body); alteredJSONBody = isSnapshotsEndpoint - ? jsonBody.map((snapshot: unknown) => appendLocationToSnapshot(snapshot, snapshotCDN)) - : appendLocationToSnapshot(jsonBody, snapshotCDN); + ? jsonBody.map((snapshot: unknown) => + appendLocationToSnapshot(snapshot, snapshotMirrors), + ) + : appendLocationToSnapshot(jsonBody, snapshotMirrors); // When replying with a body of a different length it is necessary to remove the content-length header. reply.removeHeader('content-length'); diff --git a/src/types/common.ts b/src/types/common.ts index 81ae15f4..de6b6dde 100644 --- a/src/types/common.ts +++ b/src/types/common.ts @@ -35,3 +35,8 @@ type OnchainMetadataItem = components['schemas']['asset']['onchain_metadata']; export type OnchainMetadata = { version?: number; } & Record>; + +export interface SnapshotMirror { + originalUrl: string; + mirrorUrl: string; +} diff --git a/src/utils/mithril.ts b/src/utils/mithril.ts index bfb59de2..022f0409 100644 --- a/src/utils/mithril.ts +++ b/src/utils/mithril.ts @@ -1,10 +1,10 @@ -export const appendLocationToSnapshot = (snapshot: unknown, baseSnapshotURL: string) => { +import { SnapshotMirror } from '../types/common.js'; + +export const appendLocationToSnapshot = (snapshot: unknown, snapshotMirrors: SnapshotMirror[]) => { if ( typeof snapshot !== 'object' || snapshot === null || - !('digest' in snapshot) || !('locations' in snapshot) || - typeof snapshot.digest !== 'string' || !Array.isArray(snapshot.locations) ) { console.error('Could not append URL to snapshot locations. Invalid data format.', snapshot); @@ -13,8 +13,28 @@ export const appendLocationToSnapshot = (snapshot: unknown, baseSnapshotURL: str const snapshotCopy = { ...snapshot }; - const additionalSnapshotUrl = new URL(snapshot.digest, baseSnapshotURL); + if (!Array.isArray(snapshotCopy.locations)) { + throw new TypeError('Invalid snapshot format'); + } + + const mirroredUrls: string[] = []; + + // Create mirror url for every snapshot location matching snapshotMirror configuration + for (const snapshotLocation of snapshotCopy.locations) { + if (typeof snapshotLocation !== 'string') { + continue; + } + + for (const snapshotMirror of snapshotMirrors) { + const mirroredUrl = snapshotLocation.replace( + snapshotMirror.originalUrl, + snapshotMirror.mirrorUrl, + ); + + mirroredUrls.push(mirroredUrl); + } + } + snapshotCopy.locations.push(...mirroredUrls); - (snapshotCopy.locations as unknown[]).push(additionalSnapshotUrl); return snapshotCopy; }; diff --git a/test/unit/tests/proxies/mithril.test.ts b/test/unit/tests/proxies/mithril.test.ts index 12096dfa..da2353d4 100644 --- a/test/unit/tests/proxies/mithril.test.ts +++ b/test/unit/tests/proxies/mithril.test.ts @@ -154,7 +154,7 @@ describe('mithril proxy text', () => { created_at: '2024-05-22T15:17:47.601798793Z', locations: [ 'https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network/preview-e575-i11509.3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba.tar.zst', - 'https://dummy-mithril-snapshot-cdn.com/3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba', + 'https://dummy-mithril-snapshot-cdn.com/preview-e575-i11509.3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba.tar.zst', ], compression_algorithm: 'zstandard', cardano_node_version: '8.9.0', @@ -171,7 +171,7 @@ describe('mithril proxy text', () => { created_at: '2024-05-22T14:02:55.976983297Z', locations: [ 'https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network/preview-e575-i11508.dc55f5508a3beedf990a362037ddc21a8d39e3ed81ab81eb5fa62c0a2835c0f6.tar.zst', - 'https://dummy-mithril-snapshot-cdn.com/dc55f5508a3beedf990a362037ddc21a8d39e3ed81ab81eb5fa62c0a2835c0f6', + 'https://dummy-mithril-snapshot-cdn.com/preview-e575-i11508.dc55f5508a3beedf990a362037ddc21a8d39e3ed81ab81eb5fa62c0a2835c0f6.tar.zst', ], compression_algorithm: 'zstandard', cardano_node_version: '8.9.0', @@ -225,20 +225,20 @@ describe('mithril proxy text', () => { created_at: '2024-05-22T15:17:47.601798793Z', locations: [ 'https://storage.googleapis.com/cdn.aggregator.pre-release-preview.api.mithril.network/preview-e575-i11509.3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba.tar.zst', - 'https://dummy-mithril-snapshot-cdn.com/3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba', + 'https://dummy-mithril-snapshot-cdn.com/preview-e575-i11509.3de0e6d3fd837ae1035688623cb4de8318f6205ea02da6df2592dabecdd631ba.tar.zst', ], compression_algorithm: 'zstandard', cardano_node_version: '8.9.0', }); }); - test('DOES NOT modifies response /artifact/snapshot/:digest when mithril.snapshotCDN is not set', async () => { + test('DOES NOT modifies response /artifact/snapshot/:digest when mithril.snapshotMirrors is not set', async () => { vi.spyOn(config, 'getConfig').mockReturnValue({ ...config.mainConfig, network: 'mainnet', mithril: { ...config.mainConfig.mithril, - snapshotCDN: undefined, + snapshotMirrors: undefined, }, }); diff --git a/test/unit/tests/utils/mithril.ts b/test/unit/tests/utils/mithril.ts new file mode 100644 index 00000000..c2658637 --- /dev/null +++ b/test/unit/tests/utils/mithril.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from 'vitest'; +import { appendLocationToSnapshot } from '../../../../src/utils/mithril.js'; + +describe('mithril utils', () => { + test('appendLocationToSnapshot', () => { + expect( + appendLocationToSnapshot( + { + digest: '726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78', + beacon: { + network: 'mainnet', + epoch: 522, + immutable_file_number: 6492, + }, + certificate_hash: '95306acc7c57b20f1fb451c4abf4acdb09345edf866cb28823dc1acf3f032349', + size: 51_840_671_826, + created_at: '2024-11-17T16:45:55.499648095Z', + locations: [ + 'https://storage.googleapis.com/cdn.aggregator.release-mainnet.api.mithril.network/mainnet-e522-i6492.726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78.tar.zst', + ], + compression_algorithm: 'zstandard', + cardano_node_version: '10.1.2', + }, + [ + { + originalUrl: + 'https://storage.googleapis.com/cdn.aggregator.release-mainnet.api.mithril.network', + mirrorUrl: 'https://mirror.com', + }, + ], + ), + ).toStrictEqual({ + digest: '726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78', + beacon: { + network: 'mainnet', + epoch: 522, + immutable_file_number: 6492, + }, + certificate_hash: '95306acc7c57b20f1fb451c4abf4acdb09345edf866cb28823dc1acf3f032349', + size: 51_840_671_826, + created_at: '2024-11-17T16:45:55.499648095Z', + locations: [ + 'https://storage.googleapis.com/cdn.aggregator.release-mainnet.api.mithril.network/mainnet-e522-i6492.726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78.tar.zst', + 'https://mirror.com/mainnet-e522-i6492.726f5801f7749080ab27e184d6fbd7850d571952a6cf7a8e55e79ffc04b55a78.tar.zst', + ], + compression_algorithm: 'zstandard', + cardano_node_version: '10.1.2', + }); + }); +});