From c98f0cd6a1001933764ab2fbb4026f8824127f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Mon, 7 Oct 2024 16:06:05 -0600 Subject: [PATCH 1/6] fix: process nft and sft mints in batches (#271) * fix: process nft and sft mints in batches * fix: better batching * fix: rollbacks --- src/pg/chainhook/chainhook-pg-store.ts | 242 +++++++----------- src/pg/types.ts | 12 +- .../queue/job/process-smart-contract-job.ts | 2 +- src/token-processor/util/sip-validation.ts | 4 +- tests/chainhook/nft-events.test.ts | 26 ++ tests/helpers.ts | 2 +- 6 files changed, 128 insertions(+), 160 deletions(-) diff --git a/src/pg/chainhook/chainhook-pg-store.ts b/src/pg/chainhook/chainhook-pg-store.ts index 86cb5087..1cfc0a52 100644 --- a/src/pg/chainhook/chainhook-pg-store.ts +++ b/src/pg/chainhook/chainhook-pg-store.ts @@ -14,14 +14,7 @@ import { TokenMetadataUpdateNotification, } from '../../token-processor/util/sip-validation'; import { ContractNotFoundError } from '../errors'; -import { - DbJob, - DbSipNumber, - DbSmartContractInsert, - DbTokenInsert, - DbTokenType, - DbSmartContract, -} from '../types'; +import { DbJob, DbSipNumber, DbSmartContractInsert, DbTokenType, DbSmartContract } from '../types'; import { BlockCache, CachedEvent } from './block-cache'; import { dbSipNumberToDbTokenType } from '../../token-processor/util/helpers'; import BigNumber from 'bignumber.js'; @@ -66,13 +59,16 @@ export class ChainhookPgStore extends BasePgStoreModule { /** * Inserts new tokens and new token queue entries until `token_count` items are created, usually - * used when processing an NFT contract. + * used when processing an NFT contract that has just been deployed. */ - async insertAndEnqueueSequentialTokens(args: { - smart_contract: DbSmartContract; - token_count: bigint; - }): Promise { - const tokenValues: DbTokenInsert[] = []; + async insertAndEnqueueSequentialTokens( + sql: PgSqlClient, + args: { + smart_contract: DbSmartContract; + token_count: bigint; + } + ): Promise { + const tokenValues = []; for (let index = 1; index <= args.token_count; index++) tokenValues.push({ smart_contract_id: args.smart_contract.id, @@ -83,7 +79,25 @@ export class ChainhookPgStore extends BasePgStoreModule { tx_id: args.smart_contract.tx_id, tx_index: args.smart_contract.tx_index, }); - return this.insertAndEnqueueTokens(tokenValues); + for await (const batch of batchIterate(tokenValues, 500)) { + await sql` + WITH token_inserts AS ( + INSERT INTO tokens ${sql(batch)} + ON CONFLICT ON CONSTRAINT tokens_smart_contract_id_token_number_unique DO + UPDATE SET + uri = EXCLUDED.uri, + name = EXCLUDED.name, + symbol = EXCLUDED.symbol, + decimals = EXCLUDED.decimals, + total_supply = EXCLUDED.total_supply, + updated_at = NOW() + RETURNING id + ) + INSERT INTO jobs (token_id) (SELECT id AS token_id FROM token_inserts) + ON CONFLICT (token_id) WHERE smart_contract_id IS NULL DO + UPDATE SET updated_at = NOW(), status = 'pending' + `; + } } async applyContractDeployment( @@ -150,8 +164,8 @@ export class ChainhookPgStore extends BasePgStoreModule { await this.applyContractDeployment(sql, contract, cache); for (const notification of cache.notifications) await this.applyNotification(sql, notification, cache); - for (const mint of cache.nftMints) await this.applyNftMint(sql, mint, cache); - for (const mint of cache.sftMints) await this.applySftMint(sql, mint, cache); + await this.applyTokenMints(sql, cache.nftMints, DbTokenType.nft, cache); + await this.applyTokenMints(sql, cache.sftMints, DbTokenType.sft, cache); for (const [contract, delta] of cache.ftSupplyDelta) await this.applyFtSupplyChange(sql, contract, delta, cache); } @@ -161,8 +175,8 @@ export class ChainhookPgStore extends BasePgStoreModule { await this.rollBackContractDeployment(sql, contract, cache); for (const notification of cache.notifications) await this.rollBackNotification(sql, notification, cache); - for (const mint of cache.nftMints) await this.rollBackNftMint(sql, mint, cache); - for (const mint of cache.sftMints) await this.rollBackSftMint(sql, mint, cache); + await this.rollBackTokenMints(sql, cache.nftMints, DbTokenType.nft, cache); + await this.rollBackTokenMints(sql, cache.sftMints, DbTokenType.sft, cache); for (const [contract, delta] of cache.ftSupplyDelta) await this.applyFtSupplyChange(sql, contract, delta.negated(), cache); } @@ -223,68 +237,6 @@ export class ChainhookPgStore extends BasePgStoreModule { ); } - private async applyNftMint( - sql: PgSqlClient, - mint: CachedEvent, - cache: BlockCache - ): Promise { - try { - await this.insertAndEnqueueTokens([ - { - smart_contract_id: await this.findSmartContractId( - mint.event.contractId, - DbSipNumber.sip009 - ), - type: DbTokenType.nft, - token_number: mint.event.tokenId.toString(), - block_height: cache.block.index, - index_block_hash: cache.block.hash, - tx_id: mint.tx_id, - tx_index: mint.tx_index, - }, - ]); - logger.info( - `ChainhookPgStore apply NFT mint ${mint.event.contractId} (${mint.event.tokenId}) at block ${cache.block.index}` - ); - } catch (error) { - if (error instanceof ContractNotFoundError) - logger.warn( - `ChainhookPgStore found NFT mint for nonexisting contract ${mint.event.contractId}` - ); - else throw error; - } - } - - private async applySftMint( - sql: PgSqlClient, - mint: CachedEvent, - cache: BlockCache - ): Promise { - try { - await this.insertAndEnqueueTokens([ - { - smart_contract_id: await this.findSmartContractId( - mint.event.contractId, - DbSipNumber.sip013 - ), - type: DbTokenType.sft, - token_number: mint.event.tokenId.toString(), - block_height: cache.block.index, - index_block_hash: cache.block.hash, - tx_id: mint.tx_id, - tx_index: mint.tx_index, - }, - ]); - logger.info( - `ChainhookPgStore apply SFT mint ${mint.event.contractId} (${mint.event.tokenId}) at block ${cache.block.index}` - ); - } catch (error) { - if (error instanceof ContractNotFoundError) - logger.warn(error, `ChainhookPgStore found SFT mint for nonexisting contract`); - else throw error; - } - } - private async applyFtSupplyChange( sql: PgSqlClient, contract: string, @@ -333,64 +285,6 @@ export class ChainhookPgStore extends BasePgStoreModule { ); } - private async rollBackNftMint( - sql: PgSqlClient, - mint: CachedEvent, - cache: BlockCache - ): Promise { - try { - const smart_contract_id = await this.findSmartContractId( - mint.event.contractId, - DbSipNumber.sip009 - ); - await sql` - DELETE FROM tokens - WHERE smart_contract_id = ${smart_contract_id} AND token_number = ${mint.event.tokenId} - `; - logger.info( - `ChainhookPgStore rollback NFT mint ${mint.event.contractId} (${mint.event.tokenId}) at block ${cache.block.index}` - ); - } catch (error) { - if (error instanceof ContractNotFoundError) - logger.warn(error, `ChainhookPgStore found NFT mint for nonexisting contract`); - else throw error; - } - } - - private async rollBackSftMint( - sql: PgSqlClient, - mint: CachedEvent, - cache: BlockCache - ): Promise { - try { - const smart_contract_id = await this.findSmartContractId( - mint.event.contractId, - DbSipNumber.sip013 - ); - await sql` - DELETE FROM tokens - WHERE smart_contract_id = ${smart_contract_id} AND token_number = ${mint.event.tokenId} - `; - logger.info( - `ChainhookPgStore rollback SFT mint ${mint.event.contractId} (${mint.event.tokenId}) at block ${cache.block.index}` - ); - } catch (error) { - if (error instanceof ContractNotFoundError) - logger.warn(error, `ChainhookPgStore found SFT mint for nonexisting contract`); - else throw error; - } - } - - private async findSmartContractId(principal: string, sip: DbSipNumber): Promise { - const result = await this.sql<{ id: number }[]>` - SELECT id - FROM smart_contracts - WHERE principal = ${principal} AND sip = ${sip} - `; - if (result.count) return result[0].id; - throw new ContractNotFoundError(); - } - private async enqueueDynamicTokensDueForRefresh(): Promise { const interval = ENV.METADATA_DYNAMIC_TOKEN_REFRESH_INTERVAL.toString(); await this.sql` @@ -420,11 +314,42 @@ export class ChainhookPgStore extends BasePgStoreModule { `; } - private async insertAndEnqueueTokens(tokenValues: DbTokenInsert[]): Promise { - for await (const batch of batchIterate(tokenValues, 500)) { - await this.sql` - WITH token_inserts AS ( - INSERT INTO tokens ${this.sql(batch)} + private async applyTokenMints( + sql: PgSqlClient, + mints: CachedEvent[], + tokenType: DbTokenType, + cache: BlockCache + ): Promise { + if (mints.length == 0) return; + for await (const batch of batchIterate(mints, 500)) { + const values = batch.map(m => { + logger.info( + `ChainhookPgStore apply ${tokenType.toUpperCase()} mint ${m.event.contractId} (${ + m.event.tokenId + }) at block ${cache.block.index}` + ); + return [ + m.event.contractId, + tokenType, + m.event.tokenId.toString(), + cache.block.index, + cache.block.hash, + m.tx_id, + m.tx_index, + ]; + }); + await sql` + WITH insert_values (principal, type, token_number, block_height, index_block_hash, tx_id, + tx_index) AS (VALUES ${sql(values)}), + filtered_values AS ( + SELECT s.id AS smart_contract_id, i.type::token_type, i.token_number::bigint, + i.block_height::bigint, i.index_block_hash::text, i.tx_id::text, i.tx_index::int + FROM insert_values AS i + INNER JOIN smart_contracts AS s ON s.principal = i.principal::text + ), + token_inserts AS ( + INSERT INTO tokens (smart_contract_id, type, token_number, block_height, index_block_hash, + tx_id, tx_index) (SELECT * FROM filtered_values) ON CONFLICT ON CONSTRAINT tokens_smart_contract_id_token_number_unique DO UPDATE SET uri = EXCLUDED.uri, @@ -441,4 +366,33 @@ export class ChainhookPgStore extends BasePgStoreModule { `; } } + + private async rollBackTokenMints( + sql: PgSqlClient, + mints: CachedEvent[], + tokenType: DbTokenType, + cache: BlockCache + ): Promise { + if (mints.length == 0) return; + for await (const batch of batchIterate(mints, 500)) { + const values = batch.map(m => { + logger.info( + `ChainhookPgStore rollback ${tokenType.toUpperCase()} mint ${m.event.contractId} (${ + m.event.tokenId + }) at block ${cache.block.index}` + ); + return [m.event.contractId, m.event.tokenId.toString()]; + }); + await sql` + WITH delete_values (principal, token_number) AS (VALUES ${sql(values)}) + DELETE FROM tokens WHERE id IN ( + SELECT t.id + FROM delete_values AS d + INNER JOIN smart_contracts AS s ON s.principal = d.principal::text + INNER JOIN tokens AS t + ON t.smart_contract_id = s.id AND t.token_number = d.token_number::bigint + ) + `; + } + } } diff --git a/src/pg/types.ts b/src/pg/types.ts index 85231cd9..b84f07da 100644 --- a/src/pg/types.ts +++ b/src/pg/types.ts @@ -1,4 +1,4 @@ -import { PgJsonb, PgNumeric } from '@hirosystems/api-toolkit'; +import { PgJsonb, PgNumeric, PgSqlQuery } from '@hirosystems/api-toolkit'; import { FtOrderBy, Order } from '../api/schemas'; export enum DbSipNumber { @@ -69,16 +69,6 @@ export type DbSmartContract = { non_fungible_token_name?: string; }; -export type DbTokenInsert = { - smart_contract_id: number; - type: DbTokenType; - token_number: PgNumeric; - block_height: number; - index_block_hash: string; - tx_id: string; - tx_index: number; -}; - export type DbToken = { id: number; smart_contract_id: number; diff --git a/src/token-processor/queue/job/process-smart-contract-job.ts b/src/token-processor/queue/job/process-smart-contract-job.ts index 7d946414..1aeab0fa 100644 --- a/src/token-processor/queue/job/process-smart-contract-job.ts +++ b/src/token-processor/queue/job/process-smart-contract-job.ts @@ -74,7 +74,7 @@ export class ProcessSmartContractJob extends Job { `ProcessSmartContractJob enqueueing ${tokenCount} tokens for ${this.description()}` ); await this.db.updateSmartContractTokenCount({ id: contract.id, count: tokenCount }); - await this.db.chainhook.insertAndEnqueueSequentialTokens({ + await this.db.chainhook.insertAndEnqueueSequentialTokens(sql, { smart_contract: contract, token_count: tokenCount, }); diff --git a/src/token-processor/util/sip-validation.ts b/src/token-processor/util/sip-validation.ts index 7524a47c..d7198a0d 100644 --- a/src/token-processor/util/sip-validation.ts +++ b/src/token-processor/util/sip-validation.ts @@ -375,9 +375,7 @@ export type NftMintEvent = { tokenId: bigint; }; -export type SftMintEvent = { - contractId: string; - tokenId: bigint; +export type SftMintEvent = NftMintEvent & { amount: bigint; recipient: string; }; diff --git a/tests/chainhook/nft-events.test.ts b/tests/chainhook/nft-events.test.ts index 1e57e8db..0770f37b 100644 --- a/tests/chainhook/nft-events.test.ts +++ b/tests/chainhook/nft-events.test.ts @@ -92,6 +92,32 @@ describe('NFT events', () => { await expect(db.getToken({ id: 1 })).resolves.not.toBeUndefined(); }); + test('NFT mint is ignored if contract does not exist', async () => { + const address = 'SP1K1A1PMGW2ZJCNF46NWZWHG8TS1D23EGH1KNK60'; + const contractId = `${address}.friedger-pool-nft`; + + await db.chainhook.processPayload( + new TestChainhookPayloadBuilder() + .apply() + .block({ height: 100 }) + .transaction({ hash: '0x01', sender: address }) + .event({ + type: 'NFTMintEvent', + position: { index: 0 }, + data: { + asset_identifier: `${contractId}::crashpunks-v2`, + recipient: address, + raw_value: cvToHex(uintCV(1)), + }, + }) + .build() + ); + + const jobs = await db.getPendingJobBatch({ limit: 1 }); + expect(jobs).toHaveLength(0); + await expect(db.getToken({ id: 1 })).resolves.toBeUndefined(); + }); + test('NFT mint roll back removes token', async () => { const address = 'SP1K1A1PMGW2ZJCNF46NWZWHG8TS1D23EGH1KNK60'; const contractId = `${address}.friedger-pool-nft`; diff --git a/tests/helpers.ts b/tests/helpers.ts index 44b8dbd2..a116ce4f 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -1436,7 +1436,7 @@ export async function insertAndEnqueueTestContractWithTokens( return await db.sqlWriteTransaction(async sql => { await insertAndEnqueueTestContract(db, principal, sip, tx_id); const smart_contract = (await db.getSmartContract({ principal })) as DbSmartContract; - await db.chainhook.insertAndEnqueueSequentialTokens({ + await db.chainhook.insertAndEnqueueSequentialTokens(sql, { smart_contract, token_count, }); From ec5aef9a6b86939cfe1a5e68d52ab66f5b996f2e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 7 Oct 2024 22:07:42 +0000 Subject: [PATCH 2/6] chore(release): 1.1.5-beta.1 [skip ci] ## [1.1.5-beta.1](https://github.com/hirosystems/token-metadata-api/compare/v1.1.4...v1.1.5-beta.1) (2024-10-07) ### Bug Fixes * process nft and sft mints in batches ([#271](https://github.com/hirosystems/token-metadata-api/issues/271)) ([c98f0cd](https://github.com/hirosystems/token-metadata-api/commit/c98f0cd6a1001933764ab2fbb4026f8824127f35)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b70f00c..0f67a83d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.1.5-beta.1](https://github.com/hirosystems/token-metadata-api/compare/v1.1.4...v1.1.5-beta.1) (2024-10-07) + + +### Bug Fixes + +* process nft and sft mints in batches ([#271](https://github.com/hirosystems/token-metadata-api/issues/271)) ([c98f0cd](https://github.com/hirosystems/token-metadata-api/commit/c98f0cd6a1001933764ab2fbb4026f8824127f35)) + ## [1.1.4](https://github.com/hirosystems/token-metadata-api/compare/v1.1.3...v1.1.4) (2024-09-23) From 9b28880815a75f93f218fae69dc6e7147c908514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Mon, 21 Oct 2024 13:16:54 -0600 Subject: [PATCH 3/6] fix: allow multiple sft mints for the same token per transaction (#279) * fix: allow multiple sft mints for the same token per transaction * test: sft * fix: forof --- src/pg/chainhook/chainhook-pg-store.ts | 19 +++++++++++-------- tests/chainhook/sft-events.test.ts | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/pg/chainhook/chainhook-pg-store.ts b/src/pg/chainhook/chainhook-pg-store.ts index 1cfc0a52..cda77c3d 100644 --- a/src/pg/chainhook/chainhook-pg-store.ts +++ b/src/pg/chainhook/chainhook-pg-store.ts @@ -9,12 +9,10 @@ import { StacksEvent, StacksPayload } from '@hirosystems/chainhook-client'; import { ENV } from '../../env'; import { NftMintEvent, - SftMintEvent, SmartContractDeployment, TokenMetadataUpdateNotification, } from '../../token-processor/util/sip-validation'; -import { ContractNotFoundError } from '../errors'; -import { DbJob, DbSipNumber, DbSmartContractInsert, DbTokenType, DbSmartContract } from '../types'; +import { DbSmartContractInsert, DbTokenType, DbSmartContract } from '../types'; import { BlockCache, CachedEvent } from './block-cache'; import { dbSipNumberToDbTokenType } from '../../token-processor/util/helpers'; import BigNumber from 'bignumber.js'; @@ -322,13 +320,18 @@ export class ChainhookPgStore extends BasePgStoreModule { ): Promise { if (mints.length == 0) return; for await (const batch of batchIterate(mints, 500)) { - const values = batch.map(m => { + const tokenValues = new Map(); + for (const m of batch) { + // SFT tokens may mint one single token more than once given that it's an FT within an NFT. + // This makes sure we only keep the first occurrence. + const tokenKey = `${m.event.contractId}-${m.event.tokenId}`; + if (tokenValues.has(tokenKey)) continue; logger.info( `ChainhookPgStore apply ${tokenType.toUpperCase()} mint ${m.event.contractId} (${ m.event.tokenId }) at block ${cache.block.index}` ); - return [ + tokenValues.set(tokenKey, [ m.event.contractId, tokenType, m.event.tokenId.toString(), @@ -336,11 +339,11 @@ export class ChainhookPgStore extends BasePgStoreModule { cache.block.hash, m.tx_id, m.tx_index, - ]; - }); + ]); + } await sql` WITH insert_values (principal, type, token_number, block_height, index_block_hash, tx_id, - tx_index) AS (VALUES ${sql(values)}), + tx_index) AS (VALUES ${sql([...tokenValues.values()])}), filtered_values AS ( SELECT s.id AS smart_contract_id, i.type::token_type, i.token_number::bigint, i.block_height::bigint, i.index_block_hash::text, i.tx_id::text, i.tx_index::int diff --git a/tests/chainhook/sft-events.test.ts b/tests/chainhook/sft-events.test.ts index 02c97cd5..a2d589a4 100644 --- a/tests/chainhook/sft-events.test.ts +++ b/tests/chainhook/sft-events.test.ts @@ -52,6 +52,23 @@ describe('SFT events', () => { ), }, }) + // Try a duplicate of the same token but different amount + .event({ + type: 'SmartContractEvent', + position: { index: 1 }, + data: { + contract_identifier: contractId, + topic: 'print', + raw_value: cvToHex( + tupleCV({ + type: bufferCV(Buffer.from('sft_mint')), + recipient: bufferCV(Buffer.from(address)), + 'token-id': uintCV(3), + amount: uintCV(200), + }) + ), + }, + }) .build() ); From 638e7dd4a1872c51cc59b57a7f01c1cb3a7e884d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 21 Oct 2024 19:18:38 +0000 Subject: [PATCH 4/6] chore(release): 1.1.5-beta.2 [skip ci] ## [1.1.5-beta.2](https://github.com/hirosystems/token-metadata-api/compare/v1.1.5-beta.1...v1.1.5-beta.2) (2024-10-21) ### Bug Fixes * allow multiple sft mints for the same token per transaction ([#279](https://github.com/hirosystems/token-metadata-api/issues/279)) ([9b28880](https://github.com/hirosystems/token-metadata-api/commit/9b28880815a75f93f218fae69dc6e7147c908514)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f67a83d..7cdd4248 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.1.5-beta.2](https://github.com/hirosystems/token-metadata-api/compare/v1.1.5-beta.1...v1.1.5-beta.2) (2024-10-21) + + +### Bug Fixes + +* allow multiple sft mints for the same token per transaction ([#279](https://github.com/hirosystems/token-metadata-api/issues/279)) ([9b28880](https://github.com/hirosystems/token-metadata-api/commit/9b28880815a75f93f218fae69dc6e7147c908514)) + ## [1.1.5-beta.1](https://github.com/hirosystems/token-metadata-api/compare/v1.1.4...v1.1.5-beta.1) (2024-10-07) From b67dc8cd4e21f91a50a1c0c85880d2f265ab1d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Tue, 22 Oct 2024 08:43:19 -0600 Subject: [PATCH 5/6] fix: upgrade to new chainhook ts client (#280) --- package-lock.json | 8 +-- package.json | 2 +- src/chainhook/server.ts | 57 +++-------------- tests/chainhook/predicates.test.ts | 99 ------------------------------ 4 files changed, 15 insertions(+), 151 deletions(-) delete mode 100644 tests/chainhook/predicates.test.ts diff --git a/package-lock.json b/package-lock.json index 48d47cdf..8354c4b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@fastify/type-provider-typebox": "^3.2.0", "@google-cloud/storage": "^7.12.1", "@hirosystems/api-toolkit": "^1.7.1", - "@hirosystems/chainhook-client": "^1.12.0", + "@hirosystems/chainhook-client": "^2.0.0", "@sinclair/typebox": "^0.28.17", "@stacks/transactions": "^6.1.0", "@types/node": "^20.16.1", @@ -2700,9 +2700,9 @@ } }, "node_modules/@hirosystems/chainhook-client": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/@hirosystems/chainhook-client/-/chainhook-client-1.12.0.tgz", - "integrity": "sha512-FUlYMjnM2CGkxuBR0r+8+HPj+fhpJBJdcuS2e9YFz1NXfE7aDwM4bB5IxlcsJA2a5YAge1tZWeJUdR+TAnv/Rg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@hirosystems/chainhook-client/-/chainhook-client-2.0.0.tgz", + "integrity": "sha512-CZrByDwTr4Re5PBSlr7spHCRcdZLVf4D/wmuPmMdmJjreOohcwOBmSxKQG6kwMHbnBtcVXoI9rn1c96GoaNf6Q==", "dependencies": { "@fastify/type-provider-typebox": "^3.2.0", "fastify": "^4.15.0", diff --git a/package.json b/package.json index c81d52fa..a7f06312 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "@fastify/type-provider-typebox": "^3.2.0", "@google-cloud/storage": "^7.12.1", "@hirosystems/api-toolkit": "^1.7.1", - "@hirosystems/chainhook-client": "^1.12.0", + "@hirosystems/chainhook-client": "^2.0.0", "@sinclair/typebox": "^0.28.17", "@stacks/transactions": "^6.1.0", "@types/node": "^20.16.1", diff --git a/src/chainhook/server.ts b/src/chainhook/server.ts index 570efd02..f66a79b1 100644 --- a/src/chainhook/server.ts +++ b/src/chainhook/server.ts @@ -1,55 +1,23 @@ -import * as fs from 'fs'; import { ChainhookEventObserver, ChainhookNodeOptions, + EventObserverOptions, + EventObserverPredicate, Payload, - ServerOptions, - ServerPredicate, StacksPayload, } from '@hirosystems/chainhook-client'; import { PgStore } from '../pg/pg-store'; import { ENV } from '../env'; import { logger } from '@hirosystems/api-toolkit'; -import { randomUUID } from 'node:crypto'; - -export function getPersistedPredicateFromDisk(): ServerPredicate | undefined { - const predicatePath = `${ENV.CHAINHOOK_PREDICATE_PATH}/predicate.json`; - try { - if (!fs.existsSync(predicatePath)) { - return; - } - const fileData = fs.readFileSync(predicatePath, 'utf-8'); - return JSON.parse(fileData) as ServerPredicate; - } catch (error) { - logger.error(error, `ChainhookServer unable to get persisted predicate`); - } -} - -export function persistPredicateToDisk(predicate: ServerPredicate) { - const predicatePath = `${ENV.CHAINHOOK_PREDICATE_PATH}/predicate.json`; - try { - fs.mkdirSync(ENV.CHAINHOOK_PREDICATE_PATH, { recursive: true }); - fs.writeFileSync(predicatePath, JSON.stringify(predicate, null, 2)); - } catch (error) { - logger.error(error, `ChainhookServer unable to persist predicate to disk`); - } -} export async function startChainhookServer(args: { db: PgStore }): Promise { const blockHeight = await args.db.getChainTipBlockHeight(); logger.info(`ChainhookServer is at block ${blockHeight}`); - const predicates: ServerPredicate[] = []; + const predicates: EventObserverPredicate[] = []; if (ENV.CHAINHOOK_AUTO_PREDICATE_REGISTRATION) { - const existingPredicate = getPersistedPredicateFromDisk(); - if (existingPredicate) { - logger.info( - `ChainhookServer will attempt to resume existing predicate ${existingPredicate.uuid}` - ); - } const header = { - uuid: existingPredicate?.uuid ?? randomUUID(), - name: 'block', + name: 'metadata-api-blocks', version: 1, chain: 'stacks', }; @@ -87,7 +55,7 @@ export async function startChainhookServer(args: { db: PgStore }): Promise { + const server = new ChainhookEventObserver(observer, chainhook); + await server.start(predicates, async (payload: Payload) => { logger.info( `ChainhookServer received ${ payload.chainhook.is_streaming_blocks ? 'streamed' : 'replay' - } payload from predicate ${uuid}` + } payload from predicate ${payload.chainhook.uuid}` ); await args.db.chainhook.processPayload(payload as StacksPayload); }); - if (predicates.length) persistPredicateToDisk(predicates[0]); return server; } export async function closeChainhookServer(server: ChainhookEventObserver) { - try { - const predicatePath = `${ENV.CHAINHOOK_PREDICATE_PATH}/predicate.json`; - if (fs.existsSync(predicatePath)) fs.rmSync(predicatePath); - } catch (error) { - logger.error(error, `ChainhookServer unable to delete persisted predicate`); - } await server.close(); } diff --git a/tests/chainhook/predicates.test.ts b/tests/chainhook/predicates.test.ts deleted file mode 100644 index f0374af2..00000000 --- a/tests/chainhook/predicates.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { Interceptable, MockAgent, setGlobalDispatcher } from 'undici'; -import { ENV } from '../../src/env'; -import { - closeChainhookServer, - getPersistedPredicateFromDisk, - persistPredicateToDisk, - startChainhookServer, -} from '../../src/chainhook/server'; -import { MIGRATIONS_DIR, PgStore } from '../../src/pg/pg-store'; -import { cycleMigrations } from '@hirosystems/api-toolkit'; -import { ChainhookEventObserver } from '@hirosystems/chainhook-client'; - -describe('predicates', () => { - let db: PgStore; - let mockAgent: MockAgent; - let mockClient: Interceptable; - let server: ChainhookEventObserver; - - beforeAll(async () => { - ENV.CHAINHOOK_PREDICATE_PATH = './tmp'; - ENV.CHAINHOOK_AUTO_PREDICATE_REGISTRATION = true; - ENV.PGDATABASE = 'postgres'; - db = await PgStore.connect({ skipMigrations: true }); - await cycleMigrations(MIGRATIONS_DIR); - }); - - afterAll(async () => { - await db.close(); - }); - - beforeEach(() => { - mockAgent = new MockAgent(); - mockAgent.disableNetConnect(); - mockClient = mockAgent.get('http://127.0.0.1:20456'); - mockClient - .intercept({ - path: '/ping', - method: 'GET', - }) - .reply(200); - setGlobalDispatcher(mockAgent); - }); - - afterEach(async () => { - mockClient - .intercept({ - path: /\/v1\/chainhooks\/stacks\/(.*)/, - method: 'DELETE', - }) - .reply(200); - await closeChainhookServer(server); - await mockAgent.close(); - }); - - test('registers and persists new predicate to disk', async () => { - mockClient - .intercept({ - path: /\/v1\/chainhooks\/(.*)/, - method: 'GET', - }) - .reply(200, { status: 404 }); // New predicate - mockClient - .intercept({ - path: '/v1/chainhooks', - method: 'POST', - }) - .reply(200); - server = await startChainhookServer({ db }); - expect(getPersistedPredicateFromDisk()).not.toBeUndefined(); - mockAgent.assertNoPendingInterceptors(); - }); - - test('resumes predicate stored on disk', async () => { - persistPredicateToDisk({ - uuid: 'e2777d77-473a-4c1d-9012-152deb36bf4c', - name: 'test', - version: 1, - chain: 'stacks', - networks: { - mainnet: { - start_block: 1, - include_contract_abi: true, - if_this: { - scope: 'block_height', - higher_than: 1, - }, - }, - }, - }); - mockClient - .intercept({ - path: '/v1/chainhooks/e2777d77-473a-4c1d-9012-152deb36bf4c', - method: 'GET', - }) - .reply(200, { result: { enabled: true, status: { type: 'scanning' } }, status: 200 }); - server = await startChainhookServer({ db }); - mockAgent.assertNoPendingInterceptors(); - }); -}); From a652d6d75d1cd1e1c497d4fae50bf444fdda0446 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 22 Oct 2024 14:45:22 +0000 Subject: [PATCH 6/6] chore(release): 1.1.5-beta.3 [skip ci] ## [1.1.5-beta.3](https://github.com/hirosystems/token-metadata-api/compare/v1.1.5-beta.2...v1.1.5-beta.3) (2024-10-22) ### Bug Fixes * upgrade to new chainhook ts client ([#280](https://github.com/hirosystems/token-metadata-api/issues/280)) ([b67dc8c](https://github.com/hirosystems/token-metadata-api/commit/b67dc8cd4e21f91a50a1c0c85880d2f265ab1d51)) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cdd4248..c9977c02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.1.5-beta.3](https://github.com/hirosystems/token-metadata-api/compare/v1.1.5-beta.2...v1.1.5-beta.3) (2024-10-22) + + +### Bug Fixes + +* upgrade to new chainhook ts client ([#280](https://github.com/hirosystems/token-metadata-api/issues/280)) ([b67dc8c](https://github.com/hirosystems/token-metadata-api/commit/b67dc8cd4e21f91a50a1c0c85880d2f265ab1d51)) + ## [1.1.5-beta.2](https://github.com/hirosystems/token-metadata-api/compare/v1.1.5-beta.1...v1.1.5-beta.2) (2024-10-21)