From 65113ab9a5350484685dd1319ec67bac125065e0 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 24 Sep 2024 11:27:16 +0100 Subject: [PATCH] feat: decode with coreDiscoveryId or coreDiscoveryKey Allow decode to work when a coreDiscoveryId is passed, instead of coreDiscoveryKey to match an upstream change in the multi-core-indexer to use coreDiscoveryId rather than coreDiscoveryKey. This reduces a round-trip conversion from buffer to string, and will also facilitate sending entries between threads when we run indexing in a separate worker thread. --- src/decode.ts | 9 +++---- src/index.ts | 4 +++- src/lib/decode-conversions.ts | 6 ++--- src/lib/utils.ts | 44 +++++++++++++++++++++++++---------- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/decode.ts b/src/decode.ts index c35043d..5feddbb 100644 --- a/src/decode.ts +++ b/src/decode.ts @@ -25,7 +25,7 @@ import cenc from 'compact-encoding' import { DATA_TYPE_ID_BYTES, SCHEMA_VERSION_BYTES } from './constants.js' import { ExhaustivenessError, - VersionIdObject, + VersionObject, getOwn, getProtoTypeName, } from './lib/utils.js' @@ -42,12 +42,9 @@ for (const [schemaName, dataTypeId] of Object.entries(dataTypeIds) as Array< * Decode a Buffer as an object validated against the corresponding schema * * @param buf Buffer to be decoded - * @param versionObj public key (coreKey) of the core where this block is stored, and the index of the block in the core. + * @param versionObj discovery key or discovery id of the core where this block is stored, and the index of the block in the core. * */ -export function decode( - buf: Buffer, - versionObj: VersionIdObject -): MapeoDocDecode { +export function decode(buf: Buffer, versionObj: VersionObject): MapeoDocDecode { const schemaDef = decodeBlockPrefix(buf) const encodedMsg = buf.subarray( diff --git a/src/index.ts b/src/index.ts index f3ce288..e05f37e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,9 @@ export { getVersionId, parseVersionId, valueOf, - type VersionIdObject, + /** Deprecated: rename VersionIdObject to VersionObject */ + type VersionObject as VersionIdObject, + type VersionObject, } from './lib/utils.js' export * from './schema/index.js' diff --git a/src/lib/decode-conversions.ts b/src/lib/decode-conversions.ts index 601d496..1636535 100644 --- a/src/lib/decode-conversions.ts +++ b/src/lib/decode-conversions.ts @@ -39,14 +39,14 @@ import { assert, ExhaustivenessError, getVersionId, - VersionIdObject, + VersionObject, } from './utils.js' /** Function type for converting a protobuf type of any version for a particular * schema name, and returning the most recent JSONSchema type */ type ConvertFunction = ( message: Extract, - versionObj: VersionIdObject + versionObj: VersionObject ) => FilterBySchemaName function ensure( @@ -465,7 +465,7 @@ function convertTagPrimitive({ function convertCommon( common: ProtoTypesWithSchemaInfo['common'], - versionObj: VersionIdObject + versionObj: VersionObject ): Omit { if ( !common || diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 9add5bb..153ac62 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -37,28 +37,48 @@ function capitalize(str: T): Capitalize { return (str.charAt(0).toUpperCase() + str.slice(1)) as any } -export type VersionIdObject = { +export type VersionObject = VersionDiscoveryKeyObject | VersionDiscoveryIdObject + +export type VersionDiscoveryKeyObject = { coreDiscoveryKey: Buffer index: number } +export type VersionDiscoveryIdObject = { + coreDiscoveryId: string + index: number +} + +export const VERSION_ID_SEPARATOR = '/' + /** - * Get a string versionId from a core key and index in that core. A versionId - * uniquely identifies a record in the underlying Hypercore storage used by - * Mapeo + * Get a string versionId from a core discovery key or discovery id and index in + * that core. A versionId uniquely identifies a record in the underlying + * Hypercore storage used by Mapeo * @param versionIdObject * @returns versionId string */ -export function getVersionId({ coreDiscoveryKey, index }: VersionIdObject) { - assert( - coreDiscoveryKey.byteLength >= 32, - 'version ID core discovery key must be have at least 32 bytes' - ) +export function getVersionId(versionIdObject: VersionObject) { + const { index } = versionIdObject assert( Number.isSafeInteger(index) && index >= 0, 'version ID index must be a non-negative integer' ) - return coreDiscoveryKey.toString('hex') + '/' + index + let discoveryId + if ('coreDiscoveryId' in versionIdObject) { + discoveryId = versionIdObject.coreDiscoveryId + assert( + discoveryId.length > 0, + 'version ID core discovery id must be non-empty string' + ) + } else { + assert( + versionIdObject.coreDiscoveryKey.byteLength >= 32, + 'version ID core discovery key must be have at least 32 bytes' + ) + discoveryId = versionIdObject.coreDiscoveryKey.toString('hex') + } + return discoveryId + VERSION_ID_SEPARATOR + index } /** @@ -68,8 +88,8 @@ export function getVersionId({ coreDiscoveryKey, index }: VersionIdObject) { * @param versionId hex-encoded 32-byte core key and index in the core, separated with `/` * @returns coreKey as a Buffer and index in the core */ -export function parseVersionId(versionId: string): VersionIdObject { - const items = versionId.split('/') +export function parseVersionId(versionId: string): VersionDiscoveryKeyObject { + const items = versionId.split(VERSION_ID_SEPARATOR) if (!items[0] || !items[1]) throw new Error('Invalid versionId') const coreDiscoveryKey = Buffer.from(items[0], 'hex') if (coreDiscoveryKey.length !== 32) throw new Error('Invalid versionId')