From 331e6f596ec5ef23afe9cab254eab8fe49e1b33d Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 24 Sep 2024 11:27:16 +0100 Subject: [PATCH] feat!: decode with coreDiscoveryId rather than coreDiscoveryKey BREAKING CHANGE: this changes the expected parameter for decode() to be coreDiscoveryId, to match an upstream change in the multi-core-indexer to use coreDiscoveryId rather than coreDiscoveryKey. This reduces a round-trip conversion from buffer to string, and will also facilitate sending entries between threads when we run indexing in a separate worker thread. --- src/decode.ts | 25 +++++++++-------- src/lib/decode-conversions.ts | 53 +++++++++++++++-------------------- src/lib/utils.ts | 6 ++-- test/index.test.js | 7 ++++- 4 files changed, 46 insertions(+), 45 deletions(-) diff --git a/src/decode.ts b/src/decode.ts index c35043d..9775d17 100644 --- a/src/decode.ts +++ b/src/decode.ts @@ -25,7 +25,7 @@ import cenc from 'compact-encoding' import { DATA_TYPE_ID_BYTES, SCHEMA_VERSION_BYTES } from './constants.js' import { ExhaustivenessError, - VersionIdObject, + VERSION_ID_SEPARATOR, getOwn, getProtoTypeName, } from './lib/utils.js' @@ -46,9 +46,10 @@ for (const [schemaName, dataTypeId] of Object.entries(dataTypeIds) as Array< * */ export function decode( buf: Buffer, - versionObj: VersionIdObject + { index, coreDiscoveryId }: { index: number; coreDiscoveryId: string } ): MapeoDocDecode { const schemaDef = decodeBlockPrefix(buf) + const versionId = coreDiscoveryId + VERSION_ID_SEPARATOR + index const encodedMsg = buf.subarray( DATA_TYPE_ID_BYTES + SCHEMA_VERSION_BYTES, @@ -61,25 +62,25 @@ export function decode( switch (message.schemaName) { case 'projectSettings': - return convertProjectSettings(message, versionObj) + return convertProjectSettings(message, versionId) case 'observation': - return convertObservation(message, versionObj) + return convertObservation(message, versionId) case 'field': - return convertField(message, versionObj) + return convertField(message, versionId) case 'preset': - return convertPreset(message, versionObj) + return convertPreset(message, versionId) case 'role': - return convertRole(message, versionObj) + return convertRole(message, versionId) case 'deviceInfo': - return convertDeviceInfo(message, versionObj) + return convertDeviceInfo(message, versionId) case 'coreOwnership': - return convertCoreOwnership(message, versionObj) + return convertCoreOwnership(message, versionId) case 'icon': - return convertIcon(message, versionObj) + return convertIcon(message, versionId) case 'translation': - return convertTranslation(message, versionObj) + return convertTranslation(message, versionId) case 'track': - return convertTrack(message, versionObj) + return convertTrack(message, versionId) default: throw new ExhaustivenessError(message) } diff --git a/src/lib/decode-conversions.ts b/src/lib/decode-conversions.ts index 601d496..f8a1fd7 100644 --- a/src/lib/decode-conversions.ts +++ b/src/lib/decode-conversions.ts @@ -35,18 +35,13 @@ import type { Track_1_Position } from '../proto/track/v1.js' import { ProjectSettings_1_ConfigMetadata } from '../proto/projectSettings/v1.js' import { ProjectSettings } from '../schema/projectSettings.js' import type { Position } from '../schema/observation.js' -import { - assert, - ExhaustivenessError, - getVersionId, - VersionIdObject, -} from './utils.js' +import { assert, ExhaustivenessError, getVersionId } from './utils.js' /** Function type for converting a protobuf type of any version for a particular * schema name, and returning the most recent JSONSchema type */ type ConvertFunction = ( message: Extract, - versionObj: VersionIdObject + versionId: string ) => FilterBySchemaName function ensure( @@ -59,10 +54,10 @@ function ensure( export const convertProjectSettings: ConvertFunction<'projectSettings'> = ( message, - versionObj + versionId ) => { const { common, schemaVersion, defaultPresets, ...rest } = message - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) let configMetadata: undefined | ProjectSettings['configMetadata'] if (rest.configMetadata) { @@ -101,10 +96,10 @@ function convertConfigMetadata( export const convertObservation: ConvertFunction<'observation'> = ( message, - versionObj + versionId ) => { const { common, metadata, schemaVersion, ...rest } = message - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) let presetRef if (rest.presetRef) { @@ -138,9 +133,9 @@ export const convertObservation: ConvertFunction<'observation'> = ( type FieldOptions = FilterBySchemaName['options'] -export const convertField: ConvertFunction<'field'> = (message, versionObj) => { +export const convertField: ConvertFunction<'field'> = (message, versionId) => { const { common, schemaVersion, tagKey, type, label, ...rest } = message - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) ensure(tagKey, 'field', 'tagKey') return { ...jsonSchemaCommon, @@ -172,10 +167,10 @@ type JsonSchemaPresetGeomItem = FilterBySchemaName< export const convertPreset: ConvertFunction<'preset'> = ( message, - versionObj + versionId ) => { const { common, schemaVersion, ...rest } = message - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) const geometry = rest.geometry.filter( (geomType): geomType is JsonSchemaPresetGeomItem => @@ -209,11 +204,11 @@ export const convertPreset: ConvertFunction<'preset'> = ( } } -export const convertRole: ConvertFunction<'role'> = (message, versionObj) => { +export const convertRole: ConvertFunction<'role'> = (message, versionId) => { const { common, schemaVersion, fromIndex, roleId, ...rest } = message ensure(roleId.length, 'role', 'roleId') ensure(typeof fromIndex === 'number', 'role', 'fromIndex') - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) return { ...jsonSchemaCommon, ...rest, @@ -224,10 +219,10 @@ export const convertRole: ConvertFunction<'role'> = (message, versionObj) => { export const convertDeviceInfo: ConvertFunction<'deviceInfo'> = ( message, - versionObj + versionId ) => { const { common, schemaVersion, name, ...rest } = message - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) return { ...jsonSchemaCommon, ...rest, @@ -237,7 +232,7 @@ export const convertDeviceInfo: ConvertFunction<'deviceInfo'> = ( export const convertCoreOwnership: ConvertFunction<'coreOwnership'> = ( message, - versionObj + versionId ) => { const { common, @@ -256,7 +251,7 @@ export const convertCoreOwnership: ConvertFunction<'coreOwnership'> = ( ensure(dataCoreId.byteLength, 'coreOwnership', 'dataCoreId') ensure(blobCoreId.byteLength, 'coreOwnership', 'blobCoreId') ensure(blobIndexCoreId.byteLength, 'coreOwnership', 'blobIndexCoreId') - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) return { ...jsonSchemaCommon, ...rest, @@ -269,9 +264,9 @@ export const convertCoreOwnership: ConvertFunction<'coreOwnership'> = ( } } -export const convertIcon: ConvertFunction<'icon'> = (message, versionObj) => { +export const convertIcon: ConvertFunction<'icon'> = (message, versionId) => { const { common, schemaVersion, ...rest } = message - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) const variants: Icon['variants'] = [] for (const variant of message.variants) { @@ -288,7 +283,7 @@ export const convertIcon: ConvertFunction<'icon'> = (message, versionObj) => { export const convertTranslation: ConvertFunction<'translation'> = ( message, - versionObj + versionId ) => { const { common, @@ -298,7 +293,7 @@ export const convertTranslation: ConvertFunction<'translation'> = ( regionCode, ...rest } = message - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) ensure(message.docRef, 'translation', 'docRef') ensure(message.docRef.versionId, 'translation.docRef', 'versionId') ensure(propertyRef, 'translation', 'propertyRef') @@ -316,9 +311,9 @@ export const convertTranslation: ConvertFunction<'translation'> = ( } } -export const convertTrack: ConvertFunction<'track'> = (message, versionObj) => { +export const convertTrack: ConvertFunction<'track'> = (message, versionId) => { const { common, schemaVersion, ...rest } = message - const jsonSchemaCommon = convertCommon(common, versionObj) + const jsonSchemaCommon = convertCommon(common, versionId) const locations = message.locations.map(convertTrackPosition) const observationRefs: Track['observationRefs'] = [] @@ -465,7 +460,7 @@ function convertTagPrimitive({ function convertCommon( common: ProtoTypesWithSchemaInfo['common'], - versionObj: VersionIdObject + versionId: string ): Omit { if ( !common || @@ -476,8 +471,6 @@ function convertCommon( throw new Error('Missing required common properties') } - const versionId = getVersionId(versionObj) - /** @type {string} */ let originalVersionId if (common.originalVersionId) { originalVersionId = getVersionId(common.originalVersionId) diff --git a/src/lib/utils.ts b/src/lib/utils.ts index 9add5bb..964d262 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -42,6 +42,8 @@ export type VersionIdObject = { index: number } +export const VERSION_ID_SEPARATOR = '/' + /** * Get a string versionId from a core key and index in that core. A versionId * uniquely identifies a record in the underlying Hypercore storage used by @@ -58,7 +60,7 @@ export function getVersionId({ coreDiscoveryKey, index }: VersionIdObject) { Number.isSafeInteger(index) && index >= 0, 'version ID index must be a non-negative integer' ) - return coreDiscoveryKey.toString('hex') + '/' + index + return coreDiscoveryKey.toString('hex') + VERSION_ID_SEPARATOR + index } /** @@ -69,7 +71,7 @@ export function getVersionId({ coreDiscoveryKey, index }: VersionIdObject) { * @returns coreKey as a Buffer and index in the core */ export function parseVersionId(versionId: string): VersionIdObject { - const items = versionId.split('/') + const items = versionId.split(VERSION_ID_SEPARATOR) if (!items[0] || !items[1]) throw new Error('Invalid versionId') const coreDiscoveryKey = Buffer.from(items[0], 'hex') if (coreDiscoveryKey.length !== 32) throw new Error('Invalid versionId') diff --git a/test/index.test.js b/test/index.test.js index b28f4f6..5bb2699 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -6,7 +6,6 @@ import { encode, decode, decodeBlockPrefix, - parseVersionId, validate, valueOf, } from '../dist/index.js' @@ -20,6 +19,7 @@ import { badDocs, } from './fixtures/index.js' import { cachedValues } from './fixtures/cached.js' +import { VERSION_ID_SEPARATOR } from '../dist/lib/utils.js' /** @import { SchemaName } from '../dist/types.js' */ @@ -341,3 +341,8 @@ function removeBlockPrefix(buf) { const blockPrefix = encodeBlockPrefix(decodeBlockPrefix(buf)) return buf.slice(blockPrefix.byteLength) } + +function parseVersionId(versionId) { + const [coreDiscoveryId, index] = versionId.split(VERSION_ID_SEPARATOR) + return { coreDiscoveryId, index: Number.parseInt(index) } +}