From 54bda4e1ffaa6e73e0831c13c32b0db53e504ae9 Mon Sep 17 00:00:00 2001 From: Jason Paulos Date: Wed, 4 May 2022 17:01:10 -0700 Subject: [PATCH] Add bigint support --- README.md | 63 +++----- src/Decoder.ts | 27 ++-- src/Encoder.ts | 44 +++++ src/decode.ts | 10 ++ src/decodeAsync.ts | 3 + src/index.ts | 2 + src/timestamp.ts | 4 +- src/utils/int.ts | 98 ++++++++++- test/codec-bigint.test.ts | 277 ++++++++++++++++++++++++++++++-- test/codec-int.test.ts | 23 ++- test/encode.test.ts | 34 +++- test/msgpack-test-suite.test.ts | 25 ++- 12 files changed, 526 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 19854d60..3d606e57 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ deepStrictEqual(decode(encoded), object); - [`EncodeOptions`](#encodeoptions) - [`decode(buffer: ArrayLike | BufferSource, options?: DecodeOptions): unknown`](#decodebuffer-arraylikenumber--buffersource-options-decodeoptions-unknown) - [`DecodeOptions`](#decodeoptions) + - [`IntMode`](#intmode) - [`decodeMulti(buffer: ArrayLike | BufferSource, options?: DecodeOptions): Generator`](#decodemultibuffer-arraylikenumber--buffersource-options-decodeoptions-generatorunknown-void-unknown) - [`decodeAsync(stream: ReadableStreamLike | BufferSource>, options?: DecodeAsyncOptions): Promise`](#decodeasyncstream-readablestreamlikearraylikenumber--buffersource-options-decodeasyncoptions-promiseunknown) - [`decodeArrayStream(stream: ReadableStreamLike | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable`](#decodearraystreamstream-readablestreamlikearraylikenumber--buffersource-options-decodeasyncoptions-asynciterableunknown) @@ -49,7 +50,6 @@ deepStrictEqual(decode(encoded), object); - [Reusing Encoder and Decoder instances](#reusing-encoder-and-decoder-instances) - [Extension Types](#extension-types) - [ExtensionCodec context](#extensioncodec-context) - - [Handling BigInt with ExtensionCodec](#handling-bigint-with-extensioncodec) - [The temporal module as timestamp extensions](#the-temporal-module-as-timestamp-extensions) - [Decoding a Blob](#decoding-a-blob) - [MessagePack Specification](#messagepack-specification) @@ -148,10 +148,22 @@ maxBinLength | number | `4_294_967_295` (UINT32_MAX) maxArrayLength | number | `4_294_967_295` (UINT32_MAX) maxMapLength | number | `4_294_967_295` (UINT32_MAX) maxExtLength | number | `4_294_967_295` (UINT32_MAX) +intMode | `IntMode` | `IntMode.UNSAFE_NUMBER` context | user-defined | - You can use `max${Type}Length` to limit the length of each type decoded. +`intMode` determines whether decoded integers should be returned as numbers or bigints. The possible values are described below. + +##### `IntMode` + +The `IntMode` enum defines different options for decoding integers. They are described below: + +- `IntMode.UNSAFE_NUMBER`: Always returns the value as a number. Be aware that there will be a loss of precision if the value is outside the range of `Number.MIN_SAFE_INTEGER` to `Number.MAX_SAFE_INTEGER`. +- `IntMode.SAFE_NUMBER`: Always returns the value as a number, but throws an error if the value is outside of the range of `Number.MIN_SAFE_INTEGER` to `Number.MAX_SAFE_INTEGER`. +- `IntMode.MIXED`: Returns all values inside the range of `Number.MIN_SAFE_INTEGER` to `Number.MAX_SAFE_INTEGER` as numbers and all values outside that range as bigints. +- `IntMode.BIGINT`: Always returns the value as a bigint, even if it is small enough to safely fit in a number. + ### `decodeMulti(buffer: ArrayLike | BufferSource, options?: DecodeOptions): Generator` It decodes `buffer` that includes multiple MessagePack-encoded objects, and returns decoded objects as a generator. See also `decodeMultiStream()`, which is an asynchronous variant of this function. @@ -346,39 +358,6 @@ const encoded = = encode({myType: new MyType()}, { extensionCodec, context const decoded = decode(encoded, { extensionCodec, context }); ``` -#### Handling BigInt with ExtensionCodec - -This library does not handle BigInt by default, but you can handle it with `ExtensionCodec` like this: - -```typescript -import { deepStrictEqual } from "assert"; -import { encode, decode, ExtensionCodec } from "@msgpack/msgpack"; - -const BIGINT_EXT_TYPE = 0; // Any in 0-127 -const extensionCodec = new ExtensionCodec(); -extensionCodec.register({ - type: BIGINT_EXT_TYPE, - encode: (input: unknown) => { - if (typeof input === "bigint") { - if (input <= Number.MAX_SAFE_INTEGER && input >= Number.MIN_SAFE_INTEGER) { - return encode(parseInt(input.toString(), 10)); - } else { - return encode(input.toString()); - } - } else { - return null; - } - }, - decode: (data: Uint8Array) => { - return BigInt(decode(data)); - }, -}); - -const value = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); -const encoded: = encode(value, { extensionCodec }); -deepStrictEqual(decode(encoded, { extensionCodec }), value); -``` - #### The temporal module as timestamp extensions There is a proposal for a new date/time representations in JavaScript: @@ -468,18 +447,20 @@ Source Value|MessagePack Format|Value Decoded ----|----|---- null, undefined|nil|null (*1) boolean (true, false)|bool family|boolean (true, false) -number (53-bit int)|int family|number (53-bit int) +number (53-bit int)|int family|number or bigint (*2) number (64-bit float)|float family|number (64-bit float) +bigint|int family|number or bigint (*2) string|str family|string -ArrayBufferView |bin family|Uint8Array (*2) +ArrayBufferView |bin family|Uint8Array (*3) Array|array family|Array -Object|map family|Object (*3) -Date|timestamp ext family|Date (*4) +Object|map family|Object (*4) +Date|timestamp ext family|Date (*5) * *1 Both `null` and `undefined` are mapped to `nil` (`0xC0`) type, and are decoded into `null` -* *2 Any `ArrayBufferView`s including NodeJS's `Buffer` are mapped to `bin` family, and are decoded into `Uint8Array` -* *3 In handling `Object`, it is regarded as `Record` in terms of TypeScript -* *4 MessagePack timestamps may have nanoseconds, which will lost when it is decoded into JavaScript `Date`. This behavior can be overridden by registering `-1` for the extension codec. +* *2 MessagePack ints are decoded as either numbers or bigints depending on the [IntMode](#intmode) used during decoding. +* *3 Any `ArrayBufferView`s including NodeJS's `Buffer` are mapped to `bin` family, and are decoded into `Uint8Array` +* *4 In handling `Object`, it is regarded as `Record` in terms of TypeScript +* *5 MessagePack timestamps may have nanoseconds, which will lost when it is decoded into JavaScript `Date`. This behavior can be overridden by registering `-1` for the extension codec. ## Prerequisites diff --git a/src/Decoder.ts b/src/Decoder.ts index eb6fa157..ead73248 100644 --- a/src/Decoder.ts +++ b/src/Decoder.ts @@ -1,6 +1,6 @@ import { prettyByte } from "./utils/prettyByte"; import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec"; -import { getInt64, getUint64, UINT32_MAX } from "./utils/int"; +import { IntMode, getInt64, getUint64, convertSafeIntegerToMode, UINT32_MAX } from "./utils/int"; import { utf8DecodeJs, TEXT_DECODER_THRESHOLD, utf8DecodeTD } from "./utils/utf8"; import { createDataView, ensureUint8Array } from "./utils/typedArrays"; import { CachedKeyDecoder, KeyDecoder } from "./CachedKeyDecoder"; @@ -76,6 +76,7 @@ export class Decoder { private readonly maxArrayLength = UINT32_MAX, private readonly maxMapLength = UINT32_MAX, private readonly maxExtLength = UINT32_MAX, + private readonly intMode = IntMode.UNSAFE_NUMBER, private readonly keyDecoder: KeyDecoder | null = sharedCachedKeyDecoder, ) {} @@ -274,25 +275,25 @@ export class Decoder { object = this.readF64(); } else if (headByte === 0xcc) { // uint 8 - object = this.readU8(); + object = this.convertNumber(this.readU8()); } else if (headByte === 0xcd) { // uint 16 - object = this.readU16(); + object = this.convertNumber(this.readU16()); } else if (headByte === 0xce) { // uint 32 - object = this.readU32(); + object = this.convertNumber(this.readU32()); } else if (headByte === 0xcf) { // uint 64 object = this.readU64(); } else if (headByte === 0xd0) { // int 8 - object = this.readI8(); + object = this.convertNumber(this.readI8()); } else if (headByte === 0xd1) { // int 16 - object = this.readI16(); + object = this.convertNumber(this.readI16()); } else if (headByte === 0xd2) { // int 32 - object = this.readI32(); + object = this.convertNumber(this.readI32()); } else if (headByte === 0xd3) { // int 64 object = this.readI64(); @@ -553,6 +554,10 @@ export class Decoder { return this.extensionCodec.decode(data, extType, this.context); } + private convertNumber(value: number): number | bigint { + return convertSafeIntegerToMode(value, this.intMode); + } + private lookU8() { return this.view.getUint8(this.pos); } @@ -601,14 +606,14 @@ export class Decoder { return value; } - private readU64(): number { - const value = getUint64(this.view, this.pos); + private readU64(): number | bigint { + const value = getUint64(this.view, this.pos, this.intMode); this.pos += 8; return value; } - private readI64(): number { - const value = getInt64(this.view, this.pos); + private readI64(): number | bigint { + const value = getInt64(this.view, this.pos, this.intMode); this.pos += 8; return value; } diff --git a/src/Encoder.ts b/src/Encoder.ts index afea365c..0bfaea6b 100644 --- a/src/Encoder.ts +++ b/src/Encoder.ts @@ -144,6 +144,32 @@ export class Encoder { } } + private encodeBigint(object: bigint) { + if (object >= 0) { + if (object < 0x100000000 || this.forceIntegerToFloat) { + // uint 32 or lower, or force to float + this.encodeNumber(Number(object)) + } else if (object < BigInt("0x10000000000000000")) { + // uint 64 + this.writeU8(0xcf); + this.writeUBig(object); + } else { + throw new Error(`Bigint is too large for uint64: ${object}`); + } + } else { + if (object >= -0x80000000 || this.forceIntegerToFloat) { + // int 32 or lower, or force to float + this.encodeNumber(Number(object)); + } else if (object >= BigInt(-1) * BigInt("0x8000000000000000")) { + // int 64 + this.writeU8(0xd3); + this.writeIBig(object); + } else { + throw new Error(`Bigint is too small for int64: ${object}`); + } + } + } + private writeStringHeader(byteLength: number) { if (byteLength < 32) { // fixstr @@ -189,6 +215,10 @@ export class Encoder { const ext = this.extensionCodec.tryToEncode(object, this.context); if (ext != null) { this.encodeExtension(ext); + } else if (typeof object === "bigint") { + // this is here instead of in doEncode so that we can try encoding with an extension first, + // otherwise we would break existing extensions for bigints + this.encodeBigint(object); } else if (Array.isArray(object)) { this.encodeArray(object, depth); } else if (ArrayBuffer.isView(object)) { @@ -399,4 +429,18 @@ export class Encoder { setInt64(this.view, this.pos, value); this.pos += 8; } + + private writeIBig(value: bigint) { + this.ensureBufferSizeToWrite(8); + + this.view.setBigInt64(this.pos, value); + this.pos += 8; + } + + private writeUBig(value: bigint) { + this.ensureBufferSizeToWrite(8); + + this.view.setBigUint64(this.pos, value); + this.pos += 8; + } } diff --git a/src/decode.ts b/src/decode.ts index 30a88ab3..497f31a6 100644 --- a/src/decode.ts +++ b/src/decode.ts @@ -1,4 +1,5 @@ import { Decoder } from "./Decoder"; +import type { IntMode } from "./utils/int"; import type { ExtensionCodecType } from "./ExtensionCodec"; import type { ContextOf, SplitUndefined } from "./context"; @@ -36,6 +37,13 @@ export type DecodeOptions = Readonly< * Defaults to 4_294_967_295 (UINT32_MAX). */ maxExtLength: number; + /** + * Determines whether decoded integers should be returned as numbers or bigints. + * + * Defaults to IntMode.UNSAFE_NUMBER, which always returns the value as a number, even when it + * is outside the range of Number.MIN_SAFE_INTEGER to Number.MAX_SAFE_INTEGER. + */ + intMode: IntMode; }> > & ContextOf; @@ -63,6 +71,7 @@ export function decode( options.maxArrayLength, options.maxMapLength, options.maxExtLength, + options.intMode, ); return decoder.decode(buffer); } @@ -86,6 +95,7 @@ export function decodeMulti( options.maxArrayLength, options.maxMapLength, options.maxExtLength, + options.intMode, ); return decoder.decodeMulti(buffer); } diff --git a/src/decodeAsync.ts b/src/decodeAsync.ts index ee9922fa..f4f924df 100644 --- a/src/decodeAsync.ts +++ b/src/decodeAsync.ts @@ -23,6 +23,7 @@ import type { SplitUndefined } from "./context"; options.maxArrayLength, options.maxMapLength, options.maxExtLength, + options.intMode, ); return decoder.decodeAsync(stream); } @@ -45,6 +46,7 @@ import type { SplitUndefined } from "./context"; options.maxArrayLength, options.maxMapLength, options.maxExtLength, + options.intMode, ); return decoder.decodeArrayStream(stream); @@ -68,6 +70,7 @@ export function decodeMultiStream( options.maxArrayLength, options.maxMapLength, options.maxExtLength, + options.intMode, ); return decoder.decodeStream(stream); diff --git a/src/index.ts b/src/index.ts index 28560f1b..2016eb80 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,8 @@ import { decode, decodeMulti } from "./decode"; export { decode, decodeMulti }; import type { DecodeOptions } from "./decode"; export { DecodeOptions }; +import { IntMode } from './utils/int'; +export { IntMode }; import { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream } from "./decodeAsync"; export { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream }; diff --git a/src/timestamp.ts b/src/timestamp.ts index e3fe0155..2687361b 100644 --- a/src/timestamp.ts +++ b/src/timestamp.ts @@ -1,6 +1,6 @@ // https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type import { DecodeError } from "./DecodeError"; -import { getInt64, setInt64 } from "./utils/int"; +import { IntMode, getInt64, setInt64 } from "./utils/int"; export const EXT_TIMESTAMP = -1; @@ -87,7 +87,7 @@ export function decodeTimestampToTimeSpec(data: Uint8Array): TimeSpec { case 12: { // timestamp 96 = { nsec32 (unsigned), sec64 (signed) } - const sec = getInt64(view, 4); + const sec = getInt64(view, 4, IntMode.UNSAFE_NUMBER); const nsec = view.getUint32(0); return { sec, nsec }; } diff --git a/src/utils/int.ts b/src/utils/int.ts index 7fa93fb7..c85b10c4 100644 --- a/src/utils/int.ts +++ b/src/utils/int.ts @@ -1,5 +1,30 @@ // Integer Utility +/** + * An enum of different options for decoding integers. + */ +export enum IntMode { + /** + * Always returns the value as a number. Be aware that there will be a loss of precision if the + * value is outside the range of Number.MIN_SAFE_INTEGER to Number.MAX_SAFE_INTEGER. + */ + UNSAFE_NUMBER, + /** + * Always returns the value as a number, but throws an error if the value is outside of the range + * of Number.MIN_SAFE_INTEGER to Number.MAX_SAFE_INTEGER. + */ + SAFE_NUMBER, + /** + * Returns all values inside the range of Number.MIN_SAFE_INTEGER to Number.MAX_SAFE_INTEGER as + * numbers and all values outside that range as bigints. + */ + MIXED, + /** + * Always returns the value as a bigint, even if it is small enough to safely fit in a number. + */ + BIGINT, +} + export const UINT32_MAX = 0xffff_ffff; // DataView extension to handle int64 / uint64, @@ -19,14 +44,71 @@ export function setInt64(view: DataView, offset: number, value: number): void { view.setUint32(offset + 4, low); } -export function getInt64(view: DataView, offset: number): number { - const high = view.getInt32(offset); - const low = view.getUint32(offset + 4); - return high * 0x1_0000_0000 + low; +export function getInt64(view: DataView, offset: number, mode: IntMode.UNSAFE_NUMBER | IntMode.SAFE_NUMBER): number +export function getInt64(view: DataView, offset: number, mode: IntMode.BIGINT): bigint +export function getInt64(view: DataView, offset: number, mode: IntMode): number | bigint +export function getInt64(view: DataView, offset: number, mode: IntMode): number | bigint { + if (mode === IntMode.UNSAFE_NUMBER || mode === IntMode.SAFE_NUMBER) { + // for compatibility, don't use view.getBigInt64 if the user hasn't told us to use BigInts + const high = view.getInt32(offset); + const low = view.getUint32(offset + 4); + + if (mode === IntMode.SAFE_NUMBER && ( + high < Math.floor(Number.MIN_SAFE_INTEGER / 0x1_0000_0000) || + (high === Math.floor(Number.MIN_SAFE_INTEGER / 0x1_0000_0000) && low === 0) || + high > (Number.MAX_SAFE_INTEGER - low) / 0x1_0000_0000 + )) { + const hexValue = `${high < 0 ? "-" : ""}0x${Math.abs(high).toString(16)}${low.toString(16).padStart(8, "0")}`; + throw new Error(`Mode is IntMode.SAFE_NUMBER and value is not a safe integer: ${hexValue}`); + } + + return high * 0x1_0000_0000 + low; + } + + const value = view.getBigInt64(offset); + + if (mode === IntMode.MIXED && value >= Number.MIN_SAFE_INTEGER && value <= Number.MAX_SAFE_INTEGER) { + return Number(value); + } + + return value; } -export function getUint64(view: DataView, offset: number): number { - const high = view.getUint32(offset); - const low = view.getUint32(offset + 4); - return high * 0x1_0000_0000 + low; +export function getUint64(view: DataView, offset: number, mode: IntMode.UNSAFE_NUMBER | IntMode.SAFE_NUMBER): number +export function getUint64(view: DataView, offset: number, mode: IntMode.BIGINT): bigint +export function getUint64(view: DataView, offset: number, mode: IntMode): number | bigint +export function getUint64(view: DataView, offset: number, mode: IntMode): number | bigint { + if (mode === IntMode.UNSAFE_NUMBER || mode === IntMode.SAFE_NUMBER) { + // for compatibility, don't use view.getBigUint64 if the user hasn't told us to use BigInts + const high = view.getUint32(offset); + const low = view.getUint32(offset + 4); + + if (mode === IntMode.SAFE_NUMBER && high > (Number.MAX_SAFE_INTEGER - low) / 0x1_0000_0000) { + const hexValue = `0x${high.toString(16)}${low.toString(16).padStart(8, "0")}`; + throw new Error(`Mode is IntMode.SAFE_NUMBER and value is not a safe integer: ${hexValue}`); + } + + return high * 0x1_0000_0000 + low; + } + + const value = view.getBigUint64(offset); + + if (mode === IntMode.MIXED && value <= Number.MAX_SAFE_INTEGER) { + return Number(value); + } + + return value; +} + +/** + * Convert a safe integer Number (i.e. in the range Number.MIN_SAFE_INTEGER to Number.MAX_SAFE_INTEGER) + * with respect to the given IntMode. For all modes except IntMode.BIGINT, this returns the original + * Number unmodified. + */ +export function convertSafeIntegerToMode(value: number, mode: IntMode): number | bigint { + if (mode === IntMode.BIGINT) { + return BigInt(value); + } + + return value; } diff --git a/test/codec-bigint.test.ts b/test/codec-bigint.test.ts index fc649a83..ebd4d601 100644 --- a/test/codec-bigint.test.ts +++ b/test/codec-bigint.test.ts @@ -1,5 +1,6 @@ import assert from "assert"; import { encode, decode, ExtensionCodec, DecodeError } from "../src"; +import { IntMode, getInt64, getUint64 } from "../src/utils/int"; const extensionCodec = new ExtensionCodec(); extensionCodec.register({ @@ -24,6 +25,210 @@ extensionCodec.register({ }, }); +interface TestCase { + input: bigint, + expected: Map, +} + +// declared as a function to delay referencing the BigInt constructor +function BIGINTSPECS(): Record { + return { + ZERO: { + input: BigInt(0), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0], + [IntMode.SAFE_NUMBER, 0], + [IntMode.MIXED, 0], + [IntMode.BIGINT, BigInt(0)], + ]) + }, + ONE: { + input: BigInt(1), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 1], + [IntMode.SAFE_NUMBER, 1], + [IntMode.MIXED, 1], + [IntMode.BIGINT, BigInt(1)], + ]) + }, + MINUS_ONE: { + input: BigInt(-1), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, -1], + [IntMode.SAFE_NUMBER, -1], + [IntMode.MIXED, -1], + [IntMode.BIGINT, BigInt(-1)], + ]) + }, + X_FF: { + input: BigInt(0xff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0xff], + [IntMode.SAFE_NUMBER, 0xff], + [IntMode.MIXED, 0xff], + [IntMode.BIGINT, BigInt(0xff)], + ]) + }, + MINUS_X_FF: { + input: BigInt(-0xff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, -0xff], + [IntMode.SAFE_NUMBER, -0xff], + [IntMode.MIXED, -0xff], + [IntMode.BIGINT, BigInt(-0xff)], + ]) + }, + INT32_MAX: { + input: BigInt(0x7fffffff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0x7fffffff], + [IntMode.SAFE_NUMBER, 0x7fffffff], + [IntMode.MIXED, 0x7fffffff], + [IntMode.BIGINT, BigInt(0x7fffffff)], + ]) + }, + INT32_MIN: { + input: BigInt(-0x7fffffff - 1), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, -0x7fffffff - 1], + [IntMode.SAFE_NUMBER, -0x7fffffff - 1], + [IntMode.MIXED, -0x7fffffff - 1], + [IntMode.BIGINT, BigInt(-0x7fffffff - 1)], + ]) + }, + MAX_SAFE_INTEGER: { + input: BigInt(Number.MAX_SAFE_INTEGER), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, Number.MAX_SAFE_INTEGER], + [IntMode.SAFE_NUMBER, Number.MAX_SAFE_INTEGER], + [IntMode.MIXED, Number.MAX_SAFE_INTEGER], + [IntMode.BIGINT, BigInt(Number.MAX_SAFE_INTEGER)], + ]) + }, + MAX_SAFE_INTEGER_PLUS_ONE: { + input: BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)], + [IntMode.BIGINT, BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)], + ]) + }, + MIN_SAFE_INTEGER: { + input: BigInt(Number.MIN_SAFE_INTEGER), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, Number.MIN_SAFE_INTEGER], + [IntMode.SAFE_NUMBER, Number.MIN_SAFE_INTEGER], + [IntMode.MIXED, Number.MIN_SAFE_INTEGER], + [IntMode.BIGINT, BigInt(Number.MIN_SAFE_INTEGER)], + ]) + }, + MIN_SAFE_INTEGER_MINUS_ONE: { + input: BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1)], + [IntMode.BIGINT, BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1)], + ]), + }, + INT64_MAX: { + input: BigInt("0x7fffffffffffffff"), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt("0x7fffffffffffffff")], + [IntMode.BIGINT, BigInt("0x7fffffffffffffff")], + ]) + }, + INT64_MIN: { + input: BigInt(-1) * BigInt("0x8000000000000000"), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt(-1) * BigInt("0x8000000000000000")], + [IntMode.BIGINT, BigInt(-1) * BigInt("0x8000000000000000")], + ]), + }, + } +} + +// declared as a function to delay referencing the BigInt constructor +function BIGUINTSPECS(): Record { + return { + ZERO: { + input: BigInt(0), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0], + [IntMode.SAFE_NUMBER, 0], + [IntMode.MIXED, 0], + [IntMode.BIGINT, BigInt(0)], + ]) + }, + ONE: { + input: BigInt(1), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 1], + [IntMode.SAFE_NUMBER, 1], + [IntMode.MIXED, 1], + [IntMode.BIGINT, BigInt(1)], + ]) + }, + X_FF: { + input: BigInt(0xff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0xff], + [IntMode.SAFE_NUMBER, 0xff], + [IntMode.MIXED, 0xff], + [IntMode.BIGINT, BigInt(0xff)], + ]) + }, + UINT32_MAX: { + input: BigInt(0xffffffff), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, 0xffffffff], + [IntMode.SAFE_NUMBER, 0xffffffff], + [IntMode.MIXED, 0xffffffff], + [IntMode.BIGINT, BigInt(0xffffffff)], + ]) + }, + MAX_SAFE_INTEGER: { + input: BigInt(Number.MAX_SAFE_INTEGER), + expected: new Map([ + [IntMode.UNSAFE_NUMBER, Number.MAX_SAFE_INTEGER], + [IntMode.SAFE_NUMBER, Number.MAX_SAFE_INTEGER], + [IntMode.MIXED, Number.MAX_SAFE_INTEGER], + [IntMode.BIGINT, BigInt(Number.MAX_SAFE_INTEGER)], + ]) + }, + MAX_SAFE_INTEGER_PLUS_ONE: { + input: BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)], + [IntMode.BIGINT, BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)], + ]) + }, + UINT64_MAX: { + input: BigInt("0xffffffffffffffff"), + expected: new Map([ + // exclude IntMode.UNSAFE_NUMBER, behavior will not be exact + [IntMode.SAFE_NUMBER, "error"], + [IntMode.MIXED, BigInt("0xffffffffffffffff")], + [IntMode.BIGINT, BigInt("0xffffffffffffffff")], + ]) + }, + } +} + +function abs(value: bigint): bigint { + if (value < 0) { + return BigInt(-1) * value; + } + return value; +} + describe("codec BigInt", () => { before(function () { if (typeof BigInt === "undefined") { @@ -31,21 +236,67 @@ describe("codec BigInt", () => { } }); - it("encodes and decodes 0n", () => { - const value = BigInt(0); - const encoded = encode(value, { extensionCodec }); - assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + context("extension", () => { + it("encodes and decodes 0n", () => { + const value = BigInt(0); + const encoded = encode(value, { extensionCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + }); + + it("encodes and decodes MAX_SAFE_INTEGER+1", () => { + const value = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); + const encoded = encode(value, { extensionCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + }); + + it("encodes and decodes MIN_SAFE_INTEGER-1", () => { + const value = BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1); + const encoded = encode(value, { extensionCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + }); }); - it("encodes and decodes MAX_SAFE_INTEGER+1", () => { - const value = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); - const encoded = encode(value, { extensionCodec }); - assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); - }); + context("native", () => { + context("int 64", () => { + const specs = BIGINTSPECS(); - it("encodes and decodes MIN_SAFE_INTEGER-1", () => { - const value = BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1); - const encoded = encode(value, { extensionCodec }); - assert.deepStrictEqual(decode(encoded, { extensionCodec }), value); + for (const name of Object.keys(specs)) { + const testCase = specs[name]!; + + it(`sets and gets ${testCase.input} (${testCase.input < 0 ? "-" : ""}0x${abs(testCase.input).toString(16)})`, () => { + const b = new Uint8Array(8); + const view = new DataView(b.buffer); + view.setBigInt64(0, testCase.input); + for (const [mode, expected] of testCase.expected) { + if (expected === "error") { + assert.throws(() => getInt64(view, 0, mode), new RegExp(`Mode is IntMode\\.SAFE_NUMBER and value is not a safe integer: ${testCase.input < 0 ? "-" : ""}0x${abs(testCase.input).toString(16)}$`)); + continue; + } + assert.deepStrictEqual(getInt64(view, 0, mode), expected); + } + }); + } + }); + + context("uint 64", () => { + const specs = BIGUINTSPECS(); + + for (const name of Object.keys(specs)) { + const testCase = specs[name]!; + + it(`sets and gets ${testCase.input} (0x${testCase.input.toString(16)})`, () => { + const b = new Uint8Array(8); + const view = new DataView(b.buffer); + view.setBigUint64(0, testCase.input); + for (const [mode, expected] of testCase.expected) { + if (expected === "error") { + assert.throws(() => getUint64(view, 0, mode), new RegExp(`Mode is IntMode\\.SAFE_NUMBER and value is not a safe integer: 0x${testCase.input.toString(16)}$`)); + continue; + } + assert.deepStrictEqual(getUint64(view, 0, mode), expected); + } + }); + } + }); }); }); diff --git a/test/codec-int.test.ts b/test/codec-int.test.ts index 486f93b7..7a1f106b 100644 --- a/test/codec-int.test.ts +++ b/test/codec-int.test.ts @@ -1,5 +1,5 @@ import assert from "assert"; -import { setInt64, getInt64, getUint64, setUint64 } from "../src/utils/int"; +import { IntMode, setInt64, getInt64, getUint64, setUint64 } from "../src/utils/int"; const INT64SPECS = { ZERO: 0, @@ -22,7 +22,12 @@ describe("codec: int64 / uint64", () => { const b = new Uint8Array(8); const view = new DataView(b.buffer); setInt64(view, 0, value); - assert.deepStrictEqual(getInt64(view, 0), value); + assert.deepStrictEqual(getInt64(view, 0, IntMode.UNSAFE_NUMBER), value); + assert.deepStrictEqual(getInt64(view, 0, IntMode.SAFE_NUMBER), value); + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(getInt64(view, 0, IntMode.MIXED), value); + assert.deepStrictEqual(getInt64(view, 0, IntMode.BIGINT), BigInt(value)); + } }); } }); @@ -32,14 +37,24 @@ describe("codec: int64 / uint64", () => { const b = new Uint8Array(8); const view = new DataView(b.buffer); setUint64(view, 0, 0); - assert.deepStrictEqual(getUint64(view, 0), 0); + assert.deepStrictEqual(getUint64(view, 0, IntMode.UNSAFE_NUMBER), 0); + assert.deepStrictEqual(getUint64(view, 0, IntMode.SAFE_NUMBER), 0); + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(getUint64(view, 0, IntMode.MIXED), 0); + assert.deepStrictEqual(getUint64(view, 0, IntMode.BIGINT), BigInt(0)); + } }); it(`sets and gets MAX_SAFE_INTEGER`, () => { const b = new Uint8Array(8); const view = new DataView(b.buffer); setUint64(view, 0, Number.MAX_SAFE_INTEGER); - assert.deepStrictEqual(getUint64(view, 0), Number.MAX_SAFE_INTEGER); + assert.deepStrictEqual(getUint64(view, 0, IntMode.UNSAFE_NUMBER), Number.MAX_SAFE_INTEGER); + assert.deepStrictEqual(getUint64(view, 0, IntMode.SAFE_NUMBER), Number.MAX_SAFE_INTEGER); + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(getUint64(view, 0, IntMode.MIXED), Number.MAX_SAFE_INTEGER); + assert.deepStrictEqual(getUint64(view, 0, IntMode.BIGINT), BigInt(Number.MAX_SAFE_INTEGER)); + } }); }); }); diff --git a/test/encode.test.ts b/test/encode.test.ts index ca513a54..9df21a2d 100644 --- a/test/encode.test.ts +++ b/test/encode.test.ts @@ -9,7 +9,7 @@ describe("encode", () => { }); context("forceFloat32", () => { - it("encodes numbers in float64 wihout forceFloat32", () => { + it("encodes numbers in float64 without forceFloat32", () => { assert.deepStrictEqual(encode(3.14), Uint8Array.from([0xcb, 0x40, 0x9, 0x1e, 0xb8, 0x51, 0xeb, 0x85, 0x1f])); }); @@ -28,6 +28,10 @@ describe("encode", () => { context("forceFloat", () => { it("encodes integers as integers without forceIntegerToFloat", () => { assert.deepStrictEqual(encode(3), Uint8Array.from([0x3])); + + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(encode(BigInt(3)), Uint8Array.from([0x3])); + } }); it("encodes integers as floating point when forceIntegerToFloat=true", () => { @@ -35,6 +39,13 @@ describe("encode", () => { encode(3, { forceIntegerToFloat: true }), Uint8Array.from([0xcb, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), ); + + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual( + encode(BigInt(3), { forceIntegerToFloat: true }), + Uint8Array.from([0xcb, 0x40, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), + ); + } }); it("encodes integers as float32 when forceIntegerToFloat=true and forceFloat32=true", () => { @@ -42,10 +53,21 @@ describe("encode", () => { encode(3, { forceIntegerToFloat: true, forceFloat32: true }), Uint8Array.from([0xca, 0x40, 0x40, 0x00, 0x00]), ); + + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual( + encode(BigInt(3), { forceIntegerToFloat: true, forceFloat32: true }), + Uint8Array.from([0xca, 0x40, 0x40, 0x00, 0x00]), + ); + } }); it("encodes integers as integers when forceIntegerToFloat=false", () => { assert.deepStrictEqual(encode(3, { forceIntegerToFloat: false }), Uint8Array.from([0x3])); + + if (typeof BigInt !== "undefined") { + assert.deepStrictEqual(encode(BigInt(3), { forceIntegerToFloat: false }), Uint8Array.from([0x3])); + } }); }); @@ -71,4 +93,14 @@ describe("encode", () => { const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteLength); assert.deepStrictEqual(decode(arrayBuffer), decode(buffer)); }); + + context("Bigint that exceeds 64 bits", () => { + if (typeof BigInt !== "undefined") { + const MAX_UINT64_PLUS_ONE = BigInt("0x10000000000000000"); + assert.throws(() => encode(MAX_UINT64_PLUS_ONE), /Bigint is too large for uint64: 18446744073709551616$/); + + const MIN_INT64_MINUS_ONE = BigInt(-1) * BigInt("0x8000000000000001"); + assert.throws(() => encode(MIN_INT64_MINUS_ONE), /Bigint is too small for int64: -9223372036854775809$/); + } + }); }); diff --git a/test/msgpack-test-suite.test.ts b/test/msgpack-test-suite.test.ts index 6800973e..42c220c2 100644 --- a/test/msgpack-test-suite.test.ts +++ b/test/msgpack-test-suite.test.ts @@ -2,7 +2,8 @@ import assert from "assert"; import util from "util"; import { Exam } from "msgpack-test-js"; import { MsgTimestamp } from "msg-timestamp"; -import { encode, decode, ExtensionCodec, EXT_TIMESTAMP, encodeTimeSpecToTimestamp } from "@msgpack/msgpack"; +import { MsgUInt64, MsgInt64 } from "msg-int64"; +import { encode, decode, ExtensionCodec, EXT_TIMESTAMP, encodeTimeSpecToTimestamp, IntMode } from "@msgpack/msgpack"; const extensionCodec = new ExtensionCodec(); extensionCodec.register({ @@ -24,7 +25,7 @@ extensionCodec.register({ const TEST_TYPES = { array: 1, - bignum: 0, // TODO + bignum: typeof BigInt !== "undefined", binary: 1, bool: 1, map: 1, @@ -34,6 +35,22 @@ const TEST_TYPES = { timestamp: 1, }; +function convertValueForEncoding(value: unknown): unknown { + if (value instanceof MsgInt64 || value instanceof MsgUInt64) { + return BigInt(value.toString()); + } + + return value; +} + +function convertValueForDecoding(value: unknown): unknown { + if (typeof value === "bigint") { + return value.toString(); + } + + return value; +} + describe("msgpack-test-suite", () => { Exam.getExams(TEST_TYPES).forEach((exam) => { const types = exam.getTypes(TEST_TYPES); @@ -41,7 +58,7 @@ describe("msgpack-test-suite", () => { const title = `${first}: ${exam.stringify(first)}`; it(`encodes ${title}`, () => { types.forEach((type) => { - const value = exam.getValue(type); + const value = convertValueForEncoding(exam.getValue(type)); const buffer = Buffer.from(encode(value, { extensionCodec })); if (exam.matchMsgpack(buffer)) { @@ -58,7 +75,7 @@ describe("msgpack-test-suite", () => { it(`decodes ${title}`, () => { const msgpacks = exam.getMsgpacks(); msgpacks.forEach((encoded, idx) => { - const value = decode(encoded, { extensionCodec }); + const value = convertValueForDecoding(decode(encoded, { extensionCodec, intMode: IntMode.MIXED })); if (exam.matchValue(value)) { assert(true, exam.stringify(idx)); } else {