From 6d4feb53f1dd8cc216c70b09abf6f209628deeaf Mon Sep 17 00:00:00 2001 From: Joe Hildebrand Date: Mon, 15 Jan 2024 11:27:11 -0500 Subject: [PATCH 1/4] flushToZero --- src/container.ts | 2 +- src/encoder.ts | 10 ++++++++-- src/float.ts | 20 ++++++++++++++++++++ src/options.ts | 6 ++++++ test/encoder.test.js | 9 +++++++++ test/float.test.js | 17 +++++++++++++++++ web/src/index.html | 4 ++++ web/src/index.js | 5 +++-- 8 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 test/float.test.js diff --git a/src/container.ts b/src/container.ts index 764311b..4697680 100644 --- a/src/container.ts +++ b/src/container.ts @@ -135,7 +135,7 @@ export class CBORcontainer { } } if (opts.rejectLargeNegatives && - (value as number < -0x8000000000000000n)) { + (value as bigint < -0x8000000000000000n)) { throw new Error(`Invalid 65bit negative number: ${value}`); } if (opts.boxed) { diff --git a/src/encoder.ts b/src/encoder.ts index 250f04d..53ec2bc 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -3,8 +3,8 @@ import type {EncodeOptions, RequiredEncodeOptions} from './options.js'; import {MT, NUMBYTES, SIMPLE, SYMS, TAG} from './constants.js'; import {type TagNumber, type TaggedValue, type ToCBOR, Writer} from './writer.js'; +import {flushToZero, halfToUint} from './float.js'; import type {KeyValueEncoded} from './sorts.js'; -import {halfToUint} from './float.js'; import {hexToU8} from './utils.js'; export const { @@ -37,6 +37,7 @@ export const EncodeOptionsDefault: RequiredEncodeOptions = { avoidInts: false, collapseBigInts: true, float64: false, + flushToZero: false, forceEndian: null, ignoreOriginalEncoding: false, largeNegativeAsBigInt: false, @@ -245,9 +246,14 @@ export function writeBigInt( * @param opts Encoding options. */ export function writeNumber( - val: number, w: Writer, + val: number, + w: Writer, opts: RequiredEncodeOptions ): void { + if (opts.flushToZero) { + val = flushToZero(val); + } + if (Object.is(val, -0)) { if (opts.simplifyNegativeZero) { if (opts.avoidInts) { diff --git a/src/float.ts b/src/float.ts index d5fce75..66b94f2 100644 --- a/src/float.ts +++ b/src/float.ts @@ -72,3 +72,23 @@ export function halfToUint(half: number): number | null { return s16; } + +/** + * Flush subnormal numbers to 0/-0. + * + * @param n Number. + * @returns Normalized number. + */ +export function flushToZero(n: number): number { + if (n !== 0) { // Remember 0 === -0 + const a = new ArrayBuffer(8); + const dv = new DataView(a); + dv.setFloat64(0, n, false); + const b = dv.getBigUint64(0, false); + // Subnormals have an 11-bit exponent of 0 and a non-zero mantissa. + if ((b & 0x7ff0000000000000n) === 0n) { + return (b & 0x8000000000000000n) ? -0 : 0; + } + } + return n; +} diff --git a/src/options.ts b/src/options.ts index 2d6fd78..93cc1a1 100644 --- a/src/options.ts +++ b/src/options.ts @@ -240,6 +240,12 @@ export interface EncodeOptions extends WriterOptions { */ float64?: boolean; + /** + * When writing floats, first flush any subnormal numbers to zero before + * decising on encoding. + */ + flushToZero?: boolean; + /** * How to write TypedArrays? * Null to use the current platform's endian-ness. diff --git a/test/encoder.test.js b/test/encoder.test.js index 89dc7be..9199a6a 100644 --- a/test/encoder.test.js +++ b/test/encoder.test.js @@ -179,3 +179,12 @@ test('encode avoiding ints', () => { [-0, '', '0xf90000'], ], {avoidInts: true, simplifyNegativeZero: true}); }); + +test('flush to zero', () => { + testAll([ + [1e-320, '', '0x00'], + [-1e-320, '', '0xf98000'], + [Number.EPSILON, '', '0xfa25800000'], + [-Number.EPSILON, '', '0xfaa5800000'], + ], {flushToZero: true}); +}); diff --git a/test/float.test.js b/test/float.test.js new file mode 100644 index 0000000..ef574a6 --- /dev/null +++ b/test/float.test.js @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import {flushToZero} from '../lib/float.js'; +import test from 'node:test'; + +test('flushToZero', () => { + assert.equal(flushToZero(0), 0); + assert.equal(flushToZero(-0), -0); + assert.equal(flushToZero(1), 1); + assert.equal(flushToZero(1.25), 1.25); + assert.equal(flushToZero(-1), -1); + assert.equal(flushToZero(-1.25), -1.25); + assert.equal(flushToZero(Number.EPSILON), Number.EPSILON); + assert.equal(flushToZero(-Number.EPSILON), -Number.EPSILON); + assert.notEqual(0, 1e-320); + assert.equal(flushToZero(1e-320), 0); + assert.equal(flushToZero(-1e-320), -0); +}); diff --git a/web/src/index.html b/web/src/index.html index 1d419ba..315933b 100644 --- a/web/src/index.html +++ b/web/src/index.html @@ -61,6 +61,10 @@

Encoding

+
  • + + +
    • +
  • + + +
  • diff --git a/web/src/index.js b/web/src/index.js index 1166ba6..5efb913 100644 --- a/web/src/index.js +++ b/web/src/index.js @@ -56,6 +56,7 @@ const decodeOpts = { rejectNegativeZero: false, rejectSimple: false, rejectStreaming: false, + rejectSubnormals: false, rejectUndefined: false, saveOriginal: false, sortKeys: null, From 5b879fadf3298f0e545704425708b1cb0b10fc30 Mon Sep 17 00:00:00 2001 From: Joe Hildebrand Date: Sun, 21 Jan 2024 12:57:56 -0500 Subject: [PATCH 3/4] Decode with cde/dcbor --- src/box.ts | 5 +++ src/comment.ts | 2 +- src/container.ts | 75 ++++++++++++++++++++++++++---------------- src/decodeStream.ts | 12 +++++++ src/decoder.ts | 18 +++++++--- src/diagnostic.ts | 2 +- src/index.ts | 5 +++ src/options.ts | 28 +++++++++++++--- src/types.ts | 8 +++-- test/container.test.js | 2 +- test/decoder.test.js | 33 ++++++++++--------- web/src/index.html | 16 ++++++--- web/src/index.js | 65 +++++++++++++++++++++++++----------- 13 files changed, 190 insertions(+), 81 deletions(-) diff --git a/src/box.ts b/src/box.ts index 7f6474f..c98f703 100644 --- a/src/box.ts +++ b/src/box.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-redeclare */ import {SYMS} from './constants.js'; import {Tag} from './tag.js'; @@ -37,6 +38,10 @@ export function saveEncoded(obj: OriginalEncoding, orig: Uint8Array): void { }); } +export function box(value: bigint, orig: Uint8Array): BigInt; +export function box(value: string, orig: Uint8Array): String; +export function box(value: number, orig: Uint8Array): Number; + /** * Put an object wrapper around a primitive value, such as a number, boolean, * or bigint, storing the original CBOR encoding of the value for later diff --git a/src/comment.ts b/src/comment.ts index 717e454..be8da35 100644 --- a/src/comment.ts +++ b/src/comment.ts @@ -182,7 +182,7 @@ function output( } const CommentOptionsDefault: RequiredCommentOptions = { - ...CBORcontainer.defaultOptions, + ...CBORcontainer.defaultDecodeOptions, initialDepth: 0, noPrefixHex: false, minCol: 0, diff --git a/src/container.ts b/src/container.ts index 68a8d99..79472fd 100644 --- a/src/container.ts +++ b/src/container.ts @@ -1,9 +1,9 @@ +import type {DecodeOptions, MtAiValue, Parent, RequiredDecodeOptions} from './options.js'; +import {type KeyValueEncoded, sortCoreDeterministic} from './sorts.js'; import {MT, NUMBYTES} from './constants.js'; -import type {MtAiValue, Parent, RequiredDecodeOptions} from './options.js'; import {box, getEncoded, saveEncoded} from './box.js'; import {u8concat, u8toHex} from './utils.js'; import {DecodeStream} from './decodeStream.js'; -import type {KeyValueEncoded} from './sorts.js'; import {Simple} from './simple.js'; import {Tag} from './tag.js'; import {checkSubnormal} from './float.js'; @@ -19,17 +19,6 @@ const LENGTH_FOR_AI = new Map([ const EMPTY_BUF = new Uint8Array(0); -// Decide on dCBOR approach -// export const dCBORdecodeOptions: ContainerOptions = { -// rejectLargeNegatives: true, -// rejectLongLoundNaN: true, -// rejectLongNumbers: true, -// rejectNegativeZero: true, -// rejectSimple: true, -// rejectStreaming: true, -// sortKeys: sortCoreDeterministic, -// }; - /** * A CBOR data item that can contain other items. One of: * @@ -41,17 +30,19 @@ const EMPTY_BUF = new Uint8Array(0); * This is used in various decoding applications to keep track of state. */ export class CBORcontainer { - public static defaultOptions: RequiredDecodeOptions = { + public static defaultDecodeOptions: RequiredDecodeOptions = { ...DecodeStream.defaultOptions, ParentType: CBORcontainer, boxed: false, + cde: false, + dcbor: false, rejectLargeNegatives: false, rejectBigInts: false, rejectDuplicateKeys: false, rejectFloats: false, rejectInts: false, rejectLongLoundNaN: false, - rejectLongNumbers: false, + rejectLongFloats: false, rejectNegativeZero: false, rejectSimple: false, rejectStreaming: false, @@ -61,6 +52,44 @@ export class CBORcontainer { sortKeys: null, }; + /** + * Throw errors when decoding for bytes that were not encoded with {@link + * https://www.ietf.org/archive/id/draft-ietf-cbor-cde-01.html CBOR Common + * Deterministic Encoding Profile}. + * + * CDE does not mandate this checking, so it is up to the application + * whether it wants to ensure that inputs were not encoded incompetetently + * or maliciously. To turn all of these on at once, set the cbor option to + * true. + */ + public static cdeDecodeOptions: DecodeOptions = { + cde: true, + rejectStreaming: true, + requirePreferred: true, + sortKeys: sortCoreDeterministic, + }; + + /** + * Throw errors when decoding for bytes that were not encoded with {@link + * https://www.ietf.org/archive/id/draft-mcnally-deterministic-cbor-07.html + * dCBOR: A Deterministic CBOR Application Profile}. + * + * The dCBOR spec mandates that these errors be thrown when decoding dCBOR. + * Turn this on by setting the `dcbor` option to true, which also enables + * `cde` mode. + */ + public static dcborDecodeOptions: DecodeOptions = { + ...this.cdeDecodeOptions, + dcbor: true, + rejectDuplicateKeys: true, + rejectLargeNegatives: true, + rejectLongLoundNaN: true, + rejectLongFloats: true, + rejectNegativeZero: true, + rejectSimple: true, + rejectUndefined: true, + }; + public parent: Parent | undefined; public mt: number; public ai: number; @@ -126,22 +155,12 @@ export class CBORcontainer { if (opts.rejectInts) { throw new Error(`Unexpected integer: ${value}`); } - if (opts.rejectLongNumbers && (ai > NUMBYTES.ZERO)) { - // No opts needed - const buf = encode(value, {chunkSize: 9}); - - // Known safe: - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - if (buf.length < LENGTH_FOR_AI.get(ai)!) { - throw new Error(`Int should have been encoded shorter: ${value}`); - } - } if (opts.rejectLargeNegatives && (value as bigint < -0x8000000000000000n)) { throw new Error(`Invalid 65bit negative number: ${value}`); } if (opts.boxed) { - return box(value, stream.toHere(offset)); + return box(value as number, stream.toHere(offset)); } return value; case MT.SIMPLE_FLOAT: @@ -162,7 +181,7 @@ export class CBORcontainer { // Skip the size byte checkSubnormal(stream.toHere(offset + 1)); } - if (opts.rejectLongNumbers) { + if (opts.rejectLongFloats) { // No opts needed. const buf = encode(value, {chunkSize: 9}); if ((buf[0] >> 5) !== mt) { @@ -196,7 +215,7 @@ export class CBORcontainer { return new opts.ParentType(mav, Infinity, parent, opts); } if (opts.boxed) { - return box(value, stream.toHere(offset)); + return box(value as string, stream.toHere(offset)); } return value; case MT.ARRAY: diff --git a/src/decodeStream.ts b/src/decodeStream.ts index 7e50eae..350750d 100644 --- a/src/decodeStream.ts +++ b/src/decodeStream.ts @@ -22,6 +22,7 @@ export class DecodeStream implements Sliceable { public static defaultOptions: Required = { maxDepth: 1024, encoding: 'hex', + requirePreferred: false, }; #src; @@ -115,6 +116,8 @@ export class DecodeStream implements Sliceable { throw new Error(`Invalid simple encoding in extra byte: ${val}`); } simple = true; + } else if (this.#opts.requirePreferred && (val < 24)) { + throw new Error(`Unexpectedly long integer encoding (1) for ${val}`); } break; case NUMBYTES.TWO: @@ -123,6 +126,9 @@ export class DecodeStream implements Sliceable { val = parseHalf(this.#src, this.#offset); } else { val = this.#view.getUint16(this.#offset, false); + if (this.#opts.requirePreferred && (val <= 0xff)) { + throw new Error(`Unexpectedly long integer encoding (2) for ${val}`); + } } break; case NUMBYTES.FOUR: @@ -131,6 +137,9 @@ export class DecodeStream implements Sliceable { val = this.#view.getFloat32(this.#offset, false); } else { val = this.#view.getUint32(this.#offset, false); + if (this.#opts.requirePreferred && (val <= 0xffff)) { + throw new Error(`Unexpectedly long integer encoding (4) for ${val}`); + } } break; case NUMBYTES.EIGHT: { @@ -142,6 +151,9 @@ export class DecodeStream implements Sliceable { if (val <= Number.MAX_SAFE_INTEGER) { val = Number(val); } + if (this.#opts.requirePreferred && (val <= 0xffffffff)) { + throw new Error(`Unexpectedly long integer encoding (8) for ${val}`); + } } break; } diff --git a/src/decoder.ts b/src/decoder.ts index 506f8f7..e85f0af 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -13,12 +13,20 @@ import {SYMS} from './constants.js'; */ export function decode( src: Uint8Array | string, - options?: DecodeOptions + options: DecodeOptions = {} ): T { - const opts: Required = { - ...CBORcontainer.defaultOptions, - ...options, - }; + const opts = {...CBORcontainer.defaultDecodeOptions}; + if (options.dcbor) { + Object.assign(opts, CBORcontainer.dcborDecodeOptions); + } else if (options.cde) { + Object.assign(opts, CBORcontainer.cdeDecodeOptions); + } + Object.assign(opts, options); + + if (Object.hasOwn(opts, 'rejectLongNumbers')) { + throw new TypeError('rejectLongNumbers has changed to requirePreferred'); + } + if (opts.boxed) { opts.saveOriginal = true; } diff --git a/src/diagnostic.ts b/src/diagnostic.ts index 2b22d81..c3a2895 100644 --- a/src/diagnostic.ts +++ b/src/diagnostic.ts @@ -67,7 +67,7 @@ export function diagnose( options?: DecodeOptions ): string { const opts: Required = { - ...CBORcontainer.defaultOptions, + ...CBORcontainer.defaultDecodeOptions, ...options, ParentType: DiagContainer, }; diff --git a/src/index.ts b/src/index.ts index 1eefa5f..f78cb21 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,7 @@ */ import './types.js'; +import {CBORcontainer} from './container.js'; export type {DecodeStream, ValueGenerator} from './decodeStream.js'; export type { CommentOptions, @@ -35,3 +36,7 @@ export {Simple} from './simple.js'; export {Tag} from './tag.js'; export type {TagNumber, TaggedValue, ToCBOR, Writer} from './writer.js'; export {unbox, getEncoded} from './box.js'; + +export const defaultDecodeOptions = CBORcontainer.defaultOptions; +export const cdeDecodeOptions = CBORcontainer.cdeOptions; +export const dcborDecodeOptions = CBORcontainer.dcborOptions; diff --git a/src/options.ts b/src/options.ts index b083120..46823e2 100644 --- a/src/options.ts +++ b/src/options.ts @@ -19,6 +19,14 @@ export interface DecodeStreamOptions { * @default null */ encoding?: 'base64' | 'hex' | null; + + /** + * Reject integers and lengths that could have been encoded in a smaller + * encoding. + * + * @default false + */ + requirePreferred?: boolean; } /** @@ -97,6 +105,16 @@ export interface DecodeOptions extends DecodeStreamOptions { */ boxed?: boolean; + /** + * Turn on options for draft-ietf-cbor-cde-01. + */ + cde?: boolean; + + /** + * Turn on options for draft-mcnally-deterministic-cbor-07. + */ + dcbor?: boolean; + /** * Reject negative integers in the range [CBOR_NEGATIVE_INT_MAX ... * STANDARD_NEGATIVE_INT_MAX - 1]. @@ -134,16 +152,16 @@ export interface DecodeOptions extends DecodeStreamOptions { rejectInts?: boolean; /** - * Reject NaNs that are not encoded as 0x7e00. - * @default false + * Reject floating point numbers that should have been encoded in shorter + * form, including having been encoded as an integer. */ - rejectLongLoundNaN?: boolean; + rejectLongFloats?: boolean; /** - * Reject numbers that could have been encoded in a smaller encoding. + * Reject NaNs that are not encoded as 0x7e00. * @default false */ - rejectLongNumbers?: boolean; + rejectLongLoundNaN?: boolean; /** * If negative zero (-0) is received, throw an error. diff --git a/src/types.ts b/src/types.ts index 3e301c8..6e543e2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -125,16 +125,20 @@ function u8toBigInt( if (opts.rejectBigInts) { throw new Error(`Decoding unwanted big integer: ${tag}(h'${u8toHex(tag.contents)}')`); } - if (opts.rejectLongNumbers && tag.contents[0] === 0) { + if (opts.requirePreferred && tag.contents[0] === 0) { + // The preferred serialization of the byte string is to leave out any + // leading zeroes throw new Error(`Decoding overly-large bigint: ${tag}(h'${u8toHex(tag.contents)}`); } let bi = tag.contents.reduce((t, v) => (t << 8n) | BigInt(v), 0n); if (neg) { bi = -1n - bi; } - if (opts.rejectLongNumbers && + if (opts.requirePreferred && (bi >= Number.MIN_SAFE_INTEGER) && (bi <= Number.MAX_SAFE_INTEGER)) { + // The preferred serialization of an integer that can be represented using + // major type 0 or 1 is to encode it this way instead of as a bignum throw new Error(`Decoding bigint that could have been int: ${bi}n`); } if (opts.boxed) { diff --git a/test/container.test.js b/test/container.test.js index a6960df..9b4830c 100644 --- a/test/container.test.js +++ b/test/container.test.js @@ -4,7 +4,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; test('create container', () => { - const opts = CBORcontainer.defaultOptions; + const opts = CBORcontainer.defaultDecodeOptions; const c = CBORcontainer.create([MT.POS_INT, 0, 0], undefined, opts); assert(!(c instanceof CBORcontainer)); assert.throws(() => CBORcontainer.create([-1, 0, 0], undefined, opts)); diff --git a/test/decoder.test.js b/test/decoder.test.js index 1706d76..53140ef 100644 --- a/test/decoder.test.js +++ b/test/decoder.test.js @@ -4,21 +4,9 @@ import {Tag} from '../lib/tag.js'; import assert from 'node:assert/strict'; import {decode} from '../lib/decoder.js'; import {hexToU8} from '../lib/utils.js'; -import {sortCoreDeterministic} from '../lib/sorts.js'; import test from 'node:test'; import {unbox} from '../lib/box.js'; -const dCBORdecodeOptions = { - rejectLargeNegatives: true, - rejectLongLoundNaN: true, - rejectLongNumbers: true, - rejectNegativeZero: true, - rejectSimple: true, - rejectStreaming: true, - rejectUndefined: true, - sortKeys: sortCoreDeterministic, -}; - function testAll(list, opts) { let count = 0; for (const [orig, diag, commented] of list) { @@ -53,10 +41,18 @@ test('decode bad tags', () => { failAll(cases.decodeBadTags); }); +test('decode with cde', () => { + testAll(cases.goodNumbers, {cde: true}); + testAll(cases.good.filter(([o]) => o instanceof Map), {cde: true}); + failAll([ + '0x1817', + ], {cde: true}); +}); + test('decode with dCBOR', () => { - failAll(cases.decodeBadDcbor, dCBORdecodeOptions); - testAll(cases.goodNumbers, dCBORdecodeOptions); - testAll(cases.good.filter(([o]) => o instanceof Map), dCBORdecodeOptions); + failAll(cases.decodeBadDcbor, {dcbor: true}); + testAll(cases.goodNumbers, {dcbor: true}); + testAll(cases.good.filter(([o]) => o instanceof Map), {dcbor: true}); failAll([ '0xa280008001', '0xa200010002', @@ -170,3 +166,10 @@ test('rejectSubnormals', () => { '0xf98001', ], {rejectSubnormals: true}); }); + +test('fail on old prefs', () => { + assert.throws( + () => decode('f93d00', {rejectLongNumbers: true}), + /rejectLongNumbers has changed to requirePreferred/ + ); +}); diff --git a/web/src/index.html b/web/src/index.html index ecc5bbc..bc3efee 100644 --- a/web/src/index.html +++ b/web/src/index.html @@ -120,6 +120,14 @@

    Decoding

    • +
  • + + +
    • +
  • + + +
  • @@ -144,10 +152,6 @@

    Decoding

    • -
  • - - -
  • @@ -168,6 +172,10 @@

    Decoding

    • +
  • + + +
  • diff --git a/web/src/index.js b/web/src/index.js index 5efb913..19be3de 100644 --- a/web/src/index.js +++ b/web/src/index.js @@ -1,5 +1,15 @@ import './style.css'; -import {Simple, Tag, comment, decode, diagnose, encode} from 'cbor2'; +import { + Simple, + Tag, + cdeDecodeOptions, + comment, + dcborDecodeOptions, + decode, + defaultDecodeOptions, + diagnose, + encode, +} from 'cbor2'; import {base64ToBytes, hexToU8, u8toHex} from 'cbor2/utils'; import {sortCoreDeterministic, sortLengthFirstDeterministic} from 'cbor2/sorts'; import {inspect} from 'node-inspect-extracted'; @@ -9,6 +19,15 @@ const otxt = document.getElementById('output-text'); const itxt = document.getElementById('input-text'); const ifmt = document.getElementById('input-fmt'); const copy = document.getElementById('copy'); +const sortKeysDecode = document.querySelector('#sortKeysDecode'); + +const notCdeDecodeOptions = Object.fromEntries( + Object.entries(cdeDecodeOptions).map(([k, v]) => [k, !v]) +); + +const notDcborDecodeOptions = Object.fromEntries( + Object.entries(dcborDecodeOptions).map(([k, v]) => [k, !v]) +); /** * Encode Uint8Array to base64. @@ -44,23 +63,7 @@ const encodeOpts = { sortKeys: null, }; -const decodeOpts = { - boxed: false, - rejectLargeNegatives: false, - rejectBigInts: false, - rejectDuplicateKeys: false, - rejectFloats: false, - rejectInts: false, - rejectLongLoundNaN: false, - rejectLongNumbers: false, - rejectNegativeZero: false, - rejectSimple: false, - rejectStreaming: false, - rejectSubnormals: false, - rejectUndefined: false, - saveOriginal: false, - sortKeys: null, -}; +const decodeOpts = defaultDecodeOptions; // Convert any input to a buffer function input() { @@ -175,6 +178,31 @@ sortKeysEncode.value = 'null'; function changeDecodeOption({target}) { decodeOpts[target.id] = target.checked; + let modified = false; + if (target.id === 'dcbor') { + modified = true; + Object.assign(decodeOpts, target.checked ? + dcborDecodeOptions : + notDcborDecodeOptions); + } else if (target.id === 'cde') { + modified = true; + Object.assign(decodeOpts, target.checked ? + cdeDecodeOptions : + notCdeDecodeOptions); + } + if (modified) { + for (const inp of document.querySelectorAll('#decodingOpts input')) { + inp.checked = decodeOpts[inp.id]; + } + if (decodeOpts.sortKeys === sortCoreDeterministic) { + sortKeysDecode.value = 'coreDeterministic'; + } else if (decodeOpts.sortKeys === sortLengthFirstDeterministic) { + sortKeysDecode.value = 'lengthFirstDeterministic'; + } else { + sortKeysDecode.value = 'null'; + } + } + convert(); } @@ -183,7 +211,6 @@ for (const inp of document.querySelectorAll('#decodingOpts input')) { inp.checked = decodeOpts[inp.id]; } -const sortKeysDecode = document.querySelector('#sortKeysDecode'); sortKeysDecode.onchange = () => { encodeOpts.sortKeys = { null: null, From 1c9ca8400aa9326e3f49dce31c2d613fd0fdc5e7 Mon Sep 17 00:00:00 2001 From: Joe Hildebrand Date: Mon, 22 Jan 2024 13:53:03 -0500 Subject: [PATCH 4/4] Encode with cde/dcbor --- README.md | 9 ++-- src/encoder.ts | 67 +++++++++++++++++++++--------- src/index.ts | 16 +++++--- src/options.ts | 10 +++++ test/encoder.test.js | 25 +++++------- web/src/index.html | 12 +++++- web/src/index.js | 97 ++++++++++++++++++++++++++++---------------- 7 files changed, 154 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 84886b2..a5e5a6d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ format ([RFC8949](https://www.rfc-editor.org/rfc/rfc8949.html)). This package supersedes [node-cbor](https://github.com/hildjj/node-cbor/tree/main/packages/cbor), with the following goals: -- Web-first. Usable in node. +- Web-first. Usable in Node and Deno. - Simpler where possible. Remove non-core features in favor of extensibility. - Synchronous decoding. Removes streaming capabilities that are rarely used. - Complete break of API compatibility, allowing use of newer JavaScript constructs. @@ -13,12 +13,9 @@ This package supersedes [node-cbor](https://github.com/hildjj/node-cbor/tree/mai ## Supported Node.js versions -This project now only supports versions of Node that the Node team is +This project now only supports versions of Node.js that the Node team is [currently supporting](https://github.com/nodejs/Release#release-schedule). -Ava's [support -statement](https://github.com/avajs/ava/blob/main/docs/support-statement.md) -is what we will be using as well. Since the first release will not be soon, -that means Node `18`+ is required. +Currently, that means Node `18`+ is required. ## Installation diff --git a/src/encoder.ts b/src/encoder.ts index 53ec2bc..99de0c6 100644 --- a/src/encoder.ts +++ b/src/encoder.ts @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/no-use-before-define */ import type {EncodeOptions, RequiredEncodeOptions} from './options.js'; +import {type KeyValueEncoded, sortCoreDeterministic} from './sorts.js'; import {MT, NUMBYTES, SIMPLE, SYMS, TAG} from './constants.js'; import {type TagNumber, type TaggedValue, type ToCBOR, Writer} from './writer.js'; import {flushToZero, halfToUint} from './float.js'; -import type {KeyValueEncoded} from './sorts.js'; import {hexToU8} from './utils.js'; export const { @@ -20,22 +20,12 @@ const UNDEFINED = (MT.SIMPLE_FLOAT << 5) | SIMPLE.UNDEFINED; const NULL = (MT.SIMPLE_FLOAT << 5) | SIMPLE.NULL; const TE = new TextEncoder(); -// Decide on dCBOR approach -// export const dCBORencodeOptions: EncodeOptions = { -// // Default: collapseBigInts: true, -// ignoreOriginalEncoding: true, -// largeNegativeAsBigInt: true, -// rejectCustomSimples: true, -// rejectDuplicateKeys: true, -// rejectUndefined: true, -// simplifyNegativeZero: true, -// sortKeys: sortCoreDeterministic, -// }; - -export const EncodeOptionsDefault: RequiredEncodeOptions = { +export const defaultEncodeOptions: RequiredEncodeOptions = { ...Writer.defaultOptions, avoidInts: false, + cde: false, collapseBigInts: true, + dcbor: false, float64: false, flushToZero: false, forceEndian: null, @@ -50,6 +40,41 @@ export const EncodeOptionsDefault: RequiredEncodeOptions = { sortKeys: null, }; +/** + * Encode with CDE ({@link + * https://www.ietf.org/archive/id/draft-ietf-cbor-cde-01.html CBOR Common + * Deterministic Encoding Profile}). Eable this set of options by setting + * `cde` to true. + * + * Since cbor2 always uses preferred encoding, this option only sets the + * sort algorithm for map/object keys, and ensures that any original + * encoding information (from decoding with saveOriginal) is ignored. + */ +export const cdeEncodeOptions: EncodeOptions = { + cde: true, + ignoreOriginalEncoding: true, + sortKeys: sortCoreDeterministic, +}; + +/** + * Encode with CDE and dCBOR ({@link + * https://www.ietf.org/archive/id/draft-mcnally-deterministic-cbor-07.html + * dCBOR: A Deterministic CBOR Application Profile}). Enable this set of + * options by setting `dcbor` to true. + * + * Several of these options can cause errors to be thrown for inputs that + * would have otherwise generated valid CBOR (e.g. `undefined`). + */ +export const dcborEncodeOptions: EncodeOptions = { + ...cdeEncodeOptions, + dcbor: true, + largeNegativeAsBigInt: true, + rejectCustomSimples: true, + rejectDuplicateKeys: true, + rejectUndefined: true, + simplifyNegativeZero: true, +}; + /** * Any class. Ish. */ @@ -435,11 +460,15 @@ export function writeUnknown( * @param options Tweak the conversion process. * @returns Bytes in a Uint8Array buffer. */ -export function encode(val: unknown, options?: EncodeOptions): Uint8Array { - const opts: RequiredEncodeOptions = { - ...EncodeOptionsDefault, - ...options, - }; +export function encode(val: unknown, options: EncodeOptions = {}): Uint8Array { + const opts: RequiredEncodeOptions = {...defaultEncodeOptions}; + if (options.dcbor) { + Object.assign(opts, dcborEncodeOptions); + } else if (options.cde) { + Object.assign(opts, cdeEncodeOptions); + } + Object.assign(opts, options); + const w = new Writer(opts); writeUnknown(val, w, opts); return w.read(); diff --git a/src/index.ts b/src/index.ts index f78cb21..759816d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,12 +31,18 @@ export type { export {decode} from './decoder.js'; export {diagnose} from './diagnostic.js'; export {comment} from './comment.js'; -export {encode} from './encoder.js'; +export { + cdeEncodeOptions, + defaultEncodeOptions, + dcborEncodeOptions, + encode, +} from './encoder.js'; export {Simple} from './simple.js'; export {Tag} from './tag.js'; export type {TagNumber, TaggedValue, ToCBOR, Writer} from './writer.js'; export {unbox, getEncoded} from './box.js'; - -export const defaultDecodeOptions = CBORcontainer.defaultOptions; -export const cdeDecodeOptions = CBORcontainer.cdeOptions; -export const dcborDecodeOptions = CBORcontainer.dcborOptions; +export const { + cdeDecodeOptions, + dcborDecodeOptions, + defaultDecodeOptions, +} = CBORcontainer; diff --git a/src/options.ts b/src/options.ts index 46823e2..8ae0eee 100644 --- a/src/options.ts +++ b/src/options.ts @@ -250,6 +250,11 @@ export interface EncodeOptions extends WriterOptions { */ avoidInts?: boolean; + /** + * Turn on options for draft-ietf-cbor-cde-01. + */ + cde?: boolean; + /** * Should bigints that can fit into normal integers be collapsed into * normal integers? @@ -257,6 +262,11 @@ export interface EncodeOptions extends WriterOptions { */ collapseBigInts?: boolean; + /** + * Turn on options for draft-mcnally-deterministic-cbor-07. + */ + dcbor?: boolean; + /** * When writing floats, always use the 64-bit version. Often combined with * `avoidInts`. diff --git a/test/encoder.test.js b/test/encoder.test.js index 9199a6a..fd56cd5 100644 --- a/test/encoder.test.js +++ b/test/encoder.test.js @@ -5,23 +5,12 @@ import { clearEncoder, encode, registerEncoder, writeInt, } from '../lib/encoder.js'; import {isBigEndian, u8toHex} from '../lib/utils.js'; -import {sortCoreDeterministic, sortLengthFirstDeterministic} from '../lib/sorts.js'; import {Writer} from '../lib/writer.js'; import assert from 'node:assert/strict'; +import {sortLengthFirstDeterministic} from '../lib/sorts.js'; import test from 'node:test'; import util from 'node:util'; -export const dCBORencodeOptions = { - // Default: collapseBigInts: true, - ignoreOriginalEncoding: true, - largeNegativeAsBigInt: true, - rejectCustomSimples: true, - rejectDuplicateKeys: true, - rejectUndefined: true, - simplifyNegativeZero: true, - sortKeys: sortCoreDeterministic, -}; - const BE = isBigEndian(); function testAll(list, opts = undefined) { @@ -140,9 +129,15 @@ test('deterministic sorting', () => { ); }); +test('encode cde', () => { + testAll([ + [new Map([[100, 0], [10, 1]]), '', '0xa20a01186400'], + ], {cde: true}); +}); + test('encode dCBOR', () => { - failAll(cases.encodeBadDCBOR, dCBORencodeOptions); - testAll(cases.good.filter(([o]) => o instanceof Map), dCBORencodeOptions); + failAll(cases.encodeBadDCBOR, {dcbor: true}); + testAll(cases.good.filter(([o]) => o instanceof Map), {dcbor: true}); const dv = new DataView(new ArrayBuffer(4)); dv.setFloat32(0, NaN); dv.setUint8(3, 1); @@ -152,7 +147,7 @@ test('encode dCBOR', () => { [n, '', '0xf97e00'], [-0x8000000000000000n, '', '0x3b7fffffffffffffff'], [-0x8000000000000001n, '', '0xc3488000000000000000'], - ], dCBORencodeOptions); + ], {dcbor: true}); }); test('encode rejections', () => { diff --git a/web/src/index.html b/web/src/index.html index bc3efee..34791a9 100644 --- a/web/src/index.html +++ b/web/src/index.html @@ -48,15 +48,23 @@

    Output

    Options

     

    Encoding

    -
      +
        • +
      • + + +
        • +
      • + + +
      • @@ -115,7 +123,7 @@

        Encoding

      Decoding

      -
        +
        • diff --git a/web/src/index.js b/web/src/index.js index 19be3de..b28179c 100644 --- a/web/src/index.js +++ b/web/src/index.js @@ -3,10 +3,13 @@ import { Simple, Tag, cdeDecodeOptions, + cdeEncodeOptions, comment, dcborDecodeOptions, + dcborEncodeOptions, decode, defaultDecodeOptions, + defaultEncodeOptions, diagnose, encode, } from 'cbor2'; @@ -19,8 +22,17 @@ const otxt = document.getElementById('output-text'); const itxt = document.getElementById('input-text'); const ifmt = document.getElementById('input-fmt'); const copy = document.getElementById('copy'); +const sortKeysEncode = document.querySelector('#sortKeysEncode'); const sortKeysDecode = document.querySelector('#sortKeysDecode'); +const notCdeEncodeOptions = Object.fromEntries( + Object.entries(cdeEncodeOptions).map(([k, v]) => [k, !v]) +); + +const notDcborEncodeOptions = Object.fromEntries( + Object.entries(dcborEncodeOptions).map(([k, v]) => [k, !v]) +); + const notCdeDecodeOptions = Object.fromEntries( Object.entries(cdeDecodeOptions).map(([k, v]) => [k, !v]) ); @@ -46,25 +58,35 @@ function error(e) { otxt.value = e.toString(); } -const encodeOpts = { - avoidInts: false, - collapseBigInts: true, - float64: false, - flushToZero: false, - forceEndian: null, - ignoreOriginalEncoding: false, - largeNegativeAsBigInt: false, - rejectBigInts: false, - rejectCustomSimples: false, - rejectDuplicateKeys: false, - rejectFloats: false, - rejectUndefined: false, - simplifyNegativeZero: false, - sortKeys: null, -}; - +const encodeOpts = defaultEncodeOptions; const decodeOpts = defaultDecodeOptions; +function showEncodeOpts() { + for (const inp of document.querySelectorAll('#encodeOpts input')) { + inp.checked = encodeOpts[inp.id.replace(/Encode$/, '')]; + } + if (encodeOpts.sortKeys === sortCoreDeterministic) { + sortKeysEncode.value = 'coreDeterministic'; + } else if (encodeOpts.sortKeys === sortLengthFirstDeterministic) { + sortKeysEncode.value = 'lengthFirstDeterministic'; + } else { + sortKeysEncode.value = 'null'; + } +} + +function showDecodeOpts() { + for (const inp of document.querySelectorAll('#decodeOpts input')) { + inp.checked = decodeOpts[inp.id]; + } + if (decodeOpts.sortKeys === sortCoreDeterministic) { + sortKeysDecode.value = 'coreDeterministic'; + } else if (decodeOpts.sortKeys === sortLengthFirstDeterministic) { + sortKeysDecode.value = 'lengthFirstDeterministic'; + } else { + sortKeysDecode.value = 'null'; + } +} + // Convert any input to a buffer function input() { const inp = ifmt.selectedOptions[0].label; @@ -146,14 +168,24 @@ function convert() { function changeEncodeOption({target}) { const opt = target.id.replace(/Encode$/, ''); encodeOpts[opt] = target.checked; + let modified = false; + if (opt === 'dcbor') { + Object.assign(encodeOpts, target.checked ? + dcborEncodeOptions : + notDcborEncodeOptions); + modified = true; + } else if (opt === 'cde') { + Object.assign(encodeOpts, target.checked ? + cdeEncodeOptions : + notCdeEncodeOptions); + modified = true; + } + if (modified) { + showEncodeOpts(); + } convert(); } -for (const inp of document.querySelectorAll('#encodingOpts input')) { - inp.onchange = changeEncodeOption; - inp.checked = encodeOpts[inp.id.replace(/Encode$/, '')]; -} - const forceEndian = document.querySelector('#forceEndian'); forceEndian.onchange = () => { encodeOpts.forceEndian = { @@ -165,7 +197,6 @@ forceEndian.onchange = () => { }; forceEndian.value = 'null'; -const sortKeysEncode = document.querySelector('#sortKeysEncode'); sortKeysEncode.onchange = () => { encodeOpts.sortKeys = { null: null, @@ -191,24 +222,20 @@ function changeDecodeOption({target}) { notCdeDecodeOptions); } if (modified) { - for (const inp of document.querySelectorAll('#decodingOpts input')) { - inp.checked = decodeOpts[inp.id]; - } - if (decodeOpts.sortKeys === sortCoreDeterministic) { - sortKeysDecode.value = 'coreDeterministic'; - } else if (decodeOpts.sortKeys === sortLengthFirstDeterministic) { - sortKeysDecode.value = 'lengthFirstDeterministic'; - } else { - sortKeysDecode.value = 'null'; - } + showDecodeOpts(); } convert(); } -for (const inp of document.querySelectorAll('#decodingOpts input')) { +showEncodeOpts(); +for (const inp of document.querySelectorAll('#encodeOpts input')) { + inp.onchange = changeEncodeOption; +} + +showDecodeOpts(); +for (const inp of document.querySelectorAll('#decodeOpts input')) { inp.onchange = changeDecodeOption; - inp.checked = decodeOpts[inp.id]; } sortKeysDecode.onchange = () => {