From cf9083377595a34a82e9b745e6c4f1c105d1848a Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 01:13:15 -0700 Subject: [PATCH 01/11] syntax: add datetime validator (and interop tests) --- .../syntax/datetime_parse_invalid.txt | 7 ++ .../syntax/datetime_syntax_invalid.txt | 64 +++++++++++++++++ .../syntax/datetime_syntax_valid.txt | 34 +++++++++ packages/syntax/src/datetime.ts | 56 +++++++++++++++ packages/syntax/src/index.ts | 1 + packages/syntax/tests/datetime.test.ts | 72 +++++++++++++++++++ .../interop-files/datetime_parse_invalid.txt | 1 + .../interop-files/datetime_syntax_invalid.txt | 1 + .../interop-files/datetime_syntax_valid.txt | 1 + 9 files changed, 237 insertions(+) create mode 100644 interop-test-files/syntax/datetime_parse_invalid.txt create mode 100644 interop-test-files/syntax/datetime_syntax_invalid.txt create mode 100644 interop-test-files/syntax/datetime_syntax_valid.txt create mode 100644 packages/syntax/src/datetime.ts create mode 100644 packages/syntax/tests/datetime.test.ts create mode 120000 packages/syntax/tests/interop-files/datetime_parse_invalid.txt create mode 120000 packages/syntax/tests/interop-files/datetime_syntax_invalid.txt create mode 120000 packages/syntax/tests/interop-files/datetime_syntax_valid.txt diff --git a/interop-test-files/syntax/datetime_parse_invalid.txt b/interop-test-files/syntax/datetime_parse_invalid.txt new file mode 100644 index 00000000000..3672453a29f --- /dev/null +++ b/interop-test-files/syntax/datetime_parse_invalid.txt @@ -0,0 +1,7 @@ +# superficial syntax parses ok, but are not valid datetimes for semantic reasons (eg, "month zero") +1985-00-12T23:20:50.123Z +1985-04-00T23:20:50.123Z +1985-13-12T23:20:50.123Z +1985-04-12T25:20:50.123Z +1985-04-12T23:99:50.123Z +1985-04-12T23:20:61.123Z diff --git a/interop-test-files/syntax/datetime_syntax_invalid.txt b/interop-test-files/syntax/datetime_syntax_invalid.txt new file mode 100644 index 00000000000..a686f7174c0 --- /dev/null +++ b/interop-test-files/syntax/datetime_syntax_invalid.txt @@ -0,0 +1,64 @@ + +# subtle changes to: 1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.123z +01985-04-12T23:20:50.123Z +985-04-12T23:20:50.123Z +1985-04-12T23:20:50.Z +1985-04-32T23;20:50.123Z +1985-04-32T23;20:50.123Z + +# en-dash and em-dash +1985—04-32T23;20:50.123Z +1985–04-32T23;20:50.123Z + +# whitespace + 1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.123Z +1985-04-12T 23:20:50.123Z + +# not enough zero padding +1985-4-12T23:20:50.123Z +1985-04-2T23:20:50.123Z +1985-04-12T3:20:50.123Z +1985-04-12T23:0:50.123Z +1985-04-12T23:20:5.123Z + +# too much zero padding +01985-04-12T23:20:50.123Z +1985-004-12T23:20:50.123Z +1985-04-012T23:20:50.123Z +1985-04-12T023:20:50.123Z +1985-04-12T23:020:50.123Z +1985-04-12T23:20:050.123Z + +# strict capitalization (ISO-8601) +1985-04-12t23:20:50.123Z +1985-04-12T23:20:50.123z + +# RFC-3339, but not ISO-8601 +1985-04-12T23:20:50.123-00:00 +1985-04-12_23:20:50.123Z +1985-04-12 23:20:50.123Z + +# ISO-8601, but weird +1985-04-274T23:20:50.123Z + +# timezone is required +1985-04-12T23:20:50.123 +1985-04-12T23:20:50 + +1985-04-12 +1985-04-12T23:20Z +1985-04-12T23:20:5Z +1985-04-12T23:20:50.123 ++001985-04-12T23:20:50.123Z +23:20:50.123Z + +1985-04-12T23:20:50.123+00 +1985-04-12T23:20:50.123+00:0 +1985-04-12T23:20:50.123+0:00 +1985-04-12T23:20:50.123 +1985-04-12T23:20:50.123+0000 +1985-04-12T23:20:50.123+00 +1985-04-12T23:20:50.123+ +1985-04-12T23:20:50.123- diff --git a/interop-test-files/syntax/datetime_syntax_valid.txt b/interop-test-files/syntax/datetime_syntax_valid.txt new file mode 100644 index 00000000000..79ed5cabb41 --- /dev/null +++ b/interop-test-files/syntax/datetime_syntax_valid.txt @@ -0,0 +1,34 @@ +# "preferred" +1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.000Z +2000-01-01T00:00:00.000Z +1985-04-12T23:20:50.123456Z +1985-04-12T23:20:50.120Z +1985-04-12T23:20:50.120000Z + +# "supported" +1985-04-12T23:20:50.1235678912345Z +1985-04-12T23:20:50.100Z +1985-04-12T23:20:50Z +1985-04-12T23:20:50.0Z +1985-04-12T23:20:50.123+00:00 +1985-04-12T23:20:50.123-07:00 +1985-04-12T23:20:50.123+07:00 +1985-04-12T23:20:50.123+01:45 +0985-04-12T23:20:50.123-07:00 +1985-04-12T23:20:50.123-07:00 +0123-01-01T00:00:00.000Z + +# various precisions, up through at least 12 digits +1985-04-12T23:20:50.1Z +1985-04-12T23:20:50.12Z +1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.1234Z +1985-04-12T23:20:50.12345Z +1985-04-12T23:20:50.123456Z +1985-04-12T23:20:50.1234567Z +1985-04-12T23:20:50.12345678Z +1985-04-12T23:20:50.123456789Z +1985-04-12T23:20:50.1234567890Z +1985-04-12T23:20:50.12345678901Z +1985-04-12T23:20:50.123456789012Z diff --git a/packages/syntax/src/datetime.ts b/packages/syntax/src/datetime.ts new file mode 100644 index 00000000000..bba9966d38d --- /dev/null +++ b/packages/syntax/src/datetime.ts @@ -0,0 +1,56 @@ +export const ensureValidDatetime = (dtStr: string): void => { + /* + if (!isValidISODateString(dtStr)) { + throw new InvalidDatetimeError('datetime did not parse as ISO 8601') + } + */ + const date = new Date(dtStr) + if (isNaN(date.getTime())) { + throw new InvalidDatetimeError('datetime did not parse as ISO 8601') + } + if ( + !/^[0-9]{4}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9](.[0-9]{1,20})?(Z|([+-][0-2][0-9]:[0-5][0-9]))$/.test( + dtStr, + ) + ) { + throw new InvalidDatetimeError("datetime didn't validate via regex") + } + if (dtStr.length > 64) { + throw new InvalidDatetimeError('datetime is too long (64 chars max)') + } + if (dtStr.endsWith('-00:00')) { + throw new InvalidDatetimeError( + 'datetime can not use "-00:00" for UTC timezone', + ) + } +} + +// Normalize date strings to simplified ISO so that the lexical sort preserves temporal sort. +// Rather than failing on an invalid date format, returns valid unix epoch. +export const normalizeDatetime = (dtStr: string): string => { + const date = new Date(dtStr) + if (isNaN(date.getTime())) { + return new Date(0).toISOString() + } + const iso = date.toISOString() + if (!isValidDatetime(iso)) { + // Occurs in rare cases, e.g. where resulting UTC year is negative. These also don't preserve lexical sort. + return new Date(0).toISOString() + } + return iso // YYYY-MM-DDTHH:mm:ss.sssZ +} + +export const isValidDatetime = (dtStr: string): boolean => { + try { + ensureValidDatetime(dtStr) + } catch (err) { + if (err instanceof InvalidDatetimeError) { + return false + } + throw err + } + + return true +} + +export class InvalidDatetimeError extends Error {} diff --git a/packages/syntax/src/index.ts b/packages/syntax/src/index.ts index 0b056b995ae..2a108e53795 100644 --- a/packages/syntax/src/index.ts +++ b/packages/syntax/src/index.ts @@ -2,3 +2,4 @@ export * from './handle' export * from './did' export * from './nsid' export * from './aturi' +export * from './datetime' diff --git a/packages/syntax/tests/datetime.test.ts b/packages/syntax/tests/datetime.test.ts new file mode 100644 index 00000000000..6c4cc989c94 --- /dev/null +++ b/packages/syntax/tests/datetime.test.ts @@ -0,0 +1,72 @@ +import { + isValidDatetime, + ensureValidDatetime, + normalizeDatetime, + InvalidDatetimeError, +} from '../src' +import * as readline from 'readline' +import * as fs from 'fs' + +describe('datetime validation', () => { + const expectValid = (h: string) => { + ensureValidDatetime(h) + } + const expectInvalid = (h: string) => { + expect(() => ensureValidDatetime(h)).toThrow(InvalidDatetimeError) + } + + it('conforms to interop valid datetimes', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/datetime_syntax_valid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + if (!isValidDatetime(line)) { + console.log(line) + } + expectValid(line) + }) + }) + + it('conforms to interop invalid datetimes', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/datetime_syntax_invalid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + expectInvalid(line) + }) + }) + + it('conforms to interop invalid parse (semantics) datetimes', () => { + const lineReader = readline.createInterface({ + input: fs.createReadStream( + `${__dirname}/interop-files/datetime_parse_invalid.txt`, + ), + terminal: false, + }) + lineReader.on('line', (line) => { + if (line.startsWith('#') || line.length == 0) { + return + } + expectInvalid(line) + }) + }) +}) + +describe('normalization', () => { + it('normalizes datetimes', () => { + const normalized = normalizeDatetime('1985-04-12T23:20:50.123') + expect(normalized).toMatch(/^1985-04-13T[0-9:.]+Z$/) + }) +}) diff --git a/packages/syntax/tests/interop-files/datetime_parse_invalid.txt b/packages/syntax/tests/interop-files/datetime_parse_invalid.txt new file mode 120000 index 00000000000..ef8782df266 --- /dev/null +++ b/packages/syntax/tests/interop-files/datetime_parse_invalid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/datetime_parse_invalid.txt \ No newline at end of file diff --git a/packages/syntax/tests/interop-files/datetime_syntax_invalid.txt b/packages/syntax/tests/interop-files/datetime_syntax_invalid.txt new file mode 120000 index 00000000000..948c647c88c --- /dev/null +++ b/packages/syntax/tests/interop-files/datetime_syntax_invalid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/datetime_syntax_invalid.txt \ No newline at end of file diff --git a/packages/syntax/tests/interop-files/datetime_syntax_valid.txt b/packages/syntax/tests/interop-files/datetime_syntax_valid.txt new file mode 120000 index 00000000000..9c74ded2ede --- /dev/null +++ b/packages/syntax/tests/interop-files/datetime_syntax_valid.txt @@ -0,0 +1 @@ +../../../../interop-test-files/syntax/datetime_syntax_valid.txt \ No newline at end of file From 443ef1faa13dac5c52b4147dd7bb73fb082c07cf Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 01:17:40 -0700 Subject: [PATCH 02/11] syntax: improve datetime normalization --- packages/syntax/src/datetime.ts | 13 ++++++------- packages/syntax/tests/datetime.test.ts | 12 ++++++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/syntax/src/datetime.ts b/packages/syntax/src/datetime.ts index bba9966d38d..5a11716a647 100644 --- a/packages/syntax/src/datetime.ts +++ b/packages/syntax/src/datetime.ts @@ -26,17 +26,16 @@ export const ensureValidDatetime = (dtStr: string): void => { } // Normalize date strings to simplified ISO so that the lexical sort preserves temporal sort. -// Rather than failing on an invalid date format, returns valid unix epoch. -export const normalizeDatetime = (dtStr: string): string => { +// Does *not* accept all possible strings, but will (arbitrarily) normalize no-timezone to local timezone. +export const normalizeAndEnsureValidDatetime = (dtStr: string): string => { const date = new Date(dtStr) if (isNaN(date.getTime())) { - return new Date(0).toISOString() + throw new InvalidDatetimeError( + 'datetime did not parse as any timestamp format', + ) } const iso = date.toISOString() - if (!isValidDatetime(iso)) { - // Occurs in rare cases, e.g. where resulting UTC year is negative. These also don't preserve lexical sort. - return new Date(0).toISOString() - } + ensureValidDatetime(iso) return iso // YYYY-MM-DDTHH:mm:ss.sssZ } diff --git a/packages/syntax/tests/datetime.test.ts b/packages/syntax/tests/datetime.test.ts index 6c4cc989c94..343fadec91f 100644 --- a/packages/syntax/tests/datetime.test.ts +++ b/packages/syntax/tests/datetime.test.ts @@ -1,7 +1,7 @@ import { isValidDatetime, ensureValidDatetime, - normalizeDatetime, + normalizeAndEnsureValidDatetime, InvalidDatetimeError, } from '../src' import * as readline from 'readline' @@ -66,7 +66,15 @@ describe('datetime validation', () => { describe('normalization', () => { it('normalizes datetimes', () => { - const normalized = normalizeDatetime('1985-04-12T23:20:50.123') + const normalized = normalizeAndEnsureValidDatetime( + '1985-04-12T23:20:50.123', + ) expect(normalized).toMatch(/^1985-04-13T[0-9:.]+Z$/) }) + + it('throws on invalid normalized datetimes', () => { + expect(() => normalizeAndEnsureValidDatetime('blah')).toThrow( + InvalidDatetimeError, + ) + }) }) From 1d6c337adb9247b4f3ab57e776c1981fa47e77fb Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 01:18:28 -0700 Subject: [PATCH 03/11] lexicon: stronger datetime validation (from syntax package) --- packages/lexicon/src/validators/formats.ts | 8 +++----- packages/lexicon/tests/general.test.ts | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/lexicon/src/validators/formats.ts b/packages/lexicon/src/validators/formats.ts index 63fc941628e..947d4f95d3e 100644 --- a/packages/lexicon/src/validators/formats.ts +++ b/packages/lexicon/src/validators/formats.ts @@ -1,4 +1,3 @@ -import { isValidISODateString } from 'iso-datestring-validator' import { CID } from 'multiformats/cid' import { ValidationResult, ValidationError } from '../types' import { @@ -6,19 +5,18 @@ import { ensureValidHandle, ensureValidNsid, ensureValidAtUri, + ensureValidDatetime, } from '@atproto/syntax' import { validateLanguage } from '@atproto/common-web' export function datetime(path: string, value: string): ValidationResult { try { - if (!isValidISODateString(value)) { - throw new Error() - } + ensureValidDatetime(value) } catch { return { success: false, error: new ValidationError( - `${path} must be an iso8601 formatted datetime`, + `${path} must be an valid atproto datetime (both RFC-3339 and ISO-8601)`, ), } } diff --git a/packages/lexicon/tests/general.test.ts b/packages/lexicon/tests/general.test.ts index 5217ad49c52..e615bbb2f23 100644 --- a/packages/lexicon/tests/general.test.ts +++ b/packages/lexicon/tests/general.test.ts @@ -659,7 +659,7 @@ describe('Record validation', () => { $type: 'com.example.datetime', datetime: 'bad date', }), - ).toThrow('Record/datetime must be an iso8601 formatted datetime') + ).toThrow('Record/datetime must be an valid atproto datetime (both RFC-3339 and ISO-8601)') }) it('Applies uri formatting constraint', () => { From bb9a0ed2e7fbfd1d96073954f041fcaa2557d678 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 01:27:43 -0700 Subject: [PATCH 04/11] syntax: make datetime syntax norm test more flexible --- packages/syntax/tests/datetime.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syntax/tests/datetime.test.ts b/packages/syntax/tests/datetime.test.ts index 343fadec91f..4cc9f8c0974 100644 --- a/packages/syntax/tests/datetime.test.ts +++ b/packages/syntax/tests/datetime.test.ts @@ -69,7 +69,7 @@ describe('normalization', () => { const normalized = normalizeAndEnsureValidDatetime( '1985-04-12T23:20:50.123', ) - expect(normalized).toMatch(/^1985-04-13T[0-9:.]+Z$/) + expect(normalized).toMatch(/^1985-04-1[234]T[0-9:.]+Z$/) }) it('throws on invalid normalized datetimes', () => { From 756aa576a8e0253be5ce3dc906c809fe3ecbda45 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 01:37:47 -0700 Subject: [PATCH 05/11] make fmt --- packages/lexicon/tests/general.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/lexicon/tests/general.test.ts b/packages/lexicon/tests/general.test.ts index e615bbb2f23..529b60ca9e0 100644 --- a/packages/lexicon/tests/general.test.ts +++ b/packages/lexicon/tests/general.test.ts @@ -659,7 +659,9 @@ describe('Record validation', () => { $type: 'com.example.datetime', datetime: 'bad date', }), - ).toThrow('Record/datetime must be an valid atproto datetime (both RFC-3339 and ISO-8601)') + ).toThrow( + 'Record/datetime must be an valid atproto datetime (both RFC-3339 and ISO-8601)', + ) }) it('Applies uri formatting constraint', () => { From a394556eea27e56baa1eade48c28407a56e61e6a Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Wed, 11 Oct 2023 19:01:38 -0700 Subject: [PATCH 06/11] datetime: docs, normalize and always variant --- packages/syntax/src/datetime.ts | 71 ++++++++++++++++++++------ packages/syntax/tests/datetime.test.ts | 36 +++++++++++-- 2 files changed, 86 insertions(+), 21 deletions(-) diff --git a/packages/syntax/src/datetime.ts b/packages/syntax/src/datetime.ts index 5a11716a647..c8128c54b94 100644 --- a/packages/syntax/src/datetime.ts +++ b/packages/syntax/src/datetime.ts @@ -1,13 +1,13 @@ +/* Validates datetime string against atproto Lexicon 'datetime' format. + * Syntax is described at: https://atproto.com/specs/lexicon#datetime + */ export const ensureValidDatetime = (dtStr: string): void => { - /* - if (!isValidISODateString(dtStr)) { - throw new InvalidDatetimeError('datetime did not parse as ISO 8601') - } - */ const date = new Date(dtStr) + // must parse as ISO 8601; this also verifies semantics like month is not 13 or 00 if (isNaN(date.getTime())) { throw new InvalidDatetimeError('datetime did not parse as ISO 8601') } + // regex and other checks for RFC-3339 if ( !/^[0-9]{4}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9](.[0-9]{1,20})?(Z|([+-][0-2][0-9]:[0-5][0-9]))$/.test( dtStr, @@ -25,31 +25,70 @@ export const ensureValidDatetime = (dtStr: string): void => { } } -// Normalize date strings to simplified ISO so that the lexical sort preserves temporal sort. -// Does *not* accept all possible strings, but will (arbitrarily) normalize no-timezone to local timezone. -export const normalizeAndEnsureValidDatetime = (dtStr: string): string => { +/* Same logic as ensureValidDatetime(), but returns a boolean instead of throwing an exception. + */ +export const isValidDatetime = (dtStr: string): boolean => { + try { + ensureValidDatetime(dtStr) + } catch (err) { + if (err instanceof InvalidDatetimeError) { + return false + } + throw err + } + + return true +} + +/* Takes a flexible datetime sting and normalizes representation. + * + * This function will work with any valid atproto datetime (eg, anything which isValidDatetime() is true for). It *additinally* is more flexible about accepting datetimes that don't comply to RFC 3339, or are missing timezone information, and normalizing them to a valid datetime. + * + * One use-case is a consistent, sortable string. Another is to work with older invalid createdAt datetimes. + * + * Successful output will be a valid atproto datetime with millisecond precision (3 sub-second digits) and UTC timezone with trailing 'Z' syntax. Throws `InvalidDatetimeError` if the input string could not be parsed as a datetime, even with permissive parsing. + * + * Expected output format: YYYY-MM-DDTHH:mm:ss.sssZ + */ +export const normalizeDatetime = (dtStr: string): string => { + if (isValidDatetime(dtStr)) { + const date = new Date(dtStr) + return date.toISOString() + } + + // check if this permissive datetime is missing a timezone + if (!/.*(([+-]\d\d:?\d\d)|[a-zA-Z])$/.test(dtStr)) { + const date = new Date(dtStr + 'Z') + if (!isNaN(date.getTime())) { + return date.toISOString() + } + } + + // finally try parsing as simple datetime const date = new Date(dtStr) if (isNaN(date.getTime())) { throw new InvalidDatetimeError( 'datetime did not parse as any timestamp format', ) } - const iso = date.toISOString() - ensureValidDatetime(iso) - return iso // YYYY-MM-DDTHH:mm:ss.sssZ + return date.toISOString() } -export const isValidDatetime = (dtStr: string): boolean => { +/* Variant of normalizeDatetime() which always returns a valid datetime strings. + * + * If a InvalidDatetimeError is encountered, returns the UNIX epoch time as a UTC datetime (1970-01-01T00:00:00.000Z). + */ +export const normalizeDatetimeAlways = (dtStr: string): string => { try { - ensureValidDatetime(dtStr) + return normalizeDatetime(dtStr) } catch (err) { if (err instanceof InvalidDatetimeError) { - return false + return new Date(0).toISOString() } throw err } - - return true } +/* Indicates a datetime string did not pass full atproto Lexicon datetime string format checks. + */ export class InvalidDatetimeError extends Error {} diff --git a/packages/syntax/tests/datetime.test.ts b/packages/syntax/tests/datetime.test.ts index 4cc9f8c0974..547a276c6a9 100644 --- a/packages/syntax/tests/datetime.test.ts +++ b/packages/syntax/tests/datetime.test.ts @@ -1,7 +1,8 @@ import { isValidDatetime, ensureValidDatetime, - normalizeAndEnsureValidDatetime, + normalizeDatetime, + normalizeDatetimeAlways, InvalidDatetimeError, } from '../src' import * as readline from 'readline' @@ -10,6 +11,8 @@ import * as fs from 'fs' describe('datetime validation', () => { const expectValid = (h: string) => { ensureValidDatetime(h) + normalizeDatetime(h) + normalizeDatetimeAlways(h) } const expectInvalid = (h: string) => { expect(() => ensureValidDatetime(h)).toThrow(InvalidDatetimeError) @@ -66,15 +69,38 @@ describe('datetime validation', () => { describe('normalization', () => { it('normalizes datetimes', () => { - const normalized = normalizeAndEnsureValidDatetime( - '1985-04-12T23:20:50.123', + expect(normalizeDatetime('1234-04-12T23:20:50Z')).toEqual( + '1234-04-12T23:20:50.000Z', + ) + expect(normalizeDatetime('1985-04-12T23:20:50Z')).toEqual( + '1985-04-12T23:20:50.000Z', + ) + expect(normalizeDatetime('1985-04-12T23:20:50.123')).toEqual( + '1985-04-12T23:20:50.123Z', + ) + expect(normalizeDatetime('1985-04-12 23:20:50.123')).toEqual( + '1985-04-12T23:20:50.123Z', + ) + expect(normalizeDatetime('1985-04-12T10:20:50.1+01:00')).toEqual( + '1985-04-12T09:20:50.100Z', + ) + expect(normalizeDatetime('Fri, 02 Jan 1999 12:34:56 GMT')).toEqual( + '1999-01-02T12:34:56.000Z', ) - expect(normalized).toMatch(/^1985-04-1[234]T[0-9:.]+Z$/) }) it('throws on invalid normalized datetimes', () => { - expect(() => normalizeAndEnsureValidDatetime('blah')).toThrow( + expect(() => normalizeDatetime('')).toThrow(InvalidDatetimeError) + expect(() => normalizeDatetime('blah')).toThrow(InvalidDatetimeError) + expect(() => normalizeDatetime('1999-19-39T23:20:50.123Z')).toThrow( InvalidDatetimeError, ) }) + + it('normalizes datetimes always', () => { + expect(normalizeDatetimeAlways('1985-04-12T23:20:50Z')).toEqual( + '1985-04-12T23:20:50.000Z', + ) + expect(normalizeDatetimeAlways('blah')).toEqual('1970-01-01T00:00:00.000Z') + }) }) From d5e27d640fab23d595e4e6cc842544c9b95c855d Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Wed, 11 Oct 2023 19:18:54 -0700 Subject: [PATCH 07/11] bsky replace toSimplifiedISOSafe with normalizeDatetimeAlways --- packages/bsky/src/services/indexing/plugins/block.ts | 5 ++--- .../bsky/src/services/indexing/plugins/feed-generator.ts | 5 ++--- packages/bsky/src/services/indexing/plugins/follow.ts | 5 ++--- packages/bsky/src/services/indexing/plugins/like.ts | 5 ++--- packages/bsky/src/services/indexing/plugins/list-block.ts | 5 ++--- packages/bsky/src/services/indexing/plugins/list-item.ts | 5 ++--- packages/bsky/src/services/indexing/plugins/list.ts | 5 ++--- packages/bsky/src/services/indexing/plugins/post.ts | 5 ++--- packages/bsky/src/services/indexing/plugins/repost.ts | 5 ++--- packages/bsky/src/services/indexing/plugins/thread-gate.ts | 5 ++--- packages/bsky/src/services/label/index.ts | 5 ++--- 11 files changed, 22 insertions(+), 33 deletions(-) diff --git a/packages/bsky/src/services/indexing/plugins/block.ts b/packages/bsky/src/services/indexing/plugins/block.ts index bf8ae9e5029..88e62b6f5ac 100644 --- a/packages/bsky/src/services/indexing/plugins/block.ts +++ b/packages/bsky/src/services/indexing/plugins/block.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as Block from '../../../lexicon/types/app/bsky/graph/block' import * as lex from '../../../lexicon/lexicons' @@ -27,7 +26,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, subjectDid: obj.subject, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/feed-generator.ts b/packages/bsky/src/services/indexing/plugins/feed-generator.ts index e4ae5eb4f5a..be5435966f1 100644 --- a/packages/bsky/src/services/indexing/plugins/feed-generator.ts +++ b/packages/bsky/src/services/indexing/plugins/feed-generator.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as FeedGenerator from '../../../lexicon/types/app/bsky/feed/generator' import * as lex from '../../../lexicon/lexicons' @@ -33,7 +32,7 @@ const insertFn = async ( ? JSON.stringify(obj.descriptionFacets) : undefined, avatarCid: obj.avatar?.ref.toString(), - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/follow.ts b/packages/bsky/src/services/indexing/plugins/follow.ts index e9a344db2fd..8655c7eba71 100644 --- a/packages/bsky/src/services/indexing/plugins/follow.ts +++ b/packages/bsky/src/services/indexing/plugins/follow.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as Follow from '../../../lexicon/types/app/bsky/graph/follow' import * as lex from '../../../lexicon/lexicons' @@ -28,7 +27,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, subjectDid: obj.subject, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/like.ts b/packages/bsky/src/services/indexing/plugins/like.ts index 01e0fa5c4fd..703800f67c8 100644 --- a/packages/bsky/src/services/indexing/plugins/like.ts +++ b/packages/bsky/src/services/indexing/plugins/like.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as Like from '../../../lexicon/types/app/bsky/feed/like' import * as lex from '../../../lexicon/lexicons' @@ -29,7 +28,7 @@ const insertFn = async ( creator: uri.host, subject: obj.subject.uri, subjectCid: obj.subject.cid, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/list-block.ts b/packages/bsky/src/services/indexing/plugins/list-block.ts index 33dc7cfc51a..3040f1aa3f9 100644 --- a/packages/bsky/src/services/indexing/plugins/list-block.ts +++ b/packages/bsky/src/services/indexing/plugins/list-block.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as ListBlock from '../../../lexicon/types/app/bsky/graph/listblock' import * as lex from '../../../lexicon/lexicons' @@ -27,7 +26,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, subjectUri: obj.subject, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/list-item.ts b/packages/bsky/src/services/indexing/plugins/list-item.ts index 2ab125062a7..9e08145b23e 100644 --- a/packages/bsky/src/services/indexing/plugins/list-item.ts +++ b/packages/bsky/src/services/indexing/plugins/list-item.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as ListItem from '../../../lexicon/types/app/bsky/graph/listitem' import * as lex from '../../../lexicon/lexicons' @@ -35,7 +34,7 @@ const insertFn = async ( creator: uri.host, subjectDid: obj.subject, listUri: obj.list, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/list.ts b/packages/bsky/src/services/indexing/plugins/list.ts index 293c457c4fb..0d078572501 100644 --- a/packages/bsky/src/services/indexing/plugins/list.ts +++ b/packages/bsky/src/services/indexing/plugins/list.ts @@ -1,6 +1,5 @@ import { Selectable } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { CID } from 'multiformats/cid' import * as List from '../../../lexicon/types/app/bsky/graph/list' import * as lex from '../../../lexicon/lexicons' @@ -33,7 +32,7 @@ const insertFn = async ( ? JSON.stringify(obj.descriptionFacets) : undefined, avatarCid: obj.avatar?.ref.toString(), - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/indexing/plugins/post.ts b/packages/bsky/src/services/indexing/plugins/post.ts index 7173c04a991..8a9696f800f 100644 --- a/packages/bsky/src/services/indexing/plugins/post.ts +++ b/packages/bsky/src/services/indexing/plugins/post.ts @@ -1,7 +1,6 @@ import { Insertable, Selectable, sql } from 'kysely' import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { jsonStringToLex } from '@atproto/lexicon' import { Record as PostRecord, @@ -68,7 +67,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, text: obj.text, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), replyRoot: obj.reply?.root?.uri || null, replyRootCid: obj.reply?.root?.cid || null, replyParent: obj.reply?.parent?.uri || null, diff --git a/packages/bsky/src/services/indexing/plugins/repost.ts b/packages/bsky/src/services/indexing/plugins/repost.ts index 9c46b9b3376..ea8d517dc52 100644 --- a/packages/bsky/src/services/indexing/plugins/repost.ts +++ b/packages/bsky/src/services/indexing/plugins/repost.ts @@ -1,7 +1,6 @@ import { Selectable } from 'kysely' import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import * as Repost from '../../../lexicon/types/app/bsky/feed/repost' import * as lex from '../../../lexicon/lexicons' import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' @@ -27,7 +26,7 @@ const insertFn = async ( creator: uri.host, subject: obj.subject.uri, subjectCid: obj.subject.cid, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, } const [inserted] = await Promise.all([ diff --git a/packages/bsky/src/services/indexing/plugins/thread-gate.ts b/packages/bsky/src/services/indexing/plugins/thread-gate.ts index 37f3ddb062e..9a58547f2da 100644 --- a/packages/bsky/src/services/indexing/plugins/thread-gate.ts +++ b/packages/bsky/src/services/indexing/plugins/thread-gate.ts @@ -1,6 +1,5 @@ -import { AtUri } from '@atproto/syntax' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' -import { toSimplifiedISOSafe } from '@atproto/common' import { CID } from 'multiformats/cid' import * as Threadgate from '../../../lexicon/types/app/bsky/feed/threadgate' import * as lex from '../../../lexicon/lexicons' @@ -33,7 +32,7 @@ const insertFn = async ( cid: cid.toString(), creator: uri.host, postUri: obj.post, - createdAt: toSimplifiedISOSafe(obj.createdAt), + createdAt: normalizeDatetimeAlways(obj.createdAt), indexedAt: timestamp, }) .onConflict((oc) => oc.doNothing()) diff --git a/packages/bsky/src/services/label/index.ts b/packages/bsky/src/services/label/index.ts index 7d351b95011..f44b0439ddf 100644 --- a/packages/bsky/src/services/label/index.ts +++ b/packages/bsky/src/services/label/index.ts @@ -1,6 +1,5 @@ import { sql } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { toSimplifiedISOSafe } from '@atproto/common' +import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' import { Database } from '../../db' import { Label, isSelfLabels } from '../../lexicon/types/com/atproto/label/defs' import { ids } from '../../lexicon/lexicons' @@ -166,7 +165,7 @@ export function getSelfLabels(details: { const src = new AtUri(uri).host // record creator const cts = typeof record.createdAt === 'string' - ? toSimplifiedISOSafe(record.createdAt) + ? normalizeDatetimeAlways(record.createdAt) : new Date(0).toISOString() return record.labels.values.map(({ val }) => { return { src, uri, cid, val, cts, neg: false } From 922057338bd053c90cc31fe53f6c16bd3f4e4b36 Mon Sep 17 00:00:00 2001 From: dholms Date: Tue, 21 Nov 2023 16:02:56 -0600 Subject: [PATCH 08/11] more rigorous datetime parsing on record creation --- packages/lexicon/src/validators/formats.ts | 6 ++++-- packages/pds/src/repo/prepare.ts | 20 +++++++++++++++++++- packages/pds/tests/crud.test.ts | 20 +++++++++++++++++++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/packages/lexicon/src/validators/formats.ts b/packages/lexicon/src/validators/formats.ts index 947d4f95d3e..b786c68281f 100644 --- a/packages/lexicon/src/validators/formats.ts +++ b/packages/lexicon/src/validators/formats.ts @@ -1,3 +1,4 @@ +import { isValidISODateString } from 'iso-datestring-validator' import { CID } from 'multiformats/cid' import { ValidationResult, ValidationError } from '../types' import { @@ -5,13 +6,14 @@ import { ensureValidHandle, ensureValidNsid, ensureValidAtUri, - ensureValidDatetime, } from '@atproto/syntax' import { validateLanguage } from '@atproto/common-web' export function datetime(path: string, value: string): ValidationResult { try { - ensureValidDatetime(value) + if (!isValidISODateString(value)) { + throw new Error() + } } catch { return { success: false, diff --git a/packages/pds/src/repo/prepare.ts b/packages/pds/src/repo/prepare.ts index 88201455300..06b1da95999 100644 --- a/packages/pds/src/repo/prepare.ts +++ b/packages/pds/src/repo/prepare.ts @@ -1,9 +1,10 @@ import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' +import { AtUri, ensureValidDatetime } from '@atproto/syntax' import { MINUTE, TID, dataToCborBlock } from '@atproto/common' import { LexiconDefNotFoundError, RepoRecord, + ValidationError, lexToIpld, } from '@atproto/lexicon' import { @@ -115,6 +116,7 @@ export const assertValidRecord = (record: Record) => { } try { lex.lexicons.assertValidRecord(record.$type, record) + assertValidCreatedAt(record) } catch (e) { if (e instanceof LexiconDefNotFoundError) { throw new InvalidRecordError(e.message) @@ -127,6 +129,22 @@ export const assertValidRecord = (record: Record) => { } } +// additional more rigorous check on datetimes +// this check will eventually be in the lex sdk, but this will stop the bleed until then +export const assertValidCreatedAt = (record: Record) => { + const createdAt = record['createdAt'] + if (typeof createdAt !== 'string') { + return + } + try { + ensureValidDatetime(createdAt) + } catch { + throw new ValidationError( + 'createdAt must be an valid atproto datetime (both RFC-3339 and ISO-8601)', + ) + } +} + export const setCollectionName = ( collection: string, record: RepoRecord, diff --git a/packages/pds/tests/crud.test.ts b/packages/pds/tests/crud.test.ts index 65544677ff2..f8f855ce049 100644 --- a/packages/pds/tests/crud.test.ts +++ b/packages/pds/tests/crud.test.ts @@ -13,7 +13,7 @@ import { defaultFetchHandler } from '@atproto/xrpc' import * as Post from '../src/lexicon/types/app/bsky/feed/post' import { paginateAll } from './_util' import AppContext from '../src/context' -import { ids } from '../src/lexicon/lexicons' +import { ids, lexicons } from '../src/lexicon/lexicons' const alice = { email: 'alice@test.com', @@ -579,6 +579,24 @@ describe('crud operations', () => { ) }) + it('validates datetimes more rigorously than lex sdk', async () => { + const postRecord = { + $type: 'app.bsky.feed.post', + text: 'test', + createdAt: '1985-04-12T23:20:50.123', + } + lexicons.assertValidRecord('app.bsky.feed.post', postRecord) + await expect( + aliceAgent.api.com.atproto.repo.createRecord({ + repo: alice.did, + collection: 'app.bsky.feed.post', + record: postRecord, + }), + ).rejects.toThrow( + 'Invalid app.bsky.feed.post record: createdAt must be an valid atproto datetime (both RFC-3339 and ISO-8601)', + ) + }) + describe('compare-and-swap', () => { let recordCount = 0 // Ensures unique cids const postRecord = () => ({ From eac065a9d69c4fc91b107bbfb9a338695b3758b5 Mon Sep 17 00:00:00 2001 From: dholms Date: Tue, 21 Nov 2023 16:28:57 -0600 Subject: [PATCH 09/11] handle negative dates --- interop-test-files/syntax/datetime_syntax_invalid.txt | 3 +++ packages/syntax/src/datetime.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/interop-test-files/syntax/datetime_syntax_invalid.txt b/interop-test-files/syntax/datetime_syntax_invalid.txt index a686f7174c0..d1151e21b55 100644 --- a/interop-test-files/syntax/datetime_syntax_invalid.txt +++ b/interop-test-files/syntax/datetime_syntax_invalid.txt @@ -62,3 +62,6 @@ 1985-04-12T23:20:50.123+00 1985-04-12T23:20:50.123+ 1985-04-12T23:20:50.123- + +# ISO-8601, but normalizes to a negative time +0000-01-01T00:00:00+01:00 \ No newline at end of file diff --git a/packages/syntax/src/datetime.ts b/packages/syntax/src/datetime.ts index c8128c54b94..682b4ed1fc5 100644 --- a/packages/syntax/src/datetime.ts +++ b/packages/syntax/src/datetime.ts @@ -7,6 +7,9 @@ export const ensureValidDatetime = (dtStr: string): void => { if (isNaN(date.getTime())) { throw new InvalidDatetimeError('datetime did not parse as ISO 8601') } + if (date.toISOString().startsWith('-')) { + throw new InvalidDatetimeError('datetime normalized to a negative time') + } // regex and other checks for RFC-3339 if ( !/^[0-9]{4}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9](.[0-9]{1,20})?(Z|([+-][0-2][0-9]:[0-5][0-9]))$/.test( From 91f5d93c22ca2bef413b7243376c365612f35656 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 21 Nov 2023 15:35:07 -0800 Subject: [PATCH 10/11] syntax: disallow datetimes before year 0010 --- .../syntax/datetime_syntax_invalid.txt | 3 ++- interop-test-files/syntax/datetime_syntax_valid.txt | 6 ++++++ packages/syntax/src/datetime.ts | 3 +++ packages/syntax/tests/datetime.test.ts | 12 ++++++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/interop-test-files/syntax/datetime_syntax_invalid.txt b/interop-test-files/syntax/datetime_syntax_invalid.txt index d1151e21b55..6702e43e8e7 100644 --- a/interop-test-files/syntax/datetime_syntax_invalid.txt +++ b/interop-test-files/syntax/datetime_syntax_invalid.txt @@ -64,4 +64,5 @@ 1985-04-12T23:20:50.123- # ISO-8601, but normalizes to a negative time -0000-01-01T00:00:00+01:00 \ No newline at end of file +0000-01-01T00:00:00+01:00 +-000001-12-31T23:00:00.000Z diff --git a/interop-test-files/syntax/datetime_syntax_valid.txt b/interop-test-files/syntax/datetime_syntax_valid.txt index 79ed5cabb41..f47d539c2ff 100644 --- a/interop-test-files/syntax/datetime_syntax_valid.txt +++ b/interop-test-files/syntax/datetime_syntax_valid.txt @@ -32,3 +32,9 @@ 1985-04-12T23:20:50.1234567890Z 1985-04-12T23:20:50.12345678901Z 1985-04-12T23:20:50.123456789012Z + +# extreme but currently allowed +0010-12-31T23:00:00.000Z +1000-12-31T23:00:00.000Z +1900-12-31T23:00:00.000Z +3001-12-31T23:00:00.000Z diff --git a/packages/syntax/src/datetime.ts b/packages/syntax/src/datetime.ts index 682b4ed1fc5..a5ec3b2fad3 100644 --- a/packages/syntax/src/datetime.ts +++ b/packages/syntax/src/datetime.ts @@ -26,6 +26,9 @@ export const ensureValidDatetime = (dtStr: string): void => { 'datetime can not use "-00:00" for UTC timezone', ) } + if (dtStr.startsWith('000')) { + throw new InvalidDatetimeError('datetime so close to year zero not allowed') + } } /* Same logic as ensureValidDatetime(), but returns a boolean instead of throwing an exception. diff --git a/packages/syntax/tests/datetime.test.ts b/packages/syntax/tests/datetime.test.ts index 547a276c6a9..15fdc8dc6e2 100644 --- a/packages/syntax/tests/datetime.test.ts +++ b/packages/syntax/tests/datetime.test.ts @@ -95,6 +95,15 @@ describe('normalization', () => { expect(() => normalizeDatetime('1999-19-39T23:20:50.123Z')).toThrow( InvalidDatetimeError, ) + expect(() => normalizeDatetime('-000001-12-31T23:00:00.000Z')).toThrow( + InvalidDatetimeError, + ) + expect(() => normalizeDatetime('0000-01-01T00:00:00+01:00')).toThrow( + InvalidDatetimeError, + ) + expect(() => normalizeDatetime('0001-01-01T00:00:00+01:00')).toThrow( + InvalidDatetimeError, + ) }) it('normalizes datetimes always', () => { @@ -102,5 +111,8 @@ describe('normalization', () => { '1985-04-12T23:20:50.000Z', ) expect(normalizeDatetimeAlways('blah')).toEqual('1970-01-01T00:00:00.000Z') + expect(normalizeDatetimeAlways('0000-01-01T00:00:00+01:00')).toEqual( + '1970-01-01T00:00:00.000Z', + ) }) }) From 9f1c3826f34fdc94325d38ce866121e356a0d0a5 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 21 Nov 2023 15:35:31 -0800 Subject: [PATCH 11/11] syntax: datetime normalization functions validate output --- packages/syntax/src/datetime.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/syntax/src/datetime.ts b/packages/syntax/src/datetime.ts index a5ec3b2fad3..96643271c8d 100644 --- a/packages/syntax/src/datetime.ts +++ b/packages/syntax/src/datetime.ts @@ -58,15 +58,20 @@ export const isValidDatetime = (dtStr: string): boolean => { */ export const normalizeDatetime = (dtStr: string): string => { if (isValidDatetime(dtStr)) { - const date = new Date(dtStr) - return date.toISOString() + const outStr = new Date(dtStr).toISOString() + if (isValidDatetime(outStr)) { + return outStr + } } // check if this permissive datetime is missing a timezone if (!/.*(([+-]\d\d:?\d\d)|[a-zA-Z])$/.test(dtStr)) { const date = new Date(dtStr + 'Z') if (!isNaN(date.getTime())) { - return date.toISOString() + const tzStr = date.toISOString() + if (isValidDatetime(tzStr)) { + return tzStr + } } } @@ -77,7 +82,14 @@ export const normalizeDatetime = (dtStr: string): string => { 'datetime did not parse as any timestamp format', ) } - return date.toISOString() + const isoStr = date.toISOString() + if (isValidDatetime(isoStr)) { + return isoStr + } else { + throw new InvalidDatetimeError( + 'datetime normalized to invalid timestamp string', + ) + } } /* Variant of normalizeDatetime() which always returns a valid datetime strings.