Merge remote-tracking branch 'origin/main' into multi-pds-auth

.changeset/brave-swans-kiss.md

@@ -0,0 +1,5 @@
+---
+'@atproto/syntax': patch
+---
+
+prevent unnecessary throw/catch on uri syntax

interop-test-files/syntax/datetime_parse_invalid.txt

@@ -0,0 +1,7 @@
+# superficial syntax parses ok, but are not valid datetimes for semantic reasons (eg, "month zero")
+1985-00-12T23:20:50.123Z
+1985-04-00T23:20:50.123Z
+1985-13-12T23:20:50.123Z
+1985-04-12T25:20:50.123Z
+1985-04-12T23:99:50.123Z
+1985-04-12T23:20:61.123Z

interop-test-files/syntax/datetime_syntax_invalid.txt

@@ -0,0 +1,68 @@
+
+# subtle changes to: 1985-04-12T23:20:50.123Z
+1985-04-12T23:20:50.123z
+01985-04-12T23:20:50.123Z
+985-04-12T23:20:50.123Z
+1985-04-12T23:20:50.Z
+1985-04-32T23;20:50.123Z
+1985-04-32T23;20:50.123Z
+
+# en-dash and em-dash
+1985—04-32T23;20:50.123Z
+1985–04-32T23;20:50.123Z
+
+# whitespace
+ 1985-04-12T23:20:50.123Z
+1985-04-12T23:20:50.123Z 
+1985-04-12T 23:20:50.123Z
+
+# not enough zero padding
+1985-4-12T23:20:50.123Z
+1985-04-2T23:20:50.123Z
+1985-04-12T3:20:50.123Z
+1985-04-12T23:0:50.123Z
+1985-04-12T23:20:5.123Z
+
+# too much zero padding
+01985-04-12T23:20:50.123Z
+1985-004-12T23:20:50.123Z
+1985-04-012T23:20:50.123Z
+1985-04-12T023:20:50.123Z
+1985-04-12T23:020:50.123Z
+1985-04-12T23:20:050.123Z
+
+# strict capitalization (ISO-8601)
+1985-04-12t23:20:50.123Z
+1985-04-12T23:20:50.123z
+
+# RFC-3339, but not ISO-8601
+1985-04-12T23:20:50.123-00:00
+1985-04-12_23:20:50.123Z
+1985-04-12 23:20:50.123Z
+
+# ISO-8601, but weird
+1985-04-274T23:20:50.123Z
+
+# timezone is required
+1985-04-12T23:20:50.123
+1985-04-12T23:20:50
+
+1985-04-12
+1985-04-12T23:20Z
+1985-04-12T23:20:5Z
+1985-04-12T23:20:50.123
++001985-04-12T23:20:50.123Z
+23:20:50.123Z
+
+1985-04-12T23:20:50.123+00
+1985-04-12T23:20:50.123+00:0
+1985-04-12T23:20:50.123+0:00
+1985-04-12T23:20:50.123
+1985-04-12T23:20:50.123+0000
+1985-04-12T23:20:50.123+00
+1985-04-12T23:20:50.123+
+1985-04-12T23:20:50.123-
+
+# ISO-8601, but normalizes to a negative time
+0000-01-01T00:00:00+01:00
+-000001-12-31T23:00:00.000Z

interop-test-files/syntax/datetime_syntax_valid.txt

@@ -0,0 +1,40 @@
+# "preferred"
+1985-04-12T23:20:50.123Z
+1985-04-12T23:20:50.000Z
+2000-01-01T00:00:00.000Z
+1985-04-12T23:20:50.123456Z
+1985-04-12T23:20:50.120Z
+1985-04-12T23:20:50.120000Z
+
+# "supported"
+1985-04-12T23:20:50.1235678912345Z
+1985-04-12T23:20:50.100Z
+1985-04-12T23:20:50Z
+1985-04-12T23:20:50.0Z
+1985-04-12T23:20:50.123+00:00
+1985-04-12T23:20:50.123-07:00
+1985-04-12T23:20:50.123+07:00
+1985-04-12T23:20:50.123+01:45
+0985-04-12T23:20:50.123-07:00
+1985-04-12T23:20:50.123-07:00
+0123-01-01T00:00:00.000Z
+
+# various precisions, up through at least 12 digits
+1985-04-12T23:20:50.1Z
+1985-04-12T23:20:50.12Z
+1985-04-12T23:20:50.123Z
+1985-04-12T23:20:50.1234Z
+1985-04-12T23:20:50.12345Z
+1985-04-12T23:20:50.123456Z
+1985-04-12T23:20:50.1234567Z
+1985-04-12T23:20:50.12345678Z
+1985-04-12T23:20:50.123456789Z
+1985-04-12T23:20:50.1234567890Z
+1985-04-12T23:20:50.12345678901Z
+1985-04-12T23:20:50.123456789012Z
+
+# extreme but currently allowed
+0010-12-31T23:00:00.000Z
+1000-12-31T23:00:00.000Z
+1900-12-31T23:00:00.000Z
+3001-12-31T23:00:00.000Z

interop-test-files/syntax/tid_syntax_invalid.txt

@@ -0,0 +1,15 @@
+
+# not base32
+3jzfcijpj2z21
+0000000000000
+
+# too long/short
+3jzfcijpj2z2aa
+3jzfcijpj2z2
+
+# old dashes syntax not actually supported (TTTT-TTT-TTTT-CC)
+3jzf-cij-pj2z-2a
+
+# high bit can't be high
+zzzzzzzzzzzzz
+kjzfcijpj2z2a

interop-test-files/syntax/tid_syntax_valid.txt

@@ -0,0 +1,6 @@
+# 13 digits
+# 234567abcdefghijklmnopqrstuvwxyz
+
+3jzfcijpj2z2a
+7777777777777
+3zzzzzzzzzzzz

lexicons/app/bsky/graph/defs.json

@@ -40,8 +40,9 @@
     },
     "listItemView": {
       "type": "object",
-      "required": ["subject"],
+      "required": ["uri", "subject"],
       "properties": {
+        "uri": { "type": "string", "format": "at-uri" },
         "subject": { "type": "ref", "ref": "app.bsky.actor.defs#profileView" }
       }
     },

packages/api/src/client/lexicons.ts

@@ -6446,8 +6446,12 @@ export const schemaDict = {
       },
       listItemView: {
         type: 'object',
-        required: ['subject'],
+        required: ['uri', 'subject'],
         properties: {
+          uri: {
+            type: 'string',
+            format: 'at-uri',
+          },
           subject: {
             type: 'ref',
             ref: 'lex:app.bsky.actor.defs#profileView',

packages/api/src/client/types/app/bsky/graph/defs.ts

@@ -58,6 +58,7 @@ export function validateListView(v: unknown): ValidationResult {
 }
 
 export interface ListItemView {
+  uri: string
   subject: AppBskyActorDefs.ProfileView
   [k: string]: unknown
 }

packages/bsky/src/api/app/bsky/graph/getList.ts

@@ -104,7 +104,7 @@ const presentation = (state: HydrationState, ctx: Context) => {
   const items = mapDefined(listItems, (item) => {
     const subject = actors[item.did]
     if (!subject) return
-    return { subject }
+    return { uri: item.uri, subject }
   })
   return { list: listView, items, cursor }
 }
@@ -122,7 +122,7 @@ type Params = QueryParams & {
 type SkeletonState = {
   params: Params
   list: Actor & ListInfo
-  listItems: (Actor & { cid: string; sortAt: string })[]
+  listItems: (Actor & { uri: string; cid: string; sortAt: string })[]
   cursor?: string
 }
 

packages/bsky/src/auth.ts

@@ -7,37 +7,41 @@ import { ServerConfig } from './config'
 const BASIC = 'Basic '
 const BEARER = 'Bearer '
 
-export const authVerifier =
-  (idResolver: IdResolver, opts: { aud: string | null }) =>
-  async (reqCtx: { req: express.Request; res: express.Response }) => {
+export const authVerifier = (
+  idResolver: IdResolver,
+  opts: { aud: string | null },
+) => {
+  const getSigningKey = async (
+    did: string,
+    forceRefresh: boolean,
+  ): Promise<string> => {
+    const atprotoData = await idResolver.did.resolveAtprotoData(
+      did,
+      forceRefresh,
+    )
+    return atprotoData.signingKey
+  }
+
+  return async (reqCtx: { req: express.Request; res: express.Response }) => {
     const jwtStr = getJwtStrFromReq(reqCtx.req)
     if (!jwtStr) {
       throw new AuthRequiredError('missing jwt', 'MissingJwt')
     }
-    const payload = await verifyJwt(
-      jwtStr,
-      opts.aud,
-      async (did, forceRefresh) => {
-        const atprotoData = await idResolver.did.resolveAtprotoData(
-          did,
-          forceRefresh,
-        )
-        return atprotoData.signingKey
-      },
-    )
+    const payload = await verifyJwt(jwtStr, opts.aud, getSigningKey)
     return { credentials: { did: payload.iss }, artifacts: { aud: opts.aud } }
   }
+}
 
 export const authOptionalVerifier = (
   idResolver: IdResolver,
   opts: { aud: string | null },
 ) => {
-  const verify = authVerifier(idResolver, opts)
+  const verifyAccess = authVerifier(idResolver, opts)
   return async (reqCtx: { req: express.Request; res: express.Response }) => {
     if (!reqCtx.req.headers.authorization) {
       return { credentials: { did: null } }
     }
-    return verify(reqCtx)
+    return verifyAccess(reqCtx)
   }
 }
 
@@ -131,9 +135,9 @@ export const buildBasicAuth = (username: string, password: string): string => {
 }
 
 export const getJwtStrFromReq = (req: express.Request): string | null => {
-  const { authorization = '' } = req.headers
-  if (!authorization.startsWith(BEARER)) {
+  const { authorization } = req.headers
+  if (!authorization?.startsWith(BEARER)) {
     return null
   }
-  return authorization.replace(BEARER, '').trim()
+  return authorization.slice(BEARER.length).trim()
 }

packages/bsky/src/lexicon/lexicons.ts

@@ -6446,8 +6446,12 @@ export const schemaDict = {
       },
       listItemView: {
         type: 'object',
-        required: ['subject'],
+        required: ['uri', 'subject'],
         properties: {
+          uri: {
+            type: 'string',
+            format: 'at-uri',
+          },
           subject: {
             type: 'ref',
             ref: 'lex:app.bsky.actor.defs#profileView',

packages/bsky/src/lexicon/types/app/bsky/graph/defs.ts

@@ -58,6 +58,7 @@ export function validateListView(v: unknown): ValidationResult {
 }
 
 export interface ListItemView {
+  uri: string
   subject: AppBskyActorDefs.ProfileView
   [k: string]: unknown
 }

packages/bsky/src/services/graph/index.ts

@@ -109,7 +109,11 @@ export class GraphService {
       .selectFrom('list_item')
       .innerJoin('actor as subject', 'subject.did', 'list_item.subjectDid')
       .selectAll('subject')
-      .select(['list_item.cid as cid', 'list_item.sortAt as sortAt'])
+      .select([
+        'list_item.uri as uri',
+        'list_item.cid as cid',
+        'list_item.sortAt as sortAt',
+      ])
   }
 
   async getBlockAndMuteState(

packages/bsky/src/services/indexing/plugins/block.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as Block from '../../../lexicon/types/app/bsky/graph/block'
 import * as lex from '../../../lexicon/lexicons'
@@ -27,7 +26,7 @@ const insertFn = async (
       cid: cid.toString(),
       creator: uri.host,
       subjectDid: obj.subject,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

packages/bsky/src/services/indexing/plugins/feed-generator.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as FeedGenerator from '../../../lexicon/types/app/bsky/feed/generator'
 import * as lex from '../../../lexicon/lexicons'
@@ -33,7 +32,7 @@ const insertFn = async (
         ? JSON.stringify(obj.descriptionFacets)
         : undefined,
       avatarCid: obj.avatar?.ref.toString(),
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

packages/bsky/src/services/indexing/plugins/follow.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as Follow from '../../../lexicon/types/app/bsky/graph/follow'
 import * as lex from '../../../lexicon/lexicons'
@@ -28,7 +27,7 @@ const insertFn = async (
       cid: cid.toString(),
       creator: uri.host,
       subjectDid: obj.subject,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

packages/bsky/src/services/indexing/plugins/like.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as Like from '../../../lexicon/types/app/bsky/feed/like'
 import * as lex from '../../../lexicon/lexicons'
@@ -29,7 +28,7 @@ const insertFn = async (
       creator: uri.host,
       subject: obj.subject.uri,
       subjectCid: obj.subject.cid,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())

packages/bsky/src/services/indexing/plugins/list-block.ts

@@ -1,6 +1,5 @@
 import { Selectable } from 'kysely'
-import { AtUri } from '@atproto/syntax'
-import { toSimplifiedISOSafe } from '@atproto/common'
+import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax'
 import { CID } from 'multiformats/cid'
 import * as ListBlock from '../../../lexicon/types/app/bsky/graph/listblock'
 import * as lex from '../../../lexicon/lexicons'
@@ -27,7 +26,7 @@ const insertFn = async (
       cid: cid.toString(),
       creator: uri.host,
       subjectUri: obj.subject,
-      createdAt: toSimplifiedISOSafe(obj.createdAt),
+      createdAt: normalizeDatetimeAlways(obj.createdAt),
       indexedAt: timestamp,
     })
     .onConflict((oc) => oc.doNothing())