Merge remote-tracking branch 'origin/main' into xrpc-passthru

bluesky-social · Oct 24, 2023 · b200007 · b200007
2 parents c043f1e + 825b2a0
commit b200007
Show file tree

Hide file tree

Showing 106 changed files with 692 additions and 573 deletions.
diff --git a/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts b/packages/bsky/src/api/app/bsky/feed/getAuthorFeed.ts
@@ -89,8 +89,8 @@ export const skeleton = async (
 
   if (filter === 'posts_with_media') {
     feedItemsQb = feedItemsQb
-      // and only your own posts/reposts
-      .where('post.creator', '=', actorDid)
+      // only your own posts
+      .where('type', '=', 'post')
       // only posts with media
       .whereExists((qb) =>
         qb

diff --git a/packages/bsky/src/api/app/bsky/notification/listNotifications.ts b/packages/bsky/src/api/app/bsky/notification/listNotifications.ts
@@ -53,10 +53,6 @@ const skeleton = async (
   }
   let notifBuilder = db.db
     .selectFrom('notification as notif')
-    .innerJoin('record', 'record.uri', 'notif.recordUri')
-    .innerJoin('actor as author', 'author.did', 'notif.author')
-    .where(notSoftDeletedClause(ref('record')))
-    .where(notSoftDeletedClause(ref('author')))
     .where('notif.did', '=', viewer)
     .where((clause) =>
       clause
@@ -69,23 +65,20 @@ const skeleton = async (
         ),
     )
     .select([
+      'notif.author as authorDid',
       'notif.recordUri as uri',
       'notif.recordCid as cid',
-      'author.did as authorDid',
-      'author.handle as authorHandle',
-      'author.indexedAt as authorIndexedAt',
-      'author.takedownId as authorTakedownId',
       'notif.reason as reason',
       'notif.reasonSubject as reasonSubject',
       'notif.sortAt as indexedAt',
-      'record.json as recordJson',
     ])
 
   const keyset = new NotifsKeyset(ref('notif.sortAt'), ref('notif.recordCid'))
   notifBuilder = paginate(notifBuilder, {
     cursor,
     limit,
     keyset,
+    tryIndex: true,
   })
 
   const actorStateQuery = db.db
@@ -107,17 +100,18 @@ const skeleton = async (
 }
 
 const hydration = async (state: SkeletonState, ctx: Context) => {
-  const { graphService, actorService, labelService } = ctx
+  const { graphService, actorService, labelService, db } = ctx
   const { params, notifs } = state
   const { viewer } = params
   const dids = notifs.map((notif) => notif.authorDid)
   const uris = notifs.map((notif) => notif.uri)
-  const [actors, labels, bam] = await Promise.all([
+  const [actors, records, labels, bam] = await Promise.all([
     actorService.views.profiles(dids, viewer),
+    getRecordMap(db, uris),
     labelService.getLabelsForUris(uris),
     graphService.getBlockAndMuteState(dids.map((did) => [viewer, did])),
   ])
-  return { ...state, actors, labels, bam }
+  return { ...state, actors, records, labels, bam }
 }
 
 const noBlockOrMutes = (state: HydrationState) => {
@@ -131,11 +125,11 @@ const noBlockOrMutes = (state: HydrationState) => {
 }
 
 const presentation = (state: HydrationState) => {
-  const { notifs, cursor, actors, labels, lastSeenNotifs } = state
+  const { notifs, cursor, actors, records, labels, lastSeenNotifs } = state
   const notifications = mapDefined(notifs, (notif) => {
     const author = actors[notif.authorDid]
-    if (!author) return undefined
-    const record = jsonStringToLex(notif.recordJson) as Record<string, unknown>
+    const record = records[notif.uri]
+    if (!author || !record) return undefined
     const recordLabels = labels[notif.uri] ?? []
     const recordSelfLabels = getSelfLabels({
       uri: notif.uri,
@@ -157,6 +151,24 @@ const presentation = (state: HydrationState) => {
   return { notifications, cursor }
 }
 
+const getRecordMap = async (
+  db: Database,
+  uris: string[],
+): Promise<RecordMap> => {
+  if (!uris.length) return {}
+  const { ref } = db.db.dynamic
+  const recordRows = await db.db
+    .selectFrom('record')
+    .select(['uri', 'json'])
+    .where('uri', 'in', uris)
+    .where(notSoftDeletedClause(ref('record')))
+    .execute()
+  return recordRows.reduce((acc, { uri, json }) => {
+    acc[uri] = jsonStringToLex(json) as Record<string, unknown>
+    return acc
+  }, {} as RecordMap)
+}
+
 type Context = {
   db: Database
   actorService: ActorService
@@ -178,20 +190,19 @@ type SkeletonState = {
 type HydrationState = SkeletonState & {
   bam: BlockAndMuteState
   actors: ActorInfoMap
+  records: RecordMap
   labels: Labels
 }
 
+type RecordMap = { [uri: string]: Record<string, unknown> }
+
 type NotifRow = {
-  indexedAt: string
-  cid: string
-  uri: string
   authorDid: string
-  authorHandle: string | null
-  authorIndexedAt: string
-  authorTakedownId: number | null
+  uri: string
+  cid: string
   reason: string
   reasonSubject: string | null
-  recordJson: string
+  indexedAt: string
 }
 
 class NotifsKeyset extends TimeCidKeyset<NotifRow> {

diff --git a/packages/bsky/src/logger.ts b/packages/bsky/src/logger.ts
@@ -12,4 +12,12 @@ export const httpLogger: ReturnType<typeof subsystemLogger> =
 
 export const loggerMiddleware = pinoHttp({
   logger: httpLogger,
+  serializers: {
+    err: (err) => {
+      return {
+        code: err?.code,
+        message: err?.message,
+      }
+    },
+  },
 })
diff --git a/packages/bsky/src/services/actor/index.ts b/packages/bsky/src/services/actor/index.ts
@@ -66,7 +66,13 @@ export class ActorService {
           qb = qb.orWhere('actor.did', 'in', dids)
         }
         if (handles.length) {
-          qb = qb.orWhere('actor.handle', 'in', handles)
+          qb = qb.orWhere(
+            'actor.handle',
+            'in',
+            handles.length === 1
+              ? [handles[0], handles[0]] // a silly (but worthwhile) optimization to avoid usage of actor_handle_tgrm_idx
+              : handles,
+          )
         }
         return qb
       })

diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json
@@ -36,6 +36,7 @@
     "@atproto/xrpc-server": "workspace:^",
     "@did-plc/lib": "^0.0.1",
     "@did-plc/server": "^0.0.1",
+    "axios": "^0.27.2",
     "better-sqlite3": "^7.6.2",
     "chalk": "^5.0.1",
     "dotenv": "^16.0.3",

diff --git a/packages/dev-env/src/bsky.ts b/packages/dev-env/src/bsky.ts
@@ -81,7 +81,7 @@ export class TestBsky {
     })
     // indexer
     const ns = cfg.dbPostgresSchema
-      ? await randomIntFromSeed(cfg.dbPostgresSchema, 10000)
+      ? await randomIntFromSeed(cfg.dbPostgresSchema, 1000000)
       : undefined
     const indexerCfg = new bsky.IndexerConfig({
       version: '0.0.0',
@@ -200,7 +200,7 @@ export async function getIngester(
   opts: { name: string } & Partial<bsky.IngesterConfigValues>,
 ) {
   const { name, ...config } = opts
-  const ns = name ? await randomIntFromSeed(name, 10000) : undefined
+  const ns = name ? await randomIntFromSeed(name, 1000000) : undefined
   const cfg = new bsky.IngesterConfig({
     version: '0.0.0',
     redisHost: process.env.REDIS_HOST || '',
@@ -234,7 +234,7 @@ export async function getIndexers(
   },
 ): Promise<BskyIndexers> {
   const { name, ...config } = opts
-  const ns = name ? await randomIntFromSeed(name, 10000) : undefined
+  const ns = name ? await randomIntFromSeed(name, 1000000) : undefined
   const baseCfg: bsky.IndexerConfigValues = {
     version: '0.0.0',
     didCacheStaleTTL: HOUR,

diff --git a/packages/dev-env/src/util.ts b/packages/dev-env/src/util.ts
@@ -1,3 +1,4 @@
+import axios from 'axios'
 import { IdResolver } from '@atproto/identity'
 import { TestPds } from './pds'
 import { TestBsky } from './bsky'
@@ -39,8 +40,8 @@ export const mockResolvers = (idResolver: IdResolver, pds: TestPds) => {
 
     const url = `${pds.url}/.well-known/atproto-did`
     try {
-      const res = await fetch(url, { headers: { host: handle } })
-      return await res.text()
+      const res = await axios.get(url, { headers: { host: handle } })
+      return res.data
     } catch (err) {
       return undefined
     }

diff --git a/packages/dev-infra/_common.sh b/packages/dev-infra/_common.sh
@@ -8,7 +8,9 @@ get_container_id() {
     exit 1
   fi
 
+ # first line of jq normalizes for docker compose breaking change, see docker/compose#10958
   docker compose -f $compose_file ps --format json --status running \
+    | jq -sc '.[] | if type=="array" then .[] else . end' | jq -s \
     | jq -r '.[]? | select(.Service == "'${service}'") | .ID'
 }
 

diff --git a/packages/pds/src/api/app/bsky/actor/getPreferences.ts b/packages/pds/src/api/app/bsky/actor/getPreferences.ts
@@ -1,10 +1,10 @@
 import { Server } from '../../../../lexicon'
 import AppContext from '../../../../context'
-import { AuthScope } from '../../../../auth'
+import { AuthScope } from '../../../../auth-verifier'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.actor.getPreferences({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async ({ auth }) => {
       const requester = auth.credentials.did
       const { services, db } = ctx

diff --git a/packages/pds/src/api/app/bsky/actor/getProfile.ts b/packages/pds/src/api/app/bsky/actor/getProfile.ts
@@ -7,7 +7,7 @@ import { LocalRecords } from '../../../../services/local'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.actor.getProfile({
-    auth: ctx.accessOrRoleVerifier,
+    auth: ctx.authVerifier.accessOrRole,
     handler: async ({ req, auth, params }) => {
       const requester =
         auth.credentials.type === 'access' ? auth.credentials.did : null

diff --git a/packages/pds/src/api/app/bsky/actor/getProfiles.ts b/packages/pds/src/api/app/bsky/actor/getProfiles.ts
@@ -6,7 +6,7 @@ import { handleReadAfterWrite } from '../util/read-after-write'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.actor.getProfiles({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async ({ auth, params }) => {
       const requester = auth.credentials.did
       const res = await ctx.appViewAgent.api.app.bsky.actor.getProfiles(

diff --git a/packages/pds/src/api/app/bsky/actor/getSuggestions.ts b/packages/pds/src/api/app/bsky/actor/getSuggestions.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.actor.getSuggestions({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/actor/putPreferences.ts b/packages/pds/src/api/app/bsky/actor/putPreferences.ts
@@ -5,7 +5,7 @@ import { InvalidRequestError } from '@atproto/xrpc-server'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.actor.putPreferences({
-    auth: ctx.accessVerifierCheckTakedown,
+    auth: ctx.authVerifier.accessCheckTakedown,
     handler: async ({ auth, input }) => {
       const { preferences } = input.body
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/actor/searchActors.ts b/packages/pds/src/api/app/bsky/actor/searchActors.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.actor.searchActors({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/actor/searchActorsTypeahead.ts b/packages/pds/src/api/app/bsky/actor/searchActorsTypeahead.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.actor.searchActorsTypeahead({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/feed/getActorFeeds.ts b/packages/pds/src/api/app/bsky/feed/getActorFeeds.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getActorFeeds({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/feed/getActorLikes.ts b/packages/pds/src/api/app/bsky/feed/getActorLikes.ts
@@ -7,7 +7,7 @@ import { LocalRecords } from '../../../../services/local'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getActorLikes({
-    auth: ctx.accessOrRoleVerifier,
+    auth: ctx.authVerifier.accessOrRole,
     handler: async ({ req, params, auth }) => {
       const requester =
         auth.credentials.type === 'access' ? auth.credentials.did : null

diff --git a/packages/pds/src/api/app/bsky/feed/getAuthorFeed.ts b/packages/pds/src/api/app/bsky/feed/getAuthorFeed.ts
@@ -8,7 +8,7 @@ import { isReasonRepost } from '../../../../lexicon/types/app/bsky/feed/defs'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getAuthorFeed({
-    auth: ctx.accessOrRoleVerifier,
+    auth: ctx.authVerifier.accessOrRole,
     handler: async ({ req, params, auth }) => {
       const requester =
         auth.credentials.type === 'access' ? auth.credentials.did : null

diff --git a/packages/pds/src/api/app/bsky/feed/getFeed.ts b/packages/pds/src/api/app/bsky/feed/getFeed.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getFeed({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { params, auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/feed/getFeedGenerator.ts b/packages/pds/src/api/app/bsky/feed/getFeedGenerator.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getFeedGenerator({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/feed/getFeedGenerators.ts b/packages/pds/src/api/app/bsky/feed/getFeedGenerators.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getFeedGenerators({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/feed/getLikes.ts b/packages/pds/src/api/app/bsky/feed/getLikes.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getLikes({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/feed/getListFeed.ts b/packages/pds/src/api/app/bsky/feed/getListFeed.ts
@@ -4,7 +4,7 @@ import { proxy } from '@atproto/xrpc-server'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getListFeed({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/feed/getPostThread.ts b/packages/pds/src/api/app/bsky/feed/getPostThread.ts
@@ -26,7 +26,7 @@ import {
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getPostThread({
-    auth: ctx.accessOrRoleVerifier,
+    auth: ctx.authVerifier.accessOrRole,
     handler: async (request) => {
       const { params, auth } = request
       const requester =

diff --git a/packages/pds/src/api/app/bsky/feed/getPosts.ts b/packages/pds/src/api/app/bsky/feed/getPosts.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getPosts({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did

diff --git a/packages/pds/src/api/app/bsky/feed/getRepostedBy.ts b/packages/pds/src/api/app/bsky/feed/getRepostedBy.ts
@@ -4,7 +4,7 @@ import AppContext from '../../../../context'
 
 export default function (server: Server, ctx: AppContext) {
   server.app.bsky.feed.getRepostedBy({
-    auth: ctx.accessVerifier,
+    auth: ctx.authVerifier.access,
     handler: async (request) => {
       const { auth } = request
       const requester = auth.credentials.did