From 396e210d6fc0d571f5b3744511140be7ed4392b6 Mon Sep 17 00:00:00 2001 From: dholms Date: Tue, 12 Dec 2023 16:26:03 -0600 Subject: [PATCH] feed gen routes --- .../src/api/app/bsky/feed/getActorFeeds.ts | 116 ++++++++++------- .../src/api/app/bsky/feed/getFeedGenerator.ts | 19 +-- .../api/app/bsky/feed/getFeedGenerators.ts | 119 +++++++++++------- .../api/app/bsky/feed/getSuggestedFeeds.ts | 31 ++--- packages/bsky/src/hydration/feed.ts | 39 ++++-- packages/bsky/src/hydration/graph.ts | 21 +++- packages/bsky/src/hydration/util.ts | 8 +- .../feed-generation.test.ts.snap | 36 ++++-- 8 files changed, 234 insertions(+), 155 deletions(-) diff --git a/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts b/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts index 7a28e4efe67..baec6b8efc9 100644 --- a/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts +++ b/packages/bsky/src/api/app/bsky/feed/getActorFeeds.ts @@ -1,63 +1,87 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' +import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getActorFeeds' import AppContext from '../../../../context' -import { TimeCidKeyset, paginate } from '../../../../db/pagination' +import { createPipelineNew, noRulesNew } from '../../../../pipeline' +import { HydrationState, Hydrator } from '../../../../hydration/hydrator' +import { Views } from '../../../../views' +import { DataPlaneClient } from '../../../../data-plane' +import { parseString } from '../../../../hydration/util' export default function (server: Server, ctx: AppContext) { + const getActorFeeds = createPipelineNew( + skeleton, + hydration, + noRulesNew, + presentation, + ) server.app.bsky.feed.getActorFeeds({ auth: ctx.authOptionalVerifier, handler: async ({ auth, params }) => { - const { actor, limit, cursor } = params const viewer = auth.credentials.did - - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const feedService = ctx.services.feed(db) - - const creatorRes = await actorService.getActor(actor) - if (!creatorRes) { - throw new InvalidRequestError(`Actor not found: ${actor}`) + const result = await getActorFeeds({ ...params, viewer }, ctx) + return { + encoding: 'application/json', + body: result, } + }, + }) +} - const { ref } = db.db.dynamic - let feedsQb = feedService - .selectFeedGeneratorQb(viewer) - .where('feed_generator.creator', '=', creatorRes.did) +const skeleton = async (inputs: { + ctx: Context + params: Params +}): Promise => { + const { ctx, params } = inputs + const [did] = await ctx.hydrator.actor.getDids([params.actor]) + if (!did) { + throw new InvalidRequestError('Profile not found') + } + const feedsRes = await ctx.dataplane.getActorFeeds({ + actorDid: did, + cursor: params.cursor, + limit: params.limit, + }) + return { + feedUris: feedsRes.uris, + cursor: parseString(feedsRes.cursor), + } +} - const keyset = new TimeCidKeyset( - ref('feed_generator.createdAt'), - ref('feed_generator.cid'), - ) - feedsQb = paginate(feedsQb, { - limit, - cursor, - keyset, - }) +const hydration = async (inputs: { + ctx: Context + params: Params + skeleton: Skeleton +}) => { + const { ctx, params, skeleton } = inputs + return await ctx.hydrator.hydrateFeedGens(skeleton.feedUris, params.viewer) +} - const [feedsRes, profiles] = await Promise.all([ - feedsQb.execute(), - actorService.views.profiles([creatorRes], viewer), - ]) - if (!profiles[creatorRes.did]) { - throw new InvalidRequestError(`Actor not found: ${actor}`) - } +const presentation = (inputs: { + ctx: Context + skeleton: Skeleton + hydration: HydrationState +}) => { + const { ctx, skeleton, hydration } = inputs + const feeds = mapDefined(skeleton.feedUris, (uri) => + ctx.views.feedGenerator(uri, hydration), + ) + return { + feeds, + cursor: skeleton.cursor, + } +} - const feeds = mapDefined(feedsRes, (row) => { - const feed = { - ...row, - viewer: viewer ? { like: row.viewerLike } : undefined, - } - return feedService.views.formatFeedGeneratorView(feed, profiles) - }) +type Context = { + hydrator: Hydrator + views: Views + dataplane: DataPlaneClient +} - return { - encoding: 'application/json', - body: { - cursor: keyset.packFromResult(feedsRes), - feeds, - }, - } - }, - }) +type Params = QueryParams & { viewer: string | null } + +type Skeleton = { + feedUris: string[] + cursor?: string } diff --git a/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts b/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts index 14a5688db0d..6e2f4e57b2a 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeedGenerator.ts @@ -14,17 +14,13 @@ export default function (server: Server, ctx: AppContext) { const { feed } = params const viewer = auth.credentials.did - const db = ctx.db.getReplica() - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - - const got = await feedService.getFeedGeneratorInfos([feed], viewer) - const feedInfo = got[feed] + const hydration = await ctx.hydrator.hydrateFeedGens([feed], viewer) + const feedInfo = hydration.feedgens?.get(feed) if (!feedInfo) { throw new InvalidRequestError('could not find feed') } - const feedDid = feedInfo.feedDid + const feedDid = feedInfo.record.did let resolved: DidDocument | null try { resolved = await ctx.idResolver.did.resolve(feedDid) @@ -47,14 +43,7 @@ export default function (server: Server, ctx: AppContext) { ) } - const profiles = await actorService.views.profilesBasic( - [feedInfo.creator], - viewer, - ) - const feedView = feedService.views.formatFeedGeneratorView( - feedInfo, - profiles, - ) + const feedView = ctx.views.feedGenerator(feed, hydration) if (!feedView) { throw new InvalidRequestError('could not find feed') } diff --git a/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts b/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts index 7b571ab09f6..b424bfd96b8 100644 --- a/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts +++ b/packages/bsky/src/api/app/bsky/feed/getFeedGenerators.ts @@ -1,32 +1,23 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' +import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getFeedGenerators' import AppContext from '../../../../context' -import { FeedGenInfo, FeedService } from '../../../../services/feed' -import { createPipeline, noRules } from '../../../../pipeline' -import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { Database } from '../../../../db' +import { createPipelineNew, noRulesNew } from '../../../../pipeline' +import { HydrationState, Hydrator } from '../../../../hydration/hydrator' +import { Views } from '../../../../views' export default function (server: Server, ctx: AppContext) { - const getFeedGenerators = createPipeline( + const getFeedGenerators = createPipelineNew( skeleton, hydration, - noRules, + noRulesNew, presentation, ) server.app.bsky.feed.getFeedGenerators({ auth: ctx.authOptionalVerifier, handler: async ({ params, auth }) => { - const { feeds } = params const viewer = auth.credentials.did - const db = ctx.db.getReplica() - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - - const view = await getFeedGenerators( - { feeds, viewer }, - { db, feedService, actorService }, - ) - + const view = await getFeedGenerators({ ...params, viewer }, ctx) return { encoding: 'application/json', body: view, @@ -35,46 +26,86 @@ export default function (server: Server, ctx: AppContext) { }) } -const skeleton = async (params: Params, ctx: Context) => { - const { feedService } = ctx - const genInfos = await feedService.getFeedGeneratorInfos( - params.feeds, - params.viewer, - ) +const skeleton = async (inputs: { params: Params }): Promise => { return { - params, - generators: Object.values(genInfos), + feedUris: inputs.params.feeds, } } -const hydration = async (state: SkeletonState, ctx: Context) => { - const { actorService } = ctx - const profiles = await actorService.views.profilesBasic( - state.generators.map((gen) => gen.creator), - state.params.viewer, +const hydration = async (inputs: { + ctx: Context + params: Params + skeleton: Skeleton +}) => { + const { ctx, params, skeleton } = inputs + return await ctx.hydrator.hydrateFeedGens(skeleton.feedUris, params.viewer) +} + +const presentation = (inputs: { + ctx: Context + skeleton: Skeleton + hydration: HydrationState +}) => { + const { ctx, skeleton, hydration } = inputs + const feeds = mapDefined(skeleton.feedUris, (uri) => + ctx.views.feedGenerator(uri, hydration), ) return { - ...state, - profiles, + feeds, } } -const presentation = (state: HydrationState, ctx: Context) => { - const { feedService } = ctx - const feeds = mapDefined(state.generators, (gen) => - feedService.views.formatFeedGeneratorView(gen, state.profiles), - ) - return { feeds } +type Context = { + hydrator: Hydrator + views: Views } -type Context = { - db: Database - feedService: FeedService - actorService: ActorService +type Params = QueryParams & { viewer: string | null } + +type Skeleton = { + feedUris: string[] } -type Params = { viewer: string | null; feeds: string[] } +// const skeleton = async (params: Params, ctx: Context) => { +// const { feedService } = ctx +// const genInfos = await feedService.getFeedGeneratorInfos( +// params.feeds, +// params.viewer, +// ) +// return { +// params, +// generators: Object.values(genInfos), +// } +// } + +// const hydration = async (state: SkeletonState, ctx: Context) => { +// const { actorService } = ctx +// const profiles = await actorService.views.profilesBasic( +// state.generators.map((gen) => gen.creator), +// state.params.viewer, +// ) +// return { +// ...state, +// profiles, +// } +// } + +// const presentation = (state: HydrationState, ctx: Context) => { +// const { feedService } = ctx +// const feeds = mapDefined(state.generators, (gen) => +// feedService.views.formatFeedGeneratorView(gen, state.profiles), +// ) +// return { feeds } +// } + +// type Context = { +// db: Database +// feedService: FeedService +// actorService: ActorService +// } + +// type Params = { viewer: string | null; feeds: string[] } -type SkeletonState = { params: Params; generators: FeedGenInfo[] } +// type SkeletonState = { params: Params; generators: FeedGenInfo[] } -type HydrationState = SkeletonState & { profiles: ActorInfoMap } +// type HydrationState = SkeletonState & { profiles: ActorInfoMap } diff --git a/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts b/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts index 35fac829039..298eada8aa1 100644 --- a/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts +++ b/packages/bsky/src/api/app/bsky/feed/getSuggestedFeeds.ts @@ -1,37 +1,30 @@ import { mapDefined } from '@atproto/common' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' +import { parseString } from '../../../../hydration/util' export default function (server: Server, ctx: AppContext) { server.app.bsky.feed.getSuggestedFeeds({ auth: ctx.authOptionalVerifier, - handler: async ({ auth }) => { + handler: async ({ params, auth }) => { const viewer = auth.credentials.did - const db = ctx.db.getReplica() - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - const feedsRes = await db.db - .selectFrom('suggested_feed') - .orderBy('suggested_feed.order', 'asc') - .selectAll() - .execute() - const genInfos = await feedService.getFeedGeneratorInfos( - feedsRes.map((r) => r.uri), - viewer, - ) - const genList = feedsRes.map((r) => genInfos[r.uri]).filter(Boolean) - const creators = genList.map((gen) => gen.creator) - const profiles = await actorService.views.profilesBasic(creators, viewer) - - const feedViews = mapDefined(genList, (gen) => - feedService.views.formatFeedGeneratorView(gen, profiles), + const suggestedRes = await ctx.dataplane.getSuggestedFeeds({ + actorDid: viewer ?? undefined, + limit: params.limit, + cursor: params.cursor, + }) + const uris = suggestedRes.uris + const hydration = await ctx.hydrator.hydrateFeedGens(uris, viewer) + const feedViews = mapDefined(uris, (uri) => + ctx.views.feedGenerator(uri, hydration), ) return { encoding: 'application/json', body: { feeds: feedViews, + cursor: parseString(suggestedRes.cursor), }, } }, diff --git a/packages/bsky/src/hydration/feed.ts b/packages/bsky/src/hydration/feed.ts index c4865f3e379..6b8cb43bf28 100644 --- a/packages/bsky/src/hydration/feed.ts +++ b/packages/bsky/src/hydration/feed.ts @@ -56,11 +56,8 @@ export class FeedHydrator { async getPosts(uris: string[], includeTakedowns = false): Promise { const res = await this.dataplane.getPostRecords({ uris }) return uris.reduce((acc, uri, i) => { - const record = parseRecord(res.records[i]) - if (!record || (record.takenDown && !includeTakedowns)) { - return acc.set(uri, null) - } - return acc.set(uri, parseRecord(res.records[i]) ?? null) + const record = parseRecord(res.records[i], includeTakedowns) + return acc.set(uri, record ?? null) }, new HydrationMap()) } @@ -101,10 +98,17 @@ export class FeedHydrator { }, new HydrationMap()) } - async getFeedGens(uris: string[]): Promise { + async getFeedGens( + uris: string[], + includeTakedowns = false, + ): Promise { const res = await this.dataplane.getFeedGeneratorRecords({ uris }) return uris.reduce((acc, uri, i) => { - return acc.set(uri, parseRecord(res.records[i]) ?? null) + const record = parseRecord( + res.records[i], + includeTakedowns, + ) + return acc.set(uri, record ?? null) }, new HydrationMap()) } @@ -132,7 +136,10 @@ export class FeedHydrator { }, new HydrationMap()) } - async getThreadgatesForPosts(postUris: string[]): Promise { + async getThreadgatesForPosts( + postUris: string[], + includeTakedowns = false, + ): Promise { const uris = postUris.map((uri) => { const parsed = new AtUri(uri) return AtUri.make( @@ -143,22 +150,28 @@ export class FeedHydrator { }) const res = await this.dataplane.getThreadGateRecords({ uris }) return uris.reduce((acc, uri, i) => { - return acc.set(uri, parseRecord(res.records[i]) ?? null) + const record = parseRecord( + res.records[i], + includeTakedowns, + ) + return acc.set(uri, record ?? null) }, new HydrationMap()) } // @TODO may not be supported yet by data plane - async getLikes(uris: string[]): Promise { + async getLikes(uris: string[], includeTakedowns = false): Promise { const res = await this.dataplane.getLikeRecords({ uris }) return uris.reduce((acc, uri, i) => { - return acc.set(uri, parseRecord(res.records[i]) ?? null) + const record = parseRecord(res.records[i], includeTakedowns) + return acc.set(uri, record ?? null) }, new HydrationMap()) } - async getReposts(uris: string[]): Promise { + async getReposts(uris: string[], includeTakedowns = false): Promise { const res = await this.dataplane.getRepostRecords({ uris }) return uris.reduce((acc, uri, i) => { - return acc.set(uri, parseRecord(res.records[i]) ?? null) + const record = parseRecord(res.records[i], includeTakedowns) + return acc.set(uri, record ?? null) }, new HydrationMap()) } } diff --git a/packages/bsky/src/hydration/graph.ts b/packages/bsky/src/hydration/graph.ts index 2cfe6425583..7056263d813 100644 --- a/packages/bsky/src/hydration/graph.ts +++ b/packages/bsky/src/hydration/graph.ts @@ -66,18 +66,26 @@ export class Blocks { export class GraphHydrator { constructor(public dataplane: DataPlaneClient) {} - async getLists(uris: string[]): Promise { + async getLists(uris: string[], includeTakedowns = false): Promise { const res = await this.dataplane.getListRecords({ uris }) return uris.reduce((acc, uri, i) => { - return acc.set(uri, parseRecord(res.records[i]) ?? null) + const record = parseRecord(res.records[i], includeTakedowns) + return acc.set(uri, record ?? null) }, new HydrationMap()) } // @TODO may not be supported yet by data plane - async getListItems(uris: string[]): Promise { + async getListItems( + uris: string[], + includeTakedowns = false, + ): Promise { const res = await this.dataplane.getListItemRecords({ uris }) return uris.reduce((acc, uri, i) => { - return acc.set(uri, parseRecord(res.records[i]) ?? null) + const record = parseRecord( + res.records[i], + includeTakedowns, + ) + return acc.set(uri, record ?? null) }, new HydrationMap()) } @@ -132,10 +140,11 @@ export class GraphHydrator { return blocks } - async getFollows(uris: string[]): Promise { + async getFollows(uris: string[], includeTakedowns = false): Promise { const res = await this.dataplane.getFollowRecords({ uris }) return uris.reduce((acc, uri, i) => { - return acc.set(uri, parseRecord(res.records[i]) ?? null) + const record = parseRecord(res.records[i], includeTakedowns) + return acc.set(uri, record ?? null) }, new HydrationMap()) } diff --git a/packages/bsky/src/hydration/util.ts b/packages/bsky/src/hydration/util.ts index 80911c7d40d..73aa1f691db 100644 --- a/packages/bsky/src/hydration/util.ts +++ b/packages/bsky/src/hydration/util.ts @@ -20,7 +20,13 @@ export type RecordInfo = { takenDown: boolean } -export const parseRecord = (entry: Record): RecordInfo | undefined => { +export const parseRecord = ( + entry: Record, + includeTakedowns: boolean, +): RecordInfo | undefined => { + if (!includeTakedowns && entry.takenDown) { + return undefined + } const record = parseRecordBytes(entry.record) const cid = parseCid(entry.cid) const indexedAt = entry.indexedAt?.toDate() diff --git a/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap b/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap index ac9e0eee7a0..7dca08f1956 100644 --- a/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap +++ b/packages/bsky/tests/__snapshots__/feed-generation.test.ts.snap @@ -65,9 +65,11 @@ Object { "cid": "cids(2)", "creator": Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(4)/cids(1)@jpeg", + "description": "its me!", "did": "user(3)", "displayName": "ali", "handle": "alice.test", + "indexedAt": "1970-01-01T00:00:00.000Z", "labels": Array [ Object { "cid": "cids(3)", @@ -1457,9 +1459,11 @@ Object { "cid": "cids(0)", "creator": Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(2)/cids(1)@jpeg", + "description": "its me!", "did": "user(1)", "displayName": "ali", "handle": "alice.test", + "indexedAt": "1970-01-01T00:00:00.000Z", "labels": Array [ Object { "cid": "cids(2)", @@ -1505,9 +1509,11 @@ Object { "cid": "cids(0)", "creator": Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(2)/cids(1)@jpeg", + "description": "its me!", "did": "user(1)", "displayName": "ali", "handle": "alice.test", + "indexedAt": "1970-01-01T00:00:00.000Z", "labels": Array [ Object { "cid": "cids(2)", @@ -1533,23 +1539,23 @@ Object { "muted": false, }, }, - "description": "Provides all feed candidates", + "description": "Provides even-indexed feed candidates", "did": "user(0)", - "displayName": "All", + "displayName": "Even", "indexedAt": "1970-01-01T00:00:00.000Z", - "likeCount": 2, + "likeCount": 0, "uri": "record(0)", - "viewer": Object { - "like": "record(4)", - }, + "viewer": Object {}, }, Object { "cid": "cids(3)", "creator": Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(2)/cids(1)@jpeg", + "description": "its me!", "did": "user(1)", "displayName": "ali", "handle": "alice.test", + "indexedAt": "1970-01-01T00:00:00.000Z", "labels": Array [ Object { "cid": "cids(2)", @@ -1575,13 +1581,15 @@ Object { "muted": false, }, }, - "description": "Provides even-indexed feed candidates", + "description": "Provides all feed candidates", "did": "user(0)", - "displayName": "Even", + "displayName": "All", "indexedAt": "1970-01-01T00:00:00.000Z", - "likeCount": 0, - "uri": "record(5)", - "viewer": Object {}, + "likeCount": 2, + "uri": "record(4)", + "viewer": Object { + "like": "record(5)", + }, }, ], } @@ -1594,9 +1602,11 @@ Object { "cid": "cids(0)", "creator": Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(2)/cids(1)@jpeg", + "description": "its me!", "did": "user(1)", "displayName": "ali", "handle": "alice.test", + "indexedAt": "1970-01-01T00:00:00.000Z", "labels": Array [ Object { "cid": "cids(2)", @@ -1636,9 +1646,11 @@ Object { "cid": "cids(3)", "creator": Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(2)/cids(1)@jpeg", + "description": "its me!", "did": "user(1)", "displayName": "ali", "handle": "alice.test", + "indexedAt": "1970-01-01T00:00:00.000Z", "labels": Array [ Object { "cid": "cids(2)", @@ -1676,9 +1688,11 @@ Object { "cid": "cids(4)", "creator": Object { "avatar": "https://bsky.public.url/img/avatar/plain/user(2)/cids(1)@jpeg", + "description": "its me!", "did": "user(1)", "displayName": "ali", "handle": "alice.test", + "indexedAt": "1970-01-01T00:00:00.000Z", "labels": Array [ Object { "cid": "cids(2)",