Skip to content

Commit

Permalink
List feeds (#1557)
Browse files Browse the repository at this point in the history
* lexicons for block lists

* reorg blockset functionality into graph service, impl block/mute filtering

* apply filterBlocksAndMutes() throughout appview except feeds

* update local feeds to pass through cleanFeedSkeleton(), offload block/mute application

* impl for grabbing block/mute details by did pair

* refactor getActorInfos away, use actor service

* experiment with moving getFeedGenerators over to a pipeline

* move getPostThread over to a pipeline

* move feeds over to pipelines

* move suggestions and likes over to pipelines

* move reposted-by, follows, followers over to pipelines, tidy author feed and post thread

* remove old block/mute checks

* unify post presentation logic

* move profiles endpoints over to pipelines

* tidy

* tidy

* misc fixes

* unify some profile hydration/presentation in appview

* profile detail, split hydration and presentation, misc fixes

* unify feed hydration w/ profile hydration

* unify hydration step for embeds, tidy application of labels

* setup indexing of list-blocks in bsky appview

* apply list-blocks, impl getListBlocks, tidy getList, tests

* tidy

* update pds proxy snaps

* update pds proxy snaps

* fix snap

* make algos return feed items, save work in getFeed

* misc changes, tidy

* tidy

* fix aturi import

* lex

* list purpose

* lex gen

* add route

* add proxy route

* seed client helpers

* tests

* mutes and blocks

* proxy test

* snapshot

* hoist actors out of composeThread()

* tidy

* tidy

* run ci on all prs

* format

* format

* fix snap name

* fix snapsh

---------

Co-authored-by: Devin Ivy <[email protected]>
  • Loading branch information
dholms and devinivy committed Sep 26, 2023
1 parent f7a5db3 commit ddb79cb
Show file tree
Hide file tree
Showing 28 changed files with 2,487 additions and 9 deletions.
42 changes: 42 additions & 0 deletions lexicons/app/bsky/feed/getListFeed.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
{
"lexicon": 1,
"id": "app.bsky.feed.getListFeed",
"defs": {
"main": {
"type": "query",
"description": "A view of a recent posts from actors in a list",
"parameters": {
"type": "params",
"required": ["list"],
"properties": {
"list": { "type": "string", "format": "at-uri" },
"limit": {
"type": "integer",
"minimum": 1,
"maximum": 100,
"default": 50
},
"cursor": { "type": "string" }
}
},
"output": {
"encoding": "application/json",
"schema": {
"type": "object",
"required": ["feed"],
"properties": {
"cursor": { "type": "string" },
"feed": {
"type": "array",
"items": {
"type": "ref",
"ref": "app.bsky.feed.defs#feedViewPost"
}
}
}
}
},
"errors": [{ "name": "UnknownList" }]
}
}
}
9 changes: 8 additions & 1 deletion lexicons/app/bsky/graph/defs.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,19 @@
},
"listPurpose": {
"type": "string",
"knownValues": ["app.bsky.graph.defs#modlist"]
"knownValues": [
"app.bsky.graph.defs#modlist",
"app.bsky.graph.defs#curatelist"
]
},
"modlist": {
"type": "token",
"description": "A list of actors to apply an aggregate moderation action (mute/block) on"
},
"curatelist": {
"type": "token",
"description": "A list of actors used for curation purposes such as list feeds or interaction gating"
},
"listViewerState": {
"type": "object",
"properties": {
Expand Down
14 changes: 14 additions & 0 deletions packages/api/src/client/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ import * as AppBskyFeedGetFeedGenerator from './types/app/bsky/feed/getFeedGener
import * as AppBskyFeedGetFeedGenerators from './types/app/bsky/feed/getFeedGenerators'
import * as AppBskyFeedGetFeedSkeleton from './types/app/bsky/feed/getFeedSkeleton'
import * as AppBskyFeedGetLikes from './types/app/bsky/feed/getLikes'
import * as AppBskyFeedGetListFeed from './types/app/bsky/feed/getListFeed'
import * as AppBskyFeedGetPostThread from './types/app/bsky/feed/getPostThread'
import * as AppBskyFeedGetPosts from './types/app/bsky/feed/getPosts'
import * as AppBskyFeedGetRepostedBy from './types/app/bsky/feed/getRepostedBy'
Expand Down Expand Up @@ -226,6 +227,7 @@ export * as AppBskyFeedGetFeedGenerator from './types/app/bsky/feed/getFeedGener
export * as AppBskyFeedGetFeedGenerators from './types/app/bsky/feed/getFeedGenerators'
export * as AppBskyFeedGetFeedSkeleton from './types/app/bsky/feed/getFeedSkeleton'
export * as AppBskyFeedGetLikes from './types/app/bsky/feed/getLikes'
export * as AppBskyFeedGetListFeed from './types/app/bsky/feed/getListFeed'
export * as AppBskyFeedGetPostThread from './types/app/bsky/feed/getPostThread'
export * as AppBskyFeedGetPosts from './types/app/bsky/feed/getPosts'
export * as AppBskyFeedGetRepostedBy from './types/app/bsky/feed/getRepostedBy'
Expand Down Expand Up @@ -279,6 +281,7 @@ export const COM_ATPROTO_MODERATION = {
}
export const APP_BSKY_GRAPH = {
DefsModlist: 'app.bsky.graph.defs#modlist',
DefsCuratelist: 'app.bsky.graph.defs#curatelist',
}

export class AtpBaseClient {
Expand Down Expand Up @@ -1361,6 +1364,17 @@ export class FeedNS {
})
}

getListFeed(
params?: AppBskyFeedGetListFeed.QueryParams,
opts?: AppBskyFeedGetListFeed.CallOptions,
): Promise<AppBskyFeedGetListFeed.Response> {
return this._service.xrpc
.call('app.bsky.feed.getListFeed', params, undefined, opts)
.catch((e) => {
throw AppBskyFeedGetListFeed.toKnownErr(e)
})
}

getPostThread(
params?: AppBskyFeedGetPostThread.QueryParams,
opts?: AppBskyFeedGetPostThread.CallOptions,
Expand Down
64 changes: 63 additions & 1 deletion packages/api/src/client/lexicons.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5277,6 +5277,59 @@ export const schemaDict = {
},
},
},
AppBskyFeedGetListFeed: {
lexicon: 1,
id: 'app.bsky.feed.getListFeed',
defs: {
main: {
type: 'query',
description: 'A view of a recent posts from actors in a list',
parameters: {
type: 'params',
required: ['list'],
properties: {
list: {
type: 'string',
format: 'at-uri',
},
limit: {
type: 'integer',
minimum: 1,
maximum: 100,
default: 50,
},
cursor: {
type: 'string',
},
},
},
output: {
encoding: 'application/json',
schema: {
type: 'object',
required: ['feed'],
properties: {
cursor: {
type: 'string',
},
feed: {
type: 'array',
items: {
type: 'ref',
ref: 'lex:app.bsky.feed.defs#feedViewPost',
},
},
},
},
},
errors: [
{
name: 'UnknownList',
},
],
},
},
},
AppBskyFeedGetPostThread: {
lexicon: 1,
id: 'app.bsky.feed.getPostThread',
Expand Down Expand Up @@ -5804,13 +5857,21 @@ export const schemaDict = {
},
listPurpose: {
type: 'string',
knownValues: ['app.bsky.graph.defs#modlist'],
knownValues: [
'app.bsky.graph.defs#modlist',
'app.bsky.graph.defs#curatelist',
],
},
modlist: {
type: 'token',
description:
'A list of actors to apply an aggregate moderation action (mute/block) on',
},
curatelist: {
type: 'token',
description:
'A list of actors used for curation purposes such as list feeds or interaction gating',
},
listViewerState: {
type: 'object',
properties: {
Expand Down Expand Up @@ -6984,6 +7045,7 @@ export const ids = {
AppBskyFeedGetFeedGenerators: 'app.bsky.feed.getFeedGenerators',
AppBskyFeedGetFeedSkeleton: 'app.bsky.feed.getFeedSkeleton',
AppBskyFeedGetLikes: 'app.bsky.feed.getLikes',
AppBskyFeedGetListFeed: 'app.bsky.feed.getListFeed',
AppBskyFeedGetPostThread: 'app.bsky.feed.getPostThread',
AppBskyFeedGetPosts: 'app.bsky.feed.getPosts',
AppBskyFeedGetRepostedBy: 'app.bsky.feed.getRepostedBy',
Expand Down
46 changes: 46 additions & 0 deletions packages/api/src/client/types/app/bsky/feed/getListFeed.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/**
* GENERATED CODE - DO NOT MODIFY
*/
import { Headers, XRPCError } from '@atproto/xrpc'
import { ValidationResult, BlobRef } from '@atproto/lexicon'
import { isObj, hasProp } from '../../../../util'
import { lexicons } from '../../../../lexicons'
import { CID } from 'multiformats/cid'
import * as AppBskyFeedDefs from './defs'

export interface QueryParams {
list: string
limit?: number
cursor?: string
}

export type InputSchema = undefined

export interface OutputSchema {
cursor?: string
feed: AppBskyFeedDefs.FeedViewPost[]
[k: string]: unknown
}

export interface CallOptions {
headers?: Headers
}

export interface Response {
success: boolean
headers: Headers
data: OutputSchema
}

export class UnknownListError extends XRPCError {
constructor(src: XRPCError) {
super(src.status, src.error, src.message, src.headers)
}
}

export function toKnownErr(e: any) {
if (e instanceof XRPCError) {
if (e.error === 'UnknownList') return new UnknownListError(e)
}
return e
}
7 changes: 6 additions & 1 deletion packages/api/src/client/types/app/bsky/graph/defs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,15 @@ export function validateListItemView(v: unknown): ValidationResult {
return lexicons.validate('app.bsky.graph.defs#listItemView', v)
}

export type ListPurpose = 'app.bsky.graph.defs#modlist' | (string & {})
export type ListPurpose =
| 'app.bsky.graph.defs#modlist'
| 'app.bsky.graph.defs#curatelist'
| (string & {})

/** A list of actors to apply an aggregate moderation action (mute/block) on */
export const MODLIST = 'app.bsky.graph.defs#modlist'
/** A list of actors used for curation purposes such as list feeds or interaction gating */
export const CURATELIST = 'app.bsky.graph.defs#curatelist'

export interface ListViewerState {
muted?: boolean
Expand Down
129 changes: 129 additions & 0 deletions packages/bsky/src/api/app/bsky/feed/getListFeed.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
import { Server } from '../../../../lexicon'
import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getListFeed'
import { FeedKeyset, getFeedDateThreshold } from '../util/feed'
import { paginate } from '../../../../db/pagination'
import AppContext from '../../../../context'
import { setRepoRev } from '../../../util'
import { Database } from '../../../../db'
import {
FeedHydrationState,
FeedRow,
FeedService,
} from '../../../../services/feed'
import { ActorService } from '../../../../services/actor'
import { GraphService } from '../../../../services/graph'
import { createPipeline } from '../../../../pipeline'

export default function (server: Server, ctx: AppContext) {
const getListFeed = createPipeline(
skeleton,
hydration,
noBlocksOrMutes,
presentation,
)
server.app.bsky.feed.getListFeed({
auth: ctx.authOptionalVerifier,
handler: async ({ params, auth, res }) => {
const viewer = auth.credentials.did
const db = ctx.db.getReplica()
const actorService = ctx.services.actor(db)
const feedService = ctx.services.feed(db)
const graphService = ctx.services.graph(db)

const [result, repoRev] = await Promise.all([
getListFeed(
{ ...params, viewer },
{ db, actorService, feedService, graphService },
),
actorService.getRepoRev(viewer),
])

setRepoRev(res, repoRev)

return {
encoding: 'application/json',
body: result,
}
},
})
}

export const skeleton = async (
params: Params,
ctx: Context,
): Promise<SkeletonState> => {
const { list, cursor, limit } = params
const { db } = ctx
const { ref } = db.db.dynamic

const keyset = new FeedKeyset(ref('post.sortAt'), ref('post.cid'))
const sortFrom = keyset.unpack(cursor)?.primary

let builder = ctx.feedService
.selectPostQb()
.innerJoin('list_item', 'list_item.subjectDid', 'post.creator')
.where('list_item.listUri', '=', list)
.where('post.sortAt', '>', getFeedDateThreshold(sortFrom, 3))

builder = paginate(builder, {
limit,
cursor,
keyset,
tryIndex: true,
})
const feedItems = await builder.execute()

return {
params,
feedItems,
cursor: keyset.packFromResult(feedItems),
}
}

const hydration = async (state: SkeletonState, ctx: Context) => {
const { feedService } = ctx
const { params, feedItems } = state
const refs = feedService.feedItemRefs(feedItems)
const hydrated = await feedService.feedHydration({
...refs,
viewer: params.viewer,
})
return { ...state, ...hydrated }
}

const noBlocksOrMutes = (state: HydrationState) => {
const { viewer } = state.params
if (!viewer) return state
state.feedItems = state.feedItems.filter(
(item) =>
!state.bam.block([viewer, item.postAuthorDid]) &&
!state.bam.mute([viewer, item.postAuthorDid]),
)
return state
}

const presentation = (state: HydrationState, ctx: Context) => {
const { feedService } = ctx
const { feedItems, cursor, params } = state
const feed = feedService.views.formatFeed(feedItems, state, {
viewer: params.viewer,
})
return { feed, cursor }
}

type Context = {
db: Database
actorService: ActorService
feedService: FeedService
graphService: GraphService
}

type Params = QueryParams & { viewer: string | null }

type SkeletonState = {
params: Params
feedItems: FeedRow[]
cursor?: string
}

type HydrationState = SkeletonState & FeedHydrationState
Loading

0 comments on commit ddb79cb

Please sign in to comment.