diff --git a/packages/bsky/src/feed-gen/best-of-follows.ts b/packages/bsky/src/feed-gen/best-of-follows.ts deleted file mode 100644 index 33c70ea81a4..00000000000 --- a/packages/bsky/src/feed-gen/best-of-follows.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' -import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' -import { AlgoHandler, AlgoResponse } from './types' -import { GenericKeyset, paginate } from '../db/pagination' -import AppContext from '../context' - -const handler: AlgoHandler = async ( - ctx: AppContext, - params: SkeletonParams, - viewer: string | null, -): Promise => { - if (!viewer) { - throw new AuthRequiredError('This feed requires being logged-in') - } - - const { limit, cursor } = params - const db = ctx.db.getReplica('feed') - const feedService = ctx.services.feed(db) - const { ref } = db.db.dynamic - - // candidates are ranked within a materialized view by like count, depreciated over time. - - let builder = feedService - .selectPostQb() - .innerJoin('algo_whats_hot_view as candidate', 'candidate.uri', 'post.uri') - .where((qb) => - qb - .where('post.creator', '=', viewer) - .orWhereExists((inner) => - inner - .selectFrom('follow') - .where('follow.creator', '=', viewer) - .whereRef('follow.subjectDid', '=', 'post.creator'), - ), - ) - .select('candidate.score') - .select('candidate.cid') - - const keyset = new ScoreKeyset(ref('candidate.score'), ref('candidate.cid')) - builder = paginate(builder, { limit, cursor, keyset }) - - const feedItems = await builder.execute() - - return { - feedItems, - cursor: keyset.packFromResult(feedItems), - } -} - -export default handler - -type Result = { score: number; cid: string } -type LabeledResult = { primary: number; secondary: string } -export class ScoreKeyset extends GenericKeyset { - labelResult(result: Result) { - return { - primary: result.score, - secondary: result.cid, - } - } - labeledResultToCursor(labeled: LabeledResult) { - return { - primary: Math.round(labeled.primary).toString(), - secondary: labeled.secondary, - } - } - cursorToLabeledResult(cursor: { primary: string; secondary: string }) { - const score = parseInt(cursor.primary, 10) - if (isNaN(score)) { - throw new InvalidRequestError('Malformed cursor') - } - return { - primary: score, - secondary: cursor.secondary, - } - } -} diff --git a/packages/bsky/src/feed-gen/index.ts b/packages/bsky/src/feed-gen/index.ts index d00d22c59d9..5109d32416c 100644 --- a/packages/bsky/src/feed-gen/index.ts +++ b/packages/bsky/src/feed-gen/index.ts @@ -1,9 +1,7 @@ import { AtUri } from '@atproto/syntax' import { ids } from '../lexicon/lexicons' -import withFriends from './with-friends' import bskyTeam from './bsky-team' import hotClassic from './hot-classic' -import bestOfFollows from './best-of-follows' import mutuals from './mutuals' import { MountedAlgos } from './types' @@ -13,9 +11,7 @@ const feedgenUri = (did, name) => // These are custom algorithms that will be mounted directly onto an AppView // Feel free to remove, update to your own, or serve the following logic at a record that you control export const makeAlgos = (did: string): MountedAlgos => ({ - [feedgenUri(did, 'with-friends')]: withFriends, [feedgenUri(did, 'bsky-team')]: bskyTeam, [feedgenUri(did, 'hot-classic')]: hotClassic, - [feedgenUri(did, 'best-of-follows')]: bestOfFollows, [feedgenUri(did, 'mutuals')]: mutuals, }) diff --git a/packages/bsky/src/feed-gen/whats-hot.ts b/packages/bsky/src/feed-gen/whats-hot.ts deleted file mode 100644 index 2376b98f185..00000000000 --- a/packages/bsky/src/feed-gen/whats-hot.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { NotEmptyArray } from '@atproto/common' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' -import { AlgoHandler, AlgoResponse } from './types' -import { GenericKeyset, paginate } from '../db/pagination' -import AppContext from '../context' -import { valuesList } from '../db/util' -import { sql } from 'kysely' -import { FeedItemType } from '../services/feed/types' - -const NO_WHATS_HOT_LABELS: NotEmptyArray = [ - '!no-promote', - 'corpse', - 'self-harm', - 'porn', - 'sexual', - 'nudity', - 'underwear', -] - -const handler: AlgoHandler = async ( - ctx: AppContext, - params: SkeletonParams, - _viewer: string | null, -): Promise => { - const { limit, cursor } = params - const db = ctx.db.getReplica('feed') - - const { ref } = db.db.dynamic - - // candidates are ranked within a materialized view by like count, depreciated over time. - - let builder = db.db - .selectFrom('algo_whats_hot_view as candidate') - .innerJoin('post', 'post.uri', 'candidate.uri') - .leftJoin('post_embed_record', 'post_embed_record.postUri', 'candidate.uri') - .whereNotExists((qb) => - qb - .selectFrom('label') - .selectAll() - .whereRef('val', 'in', valuesList(NO_WHATS_HOT_LABELS)) - .where('neg', '=', false) - .where((clause) => - clause - .whereRef('label.uri', '=', ref('post.creator')) - .orWhereRef('label.uri', '=', ref('post.uri')) - .orWhereRef('label.uri', '=', ref('post_embed_record.embedUri')), - ), - ) - .select([ - sql`${'post'}`.as('type'), - 'post.uri as uri', - 'post.uri as postUri', - 'post.creator as originatorDid', - 'post.creator as postAuthorDid', - 'post.replyParent as replyParent', - 'post.replyRoot as replyRoot', - 'post.indexedAt as sortAt', - 'candidate.score', - 'candidate.cid', - ]) - - const keyset = new ScoreKeyset(ref('candidate.score'), ref('candidate.cid')) - builder = paginate(builder, { limit, cursor, keyset }) - - const feedItems = await builder.execute() - - return { - feedItems, - cursor: keyset.packFromResult(feedItems), - } -} - -export default handler - -type Result = { score: number; cid: string } -type LabeledResult = { primary: number; secondary: string } -export class ScoreKeyset extends GenericKeyset { - labelResult(result: Result) { - return { - primary: result.score, - secondary: result.cid, - } - } - labeledResultToCursor(labeled: LabeledResult) { - return { - primary: Math.round(labeled.primary).toString(), - secondary: labeled.secondary, - } - } - cursorToLabeledResult(cursor: { primary: string; secondary: string }) { - const score = parseInt(cursor.primary, 10) - if (isNaN(score)) { - throw new InvalidRequestError('Malformed cursor') - } - return { - primary: score, - secondary: cursor.secondary, - } - } -} diff --git a/packages/bsky/src/feed-gen/with-friends.ts b/packages/bsky/src/feed-gen/with-friends.ts deleted file mode 100644 index 1e6d345ffcc..00000000000 --- a/packages/bsky/src/feed-gen/with-friends.ts +++ /dev/null @@ -1,43 +0,0 @@ -import AppContext from '../context' -import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' -import { paginate } from '../db/pagination' -import { AlgoHandler, AlgoResponse } from './types' -import { FeedKeyset, getFeedDateThreshold } from '../api/app/bsky/util/feed' -import { AuthRequiredError } from '@atproto/xrpc-server' - -const handler: AlgoHandler = async ( - ctx: AppContext, - params: SkeletonParams, - viewer: string | null, -): Promise => { - if (!viewer) { - throw new AuthRequiredError('This feed requires being logged-in') - } - - const { cursor, limit = 50 } = params - const db = ctx.db.getReplica('feed') - const feedService = ctx.services.feed(db) - const { ref } = db.db.dynamic - - const keyset = new FeedKeyset(ref('post.sortAt'), ref('post.cid')) - const sortFrom = keyset.unpack(cursor)?.primary - - let postsQb = feedService - .selectPostQb() - .innerJoin('follow', 'follow.subjectDid', 'post.creator') - .innerJoin('post_agg', 'post_agg.uri', 'post.uri') - .where('post_agg.likeCount', '>=', 5) - .where('follow.creator', '=', viewer) - .where('post.sortAt', '>', getFeedDateThreshold(sortFrom)) - - postsQb = paginate(postsQb, { limit, cursor, keyset, tryIndex: true }) - - const feedItems = await postsQb.execute() - - return { - feedItems, - cursor: keyset.packFromResult(feedItems), - } -} - -export default handler diff --git a/packages/bsky/tests/algos/whats-hot.test.ts b/packages/bsky/tests/algos/whats-hot.test.ts deleted file mode 100644 index 9fb93a8ce50..00000000000 --- a/packages/bsky/tests/algos/whats-hot.test.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { HOUR } from '@atproto/common' -import AtpAgent, { AtUri } from '@atproto/api' -import { TestNetwork, SeedClient } from '@atproto/dev-env' -import basicSeed from '../seeds/basic' -import { makeAlgos } from '../../src' - -describe.skip('algo whats-hot', () => { - let network: TestNetwork - let agent: AtpAgent - let sc: SeedClient - - // account dids, for convenience - let alice: string - let bob: string - let carol: string - - const feedPublisherDid = 'did:example:feed-publisher' - const feedUri = AtUri.make( - feedPublisherDid, - 'app.bsky.feed.generator', - 'whats-hot', - ).toString() - - beforeAll(async () => { - network = await TestNetwork.create({ - dbPostgresSchema: 'bsky_algo_whats_hot', - bsky: { algos: makeAlgos(feedPublisherDid) }, - }) - agent = new AtpAgent({ service: network.bsky.url }) - sc = network.getSeedClient() - await basicSeed(sc) - - alice = sc.dids.alice - bob = sc.dids.bob - carol = sc.dids.carol - await network.processAll() - await network.bsky.processAll() - }) - - afterAll(async () => { - await network.close() - }) - - it('returns well liked posts', async () => { - const img = await sc.uploadFile( - alice, - 'tests/sample-img/key-landscape-small.jpg', - 'image/jpeg', - ) - const one = await sc.post(carol, 'carol is in the chat') - const two = await sc.post(carol, "it's me, carol") - const three = await sc.post(alice, 'first post', undefined, [img]) - const four = await sc.post(bob, 'bobby boi') - const five = await sc.post(bob, 'another one') - - for (let i = 0; i < 20; i++) { - const name = `user${i}` - await sc.createAccount(name, { - handle: `user${i}.test`, - email: `user${i}@test.com`, - password: 'password', - }) - await sc.like(sc.dids[name], three.ref) // will be down-regulated by time - if (i > 3) { - await sc.like(sc.dids[name], one.ref) - } - if (i > 5) { - await sc.like(sc.dids[name], two.ref) - } - if (i > 7) { - await sc.like(sc.dids[name], four.ref) - await sc.like(sc.dids[name], five.ref) - } - } - await network.bsky.processAll() - - // move the 3rd post 5 hours into the past to check gravity - await network.bsky.ctx.db - .getPrimary() - .db.updateTable('post') - .where('uri', '=', three.ref.uriStr) - .set({ indexedAt: new Date(Date.now() - 5 * HOUR).toISOString() }) - .execute() - - await network.bsky.ctx.db - .getPrimary() - .refreshMaterializedView('algo_whats_hot_view') - - const res = await agent.api.app.bsky.feed.getFeed( - { feed: feedUri }, - { headers: await network.serviceHeaders(alice) }, - ) - expect(res.data.feed[0].post.uri).toBe(one.ref.uriStr) - expect(res.data.feed[1].post.uri).toBe(two.ref.uriStr) - const indexOfThird = res.data.feed.findIndex( - (item) => item.post.uri === three.ref.uriStr, - ) - // doesn't quite matter where this cam in but it should be down-regulated pretty severely from gravity - expect(indexOfThird).toBeGreaterThan(3) - }) - - it('paginates', async () => { - const res = await agent.api.app.bsky.feed.getFeed( - { feed: feedUri }, - { headers: await network.serviceHeaders(alice) }, - ) - const first = await agent.api.app.bsky.feed.getFeed( - { feed: feedUri, limit: 3 }, - { headers: await network.serviceHeaders(alice) }, - ) - const second = await agent.api.app.bsky.feed.getFeed( - { feed: feedUri, cursor: first.data.cursor }, - { headers: await network.serviceHeaders(alice) }, - ) - - expect([...first.data.feed, ...second.data.feed]).toEqual(res.data.feed) - }) -}) diff --git a/packages/bsky/tests/algos/with-friends.test.ts b/packages/bsky/tests/algos/with-friends.test.ts deleted file mode 100644 index 2c5339849c8..00000000000 --- a/packages/bsky/tests/algos/with-friends.test.ts +++ /dev/null @@ -1,145 +0,0 @@ -import AtpAgent, { AtUri } from '@atproto/api' -import userSeed from '../seeds/users' -import { makeAlgos } from '../../src' -import { TestNetwork, SeedClient, RecordRef } from '@atproto/dev-env' - -describe.skip('algo with friends', () => { - let network: TestNetwork - let agent: AtpAgent - let sc: SeedClient - - // account dids, for convenience - let alice: string - let bob: string - let carol: string - let dan: string - - const feedPublisherDid = 'did:example:feed-publisher' - const feedUri = AtUri.make( - feedPublisherDid, - 'app.bsky.feed.generator', - 'with-friends', - ).toString() - - beforeAll(async () => { - network = await TestNetwork.create({ - dbPostgresSchema: 'bsky_algo_with_friends', - bsky: { algos: makeAlgos(feedPublisherDid) }, - }) - agent = new AtpAgent({ service: network.bsky.url }) - sc = network.getSeedClient() - await userSeed(sc) - - alice = sc.dids.alice - bob = sc.dids.bob - carol = sc.dids.carol - dan = sc.dids.dan - await network.processAll() - await network.bsky.processAll() - }) - - afterAll(async () => { - await network.close() - }) - - let expectedFeed: string[] - - it('setup', async () => { - for (let i = 0; i < 10; i++) { - const name = `user${i}` - await sc.createAccount(name, { - handle: `user${i}.test`, - email: `user${i}@test.com`, - password: 'password', - }) - } - - const hitLikeThreshold = async (ref: RecordRef) => { - for (let i = 0; i < 10; i++) { - const name = `user${i}` - await sc.like(sc.dids[name], ref) - } - } - - // bob and dan are mutuals of alice, all userN are out-of-network. - await sc.follow(alice, bob) - await sc.follow(alice, carol) - await sc.follow(alice, dan) - await sc.follow(bob, alice) - await sc.follow(dan, alice) - const one = await sc.post(bob, 'one') - const two = await sc.post(bob, 'two') - const three = await sc.post(carol, 'three') - const four = await sc.post(carol, 'four') - const five = await sc.post(dan, 'five') - const six = await sc.post(dan, 'six') - const seven = await sc.post(sc.dids.user0, 'seven') - const eight = await sc.post(sc.dids.user0, 'eight') - const nine = await sc.post(sc.dids.user1, 'nine') - const ten = await sc.post(sc.dids.user1, 'ten') - - // 1, 2, 3, 4, 6, 8, 10 hit like threshold - await hitLikeThreshold(one.ref) - await hitLikeThreshold(two.ref) - await hitLikeThreshold(three.ref) - await hitLikeThreshold(four.ref) - await hitLikeThreshold(six.ref) - await hitLikeThreshold(eight.ref) - await hitLikeThreshold(ten.ref) - - // 1, 4, 7, 8, 10 liked by mutual - await sc.like(bob, one.ref) - await sc.like(dan, four.ref) - await sc.like(bob, seven.ref) - await sc.like(dan, eight.ref) - await sc.like(bob, nine.ref) - await sc.like(dan, ten.ref) - - // all liked by non-mutual - await sc.like(carol, one.ref) - await sc.like(carol, two.ref) - await sc.like(carol, three.ref) - await sc.like(carol, four.ref) - await sc.like(carol, five.ref) - await sc.like(carol, six.ref) - await sc.like(carol, seven.ref) - await sc.like(carol, eight.ref) - await sc.like(carol, nine.ref) - await sc.like(carol, ten.ref) - - await network.bsky.processAll() - - expectedFeed = [ - ten.ref.uriStr, - eight.ref.uriStr, - four.ref.uriStr, - one.ref.uriStr, - ] - }) - - it('returns popular in & out of network posts', async () => { - const res = await agent.api.app.bsky.feed.getFeed( - { feed: feedUri }, - { headers: await network.serviceHeaders(alice) }, - ) - const feedUris = res.data.feed.map((i) => i.post.uri) - expect(feedUris).toEqual(expectedFeed) - }) - - it('paginates', async () => { - const res = await agent.api.app.bsky.feed.getFeed( - { feed: feedUri }, - { headers: await network.serviceHeaders(alice) }, - ) - const first = await agent.api.app.bsky.feed.getFeed( - { feed: feedUri, limit: 2 }, - { headers: await network.serviceHeaders(alice) }, - ) - const second = await agent.api.app.bsky.feed.getFeed( - { feed: feedUri, cursor: first.data.cursor }, - { headers: await network.serviceHeaders(alice) }, - ) - - expect([...first.data.feed, ...second.data.feed]).toEqual(res.data.feed) - }) -})