diff --git a/packages/mod-service/src/api/app/bsky/actor/getProfile.ts b/packages/mod-service/src/api/app/bsky/actor/getProfile.ts deleted file mode 100644 index 0dacf02bcf5..00000000000 --- a/packages/mod-service/src/api/app/bsky/actor/getProfile.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/actor/getProfile' -import { softDeleted } from '../../../../db/util' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { Actor } from '../../../../db/tables/actor' -import { - ActorService, - ProfileDetailHydrationState, -} from '../../../../services/actor' -import { setRepoRev } from '../../../util' -import { createPipeline, noRules } from '../../../../pipeline' -import { ModerationService } from '../../../../services/moderation' - -export default function (server: Server, ctx: AppContext) { - const getProfile = createPipeline(skeleton, hydration, noRules, presentation) - server.app.bsky.actor.getProfile({ - auth: ctx.authOptionalAccessOrRoleVerifier, - handler: async ({ auth, params, res }) => { - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const modService = ctx.services.moderation(ctx.db.getPrimary()) - const viewer = 'did' in auth.credentials ? auth.credentials.did : null - const canViewTakendownProfile = - auth.credentials.type === 'role' && auth.credentials.triage - - const [result, repoRev] = await Promise.allSettled([ - getProfile( - { ...params, viewer, canViewTakendownProfile }, - { db, actorService, modService }, - ), - actorService.getRepoRev(viewer), - ]) - - if (repoRev.status === 'fulfilled') { - setRepoRev(res, repoRev.value) - } - if (result.status === 'rejected') { - throw result.reason - } - - return { - encoding: 'application/json', - body: result.value, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { actorService, modService } = ctx - const { canViewTakendownProfile } = params - const actor = await actorService.getActor(params.actor, true) - if (!actor) { - throw new InvalidRequestError('Profile not found') - } - if (!canViewTakendownProfile && softDeleted(actor)) { - const isSuspended = await modService.isSubjectSuspended(actor.did) - if (isSuspended) { - throw new InvalidRequestError( - 'Account has been temporarily suspended', - 'AccountTakedown', - ) - } else { - throw new InvalidRequestError( - 'Account has been taken down', - 'AccountTakedown', - ) - } - } - return { params, actor } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { actorService } = ctx - const { params, actor } = state - const { viewer, canViewTakendownProfile } = params - const hydration = await actorService.views.profileDetailHydration( - [actor.did], - { viewer, includeSoftDeleted: canViewTakendownProfile }, - ) - return { ...state, ...hydration } -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { actorService } = ctx - const { params, actor } = state - const { viewer } = params - const profiles = actorService.views.profileDetailPresentation( - [actor.did], - state, - { viewer }, - ) - const profile = profiles[actor.did] - if (!profile) { - throw new InvalidRequestError('Profile not found') - } - return profile -} - -type Context = { - db: Database - actorService: ActorService - modService: ModerationService -} - -type Params = QueryParams & { - viewer: string | null - canViewTakendownProfile: boolean -} - -type SkeletonState = { params: Params; actor: Actor } - -type HydrationState = SkeletonState & ProfileDetailHydrationState diff --git a/packages/mod-service/src/api/app/bsky/actor/getProfiles.ts b/packages/mod-service/src/api/app/bsky/actor/getProfiles.ts deleted file mode 100644 index f2e0eb3fd50..00000000000 --- a/packages/mod-service/src/api/app/bsky/actor/getProfiles.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/actor/getProfiles' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { - ActorService, - ProfileDetailHydrationState, -} from '../../../../services/actor' -import { setRepoRev } from '../../../util' -import { createPipeline, noRules } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getProfile = createPipeline(skeleton, hydration, noRules, presentation) - server.app.bsky.actor.getProfiles({ - auth: ctx.authOptionalVerifier, - handler: async ({ auth, params, res }) => { - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const viewer = auth.credentials.did - - const [result, repoRev] = await Promise.all([ - getProfile({ ...params, viewer }, { db, actorService }), - actorService.getRepoRev(viewer), - ]) - - setRepoRev(res, repoRev) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { actorService } = ctx - const actors = await actorService.getActors(params.actors) - return { params, dids: actors.map((a) => a.did) } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { actorService } = ctx - const { params, dids } = state - const { viewer } = params - const hydration = await actorService.views.profileDetailHydration(dids, { - viewer, - }) - return { ...state, ...hydration } -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { actorService } = ctx - const { params, dids } = state - const { viewer } = params - const profiles = actorService.views.profileDetailPresentation(dids, state, { - viewer, - }) - const profileViews = mapDefined(dids, (did) => profiles[did]) - return { profiles: profileViews } -} - -type Context = { - db: Database - actorService: ActorService -} - -type Params = QueryParams & { - viewer: string | null -} - -type SkeletonState = { params: Params; dids: string[] } - -type HydrationState = SkeletonState & ProfileDetailHydrationState diff --git a/packages/mod-service/src/api/app/bsky/actor/getSuggestions.ts b/packages/mod-service/src/api/app/bsky/actor/getSuggestions.ts deleted file mode 100644 index f68ba68eb66..00000000000 --- a/packages/mod-service/src/api/app/bsky/actor/getSuggestions.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { mapDefined } from '@atproto/common' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { Actor } from '../../../../db/tables/actor' -import { notSoftDeletedClause } from '../../../../db/util' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/actor/getSuggestions' -import { createPipeline } from '../../../../pipeline' -import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { BlockAndMuteState, GraphService } from '../../../../services/graph' - -export default function (server: Server, ctx: AppContext) { - const getSuggestions = createPipeline( - skeleton, - hydration, - noBlocksOrMutes, - presentation, - ) - server.app.bsky.actor.getSuggestions({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth }) => { - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const graphService = ctx.services.graph(db) - const viewer = auth.credentials.did - - const result = await getSuggestions( - { ...params, viewer }, - { db, actorService, graphService }, - ) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { db } = ctx - const { viewer } = params - const alreadyIncluded = parseCursor(params.cursor) - const { ref } = db.db.dynamic - const suggestions = await db.db - .selectFrom('suggested_follow') - .innerJoin('actor', 'actor.did', 'suggested_follow.did') - .where(notSoftDeletedClause(ref('actor'))) - .where('suggested_follow.did', '!=', viewer ?? '') - .whereNotExists((qb) => - qb - .selectFrom('follow') - .selectAll() - .where('creator', '=', viewer ?? '') - .whereRef('subjectDid', '=', ref('actor.did')), - ) - .if(alreadyIncluded.length > 0, (qb) => - qb.where('suggested_follow.order', 'not in', alreadyIncluded), - ) - .selectAll() - .orderBy('suggested_follow.order', 'asc') - .execute() - - // always include first two - const firstTwo = suggestions.filter( - (row) => row.order === 1 || row.order === 2, - ) - const rest = suggestions.filter((row) => row.order !== 1 && row.order !== 2) - const limited = firstTwo.concat(shuffle(rest)).slice(0, params.limit) - - // if the result set ends up getting larger, consider using a seed included in the cursor for for the randomized shuffle - const cursor = - limited.length > 0 - ? limited - .map((row) => row.order.toString()) - .concat(alreadyIncluded.map((id) => id.toString())) - .join(':') - : undefined - - return { params, suggestions: limited, cursor } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { graphService, actorService } = ctx - const { params, suggestions } = state - const { viewer } = params - const [actors, bam] = await Promise.all([ - actorService.views.profiles(suggestions, viewer), - graphService.getBlockAndMuteState( - viewer ? suggestions.map((sug) => [viewer, sug.did]) : [], - ), - ]) - return { ...state, bam, actors } -} - -const noBlocksOrMutes = (state: HydrationState) => { - const { viewer } = state.params - if (!viewer) return state - state.suggestions = state.suggestions.filter( - (item) => - !state.bam.block([viewer, item.did]) && - !state.bam.mute([viewer, item.did]), - ) - return state -} - -const presentation = (state: HydrationState) => { - const { suggestions, actors, cursor } = state - const suggestedActors = mapDefined(suggestions, (sug) => actors[sug.did]) - return { actors: suggestedActors, cursor } -} - -const parseCursor = (cursor?: string): number[] => { - if (!cursor) { - return [] - } - try { - return cursor - .split(':') - .map((id) => parseInt(id, 10)) - .filter((id) => !isNaN(id)) - } catch { - return [] - } -} - -const shuffle = (arr: T[]): T[] => { - return arr - .map((value) => ({ value, sort: Math.random() })) - .sort((a, b) => a.sort - b.sort) - .map(({ value }) => value) -} - -type Context = { - db: Database - actorService: ActorService - graphService: GraphService -} - -type Params = QueryParams & { viewer: string | null } - -type SkeletonState = { params: Params; suggestions: Actor[]; cursor?: string } - -type HydrationState = SkeletonState & { - bam: BlockAndMuteState - actors: ActorInfoMap -} diff --git a/packages/mod-service/src/api/app/bsky/actor/searchActors.ts b/packages/mod-service/src/api/app/bsky/actor/searchActors.ts deleted file mode 100644 index 66e934ac0b3..00000000000 --- a/packages/mod-service/src/api/app/bsky/actor/searchActors.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { sql } from 'kysely' -import AppContext from '../../../../context' -import { Server } from '../../../../lexicon' -import { - cleanQuery, - getUserSearchQuery, - SearchKeyset, -} from '../../../../services/util/search' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.actor.searchActors({ - auth: ctx.authOptionalVerifier, - handler: async ({ auth, params }) => { - const { cursor, limit } = params - const requester = auth.credentials.did - const rawQuery = params.q ?? params.term - const query = cleanQuery(rawQuery || '') - const db = ctx.db.getReplica('search') - - let results: string[] - let resCursor: string | undefined - if (ctx.searchAgent) { - const res = - await ctx.searchAgent.api.app.bsky.unspecced.searchActorsSkeleton({ - q: query, - cursor, - limit, - }) - results = res.data.actors.map((a) => a.did) - resCursor = res.data.cursor - } else { - const res = query - ? await getUserSearchQuery(db, { query, limit, cursor }) - .select('distance') - .selectAll('actor') - .execute() - : [] - results = res.map((a) => a.did) - const keyset = new SearchKeyset(sql``, sql``) - resCursor = keyset.packFromResult(res) - } - - const actors = await ctx.services - .actor(db) - .views.profiles(results, requester) - - const SKIP = [] - const filtered = results.flatMap((did) => { - const actor = actors[did] - if (!actor) return SKIP - if (actor.viewer?.blocking || actor.viewer?.blockedBy) return SKIP - return actor - }) - - return { - encoding: 'application/json', - body: { - cursor: resCursor, - actors: filtered, - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/actor/searchActorsTypeahead.ts b/packages/mod-service/src/api/app/bsky/actor/searchActorsTypeahead.ts deleted file mode 100644 index da612edcc87..00000000000 --- a/packages/mod-service/src/api/app/bsky/actor/searchActorsTypeahead.ts +++ /dev/null @@ -1,56 +0,0 @@ -import AppContext from '../../../../context' -import { Server } from '../../../../lexicon' -import { - cleanQuery, - getUserSearchQuerySimple, -} from '../../../../services/util/search' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.actor.searchActorsTypeahead({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth }) => { - const { limit } = params - const requester = auth.credentials.did - const rawQuery = params.q ?? params.term - const query = cleanQuery(rawQuery || '') - const db = ctx.db.getReplica('search') - - let results: string[] - if (ctx.searchAgent) { - const res = - await ctx.searchAgent.api.app.bsky.unspecced.searchActorsSkeleton({ - q: query, - typeahead: true, - limit, - }) - results = res.data.actors.map((a) => a.did) - } else { - const res = query - ? await getUserSearchQuerySimple(db, { query, limit }) - .selectAll('actor') - .execute() - : [] - results = res.map((a) => a.did) - } - - const actors = await ctx.services - .actor(db) - .views.profilesBasic(results, requester) - - const SKIP = [] - const filtered = results.flatMap((did) => { - const actor = actors[did] - if (!actor) return SKIP - if (actor.viewer?.blocking || actor.viewer?.blockedBy) return SKIP - return actor - }) - - return { - encoding: 'application/json', - body: { - actors: filtered, - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/feed/describeFeedGenerator.ts b/packages/mod-service/src/api/app/bsky/feed/describeFeedGenerator.ts deleted file mode 100644 index 6ff900ca4c0..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/describeFeedGenerator.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { MethodNotImplementedError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.feed.describeFeedGenerator(async () => { - if (!ctx.cfg.feedGenDid) { - throw new MethodNotImplementedError() - } - - const feeds = Object.keys(ctx.algos).map((uri) => ({ uri })) - - return { - encoding: 'application/json', - body: { - did: ctx.cfg.feedGenDid, - feeds, - }, - } - }) -} diff --git a/packages/mod-service/src/api/app/bsky/feed/getActorFeeds.ts b/packages/mod-service/src/api/app/bsky/feed/getActorFeeds.ts deleted file mode 100644 index 7a28e4efe67..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getActorFeeds.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { mapDefined } from '@atproto/common' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { TimeCidKeyset, paginate } from '../../../../db/pagination' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.feed.getActorFeeds({ - auth: ctx.authOptionalVerifier, - handler: async ({ auth, params }) => { - const { actor, limit, cursor } = params - const viewer = auth.credentials.did - - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const feedService = ctx.services.feed(db) - - const creatorRes = await actorService.getActor(actor) - if (!creatorRes) { - throw new InvalidRequestError(`Actor not found: ${actor}`) - } - - const { ref } = db.db.dynamic - let feedsQb = feedService - .selectFeedGeneratorQb(viewer) - .where('feed_generator.creator', '=', creatorRes.did) - - const keyset = new TimeCidKeyset( - ref('feed_generator.createdAt'), - ref('feed_generator.cid'), - ) - feedsQb = paginate(feedsQb, { - limit, - cursor, - keyset, - }) - - const [feedsRes, profiles] = await Promise.all([ - feedsQb.execute(), - actorService.views.profiles([creatorRes], viewer), - ]) - if (!profiles[creatorRes.did]) { - throw new InvalidRequestError(`Actor not found: ${actor}`) - } - - const feeds = mapDefined(feedsRes, (row) => { - const feed = { - ...row, - viewer: viewer ? { like: row.viewerLike } : undefined, - } - return feedService.views.formatFeedGeneratorView(feed, profiles) - }) - - return { - encoding: 'application/json', - body: { - cursor: keyset.packFromResult(feedsRes), - feeds, - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/feed/getActorLikes.ts b/packages/mod-service/src/api/app/bsky/feed/getActorLikes.ts deleted file mode 100644 index 36e36b0100b..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getActorLikes.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getActorLikes' -import { FeedKeyset } from '../util/feed' -import { paginate } from '../../../../db/pagination' -import AppContext from '../../../../context' -import { setRepoRev } from '../../../util' -import { - FeedHydrationState, - FeedRow, - FeedService, -} from '../../../../services/feed' -import { Database } from '../../../../db' -import { ActorService } from '../../../../services/actor' -import { GraphService } from '../../../../services/graph' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getActorLikes = createPipeline( - skeleton, - hydration, - noPostBlocks, - presentation, - ) - server.app.bsky.feed.getActorLikes({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth, res }) => { - const viewer = auth.credentials.did - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const feedService = ctx.services.feed(db) - const graphService = ctx.services.graph(db) - - const [result, repoRev] = await Promise.all([ - getActorLikes( - { ...params, viewer }, - { db, actorService, feedService, graphService }, - ), - actorService.getRepoRev(viewer), - ]) - - setRepoRev(res, repoRev) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { db, actorService, feedService } = ctx - const { actor, limit, cursor, viewer } = params - const { ref } = db.db.dynamic - - const actorRes = await actorService.getActor(actor) - if (!actorRes) { - throw new InvalidRequestError('Profile not found') - } - const actorDid = actorRes.did - - if (!viewer || viewer !== actorDid) { - throw new InvalidRequestError('Profile not found') - } - - let feedItemsQb = feedService - .selectFeedItemQb() - .innerJoin('like', 'like.subject', 'feed_item.uri') - .where('like.creator', '=', actorDid) - - const keyset = new FeedKeyset(ref('like.sortAt'), ref('like.cid')) - - feedItemsQb = paginate(feedItemsQb, { - limit, - cursor, - keyset, - }) - - const feedItems = await feedItemsQb.execute() - - return { params, feedItems, cursor: keyset.packFromResult(feedItems) } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { feedService } = ctx - const { params, feedItems } = state - const refs = feedService.feedItemRefs(feedItems) - const hydrated = await feedService.feedHydration({ - ...refs, - viewer: params.viewer, - }) - return { ...state, ...hydrated } -} - -const noPostBlocks = (state: HydrationState) => { - const { viewer } = state.params - state.feedItems = state.feedItems.filter( - (item) => !viewer || !state.bam.block([viewer, item.postAuthorDid]), - ) - return state -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { feedService } = ctx - const { feedItems, cursor, params } = state - const feed = feedService.views.formatFeed(feedItems, state, params.viewer) - return { feed, cursor } -} - -type Context = { - db: Database - feedService: FeedService - actorService: ActorService - graphService: GraphService -} - -type Params = QueryParams & { viewer: string | null } - -type SkeletonState = { params: Params; feedItems: FeedRow[]; cursor?: string } - -type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/mod-service/src/api/app/bsky/feed/getAuthorFeed.ts b/packages/mod-service/src/api/app/bsky/feed/getAuthorFeed.ts deleted file mode 100644 index 342f371f18d..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getAuthorFeed.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getAuthorFeed' -import { FeedKeyset } from '../util/feed' -import { paginate } from '../../../../db/pagination' -import AppContext from '../../../../context' -import { setRepoRev } from '../../../util' -import { Database } from '../../../../db' -import { - FeedHydrationState, - FeedRow, - FeedService, -} from '../../../../services/feed' -import { ActorService } from '../../../../services/actor' -import { GraphService } from '../../../../services/graph' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getAuthorFeed = createPipeline( - skeleton, - hydration, - noBlocksOrMutedReposts, - presentation, - ) - server.app.bsky.feed.getAuthorFeed({ - auth: ctx.authOptionalAccessOrRoleVerifier, - handler: async ({ params, auth, res }) => { - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const feedService = ctx.services.feed(db) - const graphService = ctx.services.graph(db) - const viewer = - auth.credentials.type === 'access' ? auth.credentials.did : null - - const [result, repoRev] = await Promise.all([ - getAuthorFeed( - { ...params, viewer }, - { db, actorService, feedService, graphService }, - ), - actorService.getRepoRev(viewer), - ]) - - setRepoRev(res, repoRev) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -export const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { cursor, limit, actor, filter, viewer } = params - const { db, actorService, feedService, graphService } = ctx - const { ref } = db.db.dynamic - - // maybe resolve did first - const actorRes = await actorService.getActor(actor) - if (!actorRes) { - throw new InvalidRequestError('Profile not found') - } - const actorDid = actorRes.did - - // verify there is not a block between requester & subject - if (viewer !== null) { - const blocks = await graphService.getBlockState([[viewer, actorDid]]) - if (blocks.blocking([viewer, actorDid])) { - throw new InvalidRequestError( - `Requester has blocked actor: ${actor}`, - 'BlockedActor', - ) - } - if (blocks.blockedBy([viewer, actorDid])) { - throw new InvalidRequestError( - `Requester is blocked by actor: $${actor}`, - 'BlockedByActor', - ) - } - } - - // defaults to posts, reposts, and replies - let feedItemsQb = feedService - .selectFeedItemQb() - .where('originatorDid', '=', actorDid) - - if (filter === 'posts_with_media') { - feedItemsQb = feedItemsQb - // only your own posts - .where('type', '=', 'post') - // only posts with media - .whereExists((qb) => - qb - .selectFrom('post_embed_image') - .select('post_embed_image.postUri') - .whereRef('post_embed_image.postUri', '=', 'feed_item.postUri'), - ) - } else if (filter === 'posts_no_replies') { - feedItemsQb = feedItemsQb.where((qb) => - qb.where('post.replyParent', 'is', null).orWhere('type', '=', 'repost'), - ) - } else if (filter === 'posts_and_author_threads') { - feedItemsQb = feedItemsQb.where((qb) => - qb - .where('type', '=', 'repost') - .orWhere('post.replyParent', 'is', null) - .orWhere('post.replyRoot', 'like', `at://${actorDid}/%`), - ) - } - - const keyset = new FeedKeyset(ref('feed_item.sortAt'), ref('feed_item.cid')) - - feedItemsQb = paginate(feedItemsQb, { - limit, - cursor, - keyset, - }) - - const feedItems = await feedItemsQb.execute() - - return { - params, - feedItems, - cursor: keyset.packFromResult(feedItems), - } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { feedService } = ctx - const { params, feedItems } = state - const refs = feedService.feedItemRefs(feedItems) - const hydrated = await feedService.feedHydration({ - ...refs, - viewer: params.viewer, - }) - return { ...state, ...hydrated } -} - -const noBlocksOrMutedReposts = (state: HydrationState) => { - const { viewer } = state.params - state.feedItems = state.feedItems.filter((item) => { - if (!viewer) return true - return ( - !state.bam.block([viewer, item.postAuthorDid]) && - (item.type === 'post' || !state.bam.mute([viewer, item.postAuthorDid])) - ) - }) - return state -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { feedService } = ctx - const { feedItems, cursor, params } = state - const feed = feedService.views.formatFeed(feedItems, state, params.viewer) - return { feed, cursor } -} - -type Context = { - db: Database - actorService: ActorService - feedService: FeedService - graphService: GraphService -} - -type Params = QueryParams & { viewer: string | null } - -type SkeletonState = { - params: Params - feedItems: FeedRow[] - cursor?: string -} - -type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/mod-service/src/api/app/bsky/feed/getFeed.ts b/packages/mod-service/src/api/app/bsky/feed/getFeed.ts deleted file mode 100644 index a09258c3163..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getFeed.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { - InvalidRequestError, - UpstreamFailureError, - ServerTimer, - serverTimingHeader, -} from '@atproto/xrpc-server' -import { ResponseType, XRPCError } from '@atproto/xrpc' -import { - DidDocument, - PoorlyFormattedDidDocumentError, - getFeedGen, -} from '@atproto/identity' -import { AtpAgent, AppBskyFeedGetFeedSkeleton } from '@atproto/api' -import { QueryParams as GetFeedParams } from '../../../../lexicon/types/app/bsky/feed/getFeed' -import { OutputSchema as SkeletonOutput } from '../../../../lexicon/types/app/bsky/feed/getFeedSkeleton' -import { SkeletonFeedPost } from '../../../../lexicon/types/app/bsky/feed/defs' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { AlgoResponse } from '../../../../feed-gen/types' -import { Database } from '../../../../db' -import { - FeedHydrationState, - FeedRow, - FeedService, -} from '../../../../services/feed' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getFeed = createPipeline( - skeleton, - hydration, - noBlocksOrMutes, - presentation, - ) - server.app.bsky.feed.getFeed({ - auth: ctx.authOptionalVerifierAnyAudience, - handler: async ({ params, auth, req }) => { - const db = ctx.db.getReplica() - const feedService = ctx.services.feed(db) - const viewer = auth.credentials.did - - const { timerSkele, timerHydr, ...result } = await getFeed( - { ...params, viewer }, - { - db, - feedService, - appCtx: ctx, - authorization: req.headers['authorization'], - }, - ) - - return { - encoding: 'application/json', - body: result, - headers: { - 'server-timing': serverTimingHeader([timerSkele, timerHydr]), - }, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const timerSkele = new ServerTimer('skele').start() - const localAlgo = ctx.appCtx.algos[params.feed] - const feedParams: GetFeedParams = { - feed: params.feed, - limit: params.limit, - cursor: params.cursor, - } - const { feedItems, cursor, ...passthrough } = - localAlgo !== undefined - ? await localAlgo(ctx.appCtx, params, params.viewer) - : await skeletonFromFeedGen(ctx, feedParams) - return { - params, - cursor, - feedItems, - timerSkele: timerSkele.stop(), - passthrough, - } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const timerHydr = new ServerTimer('hydr').start() - const { feedService } = ctx - const { params, feedItems } = state - const refs = feedService.feedItemRefs(feedItems) - const hydrated = await feedService.feedHydration({ - ...refs, - viewer: params.viewer, - }) - return { ...state, ...hydrated, timerHydr: timerHydr.stop() } -} - -const noBlocksOrMutes = (state: HydrationState) => { - const { viewer } = state.params - state.feedItems = state.feedItems.filter((item) => { - if (!viewer) return true - return ( - !state.bam.block([viewer, item.postAuthorDid]) && - !state.bam.block([viewer, item.originatorDid]) && - !state.bam.mute([viewer, item.postAuthorDid]) && - !state.bam.mute([viewer, item.originatorDid]) - ) - }) - return state -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { feedService } = ctx - const { feedItems, cursor, passthrough, params } = state - const feed = feedService.views.formatFeed(feedItems, state, params.viewer) - return { - feed, - cursor, - timerSkele: state.timerSkele, - timerHydr: state.timerHydr, - ...passthrough, - } -} - -type Context = { - db: Database - feedService: FeedService - appCtx: AppContext - authorization?: string -} - -type Params = GetFeedParams & { viewer: string | null } - -type SkeletonState = { - params: Params - feedItems: FeedRow[] - passthrough: Record // pass through additional items in feedgen response - cursor?: string - timerSkele: ServerTimer -} - -type HydrationState = SkeletonState & - FeedHydrationState & { feedItems: FeedRow[]; timerHydr: ServerTimer } - -const skeletonFromFeedGen = async ( - ctx: Context, - params: GetFeedParams, -): Promise => { - const { db, appCtx, authorization } = ctx - const { feed } = params - // Resolve and fetch feed skeleton - const found = await db.db - .selectFrom('feed_generator') - .where('uri', '=', feed) - .select('feedDid') - .executeTakeFirst() - if (!found) { - throw new InvalidRequestError('could not find feed') - } - const feedDid = found.feedDid - - let resolved: DidDocument | null - try { - resolved = await appCtx.idResolver.did.resolve(feedDid) - } catch (err) { - if (err instanceof PoorlyFormattedDidDocumentError) { - throw new InvalidRequestError(`invalid did document: ${feedDid}`) - } - throw err - } - if (!resolved) { - throw new InvalidRequestError(`could not resolve did document: ${feedDid}`) - } - - const fgEndpoint = getFeedGen(resolved) - if (!fgEndpoint) { - throw new InvalidRequestError( - `invalid feed generator service details in did document: ${feedDid}`, - ) - } - - const agent = new AtpAgent({ service: fgEndpoint }) - - let skeleton: SkeletonOutput - try { - // @TODO currently passthrough auth headers from pds - const headers: Record = authorization - ? { authorization: authorization } - : {} - const result = await agent.api.app.bsky.feed.getFeedSkeleton(params, { - headers, - }) - skeleton = result.data - } catch (err) { - if (err instanceof AppBskyFeedGetFeedSkeleton.UnknownFeedError) { - throw new InvalidRequestError(err.message, 'UnknownFeed') - } - if (err instanceof XRPCError) { - if (err.status === ResponseType.Unknown) { - throw new UpstreamFailureError('feed unavailable') - } - if (err.status === ResponseType.InvalidResponse) { - throw new UpstreamFailureError( - 'feed provided an invalid response', - 'InvalidFeedResponse', - ) - } - } - throw err - } - - const { feed: feedSkele, ...skele } = skeleton - const feedItems = await skeletonToFeedItems( - feedSkele.slice(0, params.limit), - ctx, - ) - - return { ...skele, feedItems } -} - -const skeletonToFeedItems = async ( - skeleton: SkeletonFeedPost[], - ctx: Context, -): Promise => { - const { feedService } = ctx - const feedItemUris = skeleton.map(getSkeleFeedItemUri) - const feedItemsRaw = await feedService.getFeedItems(feedItemUris) - const results: FeedRow[] = [] - for (const skeleItem of skeleton) { - const feedItem = feedItemsRaw[getSkeleFeedItemUri(skeleItem)] - if (feedItem && feedItem.postUri === skeleItem.post) { - results.push(feedItem) - } - } - return results -} - -const getSkeleFeedItemUri = (item: SkeletonFeedPost) => { - return typeof item.reason?.repost === 'string' - ? item.reason.repost - : item.post -} diff --git a/packages/mod-service/src/api/app/bsky/feed/getFeedGenerator.ts b/packages/mod-service/src/api/app/bsky/feed/getFeedGenerator.ts deleted file mode 100644 index 14a5688db0d..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getFeedGenerator.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { - DidDocument, - PoorlyFormattedDidDocumentError, - getFeedGen, -} from '@atproto/identity' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.feed.getFeedGenerator({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth }) => { - const { feed } = params - const viewer = auth.credentials.did - - const db = ctx.db.getReplica() - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - - const got = await feedService.getFeedGeneratorInfos([feed], viewer) - const feedInfo = got[feed] - if (!feedInfo) { - throw new InvalidRequestError('could not find feed') - } - - const feedDid = feedInfo.feedDid - let resolved: DidDocument | null - try { - resolved = await ctx.idResolver.did.resolve(feedDid) - } catch (err) { - if (err instanceof PoorlyFormattedDidDocumentError) { - throw new InvalidRequestError(`invalid did document: ${feedDid}`) - } - throw err - } - if (!resolved) { - throw new InvalidRequestError( - `could not resolve did document: ${feedDid}`, - ) - } - - const fgEndpoint = getFeedGen(resolved) - if (!fgEndpoint) { - throw new InvalidRequestError( - `invalid feed generator service details in did document: ${feedDid}`, - ) - } - - const profiles = await actorService.views.profilesBasic( - [feedInfo.creator], - viewer, - ) - const feedView = feedService.views.formatFeedGeneratorView( - feedInfo, - profiles, - ) - if (!feedView) { - throw new InvalidRequestError('could not find feed') - } - - return { - encoding: 'application/json', - body: { - view: feedView, - // @TODO temporarily hard-coding to true while external feedgens catch-up on describeFeedGenerator - isOnline: true, - isValid: true, - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/feed/getFeedGenerators.ts b/packages/mod-service/src/api/app/bsky/feed/getFeedGenerators.ts deleted file mode 100644 index 7b571ab09f6..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getFeedGenerators.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { FeedGenInfo, FeedService } from '../../../../services/feed' -import { createPipeline, noRules } from '../../../../pipeline' -import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { Database } from '../../../../db' - -export default function (server: Server, ctx: AppContext) { - const getFeedGenerators = createPipeline( - skeleton, - hydration, - noRules, - presentation, - ) - server.app.bsky.feed.getFeedGenerators({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth }) => { - const { feeds } = params - const viewer = auth.credentials.did - const db = ctx.db.getReplica() - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - - const view = await getFeedGenerators( - { feeds, viewer }, - { db, feedService, actorService }, - ) - - return { - encoding: 'application/json', - body: view, - } - }, - }) -} - -const skeleton = async (params: Params, ctx: Context) => { - const { feedService } = ctx - const genInfos = await feedService.getFeedGeneratorInfos( - params.feeds, - params.viewer, - ) - return { - params, - generators: Object.values(genInfos), - } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { actorService } = ctx - const profiles = await actorService.views.profilesBasic( - state.generators.map((gen) => gen.creator), - state.params.viewer, - ) - return { - ...state, - profiles, - } -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { feedService } = ctx - const feeds = mapDefined(state.generators, (gen) => - feedService.views.formatFeedGeneratorView(gen, state.profiles), - ) - return { feeds } -} - -type Context = { - db: Database - feedService: FeedService - actorService: ActorService -} - -type Params = { viewer: string | null; feeds: string[] } - -type SkeletonState = { params: Params; generators: FeedGenInfo[] } - -type HydrationState = SkeletonState & { profiles: ActorInfoMap } diff --git a/packages/mod-service/src/api/app/bsky/feed/getFeedSkeleton.ts b/packages/mod-service/src/api/app/bsky/feed/getFeedSkeleton.ts deleted file mode 100644 index 5d65044f86f..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getFeedSkeleton.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { toSkeletonItem } from '../../../../feed-gen/types' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.feed.getFeedSkeleton({ - auth: ctx.authVerifierAnyAudience, - handler: async ({ params, auth }) => { - const { feed } = params - const viewer = auth.credentials.did - const localAlgo = ctx.algos[feed] - - if (!localAlgo) { - throw new InvalidRequestError('Unknown feed', 'UnknownFeed') - } - - const result = await localAlgo(ctx, params, viewer) - - return { - encoding: 'application/json', - body: { - // @TODO should we proactively filter blocks/mutes from the skeleton, or treat this similar to other custom feeds? - feed: result.feedItems.map(toSkeletonItem), - cursor: result.cursor, - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/feed/getLikes.ts b/packages/mod-service/src/api/app/bsky/feed/getLikes.ts deleted file mode 100644 index 893617f6bb0..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getLikes.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getLikes' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import AppContext from '../../../../context' -import { notSoftDeletedClause } from '../../../../db/util' -import { BlockAndMuteState, GraphService } from '../../../../services/graph' -import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { Actor } from '../../../../db/tables/actor' -import { Database } from '../../../../db' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getLikes = createPipeline(skeleton, hydration, noBlocks, presentation) - server.app.bsky.feed.getLikes({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth }) => { - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const graphService = ctx.services.graph(db) - const viewer = auth.credentials.did - - const result = await getLikes( - { ...params, viewer }, - { db, actorService, graphService }, - ) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { db } = ctx - const { uri, cid, limit, cursor } = params - const { ref } = db.db.dynamic - - let builder = db.db - .selectFrom('like') - .where('like.subject', '=', uri) - .innerJoin('actor as creator', 'creator.did', 'like.creator') - .where(notSoftDeletedClause(ref('creator'))) - .selectAll('creator') - .select([ - 'like.cid as cid', - 'like.createdAt as createdAt', - 'like.indexedAt as indexedAt', - 'like.sortAt as sortAt', - ]) - - if (cid) { - builder = builder.where('like.subjectCid', '=', cid) - } - - const keyset = new TimeCidKeyset(ref('like.sortAt'), ref('like.cid')) - builder = paginate(builder, { - limit, - cursor, - keyset, - }) - - const likes = await builder.execute() - - return { params, likes, cursor: keyset.packFromResult(likes) } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { graphService, actorService } = ctx - const { params, likes } = state - const { viewer } = params - const [actors, bam] = await Promise.all([ - actorService.views.profiles(likes, viewer), - graphService.getBlockAndMuteState( - viewer ? likes.map((like) => [viewer, like.did]) : [], - ), - ]) - return { ...state, bam, actors } -} - -const noBlocks = (state: HydrationState) => { - const { viewer } = state.params - if (!viewer) return state - state.likes = state.likes.filter( - (item) => !state.bam.block([viewer, item.did]), - ) - return state -} - -const presentation = (state: HydrationState) => { - const { params, likes, actors, cursor } = state - const { uri, cid } = params - const likesView = mapDefined(likes, (like) => - actors[like.did] - ? { - createdAt: like.createdAt, - indexedAt: like.indexedAt, - actor: actors[like.did], - } - : undefined, - ) - return { likes: likesView, cursor, uri, cid } -} - -type Context = { - db: Database - actorService: ActorService - graphService: GraphService -} - -type Params = QueryParams & { viewer: string | null } - -type SkeletonState = { - params: Params - likes: (Actor & { createdAt: string })[] - cursor?: string -} - -type HydrationState = SkeletonState & { - bam: BlockAndMuteState - actors: ActorInfoMap -} diff --git a/packages/mod-service/src/api/app/bsky/feed/getListFeed.ts b/packages/mod-service/src/api/app/bsky/feed/getListFeed.ts deleted file mode 100644 index fd3f0360ef3..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getListFeed.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getListFeed' -import { FeedKeyset, getFeedDateThreshold } from '../util/feed' -import { paginate } from '../../../../db/pagination' -import AppContext from '../../../../context' -import { setRepoRev } from '../../../util' -import { Database } from '../../../../db' -import { - FeedHydrationState, - FeedRow, - FeedService, -} from '../../../../services/feed' -import { ActorService } from '../../../../services/actor' -import { GraphService } from '../../../../services/graph' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getListFeed = createPipeline( - skeleton, - hydration, - noBlocksOrMutes, - presentation, - ) - server.app.bsky.feed.getListFeed({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth, res }) => { - const viewer = auth.credentials.did - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const feedService = ctx.services.feed(db) - const graphService = ctx.services.graph(db) - - const [result, repoRev] = await Promise.all([ - getListFeed( - { ...params, viewer }, - { db, actorService, feedService, graphService }, - ), - actorService.getRepoRev(viewer), - ]) - - setRepoRev(res, repoRev) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -export const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { list, cursor, limit } = params - const { db } = ctx - const { ref } = db.db.dynamic - - const keyset = new FeedKeyset(ref('post.sortAt'), ref('post.cid')) - const sortFrom = keyset.unpack(cursor)?.primary - - let builder = ctx.feedService - .selectPostQb() - .innerJoin('list_item', 'list_item.subjectDid', 'post.creator') - .where('list_item.listUri', '=', list) - .where('post.sortAt', '>', getFeedDateThreshold(sortFrom, 3)) - - builder = paginate(builder, { - limit, - cursor, - keyset, - tryIndex: true, - }) - const feedItems = await builder.execute() - - return { - params, - feedItems, - cursor: keyset.packFromResult(feedItems), - } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { feedService } = ctx - const { params, feedItems } = state - const refs = feedService.feedItemRefs(feedItems) - const hydrated = await feedService.feedHydration({ - ...refs, - viewer: params.viewer, - }) - return { ...state, ...hydrated } -} - -const noBlocksOrMutes = (state: HydrationState) => { - const { viewer } = state.params - if (!viewer) return state - state.feedItems = state.feedItems.filter( - (item) => - !state.bam.block([viewer, item.postAuthorDid]) && - !state.bam.mute([viewer, item.postAuthorDid]), - ) - return state -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { feedService } = ctx - const { feedItems, cursor, params } = state - const feed = feedService.views.formatFeed(feedItems, state, params.viewer) - return { feed, cursor } -} - -type Context = { - db: Database - actorService: ActorService - feedService: FeedService - graphService: GraphService -} - -type Params = QueryParams & { viewer: string | null } - -type SkeletonState = { - params: Params - feedItems: FeedRow[] - cursor?: string -} - -type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/mod-service/src/api/app/bsky/feed/getPostThread.ts b/packages/mod-service/src/api/app/bsky/feed/getPostThread.ts deleted file mode 100644 index 873dd311ba0..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getPostThread.ts +++ /dev/null @@ -1,331 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { AtUri } from '@atproto/syntax' -import { Server } from '../../../../lexicon' -import { - BlockedPost, - NotFoundPost, - ThreadViewPost, - isNotFoundPost, -} from '../../../../lexicon/types/app/bsky/feed/defs' -import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getPostThread' -import AppContext from '../../../../context' -import { - FeedService, - FeedRow, - FeedHydrationState, -} from '../../../../services/feed' -import { - getAncestorsAndSelfQb, - getDescendentsQb, -} from '../../../../services/util/post' -import { Database } from '../../../../db' -import { setRepoRev } from '../../../util' -import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { createPipeline, noRules } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getPostThread = createPipeline( - skeleton, - hydration, - noRules, // handled in presentation: 3p block-violating replies are turned to #blockedPost, viewer blocks turned to #notFoundPost. - presentation, - ) - server.app.bsky.feed.getPostThread({ - auth: ctx.authOptionalAccessOrRoleVerifier, - handler: async ({ params, auth, res }) => { - const viewer = 'did' in auth.credentials ? auth.credentials.did : null - const db = ctx.db.getReplica('thread') - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - - const [result, repoRev] = await Promise.allSettled([ - getPostThread({ ...params, viewer }, { db, feedService, actorService }), - actorService.getRepoRev(viewer), - ]) - - if (repoRev.status === 'fulfilled') { - setRepoRev(res, repoRev.value) - } - if (result.status === 'rejected') { - throw result.reason - } - - return { - encoding: 'application/json', - body: result.value, - } - }, - }) -} - -const skeleton = async (params: Params, ctx: Context) => { - const threadData = await getThreadData(params, ctx) - if (!threadData) { - throw new InvalidRequestError(`Post not found: ${params.uri}`, 'NotFound') - } - return { params, threadData } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { feedService } = ctx - const { - threadData, - params: { viewer }, - } = state - const relevant = getRelevantIds(threadData) - const hydrated = await feedService.feedHydration({ ...relevant, viewer }) - return { ...state, ...hydrated } -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { params, profiles } = state - const { actorService } = ctx - const actors = actorService.views.profileBasicPresentation( - Object.keys(profiles), - state, - params.viewer, - ) - const thread = composeThread( - state.threadData, - actors, - state, - ctx, - params.viewer, - ) - if (isNotFoundPost(thread)) { - // @TODO technically this could be returned as a NotFoundPost based on lexicon - throw new InvalidRequestError(`Post not found: ${params.uri}`, 'NotFound') - } - return { thread } -} - -const composeThread = ( - threadData: PostThread, - actors: ActorInfoMap, - state: HydrationState, - ctx: Context, - viewer: string | null, -) => { - const { feedService } = ctx - const { posts, threadgates, embeds, blocks, labels, lists } = state - - const post = feedService.views.formatPostView( - threadData.post.postUri, - actors, - posts, - threadgates, - embeds, - labels, - lists, - viewer, - ) - - // replies that are invalid due to reply-gating: - // a. may appear as the anchor post, but without any parent or replies. - // b. may not appear anywhere else in the thread. - const isAnchorPost = state.threadData.post.uri === threadData.post.postUri - const info = posts[threadData.post.postUri] - // @TODO re-enable invalidReplyRoot check - // const badReply = !!info?.invalidReplyRoot || !!info?.violatesThreadGate - const badReply = !!info?.violatesThreadGate - const omitBadReply = !isAnchorPost && badReply - - if (!post || blocks[post.uri]?.reply || omitBadReply) { - return { - $type: 'app.bsky.feed.defs#notFoundPost', - uri: threadData.post.postUri, - notFound: true, - } - } - - if (post.author.viewer?.blocking || post.author.viewer?.blockedBy) { - return { - $type: 'app.bsky.feed.defs#blockedPost', - uri: threadData.post.postUri, - blocked: true, - author: { - did: post.author.did, - viewer: post.author.viewer - ? { - blockedBy: post.author.viewer?.blockedBy, - blocking: post.author.viewer?.blocking, - } - : undefined, - }, - } - } - - let parent - if (threadData.parent && !badReply) { - if (threadData.parent instanceof ParentNotFoundError) { - parent = { - $type: 'app.bsky.feed.defs#notFoundPost', - uri: threadData.parent.uri, - notFound: true, - } - } else { - parent = composeThread(threadData.parent, actors, state, ctx, viewer) - } - } - - let replies: (ThreadViewPost | NotFoundPost | BlockedPost)[] | undefined - if (threadData.replies && !badReply) { - replies = threadData.replies.flatMap((reply) => { - const thread = composeThread(reply, actors, state, ctx, viewer) - // e.g. don't bother including #postNotFound reply placeholders for takedowns. either way matches api contract. - const skip = [] - return isNotFoundPost(thread) ? skip : thread - }) - } - - return { - $type: 'app.bsky.feed.defs#threadViewPost', - post, - parent, - replies, - } -} - -const getRelevantIds = ( - thread: PostThread, -): { dids: Set; uris: Set } => { - const dids = new Set() - const uris = new Set() - if (thread.parent && !(thread.parent instanceof ParentNotFoundError)) { - const fromParent = getRelevantIds(thread.parent) - fromParent.dids.forEach((did) => dids.add(did)) - fromParent.uris.forEach((uri) => uris.add(uri)) - } - if (thread.replies) { - for (const reply of thread.replies) { - const fromChild = getRelevantIds(reply) - fromChild.dids.forEach((did) => dids.add(did)) - fromChild.uris.forEach((uri) => uris.add(uri)) - } - } - dids.add(thread.post.postAuthorDid) - uris.add(thread.post.postUri) - if (thread.post.replyRoot) { - // ensure root is included for checking interactions - uris.add(thread.post.replyRoot) - dids.add(new AtUri(thread.post.replyRoot).hostname) - } - return { dids, uris } -} - -const getThreadData = async ( - params: Params, - ctx: Context, -): Promise => { - const { db, feedService } = ctx - const { uri, depth, parentHeight } = params - - const [parents, children] = await Promise.all([ - getAncestorsAndSelfQb(db.db, { uri, parentHeight }) - .selectFrom('ancestor') - .innerJoin( - feedService.selectPostQb().as('post'), - 'post.uri', - 'ancestor.uri', - ) - .selectAll('post') - .execute(), - getDescendentsQb(db.db, { uri, depth }) - .selectFrom('descendent') - .innerJoin( - feedService.selectPostQb().as('post'), - 'post.uri', - 'descendent.uri', - ) - .selectAll('post') - .orderBy('sortAt', 'desc') - .execute(), - ]) - // prevent self-referential loops - const includedPosts = new Set([uri]) - const parentsByUri = parents.reduce((acc, post) => { - return Object.assign(acc, { [post.uri]: post }) - }, {} as Record) - const childrenByParentUri = children.reduce((acc, child) => { - if (!child.replyParent) return acc - if (includedPosts.has(child.uri)) return acc - includedPosts.add(child.uri) - acc[child.replyParent] ??= [] - acc[child.replyParent].push(child) - return acc - }, {} as Record) - const post = parentsByUri[uri] - if (!post) return null - return { - post, - parent: post.replyParent - ? getParentData( - parentsByUri, - includedPosts, - post.replyParent, - parentHeight, - ) - : undefined, - replies: getChildrenData(childrenByParentUri, uri, depth), - } -} - -const getParentData = ( - postsByUri: Record, - includedPosts: Set, - uri: string, - depth: number, -): PostThread | ParentNotFoundError | undefined => { - if (depth < 1) return undefined - if (includedPosts.has(uri)) return undefined - includedPosts.add(uri) - const post = postsByUri[uri] - if (!post) return new ParentNotFoundError(uri) - return { - post, - parent: post.replyParent - ? getParentData(postsByUri, includedPosts, post.replyParent, depth - 1) - : undefined, - replies: [], - } -} - -const getChildrenData = ( - childrenByParentUri: Record, - uri: string, - depth: number, -): PostThread[] | undefined => { - if (depth === 0) return undefined - const children = childrenByParentUri[uri] ?? [] - return children.map((row) => ({ - post: row, - replies: getChildrenData(childrenByParentUri, row.postUri, depth - 1), - })) -} - -class ParentNotFoundError extends Error { - constructor(public uri: string) { - super(`Parent not found: ${uri}`) - } -} - -type PostThread = { - post: FeedRow - parent?: PostThread | ParentNotFoundError - replies?: PostThread[] -} - -type Context = { - db: Database - feedService: FeedService - actorService: ActorService -} - -type Params = QueryParams & { viewer: string | null } - -type SkeletonState = { - params: Params - threadData: PostThread -} - -type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/mod-service/src/api/app/bsky/feed/getPosts.ts b/packages/mod-service/src/api/app/bsky/feed/getPosts.ts deleted file mode 100644 index 5ec4807accb..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getPosts.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { dedupeStrs } from '@atproto/common' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getPosts' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { - FeedHydrationState, - FeedRow, - FeedService, -} from '../../../../services/feed' -import { createPipeline } from '../../../../pipeline' -import { ActorService } from '../../../../services/actor' - -export default function (server: Server, ctx: AppContext) { - const getPosts = createPipeline(skeleton, hydration, noBlocks, presentation) - server.app.bsky.feed.getPosts({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth }) => { - const db = ctx.db.getReplica() - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - const viewer = auth.credentials.did - - const results = await getPosts( - { ...params, viewer }, - { db, feedService, actorService }, - ) - - return { - encoding: 'application/json', - body: results, - } - }, - }) -} - -const skeleton = async (params: Params, ctx: Context) => { - const deduped = dedupeStrs(params.uris) - const feedItems = await ctx.feedService.postUrisToFeedItems(deduped) - return { params, feedItems } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { feedService } = ctx - const { params, feedItems } = state - const refs = feedService.feedItemRefs(feedItems) - const hydrated = await feedService.feedHydration({ - ...refs, - viewer: params.viewer, - }) - return { ...state, ...hydrated } -} - -const noBlocks = (state: HydrationState) => { - const { viewer } = state.params - state.feedItems = state.feedItems.filter((item) => { - if (!viewer) return true - return !state.bam.block([viewer, item.postAuthorDid]) - }) - return state -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { feedService, actorService } = ctx - const { feedItems, profiles, params } = state - const SKIP = [] - const actors = actorService.views.profileBasicPresentation( - Object.keys(profiles), - state, - params.viewer, - ) - const postViews = feedItems.flatMap((item) => { - const postView = feedService.views.formatPostView( - item.postUri, - actors, - state.posts, - state.threadgates, - state.embeds, - state.labels, - state.lists, - params.viewer, - ) - return postView ?? SKIP - }) - return { posts: postViews } -} - -type Context = { - db: Database - feedService: FeedService - actorService: ActorService -} - -type Params = QueryParams & { viewer: string | null } - -type SkeletonState = { - params: Params - feedItems: FeedRow[] -} - -type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/mod-service/src/api/app/bsky/feed/getRepostedBy.ts b/packages/mod-service/src/api/app/bsky/feed/getRepostedBy.ts deleted file mode 100644 index 5ca5c452b63..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getRepostedBy.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getRepostedBy' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import AppContext from '../../../../context' -import { notSoftDeletedClause } from '../../../../db/util' -import { Database } from '../../../../db' -import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { BlockAndMuteState, GraphService } from '../../../../services/graph' -import { Actor } from '../../../../db/tables/actor' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getRepostedBy = createPipeline( - skeleton, - hydration, - noBlocks, - presentation, - ) - server.app.bsky.feed.getRepostedBy({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth }) => { - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const graphService = ctx.services.graph(db) - const viewer = auth.credentials.did - - const result = await getRepostedBy( - { ...params, viewer }, - { db, actorService, graphService }, - ) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { db } = ctx - const { limit, cursor, uri, cid } = params - const { ref } = db.db.dynamic - - let builder = db.db - .selectFrom('repost') - .where('repost.subject', '=', uri) - .innerJoin('actor as creator', 'creator.did', 'repost.creator') - .where(notSoftDeletedClause(ref('creator'))) - .selectAll('creator') - .select(['repost.cid as cid', 'repost.sortAt as sortAt']) - - if (cid) { - builder = builder.where('repost.subjectCid', '=', cid) - } - - const keyset = new TimeCidKeyset(ref('repost.sortAt'), ref('repost.cid')) - builder = paginate(builder, { - limit, - cursor, - keyset, - }) - - const repostedBy = await builder.execute() - return { params, repostedBy, cursor: keyset.packFromResult(repostedBy) } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { graphService, actorService } = ctx - const { params, repostedBy } = state - const { viewer } = params - const [actors, bam] = await Promise.all([ - actorService.views.profiles(repostedBy, viewer), - graphService.getBlockAndMuteState( - viewer ? repostedBy.map((item) => [viewer, item.did]) : [], - ), - ]) - return { ...state, bam, actors } -} - -const noBlocks = (state: HydrationState) => { - const { viewer } = state.params - if (!viewer) return state - state.repostedBy = state.repostedBy.filter( - (item) => !state.bam.block([viewer, item.did]), - ) - return state -} - -const presentation = (state: HydrationState) => { - const { params, repostedBy, actors, cursor } = state - const { uri, cid } = params - const repostedByView = mapDefined(repostedBy, (item) => actors[item.did]) - return { repostedBy: repostedByView, cursor, uri, cid } -} - -type Context = { - db: Database - actorService: ActorService - graphService: GraphService -} - -type Params = QueryParams & { viewer: string | null } - -type SkeletonState = { - params: Params - repostedBy: Actor[] - cursor?: string -} - -type HydrationState = SkeletonState & { - bam: BlockAndMuteState - actors: ActorInfoMap -} diff --git a/packages/mod-service/src/api/app/bsky/feed/getSuggestedFeeds.ts b/packages/mod-service/src/api/app/bsky/feed/getSuggestedFeeds.ts deleted file mode 100644 index 35fac829039..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getSuggestedFeeds.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.feed.getSuggestedFeeds({ - auth: ctx.authOptionalVerifier, - handler: async ({ auth }) => { - const viewer = auth.credentials.did - - const db = ctx.db.getReplica() - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - const feedsRes = await db.db - .selectFrom('suggested_feed') - .orderBy('suggested_feed.order', 'asc') - .selectAll() - .execute() - const genInfos = await feedService.getFeedGeneratorInfos( - feedsRes.map((r) => r.uri), - viewer, - ) - const genList = feedsRes.map((r) => genInfos[r.uri]).filter(Boolean) - const creators = genList.map((gen) => gen.creator) - const profiles = await actorService.views.profilesBasic(creators, viewer) - - const feedViews = mapDefined(genList, (gen) => - feedService.views.formatFeedGeneratorView(gen, profiles), - ) - - return { - encoding: 'application/json', - body: { - feeds: feedViews, - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/feed/getTimeline.ts b/packages/mod-service/src/api/app/bsky/feed/getTimeline.ts deleted file mode 100644 index 18cc5c2629a..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/getTimeline.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import { FeedAlgorithm, FeedKeyset, getFeedDateThreshold } from '../util/feed' -import { paginate } from '../../../../db/pagination' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/getTimeline' -import { setRepoRev } from '../../../util' -import { - FeedHydrationState, - FeedRow, - FeedService, -} from '../../../../services/feed' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getTimeline = createPipeline( - skeleton, - hydration, - noBlocksOrMutes, - presentation, - ) - server.app.bsky.feed.getTimeline({ - auth: ctx.authVerifier, - handler: async ({ params, auth, res }) => { - const viewer = auth.credentials.did - const db = ctx.db.getReplica('timeline') - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - - const [result, repoRev] = await Promise.all([ - getTimeline({ ...params, viewer }, { db, feedService }), - actorService.getRepoRev(viewer), - ]) - - setRepoRev(res, repoRev) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -export const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { cursor, limit, algorithm, viewer } = params - const { db } = ctx - const { ref } = db.db.dynamic - - if (algorithm && algorithm !== FeedAlgorithm.ReverseChronological) { - throw new InvalidRequestError(`Unsupported algorithm: ${algorithm}`) - } - - const keyset = new FeedKeyset(ref('feed_item.sortAt'), ref('feed_item.cid')) - const sortFrom = keyset.unpack(cursor)?.primary - - let followQb = db.db - .selectFrom('feed_item') - .innerJoin('follow', 'follow.subjectDid', 'feed_item.originatorDid') - .where('follow.creator', '=', viewer) - .innerJoin('post', 'post.uri', 'feed_item.postUri') - .where('feed_item.sortAt', '>', getFeedDateThreshold(sortFrom, 2)) - .selectAll('feed_item') - .select([ - 'post.replyRoot', - 'post.replyParent', - 'post.creator as postAuthorDid', - ]) - - followQb = paginate(followQb, { - limit, - cursor, - keyset, - tryIndex: true, - }) - - let selfQb = db.db - .selectFrom('feed_item') - .innerJoin('post', 'post.uri', 'feed_item.postUri') - .where('feed_item.originatorDid', '=', viewer) - .where('feed_item.sortAt', '>', getFeedDateThreshold(sortFrom, 2)) - .selectAll('feed_item') - .select([ - 'post.replyRoot', - 'post.replyParent', - 'post.creator as postAuthorDid', - ]) - - selfQb = paginate(selfQb, { - limit: Math.min(limit, 10), - cursor, - keyset, - tryIndex: true, - }) - - const [followRes, selfRes] = await Promise.all([ - followQb.execute(), - selfQb.execute(), - ]) - - const feedItems: FeedRow[] = [...followRes, ...selfRes] - .sort((a, b) => { - if (a.sortAt > b.sortAt) return -1 - if (a.sortAt < b.sortAt) return 1 - return a.cid > b.cid ? -1 : 1 - }) - .slice(0, limit) - - return { - params, - feedItems, - cursor: keyset.packFromResult(feedItems), - } -} - -const hydration = async ( - state: SkeletonState, - ctx: Context, -): Promise => { - const { feedService } = ctx - const { params, feedItems } = state - const refs = feedService.feedItemRefs(feedItems) - const hydrated = await feedService.feedHydration({ - ...refs, - viewer: params.viewer, - }) - return { ...state, ...hydrated } -} - -const noBlocksOrMutes = (state: HydrationState): HydrationState => { - const { viewer } = state.params - state.feedItems = state.feedItems.filter( - (item) => - !state.bam.block([viewer, item.postAuthorDid]) && - !state.bam.block([viewer, item.originatorDid]) && - !state.bam.mute([viewer, item.postAuthorDid]) && - !state.bam.mute([viewer, item.originatorDid]), - ) - return state -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { feedService } = ctx - const { feedItems, cursor, params } = state - const feed = feedService.views.formatFeed(feedItems, state, params.viewer) - return { feed, cursor } -} - -type Context = { - db: Database - feedService: FeedService -} - -type Params = QueryParams & { viewer: string } - -type SkeletonState = { - params: Params - feedItems: FeedRow[] - cursor?: string -} - -type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/mod-service/src/api/app/bsky/feed/searchPosts.ts b/packages/mod-service/src/api/app/bsky/feed/searchPosts.ts deleted file mode 100644 index db143fc5b8c..00000000000 --- a/packages/mod-service/src/api/app/bsky/feed/searchPosts.ts +++ /dev/null @@ -1,130 +0,0 @@ -import AppContext from '../../../../context' -import { Server } from '../../../../lexicon' -import { InvalidRequestError } from '@atproto/xrpc-server' -import AtpAgent from '@atproto/api' -import { mapDefined } from '@atproto/common' -import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/searchPosts' -import { Database } from '../../../../db' -import { - FeedHydrationState, - FeedRow, - FeedService, -} from '../../../../services/feed' -import { ActorService } from '../../../../services/actor' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const searchPosts = createPipeline( - skeleton, - hydration, - noBlocks, - presentation, - ) - server.app.bsky.feed.searchPosts({ - auth: ctx.authOptionalVerifier, - handler: async ({ auth, params }) => { - const viewer = auth.credentials.did - const db = ctx.db.getReplica('search') - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - const searchAgent = ctx.searchAgent - if (!searchAgent) { - throw new InvalidRequestError('Search not available') - } - - const results = await searchPosts( - { ...params, viewer }, - { db, feedService, actorService, searchAgent }, - ) - - return { - encoding: 'application/json', - body: results, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const res = await ctx.searchAgent.api.app.bsky.unspecced.searchPostsSkeleton({ - q: params.q, - cursor: params.cursor, - limit: params.limit, - }) - const postUris = res.data.posts.map((a) => a.uri) - const feedItems = await ctx.feedService.postUrisToFeedItems(postUris) - return { - params, - feedItems, - cursor: res.data.cursor, - hitsTotal: res.data.hitsTotal, - } -} - -const hydration = async ( - state: SkeletonState, - ctx: Context, -): Promise => { - const { feedService } = ctx - const { params, feedItems } = state - const refs = feedService.feedItemRefs(feedItems) - const hydrated = await feedService.feedHydration({ - ...refs, - viewer: params.viewer, - }) - return { ...state, ...hydrated } -} - -const noBlocks = (state: HydrationState): HydrationState => { - const { viewer } = state.params - state.feedItems = state.feedItems.filter((item) => { - if (!viewer) return true - return !state.bam.block([viewer, item.postAuthorDid]) - }) - return state -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { feedService, actorService } = ctx - const { feedItems, profiles, params } = state - const actors = actorService.views.profileBasicPresentation( - Object.keys(profiles), - state, - params.viewer, - ) - - const postViews = mapDefined(feedItems, (item) => - feedService.views.formatPostView( - item.postUri, - actors, - state.posts, - state.threadgates, - state.embeds, - state.labels, - state.lists, - params.viewer, - ), - ) - return { posts: postViews, cursor: state.cursor, hitsTotal: state.hitsTotal } -} - -type Context = { - db: Database - feedService: FeedService - actorService: ActorService - searchAgent: AtpAgent -} - -type Params = QueryParams & { viewer: string | null } - -type SkeletonState = { - params: Params - feedItems: FeedRow[] - hitsTotal?: number - cursor?: string -} - -type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/mod-service/src/api/app/bsky/graph/getBlocks.ts b/packages/mod-service/src/api/app/bsky/graph/getBlocks.ts deleted file mode 100644 index 66b809d70ce..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/getBlocks.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { Server } from '../../../../lexicon' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import AppContext from '../../../../context' -import { notSoftDeletedClause } from '../../../../db/util' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.graph.getBlocks({ - auth: ctx.authVerifier, - handler: async ({ params, auth }) => { - const { limit, cursor } = params - const requester = auth.credentials.did - const db = ctx.db.getReplica() - const { ref } = db.db.dynamic - - let blocksReq = db.db - .selectFrom('actor_block') - .where('actor_block.creator', '=', requester) - .innerJoin('actor as subject', 'subject.did', 'actor_block.subjectDid') - .where(notSoftDeletedClause(ref('subject'))) - .selectAll('subject') - .select(['actor_block.cid as cid', 'actor_block.sortAt as sortAt']) - - const keyset = new TimeCidKeyset( - ref('actor_block.sortAt'), - ref('actor_block.cid'), - ) - blocksReq = paginate(blocksReq, { - limit, - cursor, - keyset, - }) - - const blocksRes = await blocksReq.execute() - - const actorService = ctx.services.actor(db) - const blocks = await actorService.views.profilesList(blocksRes, requester) - - return { - encoding: 'application/json', - body: { - blocks, - cursor: keyset.packFromResult(blocksRes), - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/graph/getFollowers.ts b/packages/mod-service/src/api/app/bsky/graph/getFollowers.ts deleted file mode 100644 index 1382c1f87c7..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/getFollowers.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/graph/getFollowers' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { notSoftDeletedClause } from '../../../../db/util' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import { Actor } from '../../../../db/tables/actor' -import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { BlockAndMuteState, GraphService } from '../../../../services/graph' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getFollowers = createPipeline( - skeleton, - hydration, - noBlocksInclInvalid, - presentation, - ) - server.app.bsky.graph.getFollowers({ - auth: ctx.authOptionalAccessOrRoleVerifier, - handler: async ({ params, auth }) => { - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const graphService = ctx.services.graph(db) - const viewer = 'did' in auth.credentials ? auth.credentials.did : null - const canViewTakendownProfile = - auth.credentials.type === 'role' && auth.credentials.triage - - const result = await getFollowers( - { ...params, viewer, canViewTakendownProfile }, - { db, actorService, graphService }, - ) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { db, actorService } = ctx - const { limit, cursor, actor, canViewTakendownProfile } = params - const { ref } = db.db.dynamic - - const subject = await actorService.getActor(actor, canViewTakendownProfile) - if (!subject) { - throw new InvalidRequestError(`Actor not found: ${actor}`) - } - - let followersReq = db.db - .selectFrom('follow') - .where('follow.subjectDid', '=', subject.did) - .innerJoin('actor as creator', 'creator.did', 'follow.creator') - .if(!canViewTakendownProfile, (qb) => - qb.where(notSoftDeletedClause(ref('creator'))), - ) - .selectAll('creator') - .select(['follow.cid as cid', 'follow.sortAt as sortAt']) - - const keyset = new TimeCidKeyset(ref('follow.sortAt'), ref('follow.cid')) - followersReq = paginate(followersReq, { - limit, - cursor, - keyset, - }) - - const followers = await followersReq.execute() - return { - params, - followers, - subject, - cursor: keyset.packFromResult(followers), - } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { graphService, actorService } = ctx - const { params, followers, subject } = state - const { viewer } = params - const [actors, bam] = await Promise.all([ - actorService.views.profiles([subject, ...followers], viewer), - graphService.getBlockAndMuteState( - followers.flatMap((item) => { - if (viewer) { - return [ - [viewer, item.did], - [subject.did, item.did], - ] - } - return [[subject.did, item.did]] - }), - ), - ]) - return { ...state, bam, actors } -} - -const noBlocksInclInvalid = (state: HydrationState) => { - const { subject } = state - const { viewer } = state.params - state.followers = state.followers.filter( - (item) => - !state.bam.block([subject.did, item.did]) && - (!viewer || !state.bam.block([viewer, item.did])), - ) - return state -} - -const presentation = (state: HydrationState) => { - const { params, followers, subject, actors, cursor } = state - const subjectView = actors[subject.did] - const followersView = mapDefined(followers, (item) => actors[item.did]) - if (!subjectView) { - throw new InvalidRequestError(`Actor not found: ${params.actor}`) - } - return { followers: followersView, subject: subjectView, cursor } -} - -type Context = { - db: Database - actorService: ActorService - graphService: GraphService -} - -type Params = QueryParams & { - viewer: string | null - canViewTakendownProfile: boolean -} - -type SkeletonState = { - params: Params - followers: Actor[] - subject: Actor - cursor?: string -} - -type HydrationState = SkeletonState & { - bam: BlockAndMuteState - actors: ActorInfoMap -} diff --git a/packages/mod-service/src/api/app/bsky/graph/getFollows.ts b/packages/mod-service/src/api/app/bsky/graph/getFollows.ts deleted file mode 100644 index 34b5d72a605..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/getFollows.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/graph/getFollows' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { notSoftDeletedClause } from '../../../../db/util' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import { Actor } from '../../../../db/tables/actor' -import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { BlockAndMuteState, GraphService } from '../../../../services/graph' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getFollows = createPipeline( - skeleton, - hydration, - noBlocksInclInvalid, - presentation, - ) - server.app.bsky.graph.getFollows({ - auth: ctx.authOptionalAccessOrRoleVerifier, - handler: async ({ params, auth }) => { - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const graphService = ctx.services.graph(db) - const viewer = 'did' in auth.credentials ? auth.credentials.did : null - const canViewTakendownProfile = - auth.credentials.type === 'role' && auth.credentials.triage - - const result = await getFollows( - { ...params, viewer, canViewTakendownProfile }, - { db, actorService, graphService }, - ) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { db, actorService } = ctx - const { limit, cursor, actor, canViewTakendownProfile } = params - const { ref } = db.db.dynamic - - const creator = await actorService.getActor(actor, canViewTakendownProfile) - if (!creator) { - throw new InvalidRequestError(`Actor not found: ${actor}`) - } - - let followsReq = db.db - .selectFrom('follow') - .where('follow.creator', '=', creator.did) - .innerJoin('actor as subject', 'subject.did', 'follow.subjectDid') - .if(!canViewTakendownProfile, (qb) => - qb.where(notSoftDeletedClause(ref('subject'))), - ) - .selectAll('subject') - .select(['follow.cid as cid', 'follow.sortAt as sortAt']) - - const keyset = new TimeCidKeyset(ref('follow.sortAt'), ref('follow.cid')) - followsReq = paginate(followsReq, { - limit, - cursor, - keyset, - }) - - const follows = await followsReq.execute() - - return { - params, - follows, - creator, - cursor: keyset.packFromResult(follows), - } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { graphService, actorService } = ctx - const { params, follows, creator } = state - const { viewer } = params - const [actors, bam] = await Promise.all([ - actorService.views.profiles([creator, ...follows], viewer), - graphService.getBlockAndMuteState( - follows.flatMap((item) => { - if (viewer) { - return [ - [viewer, item.did], - [creator.did, item.did], - ] - } - return [[creator.did, item.did]] - }), - ), - ]) - return { ...state, bam, actors } -} - -const noBlocksInclInvalid = (state: HydrationState) => { - const { creator } = state - const { viewer } = state.params - state.follows = state.follows.filter( - (item) => - !state.bam.block([creator.did, item.did]) && - (!viewer || !state.bam.block([viewer, item.did])), - ) - return state -} - -const presentation = (state: HydrationState) => { - const { params, follows, creator, actors, cursor } = state - const creatorView = actors[creator.did] - const followsView = mapDefined(follows, (item) => actors[item.did]) - if (!creatorView) { - throw new InvalidRequestError(`Actor not found: ${params.actor}`) - } - return { follows: followsView, subject: creatorView, cursor } -} - -type Context = { - db: Database - actorService: ActorService - graphService: GraphService -} - -type Params = QueryParams & { - viewer: string | null - canViewTakendownProfile: boolean -} - -type SkeletonState = { - params: Params - follows: Actor[] - creator: Actor - cursor?: string -} - -type HydrationState = SkeletonState & { - bam: BlockAndMuteState - actors: ActorInfoMap -} diff --git a/packages/mod-service/src/api/app/bsky/graph/getList.ts b/packages/mod-service/src/api/app/bsky/graph/getList.ts deleted file mode 100644 index 82a70848cd9..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/getList.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/graph/getList' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import { Actor } from '../../../../db/tables/actor' -import { GraphService, ListInfo } from '../../../../services/graph' -import { ActorService, ProfileHydrationState } from '../../../../services/actor' -import { createPipeline, noRules } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getList = createPipeline(skeleton, hydration, noRules, presentation) - server.app.bsky.graph.getList({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth }) => { - const db = ctx.db.getReplica() - const graphService = ctx.services.graph(db) - const actorService = ctx.services.actor(db) - const viewer = auth.credentials.did - - const result = await getList( - { ...params, viewer }, - { db, graphService, actorService }, - ) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { db, graphService } = ctx - const { list, limit, cursor, viewer } = params - const { ref } = db.db.dynamic - - const listRes = await graphService - .getListsQb(viewer) - .where('list.uri', '=', list) - .executeTakeFirst() - if (!listRes) { - throw new InvalidRequestError(`List not found: ${list}`) - } - - let itemsReq = graphService - .getListItemsQb() - .where('list_item.listUri', '=', list) - .where('list_item.creator', '=', listRes.creator) - - const keyset = new TimeCidKeyset( - ref('list_item.sortAt'), - ref('list_item.cid'), - ) - - itemsReq = paginate(itemsReq, { - limit, - cursor, - keyset, - }) - - const listItems = await itemsReq.execute() - - return { - params, - list: listRes, - listItems, - cursor: keyset.packFromResult(listItems), - } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { actorService } = ctx - const { params, list, listItems } = state - const profileState = await actorService.views.profileHydration( - [list, ...listItems].map((x) => x.did), - { viewer: params.viewer }, - ) - return { ...state, ...profileState } -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { actorService, graphService } = ctx - const { params, list, listItems, cursor, ...profileState } = state - const actors = actorService.views.profilePresentation( - Object.keys(profileState.profiles), - profileState, - params.viewer, - ) - const creator = actors[list.creator] - if (!creator) { - throw new InvalidRequestError(`Actor not found: ${list.handle}`) - } - const listView = graphService.formatListView(list, actors) - if (!listView) { - throw new InvalidRequestError('List not found') - } - const items = mapDefined(listItems, (item) => { - const subject = actors[item.did] - if (!subject) return - return { uri: item.uri, subject } - }) - return { list: listView, items, cursor } -} - -type Context = { - db: Database - actorService: ActorService - graphService: GraphService -} - -type Params = QueryParams & { - viewer: string | null -} - -type SkeletonState = { - params: Params - list: Actor & ListInfo - listItems: (Actor & { uri: string; cid: string; sortAt: string })[] - cursor?: string -} - -type HydrationState = SkeletonState & ProfileHydrationState diff --git a/packages/mod-service/src/api/app/bsky/graph/getListBlocks.ts b/packages/mod-service/src/api/app/bsky/graph/getListBlocks.ts deleted file mode 100644 index 03fd3496f97..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/getListBlocks.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/graph/getListBlocks' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { Actor } from '../../../../db/tables/actor' -import { GraphService, ListInfo } from '../../../../services/graph' -import { ActorService, ProfileHydrationState } from '../../../../services/actor' -import { createPipeline, noRules } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const getListBlocks = createPipeline( - skeleton, - hydration, - noRules, - presentation, - ) - server.app.bsky.graph.getListBlocks({ - auth: ctx.authVerifier, - handler: async ({ params, auth }) => { - const db = ctx.db.getReplica() - const graphService = ctx.services.graph(db) - const actorService = ctx.services.actor(db) - const viewer = auth.credentials.did - - const result = await getListBlocks( - { ...params, viewer }, - { db, actorService, graphService }, - ) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { db, graphService } = ctx - const { limit, cursor, viewer } = params - const { ref } = db.db.dynamic - - let listsReq = graphService - .getListsQb(viewer) - .whereExists( - db.db - .selectFrom('list_block') - .where('list_block.creator', '=', viewer) - .whereRef('list_block.subjectUri', '=', ref('list.uri')) - .selectAll(), - ) - - const keyset = new TimeCidKeyset(ref('list.createdAt'), ref('list.cid')) - - listsReq = paginate(listsReq, { - limit, - cursor, - keyset, - }) - - const listInfos = await listsReq.execute() - - return { - params, - listInfos, - cursor: keyset.packFromResult(listInfos), - } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { actorService } = ctx - const { params, listInfos } = state - const profileState = await actorService.views.profileHydration( - listInfos.map((list) => list.creator), - { viewer: params.viewer }, - ) - return { ...state, ...profileState } -} - -const presentation = (state: HydrationState, ctx: Context) => { - const { actorService, graphService } = ctx - const { params, listInfos, cursor, ...profileState } = state - const actors = actorService.views.profilePresentation( - Object.keys(profileState.profiles), - profileState, - params.viewer, - ) - const lists = mapDefined(listInfos, (list) => - graphService.formatListView(list, actors), - ) - return { lists, cursor } -} - -type Context = { - db: Database - actorService: ActorService - graphService: GraphService -} - -type Params = QueryParams & { - viewer: string -} - -type SkeletonState = { - params: Params - listInfos: (Actor & ListInfo)[] - cursor?: string -} - -type HydrationState = SkeletonState & ProfileHydrationState diff --git a/packages/mod-service/src/api/app/bsky/graph/getListMutes.ts b/packages/mod-service/src/api/app/bsky/graph/getListMutes.ts deleted file mode 100644 index ab0ac77f47c..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/getListMutes.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { Server } from '../../../../lexicon' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.graph.getListMutes({ - auth: ctx.authVerifier, - handler: async ({ params, auth }) => { - const { limit, cursor } = params - const requester = auth.credentials.did - const db = ctx.db.getReplica() - const { ref } = db.db.dynamic - - const graphService = ctx.services.graph(db) - - let listsReq = graphService - .getListsQb(requester) - .whereExists( - db.db - .selectFrom('list_mute') - .where('list_mute.mutedByDid', '=', requester) - .whereRef('list_mute.listUri', '=', ref('list.uri')) - .selectAll(), - ) - - const keyset = new TimeCidKeyset(ref('list.createdAt'), ref('list.cid')) - listsReq = paginate(listsReq, { - limit, - cursor, - keyset, - }) - const listsRes = await listsReq.execute() - - const actorService = ctx.services.actor(db) - const profiles = await actorService.views.profiles(listsRes, requester) - - const lists = mapDefined(listsRes, (row) => - graphService.formatListView(row, profiles), - ) - - return { - encoding: 'application/json', - body: { - lists, - cursor: keyset.packFromResult(listsRes), - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/graph/getLists.ts b/packages/mod-service/src/api/app/bsky/graph/getLists.ts deleted file mode 100644 index 73deb51900b..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/getLists.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.graph.getLists({ - auth: ctx.authOptionalVerifier, - handler: async ({ params, auth }) => { - const { actor, limit, cursor } = params - const requester = auth.credentials.did - const db = ctx.db.getReplica() - const { ref } = db.db.dynamic - - const actorService = ctx.services.actor(db) - const graphService = ctx.services.graph(db) - - const creatorRes = await actorService.getActor(actor) - if (!creatorRes) { - throw new InvalidRequestError(`Actor not found: ${actor}`) - } - - let listsReq = graphService - .getListsQb(requester) - .where('list.creator', '=', creatorRes.did) - - const keyset = new TimeCidKeyset(ref('list.sortAt'), ref('list.cid')) - listsReq = paginate(listsReq, { - limit, - cursor, - keyset, - }) - - const [listsRes, profiles] = await Promise.all([ - listsReq.execute(), - actorService.views.profiles([creatorRes], requester), - ]) - if (!profiles[creatorRes.did]) { - throw new InvalidRequestError(`Actor not found: ${actor}`) - } - - const lists = mapDefined(listsRes, (row) => - graphService.formatListView(row, profiles), - ) - - return { - encoding: 'application/json', - body: { - lists, - cursor: keyset.packFromResult(listsRes), - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/graph/getMutes.ts b/packages/mod-service/src/api/app/bsky/graph/getMutes.ts deleted file mode 100644 index e69803d144a..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/getMutes.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { Server } from '../../../../lexicon' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import AppContext from '../../../../context' -import { notSoftDeletedClause } from '../../../../db/util' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.graph.getMutes({ - auth: ctx.authVerifier, - handler: async ({ params, auth }) => { - const { limit, cursor } = params - const requester = auth.credentials.did - const db = ctx.db.getReplica() - const { ref } = db.db.dynamic - - let mutesReq = db.db - .selectFrom('mute') - .innerJoin('actor', 'actor.did', 'mute.subjectDid') - .where(notSoftDeletedClause(ref('actor'))) - .where('mute.mutedByDid', '=', requester) - .selectAll('actor') - .select('mute.createdAt as createdAt') - - const keyset = new CreatedAtDidKeyset( - ref('mute.createdAt'), - ref('mute.subjectDid'), - ) - mutesReq = paginate(mutesReq, { - limit, - cursor, - keyset, - }) - - const mutesRes = await mutesReq.execute() - - const actorService = ctx.services.actor(db) - - return { - encoding: 'application/json', - body: { - cursor: keyset.packFromResult(mutesRes), - mutes: await actorService.views.profilesList(mutesRes, requester), - }, - } - }, - }) -} - -export class CreatedAtDidKeyset extends TimeCidKeyset<{ - createdAt: string - did: string // dids are treated identically to cids in TimeCidKeyset -}> { - labelResult(result: { createdAt: string; did: string }) { - return { primary: result.createdAt, secondary: result.did } - } -} diff --git a/packages/mod-service/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts b/packages/mod-service/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts deleted file mode 100644 index eddf0cd5fd6..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/getSuggestedFollowsByActor.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { sql } from 'kysely' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Database } from '../../../../db' -import { ActorService } from '../../../../services/actor' - -const RESULT_LENGTH = 10 - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.graph.getSuggestedFollowsByActor({ - auth: ctx.authVerifier, - handler: async ({ auth, params }) => { - const { actor } = params - const viewer = auth.credentials.did - - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const actorDid = await actorService.getActorDid(actor) - - if (!actorDid) { - throw new InvalidRequestError('Actor not found') - } - - const skeleton = await getSkeleton( - { - actor: actorDid, - viewer, - }, - { - db, - actorService, - }, - ) - const hydrationState = await actorService.views.profileDetailHydration( - skeleton.map((a) => a.did), - { viewer }, - ) - const presentationState = actorService.views.profileDetailPresentation( - skeleton.map((a) => a.did), - hydrationState, - { viewer }, - ) - const suggestions = Object.values(presentationState).filter((profile) => { - return ( - !profile.viewer?.muted && - !profile.viewer?.mutedByList && - !profile.viewer?.blocking && - !profile.viewer?.blockedBy - ) - }) - - return { - encoding: 'application/json', - body: { suggestions }, - } - }, - }) -} - -async function getSkeleton( - params: { - actor: string - viewer: string - }, - ctx: { - db: Database - actorService: ActorService - }, -): Promise<{ did: string }[]> { - const actorsViewerFollows = ctx.db.db - .selectFrom('follow') - .where('creator', '=', params.viewer) - .select('subjectDid') - const mostLikedAccounts = await ctx.db.db - .selectFrom( - ctx.db.db - .selectFrom('like') - .where('creator', '=', params.actor) - .select(sql`split_part(subject, '/', 3)`.as('subjectDid')) - .orderBy('sortAt', 'desc') - .limit(1000) // limit to 1000 - .as('likes'), - ) - .select('likes.subjectDid as did') - .select((qb) => qb.fn.count('likes.subjectDid').as('count')) - .where('likes.subjectDid', 'not in', actorsViewerFollows) - .where('likes.subjectDid', 'not in', [params.actor, params.viewer]) - .groupBy('likes.subjectDid') - .orderBy('count', 'desc') - .limit(RESULT_LENGTH) - .execute() - const resultDids = mostLikedAccounts.map((a) => ({ did: a.did })) as { - did: string - }[] - - if (resultDids.length < RESULT_LENGTH) { - // backfill with popular accounts followed by actor - const mostPopularAccountsActorFollows = await ctx.db.db - .selectFrom('follow') - .innerJoin('profile_agg', 'follow.subjectDid', 'profile_agg.did') - .select('follow.subjectDid as did') - .where('follow.creator', '=', params.actor) - .where('follow.subjectDid', '!=', params.viewer) - .where('follow.subjectDid', 'not in', actorsViewerFollows) - .if(resultDids.length > 0, (qb) => - qb.where( - 'subjectDid', - 'not in', - resultDids.map((a) => a.did), - ), - ) - .orderBy('profile_agg.followersCount', 'desc') - .limit(RESULT_LENGTH) - .execute() - - resultDids.push(...mostPopularAccountsActorFollows) - } - - if (resultDids.length < RESULT_LENGTH) { - // backfill with suggested_follow table - const additional = await ctx.db.db - .selectFrom('suggested_follow') - .where( - 'did', - 'not in', - // exclude any we already have - resultDids.map((a) => a.did).concat([params.actor, params.viewer]), - ) - // and aren't already followed by viewer - .where('did', 'not in', actorsViewerFollows) - .selectAll() - .execute() - - resultDids.push(...additional) - } - - return resultDids -} diff --git a/packages/mod-service/src/api/app/bsky/graph/muteActor.ts b/packages/mod-service/src/api/app/bsky/graph/muteActor.ts deleted file mode 100644 index 50a3723db6e..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/muteActor.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.graph.muteActor({ - auth: ctx.authVerifier, - handler: async ({ auth, input }) => { - const { actor } = input.body - const requester = auth.credentials.did - const db = ctx.db.getPrimary() - - const subjectDid = await ctx.services.actor(db).getActorDid(actor) - if (!subjectDid) { - throw new InvalidRequestError(`Actor not found: ${actor}`) - } - if (subjectDid === requester) { - throw new InvalidRequestError('Cannot mute oneself') - } - - await ctx.services.graph(db).muteActor({ - subjectDid, - mutedByDid: requester, - }) - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/graph/muteActorList.ts b/packages/mod-service/src/api/app/bsky/graph/muteActorList.ts deleted file mode 100644 index b6b29796c5c..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/muteActorList.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import * as lex from '../../../../lexicon/lexicons' -import AppContext from '../../../../context' -import { AtUri } from '@atproto/syntax' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.graph.muteActorList({ - auth: ctx.authVerifier, - handler: async ({ auth, input }) => { - const { list } = input.body - const requester = auth.credentials.did - - const db = ctx.db.getPrimary() - - const listUri = new AtUri(list) - const collId = lex.ids.AppBskyGraphList - if (listUri.collection !== collId) { - throw new InvalidRequestError(`Invalid collection: expected: ${collId}`) - } - - await ctx.services.graph(db).muteActorList({ - list, - mutedByDid: requester, - }) - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/graph/unmuteActor.ts b/packages/mod-service/src/api/app/bsky/graph/unmuteActor.ts deleted file mode 100644 index 11af919126f..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/unmuteActor.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.graph.unmuteActor({ - auth: ctx.authVerifier, - handler: async ({ auth, input }) => { - const { actor } = input.body - const requester = auth.credentials.did - const db = ctx.db.getPrimary() - - const subjectDid = await ctx.services.actor(db).getActorDid(actor) - if (!subjectDid) { - throw new InvalidRequestError(`Actor not found: ${actor}`) - } - if (subjectDid === requester) { - throw new InvalidRequestError('Cannot mute oneself') - } - - await ctx.services.graph(db).unmuteActor({ - subjectDid, - mutedByDid: requester, - }) - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/graph/unmuteActorList.ts b/packages/mod-service/src/api/app/bsky/graph/unmuteActorList.ts deleted file mode 100644 index 8b97530c216..00000000000 --- a/packages/mod-service/src/api/app/bsky/graph/unmuteActorList.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.graph.unmuteActorList({ - auth: ctx.authVerifier, - handler: async ({ auth, input }) => { - const { list } = input.body - const requester = auth.credentials.did - const db = ctx.db.getPrimary() - - await ctx.services.graph(db).unmuteActorList({ - list, - mutedByDid: requester, - }) - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/notification/getUnreadCount.ts b/packages/mod-service/src/api/app/bsky/notification/getUnreadCount.ts deleted file mode 100644 index c23d7683abe..00000000000 --- a/packages/mod-service/src/api/app/bsky/notification/getUnreadCount.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { sql } from 'kysely' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import { countAll, notSoftDeletedClause } from '../../../../db/util' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.notification.getUnreadCount({ - auth: ctx.authVerifier, - handler: async ({ auth, params }) => { - const requester = auth.credentials.did - if (params.seenAt) { - throw new InvalidRequestError('The seenAt parameter is unsupported') - } - - const db = ctx.db.getReplica() - const { ref } = db.db.dynamic - const result = await db.db - .selectFrom('notification') - .select(countAll.as('count')) - .innerJoin('actor', 'actor.did', 'notification.did') - .leftJoin('actor_state', 'actor_state.did', 'actor.did') - .innerJoin('record', 'record.uri', 'notification.recordUri') - .where(notSoftDeletedClause(ref('actor'))) - .where(notSoftDeletedClause(ref('record'))) - // Ensure to hit notification_did_sortat_idx, handling case where lastSeenNotifs is null. - .where('notification.did', '=', requester) - .where( - 'notification.sortAt', - '>', - sql`coalesce(${ref('actor_state.lastSeenNotifs')}, ${''})`, - ) - .executeTakeFirst() - - const count = result?.count ?? 0 - - return { - encoding: 'application/json', - body: { count }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/notification/listNotifications.ts b/packages/mod-service/src/api/app/bsky/notification/listNotifications.ts deleted file mode 100644 index 672e8c0997a..00000000000 --- a/packages/mod-service/src/api/app/bsky/notification/listNotifications.ts +++ /dev/null @@ -1,212 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { jsonStringToLex } from '@atproto/lexicon' -import { mapDefined } from '@atproto/common' -import { Server } from '../../../../lexicon' -import { QueryParams } from '../../../../lexicon/types/app/bsky/notification/listNotifications' -import AppContext from '../../../../context' -import { Database } from '../../../../db' -import { notSoftDeletedClause } from '../../../../db/util' -import { paginate, TimeCidKeyset } from '../../../../db/pagination' -import { BlockAndMuteState, GraphService } from '../../../../services/graph' -import { ActorInfoMap, ActorService } from '../../../../services/actor' -import { getSelfLabels, Labels, LabelService } from '../../../../services/label' -import { createPipeline } from '../../../../pipeline' - -export default function (server: Server, ctx: AppContext) { - const listNotifications = createPipeline( - skeleton, - hydration, - noBlockOrMutes, - presentation, - ) - server.app.bsky.notification.listNotifications({ - auth: ctx.authVerifier, - handler: async ({ params, auth }) => { - const db = ctx.db.getReplica() - const actorService = ctx.services.actor(db) - const graphService = ctx.services.graph(db) - const labelService = ctx.services.label(db) - const viewer = auth.credentials.did - - const result = await listNotifications( - { ...params, viewer }, - { db, actorService, graphService, labelService }, - ) - - return { - encoding: 'application/json', - body: result, - } - }, - }) -} - -const skeleton = async ( - params: Params, - ctx: Context, -): Promise => { - const { db } = ctx - const { limit, cursor, viewer } = params - const { ref } = db.db.dynamic - if (params.seenAt) { - throw new InvalidRequestError('The seenAt parameter is unsupported') - } - let notifBuilder = db.db - .selectFrom('notification as notif') - .where('notif.did', '=', viewer) - .where((clause) => - clause - .where('reasonSubject', 'is', null) - .orWhereExists( - db.db - .selectFrom('record as subject') - .selectAll() - .whereRef('subject.uri', '=', ref('notif.reasonSubject')), - ), - ) - .select([ - 'notif.author as authorDid', - 'notif.recordUri as uri', - 'notif.recordCid as cid', - 'notif.reason as reason', - 'notif.reasonSubject as reasonSubject', - 'notif.sortAt as indexedAt', - ]) - - const keyset = new NotifsKeyset(ref('notif.sortAt'), ref('notif.recordCid')) - notifBuilder = paginate(notifBuilder, { - cursor, - limit, - keyset, - tryIndex: true, - }) - - const actorStateQuery = db.db - .selectFrom('actor_state') - .selectAll() - .where('did', '=', viewer) - - const [notifs, actorState] = await Promise.all([ - notifBuilder.execute(), - actorStateQuery.executeTakeFirst(), - ]) - - return { - params, - notifs, - cursor: keyset.packFromResult(notifs), - lastSeenNotifs: actorState?.lastSeenNotifs, - } -} - -const hydration = async (state: SkeletonState, ctx: Context) => { - const { graphService, actorService, labelService, db } = ctx - const { params, notifs } = state - const { viewer } = params - const dids = notifs.map((notif) => notif.authorDid) - const uris = notifs.map((notif) => notif.uri) - const [actors, records, labels, bam] = await Promise.all([ - actorService.views.profiles(dids, viewer), - getRecordMap(db, uris), - labelService.getLabelsForUris(uris), - graphService.getBlockAndMuteState(dids.map((did) => [viewer, did])), - ]) - return { ...state, actors, records, labels, bam } -} - -const noBlockOrMutes = (state: HydrationState) => { - const { viewer } = state.params - state.notifs = state.notifs.filter( - (item) => - !state.bam.block([viewer, item.authorDid]) && - !state.bam.mute([viewer, item.authorDid]), - ) - return state -} - -const presentation = (state: HydrationState) => { - const { notifs, cursor, actors, records, labels, lastSeenNotifs } = state - const notifications = mapDefined(notifs, (notif) => { - const author = actors[notif.authorDid] - const record = records[notif.uri] - if (!author || !record) return undefined - const recordLabels = labels[notif.uri] ?? [] - const recordSelfLabels = getSelfLabels({ - uri: notif.uri, - cid: notif.cid, - record, - }) - return { - uri: notif.uri, - cid: notif.cid, - author, - reason: notif.reason, - reasonSubject: notif.reasonSubject || undefined, - record, - isRead: lastSeenNotifs ? notif.indexedAt <= lastSeenNotifs : false, - indexedAt: notif.indexedAt, - labels: [...recordLabels, ...recordSelfLabels], - } - }) - return { notifications, cursor, seenAt: lastSeenNotifs } -} - -const getRecordMap = async ( - db: Database, - uris: string[], -): Promise => { - if (!uris.length) return {} - const { ref } = db.db.dynamic - const recordRows = await db.db - .selectFrom('record') - .select(['uri', 'json']) - .where('uri', 'in', uris) - .where(notSoftDeletedClause(ref('record'))) - .execute() - return recordRows.reduce((acc, { uri, json }) => { - acc[uri] = jsonStringToLex(json) as Record - return acc - }, {} as RecordMap) -} - -type Context = { - db: Database - actorService: ActorService - graphService: GraphService - labelService: LabelService -} - -type Params = QueryParams & { - viewer: string -} - -type SkeletonState = { - params: Params - notifs: NotifRow[] - lastSeenNotifs?: string - cursor?: string -} - -type HydrationState = SkeletonState & { - bam: BlockAndMuteState - actors: ActorInfoMap - records: RecordMap - labels: Labels -} - -type RecordMap = { [uri: string]: Record } - -type NotifRow = { - authorDid: string - uri: string - cid: string - reason: string - reasonSubject: string | null - indexedAt: string -} - -class NotifsKeyset extends TimeCidKeyset { - labelResult(result: NotifRow) { - return { primary: result.indexedAt, secondary: result.cid } - } -} diff --git a/packages/mod-service/src/api/app/bsky/notification/registerPush.ts b/packages/mod-service/src/api/app/bsky/notification/registerPush.ts deleted file mode 100644 index be7d373bcd4..00000000000 --- a/packages/mod-service/src/api/app/bsky/notification/registerPush.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { Platform } from '../../../../notifications' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.notification.registerPush({ - auth: ctx.authVerifier, - handler: async ({ auth, input }) => { - const { token, platform, serviceDid, appId } = input.body - const { - credentials: { did }, - } = auth - if (serviceDid !== auth.artifacts.aud) { - throw new InvalidRequestError('Invalid serviceDid.') - } - const { notifServer } = ctx - if (platform !== 'ios' && platform !== 'android' && platform !== 'web') { - throw new InvalidRequestError( - 'Unsupported platform: must be "ios", "android", or "web".', - ) - } - await notifServer.registerDeviceForPushNotifications( - did, - token, - platform as Platform, - appId, - ) - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/notification/updateSeen.ts b/packages/mod-service/src/api/app/bsky/notification/updateSeen.ts deleted file mode 100644 index b7c705c0889..00000000000 --- a/packages/mod-service/src/api/app/bsky/notification/updateSeen.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Server } from '../../../../lexicon' -import { InvalidRequestError } from '@atproto/xrpc-server' -import AppContext from '../../../../context' -import { excluded } from '../../../../db/util' - -export default function (server: Server, ctx: AppContext) { - server.app.bsky.notification.updateSeen({ - auth: ctx.authVerifier, - handler: async ({ input, auth }) => { - const { seenAt } = input.body - const viewer = auth.credentials.did - - let parsed: string - try { - parsed = new Date(seenAt).toISOString() - } catch (_err) { - throw new InvalidRequestError('Invalid date') - } - - const db = ctx.db.getPrimary() - - await db.db - .insertInto('actor_state') - .values({ did: viewer, lastSeenNotifs: parsed }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ - lastSeenNotifs: excluded(db.db, 'lastSeenNotifs'), - }), - ) - .executeTakeFirst() - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts b/packages/mod-service/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts deleted file mode 100644 index e135d2cb7c1..00000000000 --- a/packages/mod-service/src/api/app/bsky/unspecced/getPopularFeedGenerators.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { countAll } from '../../../../db/util' -import { GenericKeyset, paginate } from '../../../../db/pagination' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { GeneratorView } from '../../../../lexicon/types/app/bsky/feed/defs' - -// THIS IS A TEMPORARY UNSPECCED ROUTE -export default function (server: Server, ctx: AppContext) { - server.app.bsky.unspecced.getPopularFeedGenerators({ - auth: ctx.authOptionalVerifier, - handler: async ({ auth, params }) => { - const { limit, cursor, query } = params - const requester = auth.credentials.did - const db = ctx.db.getReplica() - const { ref } = db.db.dynamic - const feedService = ctx.services.feed(db) - const actorService = ctx.services.actor(db) - - let inner = db.db - .selectFrom('feed_generator') - .select([ - 'uri', - 'cid', - db.db - .selectFrom('like') - .whereRef('like.subject', '=', ref('feed_generator.uri')) - .select(countAll.as('count')) - .as('likeCount'), - ]) - - if (query) { - inner = inner.where((qb) => - qb - .where('feed_generator.displayName', 'ilike', `%${query}%`) - .orWhere('feed_generator.description', 'ilike', `%${query}%`), - ) - } - - let builder = db.db.selectFrom(inner.as('feed_gens')).selectAll() - - const keyset = new LikeCountKeyset(ref('likeCount'), ref('cid')) - builder = paginate(builder, { limit, cursor, keyset, direction: 'desc' }) - - const res = await builder.execute() - - const genInfos = await feedService.getFeedGeneratorInfos( - res.map((feed) => feed.uri), - requester, - ) - - const creators = Object.values(genInfos).map((gen) => gen.creator) - const profiles = await actorService.views.profiles(creators, requester) - - const genViews: GeneratorView[] = [] - for (const row of res) { - const gen = genInfos[row.uri] - if (!gen) continue - const view = feedService.views.formatFeedGeneratorView(gen, profiles) - if (view) { - genViews.push(view) - } - } - - return { - encoding: 'application/json', - body: { - cursor: keyset.packFromResult(res), - feeds: genViews, - }, - } - }, - }) -} - -type Result = { likeCount: number; cid: string } -type LabeledResult = { primary: number; secondary: string } -export class LikeCountKeyset extends GenericKeyset { - labelResult(result: Result) { - return { - primary: result.likeCount, - secondary: result.cid, - } - } - labeledResultToCursor(labeled: LabeledResult) { - return { - primary: labeled.primary.toString(), - secondary: labeled.secondary, - } - } - cursorToLabeledResult(cursor: { primary: string; secondary: string }) { - const likes = parseInt(cursor.primary, 10) - if (isNaN(likes)) { - throw new InvalidRequestError('Malformed cursor') - } - return { - primary: likes, - secondary: cursor.secondary, - } - } -} diff --git a/packages/mod-service/src/api/app/bsky/unspecced/getTimelineSkeleton.ts b/packages/mod-service/src/api/app/bsky/unspecced/getTimelineSkeleton.ts deleted file mode 100644 index 821eeda655f..00000000000 --- a/packages/mod-service/src/api/app/bsky/unspecced/getTimelineSkeleton.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { skeleton } from '../feed/getTimeline' -import { toSkeletonItem } from '../../../../feed-gen/types' - -// THIS IS A TEMPORARY UNSPECCED ROUTE -export default function (server: Server, ctx: AppContext) { - server.app.bsky.unspecced.getTimelineSkeleton({ - auth: ctx.authVerifier, - handler: async ({ auth, params }) => { - const db = ctx.db.getReplica('timeline') - const feedService = ctx.services.feed(db) - const viewer = auth.credentials.did - - const result = await skeleton({ ...params, viewer }, { db, feedService }) - - return { - encoding: 'application/json', - body: { - feed: result.feedItems.map(toSkeletonItem), - cursor: result.cursor, - }, - } - }, - }) -} diff --git a/packages/mod-service/src/api/app/bsky/util/feed.ts b/packages/mod-service/src/api/app/bsky/util/feed.ts deleted file mode 100644 index 769b2d7e833..00000000000 --- a/packages/mod-service/src/api/app/bsky/util/feed.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { TimeCidKeyset } from '../../../../db/pagination' -import { FeedRow } from '../../../../services/feed/types' - -export enum FeedAlgorithm { - ReverseChronological = 'reverse-chronological', -} - -export class FeedKeyset extends TimeCidKeyset { - labelResult(result: FeedRow) { - return { primary: result.sortAt, secondary: result.cid } - } -} - -// For users with sparse feeds, avoid scanning more than one week for a single page -export const getFeedDateThreshold = (from: string | undefined, days = 1) => { - const timelineDateThreshold = from ? new Date(from) : new Date() - timelineDateThreshold.setDate(timelineDateThreshold.getDate() - days) - return timelineDateThreshold.toISOString() -} diff --git a/packages/mod-service/src/api/blob-resolver.ts b/packages/mod-service/src/api/blob-resolver.ts deleted file mode 100644 index 7eb245eedd5..00000000000 --- a/packages/mod-service/src/api/blob-resolver.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { pipeline, Readable } from 'stream' -import express from 'express' -import createError from 'http-errors' -import axios, { AxiosError } from 'axios' -import { CID } from 'multiformats/cid' -import { ensureValidDid } from '@atproto/syntax' -import { forwardStreamErrors, VerifyCidTransform } from '@atproto/common' -import { IdResolver, DidNotFoundError } from '@atproto/identity' -import AppContext from '../context' -import { httpLogger as log } from '../logger' -import { retryHttp } from '../util/retry' -import { Database } from '../db' -import { sql } from 'kysely' - -// Resolve and verify blob from its origin host - -export const createRouter = (ctx: AppContext): express.Router => { - const router = express.Router() - - router.get('/blob/:did/:cid', async function (req, res, next) { - try { - const { did, cid: cidStr } = req.params - try { - ensureValidDid(did) - } catch (err) { - return next(createError(400, 'Invalid did')) - } - let cid: CID - try { - cid = CID.parse(cidStr) - } catch (err) { - return next(createError(400, 'Invalid cid')) - } - - const db = ctx.db.getReplica() - const verifiedImage = await resolveBlob(did, cid, db, ctx.idResolver) - - // Send chunked response, destroying stream early (before - // closing chunk) if the bytes don't match the expected cid. - res.statusCode = 200 - res.setHeader('content-type', verifiedImage.contentType) - res.setHeader('x-content-type-options', 'nosniff') - res.setHeader('content-security-policy', `default-src 'none'; sandbox`) - pipeline(verifiedImage.stream, res, (err) => { - if (err) { - log.warn( - { err, did, cid: cidStr, pds: verifiedImage.pds }, - 'blob resolution failed during transmission', - ) - } - }) - } catch (err) { - if (err instanceof AxiosError) { - if (err.code === AxiosError.ETIMEDOUT) { - log.warn( - { host: err.request?.host, path: err.request?.path }, - 'blob resolution timeout', - ) - return next(createError(504)) // Gateway timeout - } - if (!err.response || err.response.status >= 500) { - log.warn( - { host: err.request?.host, path: err.request?.path }, - 'blob resolution failed upstream', - ) - return next(createError(502)) - } - return next(createError(404, 'Blob not found')) - } - if (err instanceof DidNotFoundError) { - return next(createError(404, 'Blob not found')) - } - return next(err) - } - }) - - return router -} - -export async function resolveBlob( - did: string, - cid: CID, - db: Database, - idResolver: IdResolver, -) { - const cidStr = cid.toString() - - const [{ pds }, takedown] = await Promise.all([ - idResolver.did.resolveAtprotoData(did), // @TODO cache did info - db.db - .selectFrom('moderation_subject_status') - .select('id') - .where('blobCids', '@>', sql`CAST(${JSON.stringify([cidStr])} AS JSONB)`) - .where('takendown', 'is', true) - .executeTakeFirst(), - ]) - if (takedown) { - throw createError(404, 'Blob not found') - } - - const blobResult = await retryHttp(() => getBlob({ pds, did, cid: cidStr })) - const imageStream: Readable = blobResult.data - const verifyCid = new VerifyCidTransform(cid) - - forwardStreamErrors(imageStream, verifyCid) - return { - pds, - contentType: - blobResult.headers['content-type'] || 'application/octet-stream', - stream: imageStream.pipe(verifyCid), - } -} - -async function getBlob(opts: { pds: string; did: string; cid: string }) { - const { pds, did, cid } = opts - return axios.get(`${pds}/xrpc/com.atproto.sync.getBlob`, { - params: { did, cid }, - decompress: true, - responseType: 'stream', - timeout: 5000, // 5sec of inactivity on the connection - }) -} diff --git a/packages/mod-service/src/api/com/atproto/admin/emitModerationEvent.ts b/packages/mod-service/src/api/com/atproto/admin/emitModerationEvent.ts index 3483e453124..41d32f167bc 100644 --- a/packages/mod-service/src/api/com/atproto/admin/emitModerationEvent.ts +++ b/packages/mod-service/src/api/com/atproto/admin/emitModerationEvent.ts @@ -213,7 +213,7 @@ export default function (server: Server, ctx: AppContext) { return { encoding: 'application/json', - body: await moderationService.views.event(moderationEvent), + body: moderationService.views.formatEvent(moderationEvent), } }, }) diff --git a/packages/mod-service/src/api/com/atproto/admin/getRecord.ts b/packages/mod-service/src/api/com/atproto/admin/getRecord.ts index 6d1825c9914..f838c43f31b 100644 --- a/packages/mod-service/src/api/com/atproto/admin/getRecord.ts +++ b/packages/mod-service/src/api/com/atproto/admin/getRecord.ts @@ -2,28 +2,25 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' import { addAccountInfoToRepoView, getPdsAccountInfo } from './util' +import { AtUri } from '@atproto/syntax' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.getRecord({ auth: ctx.roleVerifier, handler: async ({ params, auth }) => { - const { uri, cid } = params const db = ctx.db - const result = await db.db - .selectFrom('record') - .selectAll() - .where('uri', '=', uri) - .if(!!cid, (qb) => qb.where('cid', '=', cid ?? '')) - .executeTakeFirst() - if (!result) { - throw new InvalidRequestError('Record not found', 'RecordNotFound') - } + + const uri = new AtUri(params.uri) const [record, accountInfo] = await Promise.all([ - ctx.services.moderation(db).views.recordDetail(result), - getPdsAccountInfo(ctx, result.did), + ctx.services.moderation(db).views.recordDetail(uri), + getPdsAccountInfo(ctx, uri.hostname), ]) + if (!record) { + throw new InvalidRequestError('Record not found', 'RecordNotFound') + } + record.repo = addAccountInfoToRepoView( record.repo, accountInfo, diff --git a/packages/mod-service/src/api/com/atproto/admin/getRepo.ts b/packages/mod-service/src/api/com/atproto/admin/getRepo.ts index e7180f3bb61..ba003a66386 100644 --- a/packages/mod-service/src/api/com/atproto/admin/getRepo.ts +++ b/packages/mod-service/src/api/com/atproto/admin/getRepo.ts @@ -9,14 +9,13 @@ export default function (server: Server, ctx: AppContext) { handler: async ({ params, auth }) => { const { did } = params const db = ctx.db - const result = await ctx.services.actor(db).getActor(did, true) - if (!result) { - throw new InvalidRequestError('Repo not found', 'RepoNotFound') - } const [partialRepo, accountInfo] = await Promise.all([ - ctx.services.moderation(db).views.repoDetail(result), - getPdsAccountInfo(ctx, result.did), + ctx.services.moderation(db).views.repoDetail(did), + getPdsAccountInfo(ctx, did), ]) + if (!partialRepo) { + throw new InvalidRequestError('Repo not found', 'RepoNotFound') + } const repo = addAccountInfoToRepoViewDetail( partialRepo, diff --git a/packages/mod-service/src/api/com/atproto/admin/queryModerationEvents.ts b/packages/mod-service/src/api/com/atproto/admin/queryModerationEvents.ts index a4b14a04a0b..80ccc10ee12 100644 --- a/packages/mod-service/src/api/com/atproto/admin/queryModerationEvents.ts +++ b/packages/mod-service/src/api/com/atproto/admin/queryModerationEvents.ts @@ -30,7 +30,9 @@ export default function (server: Server, ctx: AppContext) { encoding: 'application/json', body: { cursor: results.cursor, - events: await moderationService.views.event(results.events), + events: results.events.map((evt) => + moderationService.views.formatEvent(evt), + ), }, } }, diff --git a/packages/mod-service/src/api/com/atproto/admin/queryModerationStatuses.ts b/packages/mod-service/src/api/com/atproto/admin/queryModerationStatuses.ts index 5a74bfca3ae..7dbb4457b34 100644 --- a/packages/mod-service/src/api/com/atproto/admin/queryModerationStatuses.ts +++ b/packages/mod-service/src/api/com/atproto/admin/queryModerationStatuses.ts @@ -22,7 +22,7 @@ export default function (server: Server, ctx: AppContext) { limit = 50, cursor, } = params - const db = ctx.db.getPrimary() + const db = ctx.db const moderationService = ctx.services.moderation(db) const results = await moderationService.getSubjectStatuses({ reviewState: getReviewState(reviewState), @@ -40,8 +40,12 @@ export default function (server: Server, ctx: AppContext) { limit, cursor, }) - const subjectStatuses = moderationService.views.subjectStatus( - results.statuses, + const subjectStatuses = results.statuses.map( + (status) => + moderationService.views.formatSubjectStatus({ + ...status, + handle: '', + }), // @TODO fix handle ) return { encoding: 'application/json', diff --git a/packages/mod-service/src/api/com/atproto/admin/searchRepos.ts b/packages/mod-service/src/api/com/atproto/admin/searchRepos.ts index ef580f30d67..a1680ec5fdc 100644 --- a/packages/mod-service/src/api/com/atproto/admin/searchRepos.ts +++ b/packages/mod-service/src/api/com/atproto/admin/searchRepos.ts @@ -5,7 +5,7 @@ export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.searchRepos({ auth: ctx.roleVerifier, handler: async ({ params }) => { - const db = ctx.db.getPrimary() + const db = ctx.db const moderationService = ctx.services.moderation(db) const { limit, cursor } = params // prefer new 'q' query param over deprecated 'term' diff --git a/packages/mod-service/src/api/com/atproto/identity/resolveHandle.ts b/packages/mod-service/src/api/com/atproto/identity/resolveHandle.ts deleted file mode 100644 index 30c1d7f8a6f..00000000000 --- a/packages/mod-service/src/api/com/atproto/identity/resolveHandle.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import * as ident from '@atproto/syntax' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.identity.resolveHandle(async ({ req, params }) => { - const handle = ident.normalizeHandle(params.handle || req.hostname) - - const db = ctx.db.getReplica() - let did: string | undefined - const user = await ctx.services.actor(db).getActor(handle, true) - if (user) { - did = user.did - } else { - const publicHostname = ctx.cfg.publicUrl - ? new URL(ctx.cfg.publicUrl).hostname - : null - if ( - publicHostname && - (handle === publicHostname || handle.endsWith(`.${publicHostname}`)) - ) { - // Avoid resolution loop - throw new InvalidRequestError('Unable to resolve handle') - } - // this is not someone on our server, but we help with resolving anyway - did = await ctx.idResolver.handle.resolve(handle) - } - if (!did) { - throw new InvalidRequestError('Unable to resolve handle') - } - - return { - encoding: 'application/json', - body: { did }, - } - }) -} diff --git a/packages/mod-service/src/api/com/atproto/repo/getRecord.ts b/packages/mod-service/src/api/com/atproto/repo/getRecord.ts deleted file mode 100644 index c42c1fd6b4c..00000000000 --- a/packages/mod-service/src/api/com/atproto/repo/getRecord.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { InvalidRequestError } from '@atproto/xrpc-server' -import { AtUri } from '@atproto/syntax' -import { Server } from '../../../../lexicon' -import AppContext from '../../../../context' -import { jsonStringToLex } from '@atproto/lexicon' - -export default function (server: Server, ctx: AppContext) { - server.com.atproto.repo.getRecord(async ({ params }) => { - const { repo, collection, rkey, cid } = params - const db = ctx.db.getReplica() - const did = await ctx.services.actor(db).getActorDid(repo) - if (!did) { - throw new InvalidRequestError(`Could not find repo: ${repo}`) - } - - const uri = AtUri.make(did, collection, rkey) - - let builder = db.db - .selectFrom('record') - .selectAll() - .where('uri', '=', uri.toString()) - if (cid) { - builder = builder.where('cid', '=', cid) - } - - const record = await builder.executeTakeFirst() - if (!record) { - throw new InvalidRequestError(`Could not locate record: ${uri}`) - } - return { - encoding: 'application/json', - body: { - uri: record.uri, - cid: record.cid, - value: jsonStringToLex(record.json) as Record, - }, - } - }) -} diff --git a/packages/mod-service/src/api/index.ts b/packages/mod-service/src/api/index.ts index da21b582019..421b343122d 100644 --- a/packages/mod-service/src/api/index.ts +++ b/packages/mod-service/src/api/index.ts @@ -1,53 +1,11 @@ import { Server } from '../lexicon' import AppContext from '../context' -import describeFeedGenerator from './app/bsky/feed/describeFeedGenerator' -import getTimeline from './app/bsky/feed/getTimeline' -import getActorFeeds from './app/bsky/feed/getActorFeeds' -import getSuggestedFeeds from './app/bsky/feed/getSuggestedFeeds' -import getAuthorFeed from './app/bsky/feed/getAuthorFeed' -import getFeed from './app/bsky/feed/getFeed' -import getFeedGenerator from './app/bsky/feed/getFeedGenerator' -import getFeedGenerators from './app/bsky/feed/getFeedGenerators' -import getFeedSkeleton from './app/bsky/feed/getFeedSkeleton' -import getLikes from './app/bsky/feed/getLikes' -import getListFeed from './app/bsky/feed/getListFeed' -import getPostThread from './app/bsky/feed/getPostThread' -import getPosts from './app/bsky/feed/getPosts' -import searchPosts from './app/bsky/feed/searchPosts' -import getActorLikes from './app/bsky/feed/getActorLikes' -import getProfile from './app/bsky/actor/getProfile' -import getProfiles from './app/bsky/actor/getProfiles' -import getRepostedBy from './app/bsky/feed/getRepostedBy' -import getBlocks from './app/bsky/graph/getBlocks' -import getListBlocks from './app/bsky/graph/getListBlocks' -import getFollowers from './app/bsky/graph/getFollowers' -import getFollows from './app/bsky/graph/getFollows' -import getList from './app/bsky/graph/getList' -import getLists from './app/bsky/graph/getLists' -import getListMutes from './app/bsky/graph/getListMutes' -import getMutes from './app/bsky/graph/getMutes' -import muteActor from './app/bsky/graph/muteActor' -import unmuteActor from './app/bsky/graph/unmuteActor' -import muteActorList from './app/bsky/graph/muteActorList' -import unmuteActorList from './app/bsky/graph/unmuteActorList' -import getSuggestedFollowsByActor from './app/bsky/graph/getSuggestedFollowsByActor' -import searchActors from './app/bsky/actor/searchActors' -import searchActorsTypeahead from './app/bsky/actor/searchActorsTypeahead' -import getSuggestions from './app/bsky/actor/getSuggestions' -import getUnreadCount from './app/bsky/notification/getUnreadCount' -import listNotifications from './app/bsky/notification/listNotifications' -import updateSeen from './app/bsky/notification/updateSeen' -import registerPush from './app/bsky/notification/registerPush' -import getPopularFeedGenerators from './app/bsky/unspecced/getPopularFeedGenerators' -import getTimelineSkeleton from './app/bsky/unspecced/getTimelineSkeleton' import createReport from './com/atproto/moderation/createReport' import emitModerationEvent from './com/atproto/admin/emitModerationEvent' import searchRepos from './com/atproto/admin/searchRepos' import adminGetRecord from './com/atproto/admin/getRecord' import getRepo from './com/atproto/admin/getRepo' import queryModerationStatuses from './com/atproto/admin/queryModerationStatuses' -import resolveHandle from './com/atproto/identity/resolveHandle' -import getRecord from './com/atproto/repo/getRecord' import queryModerationEvents from './com/atproto/admin/queryModerationEvents' import getModerationEvent from './com/atproto/admin/getModerationEvent' import fetchLabels from './com/atproto/temp/fetchLabels' @@ -56,51 +14,7 @@ export * as health from './health' export * as wellKnown from './well-known' -export * as blobResolver from './blob-resolver' - export default function (server: Server, ctx: AppContext) { - // app.bsky - describeFeedGenerator(server, ctx) - getTimeline(server, ctx) - getActorFeeds(server, ctx) - getSuggestedFeeds(server, ctx) - getAuthorFeed(server, ctx) - getFeed(server, ctx) - getFeedGenerator(server, ctx) - getFeedGenerators(server, ctx) - getFeedSkeleton(server, ctx) - getLikes(server, ctx) - getListFeed(server, ctx) - getPostThread(server, ctx) - getPosts(server, ctx) - searchPosts(server, ctx) - getActorLikes(server, ctx) - getProfile(server, ctx) - getProfiles(server, ctx) - getRepostedBy(server, ctx) - getBlocks(server, ctx) - getListBlocks(server, ctx) - getFollowers(server, ctx) - getFollows(server, ctx) - getList(server, ctx) - getLists(server, ctx) - getListMutes(server, ctx) - getMutes(server, ctx) - muteActor(server, ctx) - unmuteActor(server, ctx) - muteActorList(server, ctx) - unmuteActorList(server, ctx) - getSuggestedFollowsByActor(server, ctx) - searchActors(server, ctx) - searchActorsTypeahead(server, ctx) - getSuggestions(server, ctx) - getUnreadCount(server, ctx) - listNotifications(server, ctx) - updateSeen(server, ctx) - registerPush(server, ctx) - getPopularFeedGenerators(server, ctx) - getTimelineSkeleton(server, ctx) - // com.atproto createReport(server, ctx) emitModerationEvent(server, ctx) searchRepos(server, ctx) @@ -109,8 +23,6 @@ export default function (server: Server, ctx: AppContext) { getModerationEvent(server, ctx) queryModerationEvents(server, ctx) queryModerationStatuses(server, ctx) - resolveHandle(server, ctx) - getRecord(server, ctx) fetchLabels(server, ctx) return server } diff --git a/packages/mod-service/src/auto-moderator/abyss.ts b/packages/mod-service/src/auto-moderator/abyss.ts deleted file mode 100644 index 4799c7067a5..00000000000 --- a/packages/mod-service/src/auto-moderator/abyss.ts +++ /dev/null @@ -1,114 +0,0 @@ -import axios from 'axios' -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import * as ui8 from 'uint8arrays' -import { resolveBlob } from '../api/blob-resolver' -import { retryHttp } from '../util/retry' -import { PrimaryDatabase } from '../db' -import { IdResolver } from '@atproto/identity' -import { labelerLogger as log } from '../logger' - -export interface ImageFlagger { - scanImage(did: string, cid: CID, uri: AtUri): Promise -} - -export class Abyss implements ImageFlagger { - protected auth: string - - constructor( - public endpoint: string, - protected password: string, - public ctx: { db: PrimaryDatabase; idResolver: IdResolver }, - ) { - this.auth = basicAuth(this.password) - } - - async scanImage(did: string, cid: CID, uri: AtUri): Promise { - const start = Date.now() - const res = await retryHttp(async () => { - try { - return await this.makeReq(did, cid, uri) - } catch (err) { - log.warn({ err, did, cid: cid.toString() }, 'abyss request failed') - throw err - } - }) - log.info( - { res, did, cid: cid.toString(), duration: Date.now() - start }, - 'abyss response', - ) - return this.parseRes(res) - } - - async makeReq(did: string, cid: CID, uri: AtUri): Promise { - const { stream, contentType } = await resolveBlob( - did, - cid, - this.ctx.db, - this.ctx.idResolver, - ) - const { data } = await axios.post( - this.getReqUrl({ did, uri: uri.toString() }), - stream, - { - headers: { - 'Content-Type': contentType, - authorization: this.auth, - }, - timeout: 10000, - }, - ) - return data - } - - parseRes(res: ScannerResp): string[] { - if (!res.match || res.match.status !== 'success') { - return [] - } - const labels: string[] = [] - for (const hit of res.match.hits) { - if (TAKEDOWN_LABELS.includes(hit.label)) { - labels.push(hit.label) - } - } - return labels - } - - getReqUrl(params: { did: string; uri: string }) { - const search = new URLSearchParams(params) - return `${ - this.endpoint - }/xrpc/com.atproto.unspecced.scanBlob?${search.toString()}` - } -} - -const TAKEDOWN_LABELS = ['csam', 'csem'] - -type ScannerResp = { - blob: unknown - match?: { - status: string - hits: ScannerHit[] - } - classify?: { - hits?: unknown[] - } - review?: { - state?: string - ticketId?: string - } -} - -type ScannerHit = { - hashType: string - hashValue: string - label: string - corpus: string -} - -const basicAuth = (password: string) => { - return ( - 'Basic ' + - ui8.toString(ui8.fromString(`admin:${password}`, 'utf8'), 'base64pad') - ) -} diff --git a/packages/mod-service/src/auto-moderator/fuzzy-matcher.ts b/packages/mod-service/src/auto-moderator/fuzzy-matcher.ts deleted file mode 100644 index 07b5fb9a85e..00000000000 --- a/packages/mod-service/src/auto-moderator/fuzzy-matcher.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { dedupeStrs } from '@atproto/common' -import * as ui8 from 'uint8arrays' - -export interface TextFlagger { - getMatches(string: string): string[] -} - -export class FuzzyMatcher implements TextFlagger { - private bannedWords: Set - private falsePositives: Set - - constructor(bannedWords: string[], falsePositives: string[] = []) { - this.bannedWords = new Set(bannedWords.map((word) => word.toLowerCase())) - this.falsePositives = new Set( - falsePositives.map((word) => word.toLowerCase()), - ) - } - - static fromB64(bannedB64: string, falsePositivesB64?: string) { - return new FuzzyMatcher( - decode(bannedB64), - falsePositivesB64 ? decode(falsePositivesB64) : undefined, - ) - } - - private normalize(domain: string): string[] { - const withoutSymbols = domain.replace(/[\W_]+/g, '') // Remove non-alphanumeric characters - const lowercase = withoutSymbols.toLowerCase() - - // Replace common leetspeak characters - const leetSpeakReplacements: { [key: string]: string[] } = { - '0': ['o'], - '8': ['b'], - '3': ['e'], - '4': ['a'], - '6': ['g'], - '1': ['i', 'l'], - '5': ['s'], - '7': ['t'], - } - - return this.generatePermutations(lowercase, leetSpeakReplacements) - } - - private generatePermutations( - domain: string, - leetSpeakReplacements: { [key: string]: string[] }, - ): string[] { - const results: string[] = [] - - const leetChars = Object.keys(leetSpeakReplacements) - const firstLeetCharIndex = [...domain].findIndex((char) => - leetChars.includes(char), - ) - - if (firstLeetCharIndex === -1) { - // No leetspeak characters left in the string - results.push(domain) - } else { - const char = domain[firstLeetCharIndex] - const beforeChar = domain.slice(0, firstLeetCharIndex) - const afterChar = domain.slice(firstLeetCharIndex + 1) - - // For each replacement, generate all possible combinations - for (const replacement of leetSpeakReplacements[char]) { - const replaced = beforeChar + replacement + afterChar - - // Recursively generate all permutations for the rest of the string - const otherPermutations = this.generatePermutations( - replaced, - leetSpeakReplacements, - ) - - // Add these permutations to the results - results.push(...otherPermutations) - } - } - - return dedupeStrs(results) - } - - public getMatches(domain: string): string[] { - const normalizedDomains = this.normalize(domain) - - const foundUnacceptableWords: string[] = [] - - for (const normalizedDomain of normalizedDomains) { - for (const word of this.bannedWords) { - const match = normalizedDomain.indexOf(word) - if (match > -1) { - let isFalsePositive = false - for (const falsePositive of this.falsePositives) { - const s_fp = falsePositive.indexOf(word) - const s_nd = match - s_fp - const wordToMatch = normalizedDomain.slice( - s_nd, - s_nd + falsePositive.length, - ) - if (wordToMatch === falsePositive) { - isFalsePositive = true - break - } - } - - if (!isFalsePositive) { - foundUnacceptableWords.push(word) - } - } - } - } - - if (foundUnacceptableWords.length > 0) { - return foundUnacceptableWords - } - - return [] - } -} - -export const decode = (encoded: string): string[] => { - return ui8.toString(ui8.fromString(encoded, 'base64'), 'utf8').split(',') -} - -export const encode = (words: string[]): string => { - return ui8.toString(ui8.fromString(words.join(','), 'utf8'), 'base64') -} diff --git a/packages/mod-service/src/auto-moderator/hive.ts b/packages/mod-service/src/auto-moderator/hive.ts deleted file mode 100644 index 51d67c1c783..00000000000 --- a/packages/mod-service/src/auto-moderator/hive.ts +++ /dev/null @@ -1,187 +0,0 @@ -import axios from 'axios' -import FormData from 'form-data' -import { CID } from 'multiformats/cid' -import { IdResolver } from '@atproto/identity' -import { PrimaryDatabase } from '../db' -import { retryHttp } from '../util/retry' -import { resolveBlob } from '../api/blob-resolver' -import { labelerLogger as log } from '../logger' - -const HIVE_ENDPOINT = 'https://api.thehive.ai/api/v2/task/sync' - -export interface ImgLabeler { - labelImg(did: string, cid: CID): Promise -} - -export class HiveLabeler implements ImgLabeler { - constructor( - public hiveApiKey: string, - protected ctx: { - db: PrimaryDatabase - idResolver: IdResolver - }, - ) {} - - async labelImg(did: string, cid: CID): Promise { - const hiveRes = await retryHttp(async () => { - try { - return await this.makeHiveReq(did, cid) - } catch (err) { - log.warn({ err, did, cid: cid.toString() }, 'hive request failed') - throw err - } - }) - log.info({ hiveRes, did, cid: cid.toString() }, 'hive response') - const classes = respToClasses(hiveRes) - return summarizeLabels(classes) - } - - async makeHiveReq(did: string, cid: CID): Promise { - const { stream } = await resolveBlob( - did, - cid, - this.ctx.db, - this.ctx.idResolver, - ) - const form = new FormData() - form.append('media', stream) - const { data } = await axios.post(HIVE_ENDPOINT, form, { - headers: { - 'Content-Type': 'multipart/form-data', - authorization: `token ${this.hiveApiKey}`, - accept: 'application/json', - }, - }) - return data - } -} - -export const respToClasses = (res: HiveResp): HiveRespClass[] => { - const classes: HiveRespClass[] = [] - for (const status of res.status) { - for (const out of status.response.output) { - for (const cls of out.classes) { - classes.push(cls) - } - } - } - return classes -} - -// Matches only one (or none) of: porn, sexual, nudity -// -// porn: sexual and nudity. including both explicit activity or full-frontal and suggestive/intent -// sexual: sexually suggestive, not explicit; may include some forms of nudity -// nudity: non-sexual nudity (eg, artistic, possibly some photographic) -// -// hive docs/definitions: https://docs.thehive.ai/docs/sexual-content -export const sexualLabels = (classes: HiveRespClass[]): string[] => { - const scores = {} - - for (const cls of classes) { - scores[cls.class] = cls.score - } - - // first check if porn... - for (const pornClass of [ - 'yes_sexual_activity', - 'animal_genitalia_and_human', - 'yes_realistic_nsfw', - ]) { - if (scores[pornClass] >= 0.9) { - return ['porn'] - } - } - if (scores['general_nsfw'] >= 0.9) { - // special case for some anime examples - if (scores['animated_animal_genitalia'] >= 0.5) { - return ['porn'] - } - // special case for some pornographic/explicit classic drawings - if (scores['yes_undressed'] >= 0.9 && scores['yes_sexual_activity'] > 0.9) { - return ['porn'] - } - } - - // then check for sexual suggestive (which may include nudity)... - for (const sexualClass of ['yes_sexual_intent', 'yes_sex_toy']) { - if (scores[sexualClass] >= 0.9) { - return ['sexual'] - } - } - if (scores['yes_undressed'] >= 0.9) { - // special case for bondage examples - if (scores['yes_sex_toy'] > 0.75) { - return ['sexual'] - } - } - - // then non-sexual nudity... - for (const nudityClass of [ - 'yes_male_nudity', - 'yes_female_nudity', - 'yes_undressed', - ]) { - if (scores[nudityClass] >= 0.9) { - return ['nudity'] - } - } - - // then finally flag remaining "underwear" images in to sexually suggestive - // (after non-sexual content already labeled above) - for (const nudityClass of ['yes_male_underwear', 'yes_female_underwear']) { - if (scores[nudityClass] >= 0.9) { - // TODO: retaining 'underwear' label for a short time to help understand - // the impact of labeling all "underwear" as "sexual". This *will* be - // pulling in somewhat non-sexual content in to "sexual" label. - return ['sexual'] - } - } - - return [] -} - -// gore and violence: https://docs.thehive.ai/docs/class-descriptions-violence-gore -const labelForClass = { - very_bloody: 'gore', - human_corpse: 'corpse', - hanging: 'corpse', -} -const labelForClassLessSensitive = { - yes_self_harm: 'self-harm', -} - -export const summarizeLabels = (classes: HiveRespClass[]): string[] => { - const labels: string[] = sexualLabels(classes) - for (const cls of classes) { - if (labelForClass[cls.class] && cls.score >= 0.9) { - labels.push(labelForClass[cls.class]) - } - } - for (const cls of classes) { - if (labelForClassLessSensitive[cls.class] && cls.score >= 0.96) { - labels.push(labelForClassLessSensitive[cls.class]) - } - } - return labels -} - -type HiveResp = { - status: HiveRespStatus[] -} - -type HiveRespStatus = { - response: { - output: HiveRespOutput[] - } -} - -type HiveRespOutput = { - time: number - classes: HiveRespClass[] -} - -type HiveRespClass = { - class: string - score: number -} diff --git a/packages/mod-service/src/auto-moderator/index.ts b/packages/mod-service/src/auto-moderator/index.ts deleted file mode 100644 index 8925314808c..00000000000 --- a/packages/mod-service/src/auto-moderator/index.ts +++ /dev/null @@ -1,302 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { AtpAgent } from '@atproto/api' -import { dedupe, getFieldsFromRecord } from './util' -import { labelerLogger as log } from '../logger' -import { PrimaryDatabase } from '../db' -import { IdResolver } from '@atproto/identity' -import { BackgroundQueue } from '../background' -import { IndexerConfig } from '../indexer/config' -import { buildBasicAuth } from '../auth' -import { CID } from 'multiformats/cid' -import { LabelService } from '../services/label' -import { ModerationService } from '../services/moderation' -import { ImageFlagger } from './abyss' -import { HiveLabeler, ImgLabeler } from './hive' -import { KeywordLabeler, TextLabeler } from './keyword' -import { ids } from '../lexicon/lexicons' -import { ImageUriBuilder } from '../image/uri' -import { ImageInvalidator } from '../image/invalidator' -import { Abyss } from './abyss' -import { FuzzyMatcher, TextFlagger } from './fuzzy-matcher' -import { - REASONOTHER, - REASONVIOLATION, -} from '../lexicon/types/com/atproto/moderation/defs' - -export class AutoModerator { - public pushAgent?: AtpAgent - public imageFlagger?: ImageFlagger - public textFlagger?: TextFlagger - public imgLabeler?: ImgLabeler - public textLabeler?: TextLabeler - - services: { - label: (db: PrimaryDatabase) => LabelService - moderation?: (db: PrimaryDatabase) => ModerationService - } - - constructor( - public ctx: { - db: PrimaryDatabase - idResolver: IdResolver - cfg: IndexerConfig - backgroundQueue: BackgroundQueue - imgUriBuilder?: ImageUriBuilder - imgInvalidator?: ImageInvalidator - }, - ) { - const { imgUriBuilder, imgInvalidator } = ctx - const { hiveApiKey, abyssEndpoint, abyssPassword } = ctx.cfg - this.services = { - label: LabelService.creator(null), - } - if (imgUriBuilder && imgInvalidator) { - this.services.moderation = ModerationService.creator( - imgUriBuilder, - imgInvalidator, - ) - } else { - log.error( - { imgUriBuilder, imgInvalidator }, - 'moderation service not properly configured', - ) - } - this.imgLabeler = hiveApiKey ? new HiveLabeler(hiveApiKey, ctx) : undefined - this.textLabeler = new KeywordLabeler(ctx.cfg.labelerKeywords) - if (abyssEndpoint && abyssPassword) { - this.imageFlagger = new Abyss(abyssEndpoint, abyssPassword, ctx) - } else { - log.error( - { abyssEndpoint, abyssPassword }, - 'abyss not properly configured', - ) - } - - if (ctx.cfg.fuzzyMatchB64) { - this.textFlagger = FuzzyMatcher.fromB64( - ctx.cfg.fuzzyMatchB64, - ctx.cfg.fuzzyFalsePositiveB64, - ) - } - - if (ctx.cfg.moderationPushUrl) { - const url = new URL(ctx.cfg.moderationPushUrl) - this.pushAgent = new AtpAgent({ service: url.origin }) - this.pushAgent.api.setHeader( - 'authorization', - buildBasicAuth(url.username, url.password), - ) - } - } - - processRecord(uri: AtUri, cid: CID, obj: unknown) { - this.ctx.backgroundQueue.add(async () => { - const { text, imgs } = getFieldsFromRecord(obj, uri) - await Promise.all([ - this.labelRecord(uri, cid, text, imgs).catch((err) => { - log.error( - { err, uri: uri.toString(), record: obj }, - 'failed to label record', - ) - }), - this.flagRecordText(uri, cid, text).catch((err) => { - log.error( - { err, uri: uri.toString(), record: obj }, - 'failed to check record for text flagging', - ) - }), - this.checkImgForTakedown(uri, cid, imgs).catch((err) => { - log.error( - { err, uri: uri.toString(), record: obj }, - 'failed to check img for takedown', - ) - }), - ]) - }) - } - - processHandle(handle: string, did: string) { - this.ctx.backgroundQueue.add(async () => { - await this.flagSubjectText(handle, { did }).catch((err) => { - log.error({ err, handle, did }, 'failed to label handle') - }) - }) - } - - async labelRecord(uri: AtUri, recordCid: CID, text: string[], imgs: CID[]) { - if (uri.collection !== ids.AppBskyFeedPost) { - // @TODO label profiles - return - } - const allLabels = await Promise.all([ - this.textLabeler?.labelText(text.join(' ')), - ...imgs.map((cid) => this.imgLabeler?.labelImg(uri.host, cid)), - ]) - const labels = dedupe(allLabels.flat()) - await this.storeLabels(uri, recordCid, labels) - } - - async flagRecordText(uri: AtUri, cid: CID, text: string[]) { - if ( - ![ - ids.AppBskyActorProfile, - ids.AppBskyGraphList, - ids.AppBskyFeedGenerator, - ].includes(uri.collection) - ) { - return - } - return this.flagSubjectText(text.join(' '), { uri, cid }) - } - - async flagSubjectText( - text: string, - subject: { did: string } | { uri: AtUri; cid: CID }, - ) { - if (!this.textFlagger) return - const matches = this.textFlagger.getMatches(text) - if (matches.length < 1) return - await this.ctx.db.transaction(async (dbTxn) => { - if (!this.services.moderation) { - log.error( - { subject, text, matches }, - 'no moderation service setup to flag record text', - ) - return - } - return this.services.moderation(dbTxn).report({ - reasonType: REASONOTHER, - reason: `Automatically flagged for possible slurs: ${matches.join( - ', ', - )}`, - subject, - reportedBy: this.ctx.cfg.labelerDid, - }) - }) - } - - async checkImgForTakedown(uri: AtUri, recordCid: CID, imgCids: CID[]) { - if (imgCids.length < 0) return - const results = await Promise.all( - imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid, uri)), - ) - const takedownCids: CID[] = [] - for (let i = 0; i < results.length; i++) { - if (results.at(i)?.length) { - takedownCids.push(imgCids[i]) - } - } - if (takedownCids.length === 0) return - try { - await this.persistTakedown( - uri, - recordCid, - takedownCids, - dedupe(results.flat()), - ) - } catch (err) { - log.error( - { - err, - uri: uri.toString(), - imgCids: imgCids.map((c) => c.toString()), - results, - }, - 'failed to persist takedown', - ) - } - } - - async persistTakedown( - uri: AtUri, - recordCid: CID, - takedownCids: CID[], - labels: string[], - ) { - const reportReason = `automated takedown (${labels.join( - ', ', - )}). account needs review and possibly additional action` - const takedownReason = `automated takedown for labels: ${labels.join(', ')}` - log.warn( - { - uri: uri.toString(), - blobCids: takedownCids, - labels, - }, - 'hard takedown of record (and blobs) based on auto-matching', - ) - - if (this.services.moderation) { - await this.ctx.db.transaction(async (dbTxn) => { - // directly/locally create report, even if we use pushAgent for the takedown. don't have acctual account credentials for pushAgent, only admin auth - if (!this.services.moderation) { - // checked above, outside the transaction - return - } - const modSrvc = this.services.moderation(dbTxn) - await modSrvc.report({ - reportedBy: this.ctx.cfg.labelerDid, - reasonType: REASONVIOLATION, - subject: { - uri: uri, - cid: recordCid, - }, - reason: reportReason, - }) - }) - } - - if (this.pushAgent) { - await this.pushAgent.com.atproto.admin.emitModerationEvent({ - event: { - $type: 'com.atproto.admin.defs#modEventTakedown', - comment: takedownReason, - }, - subject: { - $type: 'com.atproto.repo.strongRef', - uri: uri.toString(), - cid: recordCid.toString(), - }, - subjectBlobCids: takedownCids.map((c) => c.toString()), - createdBy: this.ctx.cfg.labelerDid, - }) - } else { - await this.ctx.db.transaction(async (dbTxn) => { - if (!this.services.moderation) { - throw new Error('no mod push agent or uri invalidator setup') - } - const modSrvc = this.services.moderation(dbTxn) - const action = await modSrvc.logEvent({ - event: { - $type: 'com.atproto.admin.defs#modEventTakedown', - comment: takedownReason, - }, - subject: { uri, cid: recordCid }, - subjectBlobCids: takedownCids, - createdBy: this.ctx.cfg.labelerDid, - }) - await modSrvc.takedownRecord({ - takedownId: action.id, - uri: uri, - cid: recordCid, - blobCids: takedownCids, - }) - }) - } - } - - async storeLabels(uri: AtUri, cid: CID, labels: string[]): Promise { - if (labels.length < 1) return - const labelSrvc = this.services.label(this.ctx.db) - await labelSrvc.formatAndCreate( - this.ctx.cfg.labelerDid, - uri.toString(), - cid.toString(), - { create: labels }, - ) - } - - async processAll() { - await this.ctx.backgroundQueue.processAll() - } -} diff --git a/packages/mod-service/src/auto-moderator/keyword.ts b/packages/mod-service/src/auto-moderator/keyword.ts deleted file mode 100644 index 6bc504aa142..00000000000 --- a/packages/mod-service/src/auto-moderator/keyword.ts +++ /dev/null @@ -1,25 +0,0 @@ -export interface TextLabeler { - labelText(text: string): Promise -} - -export class KeywordLabeler implements TextLabeler { - constructor(public keywords: Record) {} - - async labelText(text: string): Promise { - return keywordLabeling(this.keywords, text) - } -} - -export const keywordLabeling = ( - keywords: Record, - text: string, -): string[] => { - const lowerText = text.toLowerCase() - const labels: string[] = [] - for (const word of Object.keys(keywords)) { - if (lowerText.includes(word)) { - labels.push(keywords[word]) - } - } - return labels -} diff --git a/packages/mod-service/src/auto-moderator/util.ts b/packages/mod-service/src/auto-moderator/util.ts deleted file mode 100644 index ab1467a07f2..00000000000 --- a/packages/mod-service/src/auto-moderator/util.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import * as lex from '../lexicon/lexicons' -import { - isRecord as isPost, - Record as PostRecord, -} from '../lexicon/types/app/bsky/feed/post' -import { - isRecord as isProfile, - Record as ProfileRecord, -} from '../lexicon/types/app/bsky/actor/profile' -import { - isRecord as isList, - Record as ListRecord, -} from '../lexicon/types/app/bsky/graph/list' -import { - isRecord as isGenerator, - Record as GeneratorRecord, -} from '../lexicon/types/app/bsky/feed/generator' -import { isMain as isEmbedImage } from '../lexicon/types/app/bsky/embed/images' -import { isMain as isEmbedExternal } from '../lexicon/types/app/bsky/embed/external' -import { isMain as isEmbedRecordWithMedia } from '../lexicon/types/app/bsky/embed/recordWithMedia' - -type RecordFields = { - text: string[] - imgs: CID[] -} - -export const getFieldsFromRecord = ( - record: unknown, - uri: AtUri, -): RecordFields => { - if (isPost(record)) { - return getFieldsFromPost(record) - } else if (isProfile(record)) { - return getFieldsFromProfile(record) - } else if (isList(record)) { - return getFieldsFromList(record) - } else if (isGenerator(record)) { - return getFieldsFromGenerator(record, uri) - } else { - return { text: [], imgs: [] } - } -} - -export const getFieldsFromPost = (record: PostRecord): RecordFields => { - const text: string[] = [] - const imgs: CID[] = [] - text.push(record.text) - const embeds = separateEmbeds(record.embed) - for (const embed of embeds) { - if (isEmbedImage(embed)) { - for (const img of embed.images) { - imgs.push(img.image.ref) - text.push(img.alt) - } - } else if (isEmbedExternal(embed)) { - if (embed.external.thumb) { - imgs.push(embed.external.thumb.ref) - } - text.push(embed.external.title) - text.push(embed.external.description) - } - } - return { text, imgs } -} - -export const getFieldsFromProfile = (record: ProfileRecord): RecordFields => { - const text: string[] = [] - const imgs: CID[] = [] - if (record.displayName) { - text.push(record.displayName) - } - if (record.description) { - text.push(record.description) - } - if (record.avatar) { - imgs.push(record.avatar.ref) - } - if (record.banner) { - imgs.push(record.banner.ref) - } - return { text, imgs } -} - -export const getFieldsFromList = (record: ListRecord): RecordFields => { - const text: string[] = [] - const imgs: CID[] = [] - if (record.name) { - text.push(record.name) - } - if (record.description) { - text.push(record.description) - } - if (record.avatar) { - imgs.push(record.avatar.ref) - } - return { text, imgs } -} - -export const getFieldsFromGenerator = ( - record: GeneratorRecord, - uri: AtUri, -): RecordFields => { - const text: string[] = [] - const imgs: CID[] = [] - text.push(uri.rkey) - if (record.displayName) { - text.push(record.displayName) - } - if (record.description) { - text.push(record.description) - } - if (record.avatar) { - imgs.push(record.avatar.ref) - } - return { text, imgs } -} - -export const dedupe = (strs: (string | undefined)[]): string[] => { - const set = new Set() - for (const str of strs) { - if (str !== undefined) { - set.add(str) - } - } - return [...set] -} - -const separateEmbeds = (embed: PostRecord['embed']) => { - if (!embed) { - return [] - } - if (isEmbedRecordWithMedia(embed)) { - return [{ $type: lex.ids.AppBskyEmbedRecord, ...embed.record }, embed.media] - } - return [embed] -} diff --git a/packages/mod-service/src/cache/read-through.ts b/packages/mod-service/src/cache/read-through.ts deleted file mode 100644 index 1d1849e8451..00000000000 --- a/packages/mod-service/src/cache/read-through.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { cacheLogger as log } from '../logger' -import { Redis } from '../redis' - -export type CacheItem = { - val: T | null // null here is for negative caching - updatedAt: number -} - -export type CacheOptions = { - staleTTL: number - maxTTL: number - fetchMethod: (key: string) => Promise - fetchManyMethod?: (keys: string[]) => Promise> -} - -export class ReadThroughCache { - constructor(public redis: Redis, public opts: CacheOptions) {} - - private async _fetchMany(keys: string[]): Promise> { - let result: Record = {} - if (this.opts.fetchManyMethod) { - result = await this.opts.fetchManyMethod(keys) - } else { - const got = await Promise.all(keys.map((k) => this.opts.fetchMethod(k))) - for (let i = 0; i < keys.length; i++) { - result[keys[i]] = got[i] ?? null - } - } - // ensure caching negatives - for (const key of keys) { - result[key] ??= null - } - return result - } - - private async fetchAndCache(key: string): Promise { - const fetched = await this.opts.fetchMethod(key) - this.set(key, fetched).catch((err) => - log.error({ err, key }, 'failed to set cache value'), - ) - return fetched - } - - private async fetchAndCacheMany(keys: string[]): Promise> { - const fetched = await this._fetchMany(keys) - this.setMany(fetched).catch((err) => - log.error({ err, keys }, 'failed to set cache values'), - ) - return removeNulls(fetched) - } - - async get(key: string, opts?: { revalidate?: boolean }): Promise { - if (opts?.revalidate) { - return this.fetchAndCache(key) - } - let cached: CacheItem | null - try { - const got = await this.redis.get(key) - cached = got ? JSON.parse(got) : null - } catch (err) { - cached = null - log.warn({ key, err }, 'failed to fetch value from cache') - } - if (!cached || this.isExpired(cached)) { - return this.fetchAndCache(key) - } - if (this.isStale(cached)) { - this.fetchAndCache(key).catch((err) => - log.warn({ key, err }, 'failed to refresh stale cache value'), - ) - } - return cached.val - } - - async getMany( - keys: string[], - opts?: { revalidate?: boolean }, - ): Promise> { - if (opts?.revalidate) { - return this.fetchAndCacheMany(keys) - } - let cached: Record - try { - cached = await this.redis.getMulti(keys) - } catch (err) { - cached = {} - log.warn({ keys, err }, 'failed to fetch values from cache') - } - - const stale: string[] = [] - const toFetch: string[] = [] - const results: Record = {} - for (const key of keys) { - const val = cached[key] ? (JSON.parse(cached[key]) as CacheItem) : null - if (!val || this.isExpired(val)) { - toFetch.push(key) - continue - } - if (this.isStale(val)) { - stale.push(key) - } - if (val.val) { - results[key] = val.val - } - } - const fetched = await this.fetchAndCacheMany(toFetch) - this.fetchAndCacheMany(stale).catch((err) => - log.warn({ keys, err }, 'failed to refresh stale cache values'), - ) - return { - ...results, - ...fetched, - } - } - - async set(key: string, val: T | null) { - await this.setMany({ [key]: val }) - } - - async setMany(vals: Record) { - const items: Record = {} - for (const key of Object.keys(vals)) { - items[key] = JSON.stringify({ - val: vals[key], - updatedAt: Date.now(), - }) - } - await this.redis.setMulti(items, this.opts.maxTTL) - } - - async clearEntry(key: string) { - await this.redis.del(key) - } - - isExpired(result: CacheItem) { - return Date.now() > result.updatedAt + this.opts.maxTTL - } - - isStale(result: CacheItem) { - return Date.now() > result.updatedAt + this.opts.staleTTL - } -} - -const removeNulls = (obj: Record): Record => { - return Object.entries(obj).reduce((acc, [key, val]) => { - if (val !== null) { - acc[key] = val - } - return acc - }, {} as Record) -} diff --git a/packages/mod-service/src/feed-gen/bsky-team.ts b/packages/mod-service/src/feed-gen/bsky-team.ts deleted file mode 100644 index feb9539345e..00000000000 --- a/packages/mod-service/src/feed-gen/bsky-team.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { NotEmptyArray } from '@atproto/common' -import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' -import AppContext from '../context' -import { paginate } from '../db/pagination' -import { AlgoHandler, AlgoResponse } from './types' -import { FeedKeyset } from '../api/app/bsky/util/feed' - -const BSKY_TEAM: NotEmptyArray = [ - 'did:plc:z72i7hdynmk6r22z27h6tvur', // @bsky.app - 'did:plc:ewvi7nxzyoun6zhxrhs64oiz', // @atproto.com - 'did:plc:eon2iu7v3x2ukgxkqaf7e5np', // @safety.bsky.app -] - -const handler: AlgoHandler = async ( - ctx: AppContext, - params: SkeletonParams, - _viewer: string | null, -): Promise => { - const { limit = 50, cursor } = params - const db = ctx.db.getReplica('feed') - const feedService = ctx.services.feed(db) - - const { ref } = db.db.dynamic - - const postsQb = feedService - .selectPostQb() - .where('post.creator', 'in', BSKY_TEAM) - - const keyset = new FeedKeyset(ref('sortAt'), ref('cid')) - - let feedQb = db.db.selectFrom(postsQb.as('feed_items')).selectAll() - feedQb = paginate(feedQb, { limit, cursor, keyset }) - - const feedItems = await feedQb.execute() - - return { - feedItems, - cursor: keyset.packFromResult(feedItems), - } -} - -export default handler diff --git a/packages/mod-service/src/feed-gen/hot-classic.ts b/packages/mod-service/src/feed-gen/hot-classic.ts deleted file mode 100644 index d1595105f27..00000000000 --- a/packages/mod-service/src/feed-gen/hot-classic.ts +++ /dev/null @@ -1,55 +0,0 @@ -import AppContext from '../context' -import { NotEmptyArray } from '@atproto/common' -import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' -import { paginate } from '../db/pagination' -import { AlgoHandler, AlgoResponse } from './types' -import { FeedKeyset } from '../api/app/bsky/util/feed' -import { valuesList } from '../db/util' - -const NO_WHATS_HOT_LABELS: NotEmptyArray = ['!no-promote'] - -const handler: AlgoHandler = async ( - ctx: AppContext, - params: SkeletonParams, - _viewer: string | null, -): Promise => { - const { limit = 50, cursor } = params - const db = ctx.db.getReplica('feed') - const feedService = ctx.services.feed(db) - - const { ref } = db.db.dynamic - - const postsQb = feedService - .selectPostQb() - .leftJoin('post_agg', 'post_agg.uri', 'post.uri') - .leftJoin('post_embed_record', 'post_embed_record.postUri', 'post.uri') - .where('post_agg.likeCount', '>=', 12) - .where('post.replyParent', 'is', null) - .whereNotExists((qb) => - qb - .selectFrom('label') - .selectAll() - .whereRef('val', 'in', valuesList(NO_WHATS_HOT_LABELS)) - .where('neg', '=', false) - .where((clause) => - clause - .whereRef('label.uri', '=', ref('post.creator')) - .orWhereRef('label.uri', '=', ref('post.uri')) - .orWhereRef('label.uri', '=', ref('post_embed_record.embedUri')), - ), - ) - - const keyset = new FeedKeyset(ref('sortAt'), ref('cid')) - - let feedQb = db.db.selectFrom(postsQb.as('feed_items')).selectAll() - feedQb = paginate(feedQb, { limit, cursor, keyset }) - - const feedItems = await feedQb.execute() - - return { - feedItems, - cursor: keyset.packFromResult(feedItems), - } -} - -export default handler diff --git a/packages/mod-service/src/feed-gen/index.ts b/packages/mod-service/src/feed-gen/index.ts deleted file mode 100644 index 5109d32416c..00000000000 --- a/packages/mod-service/src/feed-gen/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { ids } from '../lexicon/lexicons' -import bskyTeam from './bsky-team' -import hotClassic from './hot-classic' -import mutuals from './mutuals' -import { MountedAlgos } from './types' - -const feedgenUri = (did, name) => - AtUri.make(did, ids.AppBskyFeedGenerator, name).toString() - -// These are custom algorithms that will be mounted directly onto an AppView -// Feel free to remove, update to your own, or serve the following logic at a record that you control -export const makeAlgos = (did: string): MountedAlgos => ({ - [feedgenUri(did, 'bsky-team')]: bskyTeam, - [feedgenUri(did, 'hot-classic')]: hotClassic, - [feedgenUri(did, 'mutuals')]: mutuals, -}) diff --git a/packages/mod-service/src/feed-gen/mutuals.ts b/packages/mod-service/src/feed-gen/mutuals.ts deleted file mode 100644 index 86583ebaa56..00000000000 --- a/packages/mod-service/src/feed-gen/mutuals.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' -import AppContext from '../context' -import { paginate } from '../db/pagination' -import { AlgoHandler, AlgoResponse } from './types' -import { FeedKeyset, getFeedDateThreshold } from '../api/app/bsky/util/feed' -import { AuthRequiredError } from '@atproto/xrpc-server' - -const handler: AlgoHandler = async ( - ctx: AppContext, - params: SkeletonParams, - viewer: string | null, -): Promise => { - if (!viewer) { - throw new AuthRequiredError('This feed requires being logged-in') - } - - const { limit = 50, cursor } = params - const db = ctx.db.getReplica('feed') - const feedService = ctx.services.feed(db) - const { ref } = db.db.dynamic - - const mutualsSubquery = db.db - .selectFrom('follow') - .where('follow.creator', '=', viewer) - .whereExists((qb) => - qb - .selectFrom('follow as follow_inner') - .whereRef('follow_inner.creator', '=', 'follow.subjectDid') - .where('follow_inner.subjectDid', '=', viewer) - .selectAll(), - ) - .select('follow.subjectDid') - - const keyset = new FeedKeyset(ref('feed_item.sortAt'), ref('feed_item.cid')) - const sortFrom = keyset.unpack(cursor)?.primary - - let feedQb = feedService - .selectFeedItemQb() - .where('feed_item.type', '=', 'post') // ensures originatorDid is post.creator - .where((qb) => - qb - .where('originatorDid', '=', viewer) - .orWhere('originatorDid', 'in', mutualsSubquery), - ) - .where('feed_item.sortAt', '>', getFeedDateThreshold(sortFrom)) - - feedQb = paginate(feedQb, { limit, cursor, keyset }) - - const feedItems = await feedQb.execute() - - return { - feedItems, - cursor: keyset.packFromResult(feedItems), - } -} - -export default handler diff --git a/packages/mod-service/src/feed-gen/types.ts b/packages/mod-service/src/feed-gen/types.ts deleted file mode 100644 index 4693d64d4dd..00000000000 --- a/packages/mod-service/src/feed-gen/types.ts +++ /dev/null @@ -1,31 +0,0 @@ -import AppContext from '../context' -import { SkeletonFeedPost } from '../lexicon/types/app/bsky/feed/defs' -import { QueryParams as SkeletonParams } from '../lexicon/types/app/bsky/feed/getFeedSkeleton' -import { FeedRow } from '../services/feed' - -export type AlgoResponse = { - feedItems: FeedRow[] - cursor?: string -} - -export type AlgoHandler = ( - ctx: AppContext, - params: SkeletonParams, - viewer: string | null, -) => Promise - -export type MountedAlgos = Record - -export const toSkeletonItem = (feedItem: { - uri: string - postUri: string -}): SkeletonFeedPost => ({ - post: feedItem.postUri, - reason: - feedItem.uri === feedItem.postUri - ? undefined - : { - $type: 'app.bsky.feed.defs#skeletonReasonRepost', - repost: feedItem.uri, - }, -}) diff --git a/packages/mod-service/src/image/index.ts b/packages/mod-service/src/image/index.ts deleted file mode 100644 index 3197b5aeb5c..00000000000 --- a/packages/mod-service/src/image/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './sharp' -export type { Options, ImageInfo } from './util' diff --git a/packages/mod-service/src/image/invalidator.ts b/packages/mod-service/src/image/invalidator.ts deleted file mode 100644 index 70bf363371d..00000000000 --- a/packages/mod-service/src/image/invalidator.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { BlobCache } from './server' -import { ImageUriBuilder } from './uri' - -// Invalidation is a general interface for propagating an image blob -// takedown through any caches where a representation of it may be stored. -// @NOTE this does not remove the blob from storage: just invalidates it from caches. -// @NOTE keep in sync with same interface in aws/src/cloudfront.ts -export interface ImageInvalidator { - invalidate(subject: string, paths: string[]): Promise -} - -export class ImageProcessingServerInvalidator implements ImageInvalidator { - constructor(private cache: BlobCache) {} - async invalidate(_subject: string, paths: string[]) { - const results = await Promise.allSettled( - paths.map(async (path) => { - const [, signature] = path.split('/') - if (!signature) throw new Error('Missing signature') - const options = ImageUriBuilder.getOptions(path) - const cacheKey = [ - options.did, - options.cid.toString(), - options.preset, - ].join('::') - await this.cache.clear(cacheKey) - }), - ) - const rejection = results.find( - (result): result is PromiseRejectedResult => result.status === 'rejected', - ) - if (rejection) throw rejection.reason - } -} diff --git a/packages/mod-service/src/image/logger.ts b/packages/mod-service/src/image/logger.ts deleted file mode 100644 index d3d25481f81..00000000000 --- a/packages/mod-service/src/image/logger.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { subsystemLogger } from '@atproto/common' - -export const logger: ReturnType = - subsystemLogger('bsky:image') - -export default logger diff --git a/packages/mod-service/src/image/server.ts b/packages/mod-service/src/image/server.ts deleted file mode 100644 index 563d9b5bf4b..00000000000 --- a/packages/mod-service/src/image/server.ts +++ /dev/null @@ -1,200 +0,0 @@ -import fs from 'fs/promises' -import fsSync from 'fs' -import os from 'os' -import path from 'path' -import { Readable } from 'stream' -import axios, { AxiosError } from 'axios' -import express, { ErrorRequestHandler, NextFunction } from 'express' -import createError, { isHttpError } from 'http-errors' -import { BlobNotFoundError } from '@atproto/repo' -import { - cloneStream, - forwardStreamErrors, - isErrnoException, -} from '@atproto/common' -import { BadPathError, ImageUriBuilder } from './uri' -import log from './logger' -import { resize } from './sharp' -import { formatsToMimes, Options } from './util' -import { retryHttp } from '../util/retry' -import { ServerConfig } from '../config' - -export class ImageProcessingServer { - app = express() - uriBuilder: ImageUriBuilder - - constructor(public cfg: ServerConfig, public cache: BlobCache) { - this.uriBuilder = new ImageUriBuilder('') - this.app.get('*', this.handler.bind(this)) - this.app.use(errorMiddleware) - } - - async handler( - req: express.Request, - res: express.Response, - next: NextFunction, - ) { - try { - const path = req.path - const options = ImageUriBuilder.getOptions(path) - const cacheKey = [ - options.did, - options.cid.toString(), - options.preset, - ].join('::') - - // Cached flow - - try { - const cachedImage = await this.cache.get(cacheKey) - res.statusCode = 200 - res.setHeader('x-cache', 'hit') - res.setHeader('content-type', getMime(options.format)) - res.setHeader('cache-control', `public, max-age=31536000`) // 1 year - res.setHeader('content-length', cachedImage.size) - forwardStreamErrors(cachedImage, res) - return cachedImage.pipe(res) - } catch (err) { - // Ignore BlobNotFoundError and move on to non-cached flow - if (!(err instanceof BlobNotFoundError)) throw err - } - - // Non-cached flow - - const { localUrl } = this.cfg - const did = options.did - const cidStr = options.cid.toString() - - const blobResult = await retryHttp(() => - getBlob({ baseUrl: localUrl, did, cid: cidStr }), - ) - - const imageStream: Readable = blobResult.data - const processedImage = await resize(imageStream, options) - - // Cache in the background - this.cache - .put(cacheKey, cloneStream(processedImage)) - .catch((err) => log.error(err, 'failed to cache image')) - // Respond - res.statusCode = 200 - res.setHeader('x-cache', 'miss') - res.setHeader('content-type', getMime(options.format)) - res.setHeader('cache-control', `public, max-age=31536000`) // 1 year - forwardStreamErrors(processedImage, res) - return ( - processedImage - // @NOTE sharp does emit this in time to be set as a header - .once('info', (info) => res.setHeader('content-length', info.size)) - .pipe(res) - ) - } catch (err: unknown) { - if (err instanceof BadPathError) { - return next(createError(400, err)) - } - if (err instanceof AxiosError) { - if (err.code === AxiosError.ETIMEDOUT) { - return next(createError(504)) // Gateway timeout - } - if (!err.response || err.response.status >= 500) { - return next(createError(502)) - } - if (err.response.status === 400) { - return next(createError(400)) - } - return next(createError(404, 'Image not found')) - } - return next(err) - } - } -} - -const errorMiddleware: ErrorRequestHandler = function (err, _req, res, next) { - if (isHttpError(err)) { - log.error(err, `error: ${err.message}`) - } else { - log.error(err, 'unhandled exception') - } - if (res.headersSent) { - return next(err) - } - const httpError = createError(err) - return res.status(httpError.status).json({ - message: httpError.expose ? httpError.message : 'Internal Server Error', - }) -} - -function getMime(format: Options['format']) { - const mime = formatsToMimes[format] - if (!mime) throw new Error('Unknown format') - return mime -} - -export interface BlobCache { - get(fileId: string): Promise - put(fileId: string, stream: Readable): Promise - clear(fileId: string): Promise - clearAll(): Promise -} - -export class BlobDiskCache implements BlobCache { - tempDir: string - constructor(basePath?: string) { - this.tempDir = basePath || path.join(os.tmpdir(), 'bsky--processed-images') - if (!path.isAbsolute(this.tempDir)) { - throw new Error('Must provide an absolute path') - } - try { - fsSync.mkdirSync(this.tempDir, { recursive: true }) - } catch (err) { - // All good if cache dir already exists - if (isErrnoException(err) && err.code === 'EEXIST') return - } - } - - async get(fileId: string) { - try { - const handle = await fs.open(path.join(this.tempDir, fileId), 'r') - const { size } = await handle.stat() - if (size === 0) { - throw new BlobNotFoundError() - } - return Object.assign(handle.createReadStream(), { size }) - } catch (err) { - if (isErrnoException(err) && err.code === 'ENOENT') { - throw new BlobNotFoundError() - } - throw err - } - } - - async put(fileId: string, stream: Readable) { - const filename = path.join(this.tempDir, fileId) - try { - await fs.writeFile(filename, stream, { flag: 'wx' }) - } catch (err) { - // Do not overwrite existing file, just ignore the error - if (isErrnoException(err) && err.code === 'EEXIST') return - throw err - } - } - - async clear(fileId: string) { - const filename = path.join(this.tempDir, fileId) - await fs.rm(filename, { force: true }) - } - - async clearAll() { - await fs.rm(this.tempDir, { recursive: true, force: true }) - } -} - -function getBlob(opts: { baseUrl: string; did: string; cid: string }) { - const { baseUrl, did, cid } = opts - const enc = encodeURIComponent - return axios.get(`${baseUrl}/blob/${enc(did)}/${enc(cid)}`, { - decompress: true, - responseType: 'stream', - timeout: 2000, // 2sec of inactivity on the connection - }) -} diff --git a/packages/mod-service/src/image/sharp.ts b/packages/mod-service/src/image/sharp.ts deleted file mode 100644 index 1edc7a58835..00000000000 --- a/packages/mod-service/src/image/sharp.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { Readable } from 'stream' -import { pipeline } from 'stream/promises' -import sharp from 'sharp' -import { errHasMsg, forwardStreamErrors } from '@atproto/common' -import { formatsToMimes, ImageInfo, Options } from './util' - -export type { Options } - -export async function resize( - stream: Readable, - options: Options, -): Promise { - const { height, width, min = false, fit = 'cover', format, quality } = options - - let processor = sharp() - - // Scale up to hit any specified minimum size - if (typeof min !== 'boolean') { - const upsizeProcessor = sharp().resize({ - fit: 'outside', - width: min.width, - height: min.height, - withoutReduction: true, - withoutEnlargement: false, - }) - forwardStreamErrors(stream, upsizeProcessor) - stream = stream.pipe(upsizeProcessor) - } - - // Scale down (or possibly up if min is true) to desired size - processor = processor.resize({ - fit, - width, - height, - withoutEnlargement: min !== true, - }) - - // Output to specified format - if (format === 'jpeg') { - processor = processor.jpeg({ quality: quality ?? 100 }) - } else if (format === 'png') { - processor = processor.png({ quality: quality ?? 100 }) - } else { - const exhaustiveCheck: never = format - throw new Error(`Unhandled case: ${exhaustiveCheck}`) - } - - forwardStreamErrors(stream, processor) - return stream.pipe(processor) -} - -export async function maybeGetInfo( - stream: Readable, -): Promise { - let metadata: sharp.Metadata - try { - const processor = sharp() - const [result] = await Promise.all([ - processor.metadata(), - pipeline(stream, processor), // Handles error propagation - ]) - metadata = result - } catch (err) { - if (errHasMsg(err, 'Input buffer contains unsupported image format')) { - return null - } - throw err - } - const { size, height, width, format } = metadata - if ( - size === undefined || - height === undefined || - width === undefined || - format === undefined - ) { - return null - } - - return { - height, - width, - size, - mime: formatsToMimes[format] ?? ('unknown' as const), - } -} - -export async function getInfo(stream: Readable): Promise { - const maybeInfo = await maybeGetInfo(stream) - if (!maybeInfo) { - throw new Error('could not obtain all image metadata') - } - return maybeInfo -} diff --git a/packages/mod-service/src/image/uri.ts b/packages/mod-service/src/image/uri.ts deleted file mode 100644 index 5e288e29d10..00000000000 --- a/packages/mod-service/src/image/uri.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { CID } from 'multiformats/cid' -import { Options } from './util' - -// @NOTE if there are any additions here, ensure to include them on ImageUriBuilder.presets -export type ImagePreset = - | 'avatar' - | 'banner' - | 'feed_thumbnail' - | 'feed_fullsize' - -const PATH_REGEX = /^\/(.+?)\/plain\/(.+?)\/(.+?)@(.+?)$/ - -export class ImageUriBuilder { - constructor(public endpoint: string) {} - - static presets: ImagePreset[] = [ - 'avatar', - 'banner', - 'feed_thumbnail', - 'feed_fullsize', - ] - - getPresetUri(id: ImagePreset, did: string, cid: string | CID): string { - const options = presets[id] - if (!options) { - throw new Error(`Unrecognized requested common uri type: ${id}`) - } - return ( - this.endpoint + - ImageUriBuilder.getPath({ - preset: id, - did, - cid: typeof cid === 'string' ? CID.parse(cid) : cid, - }) - ) - } - - static getPath(opts: { preset: ImagePreset } & BlobLocation) { - const { format } = presets[opts.preset] - return `/${opts.preset}/plain/${opts.did}/${opts.cid.toString()}@${format}` - } - - static getOptions( - path: string, - ): Options & BlobLocation & { preset: ImagePreset } { - const match = path.match(PATH_REGEX) - if (!match) { - throw new BadPathError('Invalid path') - } - const [, presetUnsafe, did, cid, formatUnsafe] = match - if (!(ImageUriBuilder.presets as string[]).includes(presetUnsafe)) { - throw new BadPathError('Invalid path: bad preset') - } - if (formatUnsafe !== 'jpeg' && formatUnsafe !== 'png') { - throw new BadPathError('Invalid path: bad format') - } - const preset = presetUnsafe as ImagePreset - const format = formatUnsafe as Options['format'] - return { - ...presets[preset], - did, - cid: CID.parse(cid), - preset, - format, - } - } -} - -type BlobLocation = { cid: CID; did: string } - -export class BadPathError extends Error {} - -export const presets: Record = { - avatar: { - format: 'jpeg', - fit: 'cover', - height: 1000, - width: 1000, - min: true, - }, - banner: { - format: 'jpeg', - fit: 'cover', - height: 1000, - width: 3000, - min: true, - }, - feed_thumbnail: { - format: 'jpeg', - fit: 'inside', - height: 2000, - width: 2000, - min: true, - }, - feed_fullsize: { - format: 'jpeg', - fit: 'inside', - height: 1000, - width: 1000, - min: true, - }, -} diff --git a/packages/mod-service/src/image/util.ts b/packages/mod-service/src/image/util.ts deleted file mode 100644 index ce18ba343d5..00000000000 --- a/packages/mod-service/src/image/util.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { FormatEnum } from 'sharp' - -export type Options = Dimensions & { - format: 'jpeg' | 'png' - // When 'cover' (default), scale to fill given dimensions, cropping if necessary. - // When 'inside', scale to fit within given dimensions. - fit?: 'cover' | 'inside' - // When false (default), do not scale up. - // When true, scale up to hit dimensions given in options. - // Otherwise, scale up to hit specified min dimensions. - min?: Dimensions | boolean - // A number 1-100 - quality?: number -} - -export type ImageInfo = Dimensions & { - size: number - mime: `image/${string}` | 'unknown' -} - -export type Dimensions = { height: number; width: number } - -export const formatsToMimes: { [s in keyof FormatEnum]?: `image/${string}` } = { - jpg: 'image/jpeg', - jpeg: 'image/jpeg', - png: 'image/png', - gif: 'image/gif', - svg: 'image/svg+xml', - tif: 'image/tiff', - tiff: 'image/tiff', - webp: 'image/webp', -} diff --git a/packages/mod-service/src/indexer/config.ts b/packages/mod-service/src/indexer/config.ts deleted file mode 100644 index dd8b9ab89d5..00000000000 --- a/packages/mod-service/src/indexer/config.ts +++ /dev/null @@ -1,263 +0,0 @@ -import assert from 'assert' -import { DAY, HOUR, parseIntWithFallback } from '@atproto/common' - -export interface IndexerConfigValues { - version: string - dbPostgresUrl: string - dbPostgresSchema?: string - redisHost?: string // either set redis host, or both sentinel name and hosts - redisSentinelName?: string - redisSentinelHosts?: string[] - redisPassword?: string - didPlcUrl: string - didCacheStaleTTL: number - didCacheMaxTTL: number - handleResolveNameservers?: string[] - labelerDid: string - hiveApiKey?: string - abyssEndpoint?: string - abyssPassword?: string - imgUriEndpoint?: string - fuzzyMatchB64?: string - fuzzyFalsePositiveB64?: string - labelerKeywords: Record - moderationPushUrl?: string - indexerConcurrency?: number - indexerPartitionIds: number[] - indexerPartitionBatchSize?: number - indexerSubLockId?: number - indexerPort?: number - ingesterPartitionCount: number - indexerNamespace?: string - pushNotificationEndpoint?: string -} - -export class IndexerConfig { - constructor(private cfg: IndexerConfigValues) {} - - static readEnv(overrides?: Partial) { - const version = process.env.BSKY_VERSION || '0.0.0' - const dbPostgresUrl = - overrides?.dbPostgresUrl || process.env.DB_PRIMARY_POSTGRES_URL - const dbPostgresSchema = - overrides?.dbPostgresSchema || process.env.DB_POSTGRES_SCHEMA - const redisHost = - overrides?.redisHost || process.env.REDIS_HOST || undefined - const redisSentinelName = - overrides?.redisSentinelName || - process.env.REDIS_SENTINEL_NAME || - undefined - const redisSentinelHosts = - overrides?.redisSentinelHosts || - (process.env.REDIS_SENTINEL_HOSTS - ? process.env.REDIS_SENTINEL_HOSTS.split(',') - : []) - const redisPassword = - overrides?.redisPassword || process.env.REDIS_PASSWORD || undefined - const didPlcUrl = process.env.DID_PLC_URL || 'http://localhost:2582' - const didCacheStaleTTL = parseIntWithFallback( - process.env.DID_CACHE_STALE_TTL, - HOUR, - ) - const didCacheMaxTTL = parseIntWithFallback( - process.env.DID_CACHE_MAX_TTL, - DAY, - ) - const handleResolveNameservers = process.env.HANDLE_RESOLVE_NAMESERVERS - ? process.env.HANDLE_RESOLVE_NAMESERVERS.split(',') - : [] - const labelerDid = process.env.LABELER_DID || 'did:example:labeler' - const moderationPushUrl = - overrides?.moderationPushUrl || - process.env.MODERATION_PUSH_URL || - undefined - const hiveApiKey = process.env.HIVE_API_KEY || undefined - const abyssEndpoint = process.env.ABYSS_ENDPOINT - const abyssPassword = process.env.ABYSS_PASSWORD - const imgUriEndpoint = process.env.IMG_URI_ENDPOINT - const indexerPartitionIds = - overrides?.indexerPartitionIds || - (process.env.INDEXER_PARTITION_IDS - ? process.env.INDEXER_PARTITION_IDS.split(',').map((n) => - parseInt(n, 10), - ) - : []) - const indexerPartitionBatchSize = maybeParseInt( - process.env.INDEXER_PARTITION_BATCH_SIZE, - ) - const indexerConcurrency = maybeParseInt(process.env.INDEXER_CONCURRENCY) - const indexerNamespace = overrides?.indexerNamespace - const indexerSubLockId = maybeParseInt(process.env.INDEXER_SUB_LOCK_ID) - const indexerPort = maybeParseInt(process.env.INDEXER_PORT) - const ingesterPartitionCount = - maybeParseInt(process.env.INGESTER_PARTITION_COUNT) ?? 64 - const labelerKeywords = {} - const fuzzyMatchB64 = process.env.FUZZY_MATCH_B64 || undefined - const fuzzyFalsePositiveB64 = - process.env.FUZZY_FALSE_POSITIVE_B64 || undefined - const pushNotificationEndpoint = process.env.PUSH_NOTIFICATION_ENDPOINT - assert(dbPostgresUrl) - assert(redisHost || (redisSentinelName && redisSentinelHosts?.length)) - assert(indexerPartitionIds.length > 0) - return new IndexerConfig({ - version, - dbPostgresUrl, - dbPostgresSchema, - redisHost, - redisSentinelName, - redisSentinelHosts, - redisPassword, - didPlcUrl, - didCacheStaleTTL, - didCacheMaxTTL, - handleResolveNameservers, - labelerDid, - moderationPushUrl, - hiveApiKey, - abyssEndpoint, - abyssPassword, - imgUriEndpoint, - indexerPartitionIds, - indexerConcurrency, - indexerPartitionBatchSize, - indexerNamespace, - indexerSubLockId, - indexerPort, - ingesterPartitionCount, - labelerKeywords, - fuzzyMatchB64, - fuzzyFalsePositiveB64, - pushNotificationEndpoint, - ...stripUndefineds(overrides ?? {}), - }) - } - - get version() { - return this.cfg.version - } - - get dbPostgresUrl() { - return this.cfg.dbPostgresUrl - } - - get dbPostgresSchema() { - return this.cfg.dbPostgresSchema - } - - get redisHost() { - return this.cfg.redisHost - } - - get redisSentinelName() { - return this.cfg.redisSentinelName - } - - get redisSentinelHosts() { - return this.cfg.redisSentinelHosts - } - - get redisPassword() { - return this.cfg.redisPassword - } - - get didPlcUrl() { - return this.cfg.didPlcUrl - } - - get didCacheStaleTTL() { - return this.cfg.didCacheStaleTTL - } - - get didCacheMaxTTL() { - return this.cfg.didCacheMaxTTL - } - - get handleResolveNameservers() { - return this.cfg.handleResolveNameservers - } - - get labelerDid() { - return this.cfg.labelerDid - } - - get moderationPushUrl() { - return this.cfg.moderationPushUrl - } - - get hiveApiKey() { - return this.cfg.hiveApiKey - } - - get abyssEndpoint() { - return this.cfg.abyssEndpoint - } - - get abyssPassword() { - return this.cfg.abyssPassword - } - - get imgUriEndpoint() { - return this.cfg.imgUriEndpoint - } - - get indexerConcurrency() { - return this.cfg.indexerConcurrency - } - - get indexerPartitionIds() { - return this.cfg.indexerPartitionIds - } - - get indexerPartitionBatchSize() { - return this.cfg.indexerPartitionBatchSize - } - - get indexerNamespace() { - return this.cfg.indexerNamespace - } - - get indexerSubLockId() { - return this.cfg.indexerSubLockId - } - - get indexerPort() { - return this.cfg.indexerPort - } - - get ingesterPartitionCount() { - return this.cfg.ingesterPartitionCount - } - - get labelerKeywords() { - return this.cfg.labelerKeywords - } - - get fuzzyMatchB64() { - return this.cfg.fuzzyMatchB64 - } - - get fuzzyFalsePositiveB64() { - return this.cfg.fuzzyFalsePositiveB64 - } - - get pushNotificationEndpoint() { - return this.cfg.pushNotificationEndpoint - } -} - -function stripUndefineds( - obj: Record, -): Record { - const result = {} - Object.entries(obj).forEach(([key, val]) => { - if (val !== undefined) { - result[key] = val - } - }) - return result -} - -function maybeParseInt(str) { - const parsed = parseInt(str) - return isNaN(parsed) ? undefined : parsed -} diff --git a/packages/mod-service/src/indexer/context.ts b/packages/mod-service/src/indexer/context.ts deleted file mode 100644 index 1ce2fbf1ea2..00000000000 --- a/packages/mod-service/src/indexer/context.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { IdResolver } from '@atproto/identity' -import { PrimaryDatabase } from '../db' -import { IndexerConfig } from './config' -import { Services } from './services' -import { BackgroundQueue } from '../background' -import DidSqlCache from '../did-cache' -import { Redis } from '../redis' -import { AutoModerator } from '../auto-moderator' - -export class IndexerContext { - constructor( - private opts: { - db: PrimaryDatabase - redis: Redis - redisCache: Redis - cfg: IndexerConfig - services: Services - idResolver: IdResolver - didCache: DidSqlCache - backgroundQueue: BackgroundQueue - autoMod: AutoModerator - }, - ) {} - - get db(): PrimaryDatabase { - return this.opts.db - } - - get redis(): Redis { - return this.opts.redis - } - - get redisCache(): Redis { - return this.opts.redisCache - } - - get cfg(): IndexerConfig { - return this.opts.cfg - } - - get services(): Services { - return this.opts.services - } - - get idResolver(): IdResolver { - return this.opts.idResolver - } - - get didCache(): DidSqlCache { - return this.opts.didCache - } - - get backgroundQueue(): BackgroundQueue { - return this.opts.backgroundQueue - } - - get autoMod(): AutoModerator { - return this.opts.autoMod - } -} - -export default IndexerContext diff --git a/packages/mod-service/src/indexer/index.ts b/packages/mod-service/src/indexer/index.ts deleted file mode 100644 index 496cff67c73..00000000000 --- a/packages/mod-service/src/indexer/index.ts +++ /dev/null @@ -1,151 +0,0 @@ -import express from 'express' -import { IdResolver } from '@atproto/identity' -import { BackgroundQueue } from '../background' -import { PrimaryDatabase } from '../db' -import DidRedisCache from '../did-cache' -import log from './logger' -import { dbLogger } from '../logger' -import { IndexerConfig } from './config' -import { IndexerContext } from './context' -import { createServices } from './services' -import { IndexerSubscription } from './subscription' -import { AutoModerator } from '../auto-moderator' -import { Redis } from '../redis' -import { NotificationServer } from '../notifications' -import { CloseFn, createServer, startServer } from './server' -import { ImageUriBuilder } from '../image/uri' -import { ImageInvalidator } from '../image/invalidator' - -export { IndexerConfig } from './config' -export type { IndexerConfigValues } from './config' - -export class BskyIndexer { - public ctx: IndexerContext - public sub: IndexerSubscription - public app: express.Application - private closeServer?: CloseFn - private dbStatsInterval: NodeJS.Timer - private subStatsInterval: NodeJS.Timer - - constructor(opts: { - ctx: IndexerContext - sub: IndexerSubscription - app: express.Application - }) { - this.ctx = opts.ctx - this.sub = opts.sub - this.app = opts.app - } - - static create(opts: { - db: PrimaryDatabase - redis: Redis - redisCache: Redis - cfg: IndexerConfig - imgInvalidator?: ImageInvalidator - }): BskyIndexer { - const { db, redis, redisCache, cfg } = opts - const didCache = new DidRedisCache(redisCache.withNamespace('did-doc'), { - staleTTL: cfg.didCacheStaleTTL, - maxTTL: cfg.didCacheMaxTTL, - }) - const idResolver = new IdResolver({ - plcUrl: cfg.didPlcUrl, - didCache, - backupNameservers: cfg.handleResolveNameservers, - }) - const backgroundQueue = new BackgroundQueue(db) - - const imgUriBuilder = cfg.imgUriEndpoint - ? new ImageUriBuilder(cfg.imgUriEndpoint) - : undefined - const imgInvalidator = opts.imgInvalidator - const autoMod = new AutoModerator({ - db, - idResolver, - cfg, - backgroundQueue, - imgUriBuilder, - imgInvalidator, - }) - - const notifServer = cfg.pushNotificationEndpoint - ? new NotificationServer(db, cfg.pushNotificationEndpoint) - : undefined - const services = createServices({ - idResolver, - autoMod, - backgroundQueue, - notifServer, - }) - const ctx = new IndexerContext({ - db, - redis, - redisCache, - cfg, - services, - idResolver, - didCache, - backgroundQueue, - autoMod, - }) - const sub = new IndexerSubscription(ctx, { - partitionIds: cfg.indexerPartitionIds, - partitionBatchSize: cfg.indexerPartitionBatchSize, - concurrency: cfg.indexerConcurrency, - subLockId: cfg.indexerSubLockId, - }) - - const app = createServer(sub, cfg) - - return new BskyIndexer({ ctx, sub, app }) - } - - async start() { - const { db, backgroundQueue } = this.ctx - const pool = db.pool - this.dbStatsInterval = setInterval(() => { - dbLogger.info( - { - idleCount: pool.idleCount, - totalCount: pool.totalCount, - waitingCount: pool.waitingCount, - }, - 'db pool stats', - ) - dbLogger.info( - { - runningCount: backgroundQueue.queue.pending, - waitingCount: backgroundQueue.queue.size, - }, - 'background queue stats', - ) - }, 10000) - this.subStatsInterval = setInterval(() => { - log.info( - { - processedCount: this.sub.processedCount, - runningCount: this.sub.repoQueue.main.pending, - waitingCount: this.sub.repoQueue.main.size, - }, - 'indexer stats', - ) - }, 500) - this.sub.run() - this.closeServer = startServer(this.app, this.ctx.cfg.indexerPort) - return this - } - - async destroy(opts?: { skipDb: boolean; skipRedis: true }): Promise { - if (this.closeServer) await this.closeServer() - await this.sub.destroy() - clearInterval(this.subStatsInterval) - await this.ctx.didCache.destroy() - if (!opts?.skipRedis) await this.ctx.redis.destroy() - if (!opts?.skipRedis) await this.ctx.redisCache.destroy() - if (!opts?.skipDb) await this.ctx.db.close() - clearInterval(this.dbStatsInterval) - } -} - -export default BskyIndexer diff --git a/packages/mod-service/src/indexer/logger.ts b/packages/mod-service/src/indexer/logger.ts deleted file mode 100644 index 45752727f99..00000000000 --- a/packages/mod-service/src/indexer/logger.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { subsystemLogger } from '@atproto/common' - -const logger: ReturnType = - subsystemLogger('bsky:indexer') - -export default logger diff --git a/packages/mod-service/src/indexer/server.ts b/packages/mod-service/src/indexer/server.ts deleted file mode 100644 index dfafb741eb4..00000000000 --- a/packages/mod-service/src/indexer/server.ts +++ /dev/null @@ -1,46 +0,0 @@ -import express from 'express' -import { IndexerSubscription } from './subscription' -import { IndexerConfig } from './config' -import { randomIntFromSeed } from '@atproto/crypto' - -export type CloseFn = () => Promise - -export const createServer = ( - sub: IndexerSubscription, - cfg: IndexerConfig, -): express.Application => { - const app = express() - app.post('/reprocess/:did', async (req, res) => { - const did = req.params.did - try { - const partition = await randomIntFromSeed(did, cfg.ingesterPartitionCount) - const supportedPartition = cfg.indexerPartitionIds.includes(partition) - if (!supportedPartition) { - return res.status(400).send(`unsupported partition: ${partition}`) - } - } catch (err) { - return res.status(500).send('could not calculate partition') - } - await sub.requestReprocess(req.params.did) - res.sendStatus(200) - }) - return app -} - -export const startServer = ( - app: express.Application, - port?: number, -): CloseFn => { - const server = app.listen(port) - return () => { - return new Promise((resolve, reject) => { - server.close((err) => { - if (err) { - reject(err) - } else { - resolve() - } - }) - }) - } -} diff --git a/packages/mod-service/src/indexer/services.ts b/packages/mod-service/src/indexer/services.ts deleted file mode 100644 index df173352046..00000000000 --- a/packages/mod-service/src/indexer/services.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { IdResolver } from '@atproto/identity' -import { PrimaryDatabase } from '../db' -import { BackgroundQueue } from '../background' -import { IndexingService } from '../services/indexing' -import { LabelService } from '../services/label' -import { NotificationServer } from '../notifications' -import { AutoModerator } from '../auto-moderator' - -export function createServices(resources: { - idResolver: IdResolver - autoMod: AutoModerator - backgroundQueue: BackgroundQueue - notifServer?: NotificationServer -}): Services { - const { idResolver, autoMod, backgroundQueue, notifServer } = resources - return { - indexing: IndexingService.creator( - idResolver, - autoMod, - backgroundQueue, - notifServer, - ), - label: LabelService.creator(null), - } -} - -export type Services = { - indexing: FromDbPrimary - label: FromDbPrimary -} - -type FromDbPrimary = (db: PrimaryDatabase) => T diff --git a/packages/mod-service/src/indexer/subscription.ts b/packages/mod-service/src/indexer/subscription.ts deleted file mode 100644 index abc672db3b0..00000000000 --- a/packages/mod-service/src/indexer/subscription.ts +++ /dev/null @@ -1,345 +0,0 @@ -import assert from 'node:assert' -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { cborDecode, wait, handleAllSettledErrors } from '@atproto/common' -import { DisconnectError } from '@atproto/xrpc-server' -import { - WriteOpAction, - readCarWithRoot, - cborToLexRecord, - def, - Commit, -} from '@atproto/repo' -import { ValidationError } from '@atproto/lexicon' -import * as message from '../lexicon/types/com/atproto/sync/subscribeRepos' -import { Leader } from '../db/leader' -import { IndexingService } from '../services/indexing' -import log from './logger' -import { - ConsecutiveItem, - ConsecutiveList, - LatestQueue, - PartitionedQueue, - PerfectMap, - ProcessableMessage, - jitter, - loggableMessage, - strToInt, -} from '../subscription/util' -import IndexerContext from './context' - -export const INDEXER_SUB_LOCK_ID = 1200 // need one per partition - -export class IndexerSubscription { - destroyed = false - leader = new Leader(this.opts.subLockId || INDEXER_SUB_LOCK_ID, this.ctx.db) - processedCount = 0 - repoQueue = new PartitionedQueue({ - concurrency: this.opts.concurrency ?? Infinity, - }) - partitions = new PerfectMap() - partitionIds = this.opts.partitionIds - indexingSvc: IndexingService - - constructor( - public ctx: IndexerContext, - public opts: { - partitionIds: number[] - subLockId?: number - concurrency?: number - partitionBatchSize?: number - }, - ) { - this.indexingSvc = ctx.services.indexing(ctx.db) - } - - async processEvents(opts: { signal: AbortSignal }) { - const done = () => this.destroyed || opts.signal.aborted - while (!done()) { - const results = await this.ctx.redis.readStreams( - this.partitionIds.map((id) => ({ - key: partitionKey(id), - cursor: this.partitions.get(id).cursor, - })), - { - blockMs: 1000, - count: this.opts.partitionBatchSize ?? 50, // events per stream - }, - ) - if (done()) break - for (const { key, messages } of results) { - const partition = this.partitions.get(partitionId(key)) - for (const msg of messages) { - const seq = strToInt(msg.cursor) - const envelope = getEnvelope(msg.contents) - partition.cursor = seq - const item = partition.consecutive.push(seq) - this.repoQueue.add(envelope.repo, async () => { - await this.handleMessage(partition, item, envelope) - }) - } - } - await this.repoQueue.main.onEmpty() // backpressure - } - } - - async run() { - while (!this.destroyed) { - try { - const { ran } = await this.leader.run(async ({ signal }) => { - // initialize cursors to 0 (read from beginning of stream) - for (const id of this.partitionIds) { - this.partitions.set(id, new Partition(id, 0)) - } - // process events - await this.processEvents({ signal }) - }) - if (ran && !this.destroyed) { - throw new Error('Indexer sub completed, but should be persistent') - } - } catch (err) { - log.error({ err }, 'indexer sub error') - } - if (!this.destroyed) { - await wait(5000 + jitter(1000)) // wait then try to become leader - } - } - } - - async requestReprocess(did: string) { - await this.repoQueue.add(did, async () => { - try { - await this.indexingSvc.indexRepo(did, undefined) - } catch (err) { - log.error({ did }, 'failed to reprocess repo') - } - }) - } - - async destroy() { - this.destroyed = true - await this.repoQueue.destroy() - await Promise.all( - [...this.partitions.values()].map((p) => p.cursorQueue.destroy()), - ) - this.leader.destroy(new DisconnectError()) - } - - async resume() { - this.destroyed = false - this.partitions = new Map() - this.repoQueue = new PartitionedQueue({ - concurrency: this.opts.concurrency ?? Infinity, - }) - await this.run() - } - - private async handleMessage( - partition: Partition, - item: ConsecutiveItem, - envelope: Envelope, - ) { - const msg = envelope.event - try { - if (message.isCommit(msg)) { - await this.handleCommit(msg) - } else if (message.isHandle(msg)) { - await this.handleUpdateHandle(msg) - } else if (message.isTombstone(msg)) { - await this.handleTombstone(msg) - } else if (message.isMigrate(msg)) { - // Ignore migrations - } else { - const exhaustiveCheck: never = msg - throw new Error(`Unhandled message type: ${exhaustiveCheck['$type']}`) - } - } catch (err) { - // We log messages we can't process and move on: - // otherwise the cursor would get stuck on a poison message. - log.error( - { err, message: loggableMessage(msg) }, - 'indexer message processing error', - ) - } finally { - this.processedCount++ - const latest = item.complete().at(-1) - if (latest !== undefined) { - partition.cursorQueue - .add(async () => { - await this.ctx.redis.trimStream(partition.key, latest + 1) - }) - .catch((err) => { - log.error({ err }, 'indexer cursor error') - }) - } - } - } - - private async handleCommit(msg: message.Commit) { - const indexRecords = async () => { - const { root, rootCid, ops } = await getOps(msg) - if (msg.tooBig) { - await this.indexingSvc.indexRepo(msg.repo, rootCid.toString()) - await this.indexingSvc.setCommitLastSeen(root, msg) - return - } - if (msg.rebase) { - const needsReindex = await this.indexingSvc.checkCommitNeedsIndexing( - root, - ) - if (needsReindex) { - await this.indexingSvc.indexRepo(msg.repo, rootCid.toString()) - } - await this.indexingSvc.setCommitLastSeen(root, msg) - return - } - for (const op of ops) { - if (op.action === WriteOpAction.Delete) { - await this.indexingSvc.deleteRecord(op.uri) - } else { - try { - await this.indexingSvc.indexRecord( - op.uri, - op.cid, - op.record, - op.action, // create or update - msg.time, - ) - } catch (err) { - if (err instanceof ValidationError) { - log.warn( - { - did: msg.repo, - commit: msg.commit.toString(), - uri: op.uri.toString(), - cid: op.cid.toString(), - }, - 'skipping indexing of invalid record', - ) - } else { - log.error( - { - err, - did: msg.repo, - commit: msg.commit.toString(), - uri: op.uri.toString(), - cid: op.cid.toString(), - }, - 'skipping indexing due to error processing record', - ) - } - } - } - } - await this.indexingSvc.setCommitLastSeen(root, msg) - } - const results = await Promise.allSettled([ - indexRecords(), - this.indexingSvc.indexHandle(msg.repo, msg.time), - ]) - handleAllSettledErrors(results) - } - - private async handleUpdateHandle(msg: message.Handle) { - await this.indexingSvc.indexHandle(msg.did, msg.time, true) - } - - private async handleTombstone(msg: message.Tombstone) { - await this.indexingSvc.tombstoneActor(msg.did) - } -} - -async function getOps( - msg: message.Commit, -): Promise<{ root: Commit; rootCid: CID; ops: PreparedWrite[] }> { - const car = await readCarWithRoot(msg.blocks as Uint8Array) - const rootBytes = car.blocks.get(car.root) - assert(rootBytes, 'Missing commit block in car slice') - - const root = def.commit.schema.parse(cborDecode(rootBytes)) - const ops: PreparedWrite[] = msg.ops.map((op) => { - const [collection, rkey] = op.path.split('/') - assert(collection && rkey) - if ( - op.action === WriteOpAction.Create || - op.action === WriteOpAction.Update - ) { - assert(op.cid) - const record = car.blocks.get(op.cid) - assert(record) - return { - action: - op.action === WriteOpAction.Create - ? WriteOpAction.Create - : WriteOpAction.Update, - cid: op.cid, - record: cborToLexRecord(record), - blobs: [], - uri: AtUri.make(msg.repo, collection, rkey), - } - } else if (op.action === WriteOpAction.Delete) { - return { - action: WriteOpAction.Delete, - uri: AtUri.make(msg.repo, collection, rkey), - } - } else { - throw new Error(`Unknown repo op action: ${op.action}`) - } - }) - - return { root, rootCid: car.root, ops } -} - -function getEnvelope(val: Record): Envelope { - assert(val.repo && val.event, 'malformed message contents') - return { - repo: val.repo.toString(), - event: cborDecode(val.event) as ProcessableMessage, - } -} - -type Envelope = { - repo: string - event: ProcessableMessage -} - -class Partition { - consecutive = new ConsecutiveList() - cursorQueue = new LatestQueue() - constructor(public id: number, public cursor: number) {} - get key() { - return partitionKey(this.id) - } -} - -function partitionId(key: string) { - assert(key.startsWith('repo:')) - return strToInt(key.replace('repo:', '')) -} - -function partitionKey(p: number) { - return `repo:${p}` -} - -type PreparedCreate = { - action: WriteOpAction.Create - uri: AtUri - cid: CID - record: Record - blobs: CID[] // differs from similar type in pds -} - -type PreparedUpdate = { - action: WriteOpAction.Update - uri: AtUri - cid: CID - record: Record - blobs: CID[] // differs from similar type in pds -} - -type PreparedDelete = { - action: WriteOpAction.Delete - uri: AtUri -} - -type PreparedWrite = PreparedCreate | PreparedUpdate | PreparedDelete diff --git a/packages/mod-service/src/ingester/config.ts b/packages/mod-service/src/ingester/config.ts deleted file mode 100644 index 969aeeff7aa..00000000000 --- a/packages/mod-service/src/ingester/config.ts +++ /dev/null @@ -1,141 +0,0 @@ -import assert from 'assert' - -export interface IngesterConfigValues { - version: string - dbPostgresUrl: string - dbPostgresSchema?: string - redisHost?: string // either set redis host, or both sentinel name and hosts - redisSentinelName?: string - redisSentinelHosts?: string[] - redisPassword?: string - repoProvider: string - ingesterPartitionCount: number - ingesterNamespace?: string - ingesterSubLockId?: number - ingesterMaxItems?: number - ingesterCheckItemsEveryN?: number - ingesterInitialCursor?: number -} - -export class IngesterConfig { - constructor(private cfg: IngesterConfigValues) {} - - static readEnv(overrides?: Partial) { - const version = process.env.BSKY_VERSION || '0.0.0' - const dbPostgresUrl = - overrides?.dbPostgresUrl || process.env.DB_PRIMARY_POSTGRES_URL - const dbPostgresSchema = - overrides?.dbPostgresSchema || process.env.DB_POSTGRES_SCHEMA - const redisHost = - overrides?.redisHost || process.env.REDIS_HOST || undefined - const redisSentinelName = - overrides?.redisSentinelName || - process.env.REDIS_SENTINEL_NAME || - undefined - const redisSentinelHosts = - overrides?.redisSentinelHosts || - (process.env.REDIS_SENTINEL_HOSTS - ? process.env.REDIS_SENTINEL_HOSTS.split(',') - : []) - const redisPassword = - overrides?.redisPassword || process.env.REDIS_PASSWORD || undefined - const repoProvider = overrides?.repoProvider || process.env.REPO_PROVIDER // E.g. ws://abc.com:4000 - const ingesterPartitionCount = - overrides?.ingesterPartitionCount || - maybeParseInt(process.env.INGESTER_PARTITION_COUNT) - const ingesterSubLockId = - overrides?.ingesterSubLockId || - maybeParseInt(process.env.INGESTER_SUB_LOCK_ID) - const ingesterMaxItems = - overrides?.ingesterMaxItems || - maybeParseInt(process.env.INGESTER_MAX_ITEMS) - const ingesterCheckItemsEveryN = - overrides?.ingesterCheckItemsEveryN || - maybeParseInt(process.env.INGESTER_CHECK_ITEMS_EVERY_N) - const ingesterInitialCursor = - overrides?.ingesterInitialCursor || - maybeParseInt(process.env.INGESTER_INITIAL_CURSOR) - const ingesterNamespace = overrides?.ingesterNamespace - assert(dbPostgresUrl) - assert(redisHost || (redisSentinelName && redisSentinelHosts?.length)) - assert(repoProvider) - assert(ingesterPartitionCount) - return new IngesterConfig({ - version, - dbPostgresUrl, - dbPostgresSchema, - redisHost, - redisSentinelName, - redisSentinelHosts, - redisPassword, - repoProvider, - ingesterPartitionCount, - ingesterSubLockId, - ingesterNamespace, - ingesterMaxItems, - ingesterCheckItemsEveryN, - ingesterInitialCursor, - }) - } - - get version() { - return this.cfg.version - } - - get dbPostgresUrl() { - return this.cfg.dbPostgresUrl - } - - get dbPostgresSchema() { - return this.cfg.dbPostgresSchema - } - - get redisHost() { - return this.cfg.redisHost - } - - get redisSentinelName() { - return this.cfg.redisSentinelName - } - - get redisSentinelHosts() { - return this.cfg.redisSentinelHosts - } - - get redisPassword() { - return this.cfg.redisPassword - } - - get repoProvider() { - return this.cfg.repoProvider - } - - get ingesterPartitionCount() { - return this.cfg.ingesterPartitionCount - } - - get ingesterMaxItems() { - return this.cfg.ingesterMaxItems - } - - get ingesterCheckItemsEveryN() { - return this.cfg.ingesterCheckItemsEveryN - } - - get ingesterInitialCursor() { - return this.cfg.ingesterInitialCursor - } - - get ingesterNamespace() { - return this.cfg.ingesterNamespace - } - - get ingesterSubLockId() { - return this.cfg.ingesterSubLockId - } -} - -function maybeParseInt(str) { - const parsed = parseInt(str) - return isNaN(parsed) ? undefined : parsed -} diff --git a/packages/mod-service/src/ingester/context.ts b/packages/mod-service/src/ingester/context.ts deleted file mode 100644 index 792d3c2015a..00000000000 --- a/packages/mod-service/src/ingester/context.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { PrimaryDatabase } from '../db' -import { Redis } from '../redis' -import { IngesterConfig } from './config' - -export class IngesterContext { - constructor( - private opts: { - db: PrimaryDatabase - redis: Redis - cfg: IngesterConfig - }, - ) {} - - get db(): PrimaryDatabase { - return this.opts.db - } - - get redis(): Redis { - return this.opts.redis - } - - get cfg(): IngesterConfig { - return this.opts.cfg - } -} - -export default IngesterContext diff --git a/packages/mod-service/src/ingester/index.ts b/packages/mod-service/src/ingester/index.ts deleted file mode 100644 index 376da2887da..00000000000 --- a/packages/mod-service/src/ingester/index.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { PrimaryDatabase } from '../db' -import log from './logger' -import { dbLogger } from '../logger' -import { Redis } from '../redis' -import { IngesterConfig } from './config' -import { IngesterContext } from './context' -import { IngesterSubscription } from './subscription' - -export { IngesterConfig } from './config' -export type { IngesterConfigValues } from './config' - -export class BskyIngester { - public ctx: IngesterContext - public sub: IngesterSubscription - private dbStatsInterval: NodeJS.Timer - private subStatsInterval: NodeJS.Timer - - constructor(opts: { ctx: IngesterContext; sub: IngesterSubscription }) { - this.ctx = opts.ctx - this.sub = opts.sub - } - - static create(opts: { - db: PrimaryDatabase - redis: Redis - cfg: IngesterConfig - }): BskyIngester { - const { db, redis, cfg } = opts - const ctx = new IngesterContext({ db, redis, cfg }) - const sub = new IngesterSubscription(ctx, { - service: cfg.repoProvider, - subLockId: cfg.ingesterSubLockId, - partitionCount: cfg.ingesterPartitionCount, - maxItems: cfg.ingesterMaxItems, - checkItemsEveryN: cfg.ingesterCheckItemsEveryN, - initialCursor: cfg.ingesterInitialCursor, - }) - return new BskyIngester({ ctx, sub }) - } - - async start() { - const { db } = this.ctx - const pool = db.pool - this.dbStatsInterval = setInterval(() => { - dbLogger.info( - { - idleCount: pool.idleCount, - totalCount: pool.totalCount, - waitingCount: pool.waitingCount, - }, - 'db pool stats', - ) - }, 10000) - this.subStatsInterval = setInterval(() => { - log.info( - { - seq: this.sub.lastSeq, - streamsLength: - this.sub.backpressure.lastTotal !== null - ? this.sub.backpressure.lastTotal - : undefined, - }, - 'ingester stats', - ) - }, 500) - this.sub.run() - return this - } - - async destroy(opts?: { skipDb: boolean }): Promise { - await this.sub.destroy() - clearInterval(this.subStatsInterval) - await this.ctx.redis.destroy() - if (!opts?.skipDb) await this.ctx.db.close() - clearInterval(this.dbStatsInterval) - } -} - -export default BskyIngester diff --git a/packages/mod-service/src/ingester/logger.ts b/packages/mod-service/src/ingester/logger.ts deleted file mode 100644 index 49855166481..00000000000 --- a/packages/mod-service/src/ingester/logger.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { subsystemLogger } from '@atproto/common' - -const logger: ReturnType = - subsystemLogger('bsky:ingester') - -export default logger diff --git a/packages/mod-service/src/ingester/subscription.ts b/packages/mod-service/src/ingester/subscription.ts deleted file mode 100644 index 14f301e07f9..00000000000 --- a/packages/mod-service/src/ingester/subscription.ts +++ /dev/null @@ -1,288 +0,0 @@ -import { - Deferrable, - cborEncode, - createDeferrable, - ui8ToBuffer, - wait, -} from '@atproto/common' -import { randomIntFromSeed } from '@atproto/crypto' -import { DisconnectError, Subscription } from '@atproto/xrpc-server' -import { OutputSchema as Message } from '../lexicon/types/com/atproto/sync/subscribeRepos' -import * as message from '../lexicon/types/com/atproto/sync/subscribeRepos' -import { ids, lexicons } from '../lexicon/lexicons' -import { Leader } from '../db/leader' -import log from './logger' -import { - LatestQueue, - ProcessableMessage, - loggableMessage, - jitter, - strToInt, -} from '../subscription/util' -import { IngesterContext } from './context' - -const METHOD = ids.ComAtprotoSyncSubscribeRepos -const CURSOR_KEY = 'ingester:cursor' -export const INGESTER_SUB_LOCK_ID = 1000 - -export class IngesterSubscription { - cursorQueue = new LatestQueue() - destroyed = false - lastSeq: number | undefined - backpressure = new Backpressure(this) - leader = new Leader(this.opts.subLockId || INGESTER_SUB_LOCK_ID, this.ctx.db) - processor = new Processor(this) - - constructor( - public ctx: IngesterContext, - public opts: { - service: string - partitionCount: number - maxItems?: number - checkItemsEveryN?: number - subLockId?: number - initialCursor?: number - }, - ) {} - - async run() { - while (!this.destroyed) { - try { - const { ran } = await this.leader.run(async ({ signal }) => { - const sub = this.getSubscription({ signal }) - for await (const msg of sub) { - const details = getMessageDetails(msg) - if ('info' in details) { - // These messages are not sequenced, we just log them and carry on - log.warn( - { provider: this.opts.service, message: loggableMessage(msg) }, - `ingester sub ${details.info ? 'info' : 'unknown'} message`, - ) - continue - } - this.processor.send(details) - await this.backpressure.ready() - } - }) - if (ran && !this.destroyed) { - throw new Error('Ingester sub completed, but should be persistent') - } - } catch (err) { - log.error({ err, provider: this.opts.service }, 'ingester sub error') - } - if (!this.destroyed) { - await wait(1000 + jitter(500)) // wait then try to become leader - } - } - } - - async destroy() { - this.destroyed = true - await this.processor.destroy() - await this.cursorQueue.destroy() - this.leader.destroy(new DisconnectError()) - } - - async resume() { - this.destroyed = false - this.processor = new Processor(this) - this.cursorQueue = new LatestQueue() - await this.run() - } - - async getCursor(): Promise { - const val = await this.ctx.redis.get(CURSOR_KEY) - const initialCursor = this.opts.initialCursor ?? 0 - return val !== null ? strToInt(val) : initialCursor - } - - async resetCursor(): Promise { - await this.ctx.redis.del(CURSOR_KEY) - } - - async setCursor(seq: number): Promise { - await this.ctx.redis.set(CURSOR_KEY, seq) - } - - private getSubscription(opts: { signal: AbortSignal }) { - return new Subscription({ - service: this.opts.service, - method: METHOD, - signal: opts.signal, - getParams: async () => { - const cursor = await this.getCursor() - return { cursor } - }, - onReconnectError: (err, reconnects, initial) => { - log.warn({ err, reconnects, initial }, 'ingester sub reconnect') - }, - validate: (value) => { - try { - return lexicons.assertValidXrpcMessage(METHOD, value) - } catch (err) { - log.warn( - { - err, - seq: ifNumber(value?.['seq']), - repo: ifString(value?.['repo']), - commit: ifString(value?.['commit']?.toString()), - time: ifString(value?.['time']), - provider: this.opts.service, - }, - 'ingester sub skipped invalid message', - ) - } - }, - }) - } -} - -function ifString(val: unknown): string | undefined { - return typeof val === 'string' ? val : undefined -} - -function ifNumber(val: unknown): number | undefined { - return typeof val === 'number' ? val : undefined -} - -function getMessageDetails(msg: Message): - | { info: message.Info | null } - | { - seq: number - repo: string - message: ProcessableMessage - } { - if (message.isCommit(msg)) { - return { seq: msg.seq, repo: msg.repo, message: msg } - } else if (message.isHandle(msg)) { - return { seq: msg.seq, repo: msg.did, message: msg } - } else if (message.isMigrate(msg)) { - return { seq: msg.seq, repo: msg.did, message: msg } - } else if (message.isTombstone(msg)) { - return { seq: msg.seq, repo: msg.did, message: msg } - } else if (message.isInfo(msg)) { - return { info: msg } - } - return { info: null } -} - -async function getPartition(did: string, n: number) { - const partition = await randomIntFromSeed(did, n) - return `repo:${partition}` -} - -class Processor { - running: Deferrable | null = null - destroyed = false - unprocessed: MessageEnvelope[] = [] - - constructor(public sub: IngesterSubscription) {} - - async handleBatch(batch: MessageEnvelope[]) { - if (!batch.length) return - const items = await Promise.all( - batch.map(async ({ seq, repo, message }) => { - const key = await getPartition(repo, this.sub.opts.partitionCount) - const fields: [string, string | Buffer][] = [ - ['repo', repo], - ['event', ui8ToBuffer(cborEncode(message))], - ] - return { key, id: seq, fields } - }), - ) - const results = await this.sub.ctx.redis.addMultiToStream(items) - results.forEach(([err], i) => { - if (err) { - // skipping over messages that have already been added or fully processed - const item = batch.at(i) - log.warn( - { seq: item?.seq, repo: item?.repo }, - 'ingester skipping message', - ) - } - }) - const lastSeq = batch[batch.length - 1].seq - this.sub.lastSeq = lastSeq - this.sub.cursorQueue.add(() => this.sub.setCursor(lastSeq)) - } - - async process() { - if (this.running || this.destroyed || !this.unprocessed.length) return - const next = this.unprocessed.splice(100) // pipeline no more than 100 - const processing = this.unprocessed - this.unprocessed = next - this.running = createDeferrable() - try { - await this.handleBatch(processing) - } catch (err) { - log.error( - { err, size: processing.length }, - 'ingester processing failed, rolling over to next batch', - ) - this.unprocessed.unshift(...processing) - } finally { - this.running.resolve() - this.running = null - this.process() - } - } - - send(envelope: MessageEnvelope) { - this.unprocessed.push(envelope) - this.process() - } - - async destroy() { - this.destroyed = true - this.unprocessed = [] - await this.running?.complete - } -} - -type MessageEnvelope = { - seq: number - repo: string - message: ProcessableMessage -} - -class Backpressure { - count = 0 - lastTotal: number | null = null - partitionCount = this.sub.opts.partitionCount - limit = this.sub.opts.maxItems ?? Infinity - checkEvery = this.sub.opts.checkItemsEveryN ?? 500 - - constructor(public sub: IngesterSubscription) {} - - async ready() { - this.count++ - const shouldCheck = - this.limit !== Infinity && - (this.count === 1 || this.count % this.checkEvery === 0) - if (!shouldCheck) return - let ready = false - const start = Date.now() - while (!ready) { - ready = await this.check() - if (!ready) { - log.warn( - { - limit: this.limit, - total: this.lastTotal, - duration: Date.now() - start, - }, - 'ingester backpressure', - ) - await wait(250) - } - } - } - - async check() { - const lens = await this.sub.ctx.redis.streamLengths( - [...Array(this.partitionCount)].map((_, i) => `repo:${i}`), - ) - this.lastTotal = lens.reduce((sum, len) => sum + len, 0) - return this.lastTotal < this.limit - } -} diff --git a/packages/mod-service/src/services/actor/index.ts b/packages/mod-service/src/services/actor/index.ts deleted file mode 100644 index 7ef61529926..00000000000 --- a/packages/mod-service/src/services/actor/index.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { sql } from 'kysely' -import { wait } from '@atproto/common' -import { Database } from '../../db' -import { notSoftDeletedClause } from '../../db/util' -import { ActorViews } from './views' -import { ImageUriBuilder } from '../../image/uri' -import { Actor } from '../../db/tables/actor' -import { TimeCidKeyset, paginate } from '../../db/pagination' -import { SearchKeyset, getUserSearchQuery } from '../util/search' -import { FromDb } from '../types' -import { GraphService } from '../graph' -import { LabelService } from '../label' - -export * from './types' - -export class ActorService { - views: ActorViews - - constructor( - public db: Database, - public imgUriBuilder: ImageUriBuilder, - private graph: FromDb, - private label: FromDb, - ) { - this.views = new ActorViews(this.db, this.imgUriBuilder, graph, label) - } - - static creator( - imgUriBuilder: ImageUriBuilder, - graph: FromDb, - label: FromDb, - ) { - return (db: Database) => new ActorService(db, imgUriBuilder, graph, label) - } - - async getActorDid(handleOrDid: string): Promise { - if (handleOrDid.startsWith('did:')) { - return handleOrDid - } - const subject = await this.getActor(handleOrDid, true) - return subject?.did ?? null - } - - async getActor( - handleOrDid: string, - includeSoftDeleted = false, - ): Promise { - const actors = await this.getActors([handleOrDid], includeSoftDeleted) - return actors[0] || null - } - - async getActors( - handleOrDids: string[], - includeSoftDeleted = false, - ): Promise { - const { ref } = this.db.db.dynamic - const dids: string[] = [] - const handles: string[] = [] - const order: Record = {} - handleOrDids.forEach((item, i) => { - if (item.startsWith('did:')) { - order[item] = i - dids.push(item) - } else { - order[item.toLowerCase()] = i - handles.push(item.toLowerCase()) - } - }) - const results = await this.db.db - .selectFrom('actor') - .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('actor'))), - ) - .where((qb) => { - if (dids.length) { - qb = qb.orWhere('actor.did', 'in', dids) - } - if (handles.length) { - qb = qb.orWhere( - 'actor.handle', - 'in', - handles.length === 1 - ? [handles[0], handles[0]] // a silly (but worthwhile) optimization to avoid usage of actor_handle_tgrm_idx - : handles, - ) - } - return qb - }) - .selectAll() - .execute() - - return results.sort((a, b) => { - const orderA = order[a.did] ?? order[a.handle?.toLowerCase() ?? ''] - const orderB = order[b.did] ?? order[b.handle?.toLowerCase() ?? ''] - return orderA - orderB - }) - } - - async getSearchResults({ - cursor, - limit = 25, - query = '', - includeSoftDeleted, - }: { - cursor?: string - limit?: number - query?: string - includeSoftDeleted?: boolean - }) { - const searchField = query.startsWith('did:') ? 'did' : 'handle' - let paginatedBuilder - const { ref } = this.db.db.dynamic - const paginationOptions = { - limit, - cursor, - direction: 'asc' as const, - } - let keyset - - if (query && searchField === 'handle') { - keyset = new SearchKeyset(sql``, sql``) - paginatedBuilder = getUserSearchQuery(this.db, { - query, - includeSoftDeleted, - ...paginationOptions, - }).select('distance') - } else { - paginatedBuilder = this.db.db - .selectFrom('actor') - .select([sql`0`.as('distance')]) - keyset = new ListKeyset(ref('indexedAt'), ref('did')) - - // When searchField === 'did', the query will always be a valid string because - // searchField is set to 'did' after checking that the query is a valid did - if (query && searchField === 'did') { - paginatedBuilder = paginatedBuilder.where('actor.did', '=', query) - } - paginatedBuilder = paginate(paginatedBuilder, { - keyset, - ...paginationOptions, - }) - } - - const results: Actor[] = await paginatedBuilder.selectAll('actor').execute() - return { results, cursor: keyset.packFromResult(results) } - } - - async getRepoRev(did: string | null): Promise { - if (did === null) return null - const res = await this.db.db - .selectFrom('actor_sync') - .select('repoRev') - .where('did', '=', did) - .executeTakeFirst() - return res?.repoRev ?? null - } - - async *all( - opts: { - batchSize?: number - forever?: boolean - cooldownMs?: number - startFromDid?: string - } = {}, - ) { - const { - cooldownMs = 1000, - batchSize = 1000, - forever = false, - startFromDid, - } = opts - const baseQuery = this.db.db - .selectFrom('actor') - .selectAll() - .orderBy('did') - .limit(batchSize) - while (true) { - let cursor = startFromDid - do { - const actors = cursor - ? await baseQuery.where('did', '>', cursor).execute() - : await baseQuery.execute() - for (const actor of actors) { - yield actor - } - cursor = actors.at(-1)?.did - } while (cursor) - if (forever) { - await wait(cooldownMs) - } else { - return - } - } - } -} - -type ActorResult = Actor -export class ListKeyset extends TimeCidKeyset<{ - indexedAt: string - did: string // handles are treated identically to cids in TimeCidKeyset -}> { - labelResult(result: { indexedAt: string; did: string }) { - return { primary: result.indexedAt, secondary: result.did } - } -} diff --git a/packages/mod-service/src/services/actor/types.ts b/packages/mod-service/src/services/actor/types.ts deleted file mode 100644 index d622e641099..00000000000 --- a/packages/mod-service/src/services/actor/types.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { ListViewBasic } from '../../lexicon/types/app/bsky/graph/defs' -import { Label } from '../../lexicon/types/com/atproto/label/defs' -import { BlockAndMuteState } from '../graph' -import { ListInfoMap } from '../graph/types' -import { Labels } from '../label' - -export type ActorInfo = { - did: string - handle: string - displayName?: string - description?: string // omitted from basic profile view - avatar?: string - indexedAt?: string // omitted from basic profile view - viewer?: { - muted?: boolean - mutedByList?: ListViewBasic - blockedBy?: boolean - blocking?: string - blockingByList?: ListViewBasic - following?: string - followedBy?: string - } - labels?: Label[] -} -export type ActorInfoMap = { [did: string]: ActorInfo } - -export type ProfileViewMap = ActorInfoMap - -export type ProfileInfo = { - did: string - handle: string | null - profileUri: string | null - profileCid: string | null - displayName: string | null - description: string | null - avatarCid: string | null - indexedAt: string | null - profileJson: string | null - viewerFollowing: string | null - viewerFollowedBy: string | null -} - -export type ProfileInfoMap = { [did: string]: ProfileInfo } - -export type ProfileHydrationState = { - profiles: ProfileInfoMap - labels: Labels - lists: ListInfoMap - bam: BlockAndMuteState -} - -export type ProfileDetailInfo = ProfileInfo & { - bannerCid: string | null - followsCount: number | null - followersCount: number | null - postsCount: number | null -} - -export type ProfileDetailInfoMap = { [did: string]: ProfileDetailInfo } - -export type ProfileDetailHydrationState = { - profilesDetailed: ProfileDetailInfoMap - labels: Labels - lists: ListInfoMap - bam: BlockAndMuteState -} - -export const toMapByDid = ( - items: T[], -): Record => { - return items.reduce((cur, item) => { - cur[item.did] = item - return cur - }, {} as Record) -} diff --git a/packages/mod-service/src/services/actor/views.ts b/packages/mod-service/src/services/actor/views.ts deleted file mode 100644 index 32e267a8868..00000000000 --- a/packages/mod-service/src/services/actor/views.ts +++ /dev/null @@ -1,375 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { INVALID_HANDLE } from '@atproto/syntax' -import { jsonStringToLex } from '@atproto/lexicon' -import { - ProfileViewDetailed, - ProfileView, -} from '../../lexicon/types/app/bsky/actor/defs' -import { Database } from '../../db' -import { noMatch, notSoftDeletedClause } from '../../db/util' -import { Actor } from '../../db/tables/actor' -import { ImageUriBuilder } from '../../image/uri' -import { LabelService, Labels, getSelfLabels } from '../label' -import { BlockAndMuteState, GraphService } from '../graph' -import { - ActorInfoMap, - ProfileDetailHydrationState, - ProfileHydrationState, - ProfileInfoMap, - ProfileViewMap, - toMapByDid, -} from './types' -import { ListInfoMap } from '../graph/types' -import { FromDb } from '../types' - -export class ActorViews { - services: { - label: LabelService - graph: GraphService - } - - constructor( - private db: Database, - private imgUriBuilder: ImageUriBuilder, - private graph: FromDb, - private label: FromDb, - ) { - this.services = { - label: label(db), - graph: graph(db), - } - } - - async profiles( - results: (ActorResult | string)[], // @TODO simplify down to just string[] - viewer: string | null, - opts?: { includeSoftDeleted?: boolean }, - ): Promise { - if (results.length === 0) return {} - const dids = results.map((res) => (typeof res === 'string' ? res : res.did)) - const hydrated = await this.profileHydration(dids, { - viewer, - ...opts, - }) - return this.profilePresentation(dids, hydrated, viewer) - } - - async profilesBasic( - results: (ActorResult | string)[], - viewer: string | null, - opts?: { includeSoftDeleted?: boolean }, - ): Promise { - if (results.length === 0) return {} - const dids = results.map((res) => (typeof res === 'string' ? res : res.did)) - const hydrated = await this.profileHydration(dids, { - viewer, - includeSoftDeleted: opts?.includeSoftDeleted, - }) - return this.profileBasicPresentation(dids, hydrated, viewer) - } - - async profilesList( - results: ActorResult[], - viewer: string | null, - opts?: { includeSoftDeleted?: boolean }, - ): Promise { - const profiles = await this.profiles(results, viewer, opts) - return mapDefined(results, (result) => profiles[result.did]) - } - - async profileDetailHydration( - dids: string[], - opts: { - viewer?: string | null - includeSoftDeleted?: boolean - }, - state?: { - bam: BlockAndMuteState - labels: Labels - }, - ): Promise { - const { viewer = null, includeSoftDeleted } = opts - const { ref } = this.db.db.dynamic - const profileInfosQb = this.db.db - .selectFrom('actor') - .where('actor.did', 'in', dids.length ? dids : ['']) - .leftJoin('profile', 'profile.creator', 'actor.did') - .leftJoin('profile_agg', 'profile_agg.did', 'actor.did') - .leftJoin('record', 'record.uri', 'profile.uri') - .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('actor'))), - ) - .select([ - 'actor.did as did', - 'actor.handle as handle', - 'profile.uri as profileUri', - 'profile.cid as profileCid', - 'profile.displayName as displayName', - 'profile.description as description', - 'profile.avatarCid as avatarCid', - 'profile.bannerCid as bannerCid', - 'profile.indexedAt as indexedAt', - 'profile_agg.followsCount as followsCount', - 'profile_agg.followersCount as followersCount', - 'profile_agg.postsCount as postsCount', - 'record.json as profileJson', - this.db.db - .selectFrom('follow') - .if(!viewer, (q) => q.where(noMatch)) - .where('creator', '=', viewer ?? '') - .whereRef('subjectDid', '=', ref('actor.did')) - .select('uri') - .as('viewerFollowing'), - this.db.db - .selectFrom('follow') - .if(!viewer, (q) => q.where(noMatch)) - .whereRef('creator', '=', ref('actor.did')) - .where('subjectDid', '=', viewer ?? '') - .select('uri') - .as('viewerFollowedBy'), - ]) - const [profiles, labels, bam] = await Promise.all([ - profileInfosQb.execute(), - this.services.label.getLabelsForSubjects(dids, state?.labels), - this.services.graph.getBlockAndMuteState( - viewer ? dids.map((did) => [viewer, did]) : [], - state?.bam, - ), - ]) - const listUris = mapDefined(profiles, ({ did }) => { - const muteList = viewer && bam.muteList([viewer, did]) - const blockList = viewer && bam.blockList([viewer, did]) - const lists: string[] = [] - if (muteList) lists.push(muteList) - if (blockList) lists.push(blockList) - return lists - }).flat() - const lists = await this.services.graph.getListViews(listUris, viewer) - return { profilesDetailed: toMapByDid(profiles), labels, bam, lists } - } - - profileDetailPresentation( - dids: string[], - state: ProfileDetailHydrationState, - opts: { - viewer?: string | null - }, - ): Record { - const { viewer } = opts - const { profilesDetailed, lists, labels, bam } = state - return dids.reduce((acc, did) => { - const prof = profilesDetailed[did] - if (!prof) return acc - const avatar = prof?.avatarCid - ? this.imgUriBuilder.getPresetUri('avatar', prof.did, prof.avatarCid) - : undefined - const banner = prof?.bannerCid - ? this.imgUriBuilder.getPresetUri('banner', prof.did, prof.bannerCid) - : undefined - const mutedByListUri = viewer && bam.muteList([viewer, did]) - const mutedByList = - mutedByListUri && lists[mutedByListUri] - ? this.services.graph.formatListViewBasic(lists[mutedByListUri]) - : undefined - const blockingByListUri = viewer && bam.blockList([viewer, did]) - const blockingByList = - blockingByListUri && lists[blockingByListUri] - ? this.services.graph.formatListViewBasic(lists[blockingByListUri]) - : undefined - const actorLabels = labels[did] ?? [] - const selfLabels = getSelfLabels({ - uri: prof.profileUri, - cid: prof.profileCid, - record: - prof.profileJson !== null - ? (jsonStringToLex(prof.profileJson) as Record) - : null, - }) - acc[did] = { - did: prof.did, - handle: prof.handle ?? INVALID_HANDLE, - displayName: prof?.displayName || undefined, - description: prof?.description || undefined, - avatar, - banner, - followsCount: prof?.followsCount ?? 0, - followersCount: prof?.followersCount ?? 0, - postsCount: prof?.postsCount ?? 0, - indexedAt: prof?.indexedAt || undefined, - viewer: viewer - ? { - muted: bam.mute([viewer, did]), - mutedByList, - blockedBy: !!bam.blockedBy([viewer, did]), - blocking: bam.blocking([viewer, did]) ?? undefined, - blockingByList, - following: - prof?.viewerFollowing && !bam.block([viewer, did]) - ? prof.viewerFollowing - : undefined, - followedBy: - prof?.viewerFollowedBy && !bam.block([viewer, did]) - ? prof.viewerFollowedBy - : undefined, - } - : undefined, - labels: [...actorLabels, ...selfLabels], - } - return acc - }, {} as Record) - } - - async profileHydration( - dids: string[], - opts: { - viewer?: string | null - includeSoftDeleted?: boolean - }, - state?: { - bam: BlockAndMuteState - labels: Labels - }, - ): Promise { - const { viewer = null, includeSoftDeleted } = opts - const { ref } = this.db.db.dynamic - const profileInfosQb = this.db.db - .selectFrom('actor') - .where('actor.did', 'in', dids.length ? dids : ['']) - .leftJoin('profile', 'profile.creator', 'actor.did') - .leftJoin('record', 'record.uri', 'profile.uri') - .if(!includeSoftDeleted, (qb) => - qb.where(notSoftDeletedClause(ref('actor'))), - ) - .select([ - 'actor.did as did', - 'actor.handle as handle', - 'profile.uri as profileUri', - 'profile.cid as profileCid', - 'profile.displayName as displayName', - 'profile.description as description', - 'profile.avatarCid as avatarCid', - 'profile.indexedAt as indexedAt', - 'record.json as profileJson', - this.db.db - .selectFrom('follow') - .if(!viewer, (q) => q.where(noMatch)) - .where('creator', '=', viewer ?? '') - .whereRef('subjectDid', '=', ref('actor.did')) - .select('uri') - .as('viewerFollowing'), - this.db.db - .selectFrom('follow') - .if(!viewer, (q) => q.where(noMatch)) - .whereRef('creator', '=', ref('actor.did')) - .where('subjectDid', '=', viewer ?? '') - .select('uri') - .as('viewerFollowedBy'), - ]) - const [profiles, labels, bam] = await Promise.all([ - profileInfosQb.execute(), - this.services.label.getLabelsForSubjects(dids, state?.labels), - this.services.graph.getBlockAndMuteState( - viewer ? dids.map((did) => [viewer, did]) : [], - state?.bam, - ), - ]) - const listUris = mapDefined(profiles, ({ did }) => { - const muteList = viewer && bam.muteList([viewer, did]) - const blockList = viewer && bam.blockList([viewer, did]) - const lists: string[] = [] - if (muteList) lists.push(muteList) - if (blockList) lists.push(blockList) - return lists - }).flat() - const lists = await this.services.graph.getListViews(listUris, viewer) - return { profiles: toMapByDid(profiles), labels, bam, lists } - } - - profilePresentation( - dids: string[], - state: { - profiles: ProfileInfoMap - lists: ListInfoMap - labels: Labels - bam: BlockAndMuteState - }, - viewer: string | null, - ): ProfileViewMap { - const { profiles, lists, labels, bam } = state - return dids.reduce((acc, did) => { - const prof = profiles[did] - if (!prof) return acc - const avatar = prof?.avatarCid - ? this.imgUriBuilder.getPresetUri('avatar', prof.did, prof.avatarCid) - : undefined - const mutedByListUri = viewer && bam.muteList([viewer, did]) - const mutedByList = - mutedByListUri && lists[mutedByListUri] - ? this.services.graph.formatListViewBasic(lists[mutedByListUri]) - : undefined - const blockingByListUri = viewer && bam.blockList([viewer, did]) - const blockingByList = - blockingByListUri && lists[blockingByListUri] - ? this.services.graph.formatListViewBasic(lists[blockingByListUri]) - : undefined - const actorLabels = labels[did] ?? [] - const selfLabels = getSelfLabels({ - uri: prof.profileUri, - cid: prof.profileCid, - record: - prof.profileJson !== null - ? (jsonStringToLex(prof.profileJson) as Record) - : null, - }) - acc[did] = { - did: prof.did, - handle: prof.handle ?? INVALID_HANDLE, - displayName: prof?.displayName || undefined, - description: prof?.description || undefined, - avatar, - indexedAt: prof?.indexedAt || undefined, - viewer: viewer - ? { - muted: bam.mute([viewer, did]), - mutedByList, - blockedBy: !!bam.blockedBy([viewer, did]), - blocking: bam.blocking([viewer, did]) ?? undefined, - blockingByList, - following: - prof?.viewerFollowing && !bam.block([viewer, did]) - ? prof.viewerFollowing - : undefined, - followedBy: - prof?.viewerFollowedBy && !bam.block([viewer, did]) - ? prof.viewerFollowedBy - : undefined, - } - : undefined, - labels: [...actorLabels, ...selfLabels], - } - return acc - }, {} as ProfileViewMap) - } - - profileBasicPresentation( - dids: string[], - state: ProfileHydrationState, - viewer: string | null, - ): ProfileViewMap { - const result = this.profilePresentation(dids, state, viewer) - return Object.values(result).reduce((acc, prof) => { - const profileBasic = { - did: prof.did, - handle: prof.handle, - displayName: prof.displayName, - avatar: prof.avatar, - viewer: prof.viewer, - labels: prof.labels, - } - acc[prof.did] = profileBasic - return acc - }, {} as ProfileViewMap) - } -} - -type ActorResult = Actor diff --git a/packages/mod-service/src/services/feed/index.ts b/packages/mod-service/src/services/feed/index.ts deleted file mode 100644 index a8768518d70..00000000000 --- a/packages/mod-service/src/services/feed/index.ts +++ /dev/null @@ -1,561 +0,0 @@ -import { sql } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { jsonStringToLex } from '@atproto/lexicon' -import { mapDefined } from '@atproto/common' -import { Database } from '../../db' -import { countAll, noMatch, notSoftDeletedClause } from '../../db/util' -import { ImageUriBuilder } from '../../image/uri' -import { ids } from '../../lexicon/lexicons' -import { - Record as PostRecord, - isRecord as isPostRecord, -} from '../../lexicon/types/app/bsky/feed/post' -import { - Record as ThreadgateRecord, - isListRule, -} from '../../lexicon/types/app/bsky/feed/threadgate' -import { isMain as isEmbedImages } from '../../lexicon/types/app/bsky/embed/images' -import { isMain as isEmbedExternal } from '../../lexicon/types/app/bsky/embed/external' -import { - isMain as isEmbedRecord, - isViewRecord, -} from '../../lexicon/types/app/bsky/embed/record' -import { isMain as isEmbedRecordWithMedia } from '../../lexicon/types/app/bsky/embed/recordWithMedia' -import { - PostInfoMap, - FeedItemType, - FeedRow, - FeedGenInfoMap, - PostEmbedViews, - RecordEmbedViewRecordMap, - PostInfo, - RecordEmbedViewRecord, - PostBlocksMap, - FeedHydrationState, - ThreadgateInfoMap, -} from './types' -import { LabelService } from '../label' -import { ActorService } from '../actor' -import { - BlockAndMuteState, - GraphService, - ListInfoMap, - RelationshipPair, -} from '../graph' -import { FeedViews } from './views' -import { threadgateToPostUri, postToThreadgateUri } from './util' -import { FromDb } from '../types' - -export * from './types' - -export class FeedService { - views: FeedViews - services: { - label: LabelService - actor: ActorService - graph: GraphService - } - - constructor( - public db: Database, - public imgUriBuilder: ImageUriBuilder, - private actor: FromDb, - private label: FromDb, - private graph: FromDb, - ) { - this.views = new FeedViews(this.db, this.imgUriBuilder, actor, graph) - this.services = { - label: label(this.db), - actor: actor(this.db), - graph: graph(this.db), - } - } - - static creator( - imgUriBuilder: ImageUriBuilder, - actor: FromDb, - label: FromDb, - graph: FromDb, - ) { - return (db: Database) => - new FeedService(db, imgUriBuilder, actor, label, graph) - } - - selectPostQb() { - return this.db.db - .selectFrom('post') - .select([ - sql`${'post'}`.as('type'), - 'post.uri as uri', - 'post.cid as cid', - 'post.uri as postUri', - 'post.creator as originatorDid', - 'post.creator as postAuthorDid', - 'post.replyParent as replyParent', - 'post.replyRoot as replyRoot', - 'post.sortAt as sortAt', - ]) - } - - selectFeedItemQb() { - return this.db.db - .selectFrom('feed_item') - .innerJoin('post', 'post.uri', 'feed_item.postUri') - .selectAll('feed_item') - .select([ - 'post.replyRoot', - 'post.replyParent', - 'post.creator as postAuthorDid', - ]) - } - - selectFeedGeneratorQb(viewer?: string | null) { - const { ref } = this.db.db.dynamic - return this.db.db - .selectFrom('feed_generator') - .innerJoin('actor', 'actor.did', 'feed_generator.creator') - .innerJoin('record', 'record.uri', 'feed_generator.uri') - .selectAll('feed_generator') - .where(notSoftDeletedClause(ref('actor'))) - .where(notSoftDeletedClause(ref('record'))) - .select((qb) => - qb - .selectFrom('like') - .whereRef('like.subject', '=', 'feed_generator.uri') - .select(countAll.as('count')) - .as('likeCount'), - ) - .select((qb) => - qb - .selectFrom('like') - .if(!viewer, (q) => q.where(noMatch)) - .where('like.creator', '=', viewer ?? '') - .whereRef('like.subject', '=', 'feed_generator.uri') - .select('uri') - .as('viewerLike'), - ) - } - - async getPostInfos( - postUris: string[], - viewer: string | null, - ): Promise { - if (postUris.length < 1) return {} - const db = this.db.db - const { ref } = db.dynamic - const posts = await db - .selectFrom('post') - .where('post.uri', 'in', postUris) - .innerJoin('actor', 'actor.did', 'post.creator') - .innerJoin('record', 'record.uri', 'post.uri') - .leftJoin('post_agg', 'post_agg.uri', 'post.uri') - .where(notSoftDeletedClause(ref('actor'))) // Ensures post reply parent/roots get omitted from views when taken down - .where(notSoftDeletedClause(ref('record'))) - .select([ - 'post.uri as uri', - 'post.cid as cid', - 'post.creator as creator', - 'post.sortAt as indexedAt', - 'post.invalidReplyRoot as invalidReplyRoot', - 'post.violatesThreadGate as violatesThreadGate', - 'record.json as recordJson', - 'post_agg.likeCount as likeCount', - 'post_agg.repostCount as repostCount', - 'post_agg.replyCount as replyCount', - 'post.tags as tags', - db - .selectFrom('repost') - .if(!viewer, (q) => q.where(noMatch)) - .where('creator', '=', viewer ?? '') - .whereRef('subject', '=', ref('post.uri')) - .select('uri') - .as('requesterRepost'), - db - .selectFrom('like') - .if(!viewer, (q) => q.where(noMatch)) - .where('creator', '=', viewer ?? '') - .whereRef('subject', '=', ref('post.uri')) - .select('uri') - .as('requesterLike'), - ]) - .execute() - return posts.reduce((acc, cur) => { - const { recordJson, ...post } = cur - const record = jsonStringToLex(recordJson) as PostRecord - const info: PostInfo = { - ...post, - invalidReplyRoot: post.invalidReplyRoot ?? false, - violatesThreadGate: post.violatesThreadGate ?? false, - record, - viewer, - } - return Object.assign(acc, { [post.uri]: info }) - }, {} as PostInfoMap) - } - - async getFeedGeneratorInfos(generatorUris: string[], viewer: string | null) { - if (generatorUris.length < 1) return {} - const feedGens = await this.selectFeedGeneratorQb(viewer) - .where('feed_generator.uri', 'in', generatorUris) - .execute() - return feedGens.reduce( - (acc, cur) => ({ - ...acc, - [cur.uri]: { - ...cur, - viewer: viewer ? { like: cur.viewerLike } : undefined, - }, - }), - {} as FeedGenInfoMap, - ) - } - - async getFeedItems(uris: string[]): Promise> { - if (uris.length < 1) return {} - const feedItems = await this.selectFeedItemQb() - .where('feed_item.uri', 'in', uris) - .execute() - return feedItems.reduce((acc, item) => { - return Object.assign(acc, { [item.uri]: item }) - }, {} as Record) - } - - async postUrisToFeedItems(uris: string[]): Promise { - const feedItems = await this.getFeedItems(uris) - return mapDefined(uris, (uri) => feedItems[uri]) - } - - feedItemRefs(items: FeedRow[]) { - const actorDids = new Set() - const postUris = new Set() - for (const item of items) { - postUris.add(item.postUri) - actorDids.add(item.postAuthorDid) - actorDids.add(item.originatorDid) - if (item.replyParent) { - postUris.add(item.replyParent) - actorDids.add(new AtUri(item.replyParent).hostname) - } - if (item.replyRoot) { - postUris.add(item.replyRoot) - actorDids.add(new AtUri(item.replyRoot).hostname) - } - } - return { dids: actorDids, uris: postUris } - } - - async feedHydration( - refs: { - dids: Set - uris: Set - viewer: string | null - }, - depth = 0, - ): Promise { - const { viewer, dids, uris } = refs - const [posts, threadgates, labels, bam] = await Promise.all([ - this.getPostInfos(Array.from(uris), viewer), - this.threadgatesByPostUri(Array.from(uris)), - this.services.label.getLabelsForSubjects([...uris, ...dids]), - this.services.graph.getBlockAndMuteState( - viewer ? [...dids].map((did) => [viewer, did]) : [], - ), - ]) - - // profileState for labels and bam handled above, profileHydration() shouldn't fetch additional - const [profileState, blocks, lists] = await Promise.all([ - this.services.actor.views.profileHydration( - Array.from(dids), - { viewer }, - { bam, labels }, - ), - this.blocksForPosts(posts, bam), - this.listsForThreadgates(threadgates, viewer), - ]) - const embeds = await this.embedsForPosts(posts, blocks, viewer, depth) - return { - posts, - threadgates, - blocks, - embeds, - labels, // includes info for profiles - bam, // includes info for profiles - profiles: profileState.profiles, - lists: Object.assign(lists, profileState.lists), - } - } - - // applies blocks for visibility to third-parties (i.e. based on post content) - async blocksForPosts( - posts: PostInfoMap, - bam?: BlockAndMuteState, - ): Promise { - const relationships: RelationshipPair[] = [] - const byPost: Record = {} - const didFromUri = (uri) => new AtUri(uri).host - for (const post of Object.values(posts)) { - // skip posts that we can't process or appear to already have been processed - if (!isPostRecord(post.record)) continue - if (byPost[post.uri]) continue - byPost[post.uri] = {} - // 3p block for replies - const parentUri = post.record.reply?.parent.uri - const parentDid = parentUri ? didFromUri(parentUri) : null - // 3p block for record embeds - const embedUris = nestedRecordUris([post.record]) - // gather actor relationships among posts - if (parentDid) { - const pair: RelationshipPair = [post.creator, parentDid] - relationships.push(pair) - byPost[post.uri].reply = pair - } - for (const embedUri of embedUris) { - const pair: RelationshipPair = [post.creator, didFromUri(embedUri)] - relationships.push(pair) - byPost[post.uri].embed = pair - } - } - // compute block state from all actor relationships among posts - const blockState = await this.services.graph.getBlockState( - relationships, - bam, - ) - const result: PostBlocksMap = {} - Object.entries(byPost).forEach(([uri, block]) => { - if (block.embed && blockState.block(block.embed)) { - result[uri] ??= {} - result[uri].embed = true - } - if (block.reply && blockState.block(block.reply)) { - result[uri] ??= {} - result[uri].reply = true - } - }) - return result - } - - async embedsForPosts( - postInfos: PostInfoMap, - blocks: PostBlocksMap, - viewer: string | null, - depth: number, - ) { - const postMap = postRecordsFromInfos(postInfos) - const posts = Object.values(postMap) - if (posts.length < 1) { - return {} - } - const recordEmbedViews = - depth > 1 ? {} : await this.nestedRecordViews(posts, viewer, depth) - - const postEmbedViews: PostEmbedViews = {} - for (const [uri, post] of Object.entries(postMap)) { - const creator = new AtUri(uri).hostname - if (!post.embed) continue - if (isEmbedImages(post.embed)) { - postEmbedViews[uri] = this.views.imagesEmbedView(creator, post.embed) - } else if (isEmbedExternal(post.embed)) { - postEmbedViews[uri] = this.views.externalEmbedView(creator, post.embed) - } else if (isEmbedRecord(post.embed)) { - if (!recordEmbedViews[post.embed.record.uri]) continue - postEmbedViews[uri] = { - $type: 'app.bsky.embed.record#view', - record: applyEmbedBlock( - uri, - blocks, - recordEmbedViews[post.embed.record.uri], - ), - } - } else if (isEmbedRecordWithMedia(post.embed)) { - const embedRecordView = recordEmbedViews[post.embed.record.record.uri] - if (!embedRecordView) continue - const formatted = this.views.getRecordWithMediaEmbedView( - creator, - post.embed, - applyEmbedBlock(uri, blocks, embedRecordView), - ) - if (formatted) { - postEmbedViews[uri] = formatted - } - } - } - return postEmbedViews - } - - async nestedRecordViews( - posts: PostRecord[], - viewer: string | null, - depth: number, - ): Promise { - const nestedUris = nestedRecordUris(posts) - if (nestedUris.length < 1) return {} - const nestedDids = new Set() - const nestedPostUris = new Set() - const nestedFeedGenUris = new Set() - const nestedListUris = new Set() - for (const uri of nestedUris) { - const parsed = new AtUri(uri) - nestedDids.add(parsed.hostname) - if (parsed.collection === ids.AppBskyFeedPost) { - nestedPostUris.add(uri) - } else if (parsed.collection === ids.AppBskyFeedGenerator) { - nestedFeedGenUris.add(uri) - } else if (parsed.collection === ids.AppBskyGraphList) { - nestedListUris.add(uri) - } - } - const [feedState, feedGenInfos, listViews] = await Promise.all([ - this.feedHydration( - { - dids: nestedDids, - uris: nestedPostUris, - viewer, - }, - depth + 1, - ), - this.getFeedGeneratorInfos([...nestedFeedGenUris], viewer), - this.services.graph.getListViews([...nestedListUris], viewer), - ]) - const actorInfos = this.services.actor.views.profileBasicPresentation( - [...nestedDids], - feedState, - viewer, - ) - const recordEmbedViews: RecordEmbedViewRecordMap = {} - for (const uri of nestedUris) { - const collection = new AtUri(uri).collection - if (collection === ids.AppBskyFeedGenerator && feedGenInfos[uri]) { - const genView = this.views.formatFeedGeneratorView( - feedGenInfos[uri], - actorInfos, - ) - if (genView) { - recordEmbedViews[uri] = { - $type: 'app.bsky.feed.defs#generatorView', - ...genView, - } - } - } else if (collection === ids.AppBskyGraphList && listViews[uri]) { - const listView = this.services.graph.formatListView( - listViews[uri], - actorInfos, - ) - if (listView) { - recordEmbedViews[uri] = { - $type: 'app.bsky.graph.defs#listView', - ...listView, - } - } - } else if (collection === ids.AppBskyFeedPost && feedState.posts[uri]) { - const formatted = this.views.formatPostView( - uri, - actorInfos, - feedState.posts, - feedState.threadgates, - feedState.embeds, - feedState.labels, - feedState.lists, - viewer, - ) - recordEmbedViews[uri] = this.views.getRecordEmbedView( - uri, - formatted, - depth > 0, - ) - } else { - recordEmbedViews[uri] = { - $type: 'app.bsky.embed.record#viewNotFound', - uri, - notFound: true, - } - } - } - return recordEmbedViews - } - - async threadgatesByPostUri(postUris: string[]): Promise { - const gates = postUris.length - ? await this.db.db - .selectFrom('record') - .where('uri', 'in', postUris.map(postToThreadgateUri)) - .select(['uri', 'cid', 'json']) - .execute() - : [] - const gatesByPostUri = gates.reduce((acc, gate) => { - const record = jsonStringToLex(gate.json) as ThreadgateRecord - const postUri = threadgateToPostUri(gate.uri) - if (record.post !== postUri) return acc // invalid, skip - acc[postUri] = { uri: gate.uri, cid: gate.cid, record } - return acc - }, {} as ThreadgateInfoMap) - return gatesByPostUri - } - - listsForThreadgates( - threadgates: ThreadgateInfoMap, - viewer: string | null, - ): Promise { - const listsUris = new Set() - Object.values(threadgates).forEach((gate) => { - gate?.record.allow?.forEach((rule) => { - if (isListRule(rule)) { - listsUris.add(rule.list) - } - }) - }) - return this.services.graph.getListViews([...listsUris], viewer) - } -} - -const postRecordsFromInfos = ( - infos: PostInfoMap, -): { [uri: string]: PostRecord } => { - const records: { [uri: string]: PostRecord } = {} - for (const [uri, info] of Object.entries(infos)) { - if (isPostRecord(info.record)) { - records[uri] = info.record - } - } - return records -} - -const nestedRecordUris = (posts: PostRecord[]): string[] => { - const uris: string[] = [] - for (const post of posts) { - if (!post.embed) continue - if (isEmbedRecord(post.embed)) { - uris.push(post.embed.record.uri) - } else if (isEmbedRecordWithMedia(post.embed)) { - uris.push(post.embed.record.record.uri) - } else { - continue - } - } - return uris -} - -type PostRelationships = { reply?: RelationshipPair; embed?: RelationshipPair } - -function applyEmbedBlock( - uri: string, - blocks: PostBlocksMap, - view: RecordEmbedViewRecord, -): RecordEmbedViewRecord { - if (isViewRecord(view) && blocks[uri]?.embed) { - return { - $type: 'app.bsky.embed.record#viewBlocked', - uri: view.uri, - blocked: true, - author: { - did: view.author.did, - viewer: view.author.viewer - ? { - blockedBy: view.author.viewer?.blockedBy, - blocking: view.author.viewer?.blocking, - } - : undefined, - }, - } - } - return view -} diff --git a/packages/mod-service/src/services/feed/types.ts b/packages/mod-service/src/services/feed/types.ts deleted file mode 100644 index 8d4bd67f6bb..00000000000 --- a/packages/mod-service/src/services/feed/types.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { Selectable } from 'kysely' -import { Record as ThreadgateRecord } from '../../lexicon/types/app/bsky/feed/threadgate' -import { View as ImagesEmbedView } from '../../lexicon/types/app/bsky/embed/images' -import { View as ExternalEmbedView } from '../../lexicon/types/app/bsky/embed/external' -import { - ViewBlocked, - ViewNotFound, - ViewRecord, - View as RecordEmbedView, -} from '../../lexicon/types/app/bsky/embed/record' -import { View as RecordWithMediaEmbedView } from '../../lexicon/types/app/bsky/embed/recordWithMedia' -import { - BlockedPost, - GeneratorView, - NotFoundPost, - PostView, -} from '../../lexicon/types/app/bsky/feed/defs' -import { FeedGenerator } from '../../db/tables/feed-generator' -import { ListView } from '../../lexicon/types/app/bsky/graph/defs' -import { ProfileHydrationState } from '../actor' -import { Labels } from '../label' -import { BlockAndMuteState } from '../graph' - -export type PostEmbedViews = { - [uri: string]: PostEmbedView -} - -export type PostEmbedView = - | ImagesEmbedView - | ExternalEmbedView - | RecordEmbedView - | RecordWithMediaEmbedView - -export type PostInfo = { - uri: string - cid: string - creator: string - record: Record - indexedAt: string - likeCount: number | null - repostCount: number | null - replyCount: number | null - requesterRepost: string | null - requesterLike: string | null - invalidReplyRoot: boolean - violatesThreadGate: boolean - viewer: string | null -} - -export type PostInfoMap = { [uri: string]: PostInfo } - -export type PostBlocksMap = { - [uri: string]: { reply?: boolean; embed?: boolean } -} - -export type ThreadgateInfo = { - uri: string - cid: string - record: ThreadgateRecord -} - -export type ThreadgateInfoMap = { - [postUri: string]: ThreadgateInfo -} - -export type FeedGenInfo = Selectable & { - likeCount: number - viewer?: { - like?: string - } -} - -export type FeedGenInfoMap = { [uri: string]: FeedGenInfo } - -export type FeedItemType = 'post' | 'repost' - -export type FeedRow = { - type: FeedItemType - uri: string - cid: string - postUri: string - postAuthorDid: string - originatorDid: string - replyParent: string | null - replyRoot: string | null - sortAt: string -} - -export type MaybePostView = PostView | NotFoundPost | BlockedPost - -export type RecordEmbedViewRecord = - | ViewRecord - | ViewNotFound - | ViewBlocked - | GeneratorView - | ListView - -export type RecordEmbedViewRecordMap = { [uri: string]: RecordEmbedViewRecord } - -export type FeedHydrationState = ProfileHydrationState & { - posts: PostInfoMap - threadgates: ThreadgateInfoMap - embeds: PostEmbedViews - labels: Labels - blocks: PostBlocksMap - bam: BlockAndMuteState -} diff --git a/packages/mod-service/src/services/feed/util.ts b/packages/mod-service/src/services/feed/util.ts deleted file mode 100644 index 83b5e59d705..00000000000 --- a/packages/mod-service/src/services/feed/util.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { sql } from 'kysely' -import { AtUri } from '@atproto/syntax' -import { - Record as PostRecord, - ReplyRef, -} from '../../lexicon/types/app/bsky/feed/post' -import { - Record as GateRecord, - isFollowingRule, - isListRule, - isMentionRule, -} from '../../lexicon/types/app/bsky/feed/threadgate' -import { isMention } from '../../lexicon/types/app/bsky/richtext/facet' -import { valuesList } from '../../db/util' -import DatabaseSchema from '../../db/database-schema' -import { ids } from '../../lexicon/lexicons' - -export const invalidReplyRoot = ( - reply: ReplyRef, - parent: { - record: PostRecord - invalidReplyRoot: boolean | null - }, -) => { - const replyRoot = reply.root.uri - const replyParent = reply.parent.uri - // if parent is not a valid reply, transitively this is not a valid one either - if (parent.invalidReplyRoot) { - return true - } - // replying to root post: ensure the root looks correct - if (replyParent === replyRoot) { - return !!parent.record.reply - } - // replying to a reply: ensure the parent is a reply for the same root post - return parent.record.reply?.root.uri !== replyRoot -} - -type ParsedThreadGate = { - canReply?: boolean - allowMentions?: boolean - allowFollowing?: boolean - allowListUris?: string[] -} - -export const parseThreadGate = ( - replierDid: string, - ownerDid: string, - rootPost: PostRecord | null, - gate: GateRecord | null, -): ParsedThreadGate => { - if (replierDid === ownerDid) { - return { canReply: true } - } - // if gate.allow is unset then *any* reply is allowed, if it is an empty array then *no* reply is allowed - if (!gate || !gate.allow) { - return { canReply: true } - } - - const allowMentions = !!gate.allow.find(isMentionRule) - const allowFollowing = !!gate.allow.find(isFollowingRule) - const allowListUris = gate.allow?.filter(isListRule).map((item) => item.list) - - // check mentions first since it's quick and synchronous - if (allowMentions) { - const isMentioned = rootPost?.facets?.some((facet) => { - return facet.features.some( - (item) => isMention(item) && item.did === replierDid, - ) - }) - if (isMentioned) { - return { canReply: true, allowMentions, allowFollowing, allowListUris } - } - } - return { allowMentions, allowFollowing, allowListUris } -} - -export const violatesThreadGate = async ( - db: DatabaseSchema, - replierDid: string, - ownerDid: string, - rootPost: PostRecord | null, - gate: GateRecord | null, -) => { - const { - canReply, - allowFollowing, - allowListUris = [], - } = parseThreadGate(replierDid, ownerDid, rootPost, gate) - if (canReply) { - return false - } - if (!allowFollowing && !allowListUris?.length) { - return true - } - const { ref } = db.dynamic - const nullResult = sql`${null}` - const check = await db - .selectFrom(valuesList([replierDid]).as(sql`subject (did)`)) - .select([ - allowFollowing - ? db - .selectFrom('follow') - .where('creator', '=', ownerDid) - .whereRef('subjectDid', '=', ref('subject.did')) - .select('creator') - .as('isFollowed') - : nullResult.as('isFollowed'), - allowListUris.length - ? db - .selectFrom('list_item') - .where('list_item.listUri', 'in', allowListUris) - .whereRef('list_item.subjectDid', '=', ref('subject.did')) - .limit(1) - .select('listUri') - .as('isInList') - : nullResult.as('isInList'), - ]) - .executeTakeFirst() - - if (allowFollowing && check?.isFollowed) { - return false - } else if (allowListUris.length && check?.isInList) { - return false - } - - return true -} - -export const postToThreadgateUri = (postUri: string) => { - const gateUri = new AtUri(postUri) - gateUri.collection = ids.AppBskyFeedThreadgate - return gateUri.toString() -} - -export const threadgateToPostUri = (gateUri: string) => { - const postUri = new AtUri(gateUri) - postUri.collection = ids.AppBskyFeedPost - return postUri.toString() -} diff --git a/packages/mod-service/src/services/feed/views.ts b/packages/mod-service/src/services/feed/views.ts deleted file mode 100644 index f013570e2d7..00000000000 --- a/packages/mod-service/src/services/feed/views.ts +++ /dev/null @@ -1,470 +0,0 @@ -import { mapDefined } from '@atproto/common' -import { AtUri } from '@atproto/syntax' -import { Database } from '../../db' -import { - FeedViewPost, - GeneratorView, - PostView, -} from '../../lexicon/types/app/bsky/feed/defs' -import { - Main as EmbedImages, - isMain as isEmbedImages, - View as EmbedImagesView, -} from '../../lexicon/types/app/bsky/embed/images' -import { - Main as EmbedExternal, - isMain as isEmbedExternal, - View as EmbedExternalView, -} from '../../lexicon/types/app/bsky/embed/external' -import { Main as EmbedRecordWithMedia } from '../../lexicon/types/app/bsky/embed/recordWithMedia' -import { - ViewBlocked, - ViewNotFound, - ViewRecord, -} from '../../lexicon/types/app/bsky/embed/record' -import { Record as PostRecord } from '../../lexicon/types/app/bsky/feed/post' -import { isListRule } from '../../lexicon/types/app/bsky/feed/threadgate' -import { - PostEmbedViews, - FeedGenInfo, - FeedRow, - MaybePostView, - PostInfoMap, - RecordEmbedViewRecord, - PostBlocksMap, - FeedHydrationState, - ThreadgateInfoMap, - ThreadgateInfo, -} from './types' -import { Labels, getSelfLabels } from '../label' -import { ImageUriBuilder } from '../../image/uri' -import { ActorInfoMap, ActorService } from '../actor' -import { ListInfoMap, GraphService } from '../graph' -import { FromDb } from '../types' -import { parseThreadGate } from './util' - -export class FeedViews { - services: { - actor: ActorService - graph: GraphService - } - - constructor( - public db: Database, - public imgUriBuilder: ImageUriBuilder, - private actor: FromDb, - private graph: FromDb, - ) { - this.services = { - actor: actor(this.db), - graph: graph(this.db), - } - } - - static creator( - imgUriBuilder: ImageUriBuilder, - actor: FromDb, - graph: FromDb, - ) { - return (db: Database) => new FeedViews(db, imgUriBuilder, actor, graph) - } - - formatFeedGeneratorView( - info: FeedGenInfo, - profiles: ActorInfoMap, - ): GeneratorView | undefined { - const profile = profiles[info.creator] - if (!profile) { - return undefined - } - return { - uri: info.uri, - cid: info.cid, - did: info.feedDid, - creator: profile, - displayName: info.displayName ?? undefined, - description: info.description ?? undefined, - descriptionFacets: info.descriptionFacets - ? JSON.parse(info.descriptionFacets) - : undefined, - avatar: info.avatarCid - ? this.imgUriBuilder.getPresetUri( - 'avatar', - info.creator, - info.avatarCid, - ) - : undefined, - likeCount: info.likeCount, - viewer: info.viewer - ? { - like: info.viewer.like ?? undefined, - } - : undefined, - indexedAt: info.indexedAt, - } - } - - formatFeed( - items: FeedRow[], - state: FeedHydrationState, - viewer: string | null, - opts?: { - usePostViewUnion?: boolean - }, - ): FeedViewPost[] { - const { posts, threadgates, profiles, blocks, embeds, labels, lists } = - state - const actors = this.services.actor.views.profileBasicPresentation( - Object.keys(profiles), - state, - viewer, - ) - const feed: FeedViewPost[] = [] - for (const item of items) { - const info = posts[item.postUri] - const post = this.formatPostView( - item.postUri, - actors, - posts, - threadgates, - embeds, - labels, - lists, - viewer, - ) - // skip over not found & blocked posts - if (!post || blocks[post.uri]?.reply) { - continue - } - const feedPost = { post } - if (item.type === 'repost') { - const originator = actors[item.originatorDid] - // skip over reposts where we don't have reposter profile - if (!originator) { - continue - } else { - feedPost['reason'] = { - $type: 'app.bsky.feed.defs#reasonRepost', - by: originator, - indexedAt: item.sortAt, - } - } - } - // posts that violate reply-gating may appear in feeds, but without any thread context - if ( - item.replyParent && - item.replyRoot && - !info?.invalidReplyRoot && - !info?.violatesThreadGate - ) { - const replyParent = this.formatMaybePostView( - item.replyParent, - actors, - posts, - threadgates, - embeds, - labels, - lists, - blocks, - viewer, - opts, - ) - const replyRoot = this.formatMaybePostView( - item.replyRoot, - actors, - posts, - threadgates, - embeds, - labels, - lists, - blocks, - viewer, - opts, - ) - if (replyRoot && replyParent) { - feedPost['reply'] = { - root: replyRoot, - parent: replyParent, - } - } - } - feed.push(feedPost) - } - return feed - } - - formatPostView( - uri: string, - actors: ActorInfoMap, - posts: PostInfoMap, - threadgates: ThreadgateInfoMap, - embeds: PostEmbedViews, - labels: Labels, - lists: ListInfoMap, - viewer: string | null, - ): PostView | undefined { - const post = posts[uri] - const gate = threadgates[uri] - const author = actors[post?.creator] - if (!post || !author) return undefined - const postLabels = labels[uri] ?? [] - const postSelfLabels = getSelfLabels({ - uri: post.uri, - cid: post.cid, - record: post.record, - }) - return { - uri: post.uri, - cid: post.cid, - author: author, - record: post.record, - embed: embeds[uri], - replyCount: post.replyCount ?? 0, - repostCount: post.repostCount ?? 0, - likeCount: post.likeCount ?? 0, - indexedAt: post.indexedAt, - viewer: post.viewer - ? { - repost: post.requesterRepost ?? undefined, - like: post.requesterLike ?? undefined, - replyDisabled: this.userReplyDisabled( - uri, - actors, - posts, - threadgates, - lists, - viewer, - ), - } - : undefined, - labels: [...postLabels, ...postSelfLabels], - threadgate: - !post.record.reply && gate - ? this.formatThreadgate(gate, lists) - : undefined, - } - } - - userReplyDisabled( - uri: string, - actors: ActorInfoMap, - posts: PostInfoMap, - threadgates: ThreadgateInfoMap, - lists: ListInfoMap, - viewer: string | null, - ): boolean | undefined { - if (viewer === null) { - return undefined - } else if (posts[uri]?.violatesThreadGate) { - return true - } - - const rootUriStr: string = - posts[uri]?.record?.['reply']?.['root']?.['uri'] ?? uri - const gate = threadgates[rootUriStr]?.record - if (!gate) { - return undefined - } - const rootPost = posts[rootUriStr]?.record as PostRecord | undefined - const ownerDid = new AtUri(rootUriStr).hostname - - const { - canReply, - allowFollowing, - allowListUris = [], - } = parseThreadGate(viewer, ownerDid, rootPost ?? null, gate ?? null) - - if (canReply) { - return false - } - if (allowFollowing && actors[ownerDid]?.viewer?.followedBy) { - return false - } - for (const listUri of allowListUris) { - const list = lists[listUri] - if (list?.viewerInList) { - return false - } - } - return true - } - - formatMaybePostView( - uri: string, - actors: ActorInfoMap, - posts: PostInfoMap, - threadgates: ThreadgateInfoMap, - embeds: PostEmbedViews, - labels: Labels, - lists: ListInfoMap, - blocks: PostBlocksMap, - viewer: string | null, - opts?: { - usePostViewUnion?: boolean - }, - ): MaybePostView | undefined { - const post = this.formatPostView( - uri, - actors, - posts, - threadgates, - embeds, - labels, - lists, - viewer, - ) - if (!post) { - if (!opts?.usePostViewUnion) return - return this.notFoundPost(uri) - } - if ( - post.author.viewer?.blockedBy || - post.author.viewer?.blocking || - blocks[uri]?.reply - ) { - if (!opts?.usePostViewUnion) return - return this.blockedPost(post) - } - return { - $type: 'app.bsky.feed.defs#postView', - ...post, - } - } - - blockedPost(post: PostView) { - return { - $type: 'app.bsky.feed.defs#blockedPost', - uri: post.uri, - blocked: true as const, - author: { - did: post.author.did, - viewer: post.author.viewer - ? { - blockedBy: post.author.viewer?.blockedBy, - blocking: post.author.viewer?.blocking, - } - : undefined, - }, - } - } - - notFoundPost(uri: string) { - return { - $type: 'app.bsky.feed.defs#notFoundPost', - uri: uri, - notFound: true as const, - } - } - - imagesEmbedView(did: string, embed: EmbedImages) { - const imgViews = embed.images.map((img) => ({ - thumb: this.imgUriBuilder.getPresetUri( - 'feed_thumbnail', - did, - img.image.ref, - ), - fullsize: this.imgUriBuilder.getPresetUri( - 'feed_fullsize', - did, - img.image.ref, - ), - alt: img.alt, - aspectRatio: img.aspectRatio, - })) - return { - $type: 'app.bsky.embed.images#view', - images: imgViews, - } - } - - externalEmbedView(did: string, embed: EmbedExternal) { - const { uri, title, description, thumb } = embed.external - return { - $type: 'app.bsky.embed.external#view', - external: { - uri, - title, - description, - thumb: thumb - ? this.imgUriBuilder.getPresetUri('feed_thumbnail', did, thumb.ref) - : undefined, - }, - } - } - - getRecordEmbedView( - uri: string, - post?: PostView, - omitEmbeds = false, - ): (ViewRecord | ViewNotFound | ViewBlocked) & { $type: string } { - if (!post) { - return { - $type: 'app.bsky.embed.record#viewNotFound', - uri, - notFound: true, - } - } - if (post.author.viewer?.blocking || post.author.viewer?.blockedBy) { - return { - $type: 'app.bsky.embed.record#viewBlocked', - uri, - blocked: true, - author: { - did: post.author.did, - viewer: post.author.viewer - ? { - blockedBy: post.author.viewer?.blockedBy, - blocking: post.author.viewer?.blocking, - } - : undefined, - }, - } - } - return { - $type: 'app.bsky.embed.record#viewRecord', - uri: post.uri, - cid: post.cid, - author: post.author, - value: post.record, - labels: post.labels, - indexedAt: post.indexedAt, - embeds: omitEmbeds ? undefined : post.embed ? [post.embed] : [], - } - } - - getRecordWithMediaEmbedView( - did: string, - embed: EmbedRecordWithMedia, - embedRecordView: RecordEmbedViewRecord, - ) { - let mediaEmbed: EmbedImagesView | EmbedExternalView - if (isEmbedImages(embed.media)) { - mediaEmbed = this.imagesEmbedView(did, embed.media) - } else if (isEmbedExternal(embed.media)) { - mediaEmbed = this.externalEmbedView(did, embed.media) - } else { - return - } - return { - $type: 'app.bsky.embed.recordWithMedia#view', - record: { - record: embedRecordView, - }, - media: mediaEmbed, - } - } - - formatThreadgate(gate: ThreadgateInfo, lists: ListInfoMap) { - return { - uri: gate.uri, - cid: gate.cid, - record: gate.record, - lists: mapDefined(gate.record.allow ?? [], (rule) => { - if (!isListRule(rule)) return - const list = lists[rule.list] - if (!list) return - return this.services.graph.formatListViewBasic(list) - }), - } - } -} diff --git a/packages/mod-service/src/services/graph/index.ts b/packages/mod-service/src/services/graph/index.ts deleted file mode 100644 index b154a8c47bb..00000000000 --- a/packages/mod-service/src/services/graph/index.ts +++ /dev/null @@ -1,381 +0,0 @@ -import { sql } from 'kysely' -import { Database } from '../../db' -import { ImageUriBuilder } from '../../image/uri' -import { valuesList } from '../../db/util' -import { ListInfo } from './types' -import { ActorInfoMap } from '../actor' -import { - ListView, - ListViewBasic, -} from '../../lexicon/types/app/bsky/graph/defs' - -export * from './types' - -export class GraphService { - constructor(public db: Database, public imgUriBuilder: ImageUriBuilder) {} - - static creator(imgUriBuilder: ImageUriBuilder) { - return (db: Database) => new GraphService(db, imgUriBuilder) - } - - async muteActor(info: { - subjectDid: string - mutedByDid: string - createdAt?: Date - }) { - const { subjectDid, mutedByDid, createdAt = new Date() } = info - await this.db - .asPrimary() - .db.insertInto('mute') - .values({ - subjectDid, - mutedByDid, - createdAt: createdAt.toISOString(), - }) - .onConflict((oc) => oc.doNothing()) - .execute() - } - - async unmuteActor(info: { subjectDid: string; mutedByDid: string }) { - const { subjectDid, mutedByDid } = info - await this.db - .asPrimary() - .db.deleteFrom('mute') - .where('subjectDid', '=', subjectDid) - .where('mutedByDid', '=', mutedByDid) - .execute() - } - - async muteActorList(info: { - list: string - mutedByDid: string - createdAt?: Date - }) { - const { list, mutedByDid, createdAt = new Date() } = info - await this.db - .asPrimary() - .db.insertInto('list_mute') - .values({ - listUri: list, - mutedByDid, - createdAt: createdAt.toISOString(), - }) - .onConflict((oc) => oc.doNothing()) - .execute() - } - - async unmuteActorList(info: { list: string; mutedByDid: string }) { - const { list, mutedByDid } = info - await this.db - .asPrimary() - .db.deleteFrom('list_mute') - .where('listUri', '=', list) - .where('mutedByDid', '=', mutedByDid) - .execute() - } - - getListsQb(viewer: string | null) { - const { ref } = this.db.db.dynamic - return this.db.db - .selectFrom('list') - .innerJoin('actor', 'actor.did', 'list.creator') - .selectAll('list') - .selectAll('actor') - .select('list.sortAt as sortAt') - .select([ - this.db.db - .selectFrom('list_mute') - .where('list_mute.mutedByDid', '=', viewer ?? '') - .whereRef('list_mute.listUri', '=', ref('list.uri')) - .select('list_mute.listUri') - .as('viewerMuted'), - this.db.db - .selectFrom('list_block') - .where('list_block.creator', '=', viewer ?? '') - .whereRef('list_block.subjectUri', '=', ref('list.uri')) - .select('list_block.uri') - .as('viewerListBlockUri'), - this.db.db - .selectFrom('list_item') - .whereRef('list_item.listUri', '=', ref('list.uri')) - .where('list_item.subjectDid', '=', viewer ?? '') - .select('list_item.uri') - .as('viewerInList'), - ]) - } - - getListItemsQb() { - return this.db.db - .selectFrom('list_item') - .innerJoin('actor as subject', 'subject.did', 'list_item.subjectDid') - .selectAll('subject') - .select([ - 'list_item.uri as uri', - 'list_item.cid as cid', - 'list_item.sortAt as sortAt', - ]) - } - - async getBlockAndMuteState( - pairs: RelationshipPair[], - bam?: BlockAndMuteState, - ) { - pairs = bam ? pairs.filter((pair) => !bam.has(pair)) : pairs - const result = bam ?? new BlockAndMuteState() - if (!pairs.length) return result - const { ref } = this.db.db.dynamic - const sourceRef = ref('pair.source') - const targetRef = ref('pair.target') - const values = valuesList(pairs.map((p) => sql`${p[0]}, ${p[1]}`)) - const items = await this.db.db - .selectFrom(values.as(sql`pair (source, target)`)) - .select([ - sql`${sourceRef}`.as('source'), - sql`${targetRef}`.as('target'), - this.db.db - .selectFrom('actor_block') - .whereRef('creator', '=', sourceRef) - .whereRef('subjectDid', '=', targetRef) - .select('uri') - .as('blocking'), - this.db.db - .selectFrom('list_item') - .innerJoin('list_block', 'list_block.subjectUri', 'list_item.listUri') - .whereRef('list_block.creator', '=', sourceRef) - .whereRef('list_item.subjectDid', '=', targetRef) - .select('list_item.listUri') - .limit(1) - .as('blockingViaList'), - this.db.db - .selectFrom('actor_block') - .whereRef('creator', '=', targetRef) - .whereRef('subjectDid', '=', sourceRef) - .select('uri') - .as('blockedBy'), - this.db.db - .selectFrom('list_item') - .innerJoin('list_block', 'list_block.subjectUri', 'list_item.listUri') - .whereRef('list_block.creator', '=', targetRef) - .whereRef('list_item.subjectDid', '=', sourceRef) - .select('list_item.listUri') - .limit(1) - .as('blockedByViaList'), - this.db.db - .selectFrom('mute') - .whereRef('mutedByDid', '=', sourceRef) - .whereRef('subjectDid', '=', targetRef) - .select(sql`${true}`.as('val')) - .as('muting'), - this.db.db - .selectFrom('list_item') - .innerJoin('list_mute', 'list_mute.listUri', 'list_item.listUri') - .whereRef('list_mute.mutedByDid', '=', sourceRef) - .whereRef('list_item.subjectDid', '=', targetRef) - .select('list_item.listUri') - .limit(1) - .as('mutingViaList'), - ]) - .selectAll() - .execute() - items.forEach((item) => result.add(item)) - return result - } - - async getBlockState(pairs: RelationshipPair[], bam?: BlockAndMuteState) { - pairs = bam ? pairs.filter((pair) => !bam.has(pair)) : pairs - const result = bam ?? new BlockAndMuteState() - if (!pairs.length) return result - const { ref } = this.db.db.dynamic - const sourceRef = ref('pair.source') - const targetRef = ref('pair.target') - const values = valuesList(pairs.map((p) => sql`${p[0]}, ${p[1]}`)) - const items = await this.db.db - .selectFrom(values.as(sql`pair (source, target)`)) - .select([ - sql`${sourceRef}`.as('source'), - sql`${targetRef}`.as('target'), - this.db.db - .selectFrom('actor_block') - .whereRef('creator', '=', sourceRef) - .whereRef('subjectDid', '=', targetRef) - .select('uri') - .as('blocking'), - this.db.db - .selectFrom('list_item') - .innerJoin('list_block', 'list_block.subjectUri', 'list_item.listUri') - .whereRef('list_block.creator', '=', sourceRef) - .whereRef('list_item.subjectDid', '=', targetRef) - .select('list_item.listUri') - .limit(1) - .as('blockingViaList'), - this.db.db - .selectFrom('actor_block') - .whereRef('creator', '=', targetRef) - .whereRef('subjectDid', '=', sourceRef) - .select('uri') - .as('blockedBy'), - this.db.db - .selectFrom('list_item') - .innerJoin('list_block', 'list_block.subjectUri', 'list_item.listUri') - .whereRef('list_block.creator', '=', targetRef) - .whereRef('list_item.subjectDid', '=', sourceRef) - .select('list_item.listUri') - .limit(1) - .as('blockedByViaList'), - ]) - .selectAll() - .execute() - items.forEach((item) => result.add(item)) - return result - } - - async getListViews(listUris: string[], requester: string | null) { - if (listUris.length < 1) return {} - const lists = await this.getListsQb(requester) - .where('list.uri', 'in', listUris) - .execute() - return lists.reduce( - (acc, cur) => ({ - ...acc, - [cur.uri]: cur, - }), - {}, - ) - } - - formatListView(list: ListInfo, profiles: ActorInfoMap): ListView | undefined { - if (!profiles[list.creator]) { - return undefined - } - return { - ...this.formatListViewBasic(list), - creator: profiles[list.creator], - description: list.description ?? undefined, - descriptionFacets: list.descriptionFacets - ? JSON.parse(list.descriptionFacets) - : undefined, - indexedAt: list.sortAt, - } - } - - formatListViewBasic(list: ListInfo): ListViewBasic { - return { - uri: list.uri, - cid: list.cid, - name: list.name, - purpose: list.purpose, - avatar: list.avatarCid - ? this.imgUriBuilder.getPresetUri( - 'avatar', - list.creator, - list.avatarCid, - ) - : undefined, - indexedAt: list.sortAt, - viewer: { - muted: !!list.viewerMuted, - blocked: list.viewerListBlockUri ?? undefined, - }, - } - } -} - -export type RelationshipPair = [didA: string, didB: string] - -export class BlockAndMuteState { - hasIdx = new Map>() // did -> did - blockIdx = new Map>() // did -> did -> block uri - blockListIdx = new Map>() // did -> did -> list uri - muteIdx = new Map>() // did -> did - muteListIdx = new Map>() // did -> did -> list uri - constructor(items: BlockAndMuteInfo[] = []) { - items.forEach((item) => this.add(item)) - } - add(item: BlockAndMuteInfo) { - if (item.source === item.target) { - return // we do not respect self-blocks or self-mutes - } - if (item.blocking) { - const map = this.blockIdx.get(item.source) ?? new Map() - map.set(item.target, item.blocking) - if (!this.blockIdx.has(item.source)) { - this.blockIdx.set(item.source, map) - } - } - if (item.blockingViaList) { - const map = this.blockListIdx.get(item.source) ?? new Map() - map.set(item.target, item.blockingViaList) - if (!this.blockListIdx.has(item.source)) { - this.blockListIdx.set(item.source, map) - } - } - if (item.blockedBy) { - const map = this.blockIdx.get(item.target) ?? new Map() - map.set(item.source, item.blockedBy) - if (!this.blockIdx.has(item.target)) { - this.blockIdx.set(item.target, map) - } - } - if (item.blockedByViaList) { - const map = this.blockListIdx.get(item.target) ?? new Map() - map.set(item.source, item.blockedByViaList) - if (!this.blockListIdx.has(item.target)) { - this.blockListIdx.set(item.target, map) - } - } - if (item.muting) { - const set = this.muteIdx.get(item.source) ?? new Set() - set.add(item.target) - if (!this.muteIdx.has(item.source)) { - this.muteIdx.set(item.source, set) - } - } - if (item.mutingViaList) { - const map = this.muteListIdx.get(item.source) ?? new Map() - map.set(item.target, item.mutingViaList) - if (!this.muteListIdx.has(item.source)) { - this.muteListIdx.set(item.source, map) - } - } - const set = this.hasIdx.get(item.source) ?? new Set() - set.add(item.target) - if (!this.hasIdx.has(item.source)) { - this.hasIdx.set(item.source, set) - } - } - block(pair: RelationshipPair): boolean { - return !!this.blocking(pair) || !!this.blockedBy(pair) - } - // block or list uri - blocking(pair: RelationshipPair): string | null { - return this.blockIdx.get(pair[0])?.get(pair[1]) ?? this.blockList(pair) - } - // block or list uri - blockedBy(pair: RelationshipPair): string | null { - return this.blocking([pair[1], pair[0]]) - } - mute(pair: RelationshipPair): boolean { - return !!this.muteIdx.get(pair[0])?.has(pair[1]) || !!this.muteList(pair) - } - // list uri - blockList(pair: RelationshipPair): string | null { - return this.blockListIdx.get(pair[0])?.get(pair[1]) ?? null - } - muteList(pair: RelationshipPair): string | null { - return this.muteListIdx.get(pair[0])?.get(pair[1]) ?? null - } - has(pair: RelationshipPair) { - return !!this.hasIdx.get(pair[0])?.has(pair[1]) - } -} - -type BlockAndMuteInfo = { - source: string - target: string - blocking?: string | null - blockingViaList?: string | null - blockedBy?: string | null - blockedByViaList?: string | null - muting?: true | null - mutingViaList?: string | null -} diff --git a/packages/mod-service/src/services/graph/types.ts b/packages/mod-service/src/services/graph/types.ts deleted file mode 100644 index 5ff254dc383..00000000000 --- a/packages/mod-service/src/services/graph/types.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Selectable } from 'kysely' -import { List } from '../../db/tables/list' - -export type ListInfo = Selectable & { - viewerMuted: string | null - viewerListBlockUri: string | null - viewerInList: string | null -} - -export type ListInfoMap = Record diff --git a/packages/mod-service/src/services/index.ts b/packages/mod-service/src/services/index.ts index 1feeb6fae87..f046df1bdbe 100644 --- a/packages/mod-service/src/services/index.ts +++ b/packages/mod-service/src/services/index.ts @@ -1,36 +1,13 @@ -import { ImageUriBuilder } from '../image/uri' -import { ActorService } from './actor' -import { FeedService } from './feed' -import { GraphService } from './graph' +import AtpAgent from '@atproto/api' import { ModerationService } from './moderation' -import { LabelCacheOpts, LabelService } from './label' -import { ImageInvalidator } from '../image/invalidator' import { FromDb } from './types' -export function createServices(resources: { - imgUriBuilder: ImageUriBuilder - imgInvalidator: ImageInvalidator - labelCacheOpts: LabelCacheOpts -}): Services { - const { imgUriBuilder, imgInvalidator, labelCacheOpts } = resources - const label = LabelService.creator(labelCacheOpts) - const graph = GraphService.creator(imgUriBuilder) - const actor = ActorService.creator(imgUriBuilder, graph, label) - const moderation = ModerationService.creator(imgUriBuilder, imgInvalidator) - const feed = FeedService.creator(imgUriBuilder, actor, label, graph) +export function createServices(appviewAgent: AtpAgent): Services { return { - actor, - feed, - moderation, - graph, - label, + moderation: ModerationService.creator(appviewAgent), } } export type Services = { - actor: FromDb - feed: FromDb - graph: FromDb moderation: FromDb - label: FromDb } diff --git a/packages/mod-service/src/services/indexing/index.ts b/packages/mod-service/src/services/indexing/index.ts deleted file mode 100644 index 44dd9c3c986..00000000000 --- a/packages/mod-service/src/services/indexing/index.ts +++ /dev/null @@ -1,450 +0,0 @@ -import { sql } from 'kysely' -import { CID } from 'multiformats/cid' -import AtpAgent, { ComAtprotoSyncGetLatestCommit } from '@atproto/api' -import { - readCarWithRoot, - WriteOpAction, - verifyRepo, - Commit, - VerifiedRepo, - getAndParseRecord, -} from '@atproto/repo' -import { AtUri } from '@atproto/syntax' -import { IdResolver, getPds } from '@atproto/identity' -import { DAY, HOUR } from '@atproto/common' -import { ValidationError } from '@atproto/lexicon' -import { PrimaryDatabase } from '../../db' -import * as Post from './plugins/post' -import * as Threadgate from './plugins/thread-gate' -import * as Like from './plugins/like' -import * as Repost from './plugins/repost' -import * as Follow from './plugins/follow' -import * as Profile from './plugins/profile' -import * as List from './plugins/list' -import * as ListItem from './plugins/list-item' -import * as ListBlock from './plugins/list-block' -import * as Block from './plugins/block' -import * as FeedGenerator from './plugins/feed-generator' -import RecordProcessor from './processor' -import { subLogger } from '../../logger' -import { retryHttp } from '../../util/retry' -import { BackgroundQueue } from '../../background' -import { NotificationServer } from '../../notifications' -import { AutoModerator } from '../../auto-moderator' -import { Actor } from '../../db/tables/actor' - -export class IndexingService { - records: { - post: Post.PluginType - threadGate: Threadgate.PluginType - like: Like.PluginType - repost: Repost.PluginType - follow: Follow.PluginType - profile: Profile.PluginType - list: List.PluginType - listItem: ListItem.PluginType - listBlock: ListBlock.PluginType - block: Block.PluginType - feedGenerator: FeedGenerator.PluginType - } - - constructor( - public db: PrimaryDatabase, - public idResolver: IdResolver, - public autoMod: AutoModerator, - public backgroundQueue: BackgroundQueue, - public notifServer?: NotificationServer, - ) { - this.records = { - post: Post.makePlugin(this.db, backgroundQueue, notifServer), - threadGate: Threadgate.makePlugin(this.db, backgroundQueue, notifServer), - like: Like.makePlugin(this.db, backgroundQueue, notifServer), - repost: Repost.makePlugin(this.db, backgroundQueue, notifServer), - follow: Follow.makePlugin(this.db, backgroundQueue, notifServer), - profile: Profile.makePlugin(this.db, backgroundQueue, notifServer), - list: List.makePlugin(this.db, backgroundQueue, notifServer), - listItem: ListItem.makePlugin(this.db, backgroundQueue, notifServer), - listBlock: ListBlock.makePlugin(this.db, backgroundQueue, notifServer), - block: Block.makePlugin(this.db, backgroundQueue, notifServer), - feedGenerator: FeedGenerator.makePlugin( - this.db, - backgroundQueue, - notifServer, - ), - } - } - - transact(txn: PrimaryDatabase) { - txn.assertTransaction() - return new IndexingService( - txn, - this.idResolver, - this.autoMod, - this.backgroundQueue, - this.notifServer, - ) - } - - static creator( - idResolver: IdResolver, - autoMod: AutoModerator, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, - ) { - return (db: PrimaryDatabase) => - new IndexingService(db, idResolver, autoMod, backgroundQueue, notifServer) - } - - async indexRecord( - uri: AtUri, - cid: CID, - obj: unknown, - action: WriteOpAction.Create | WriteOpAction.Update, - timestamp: string, - opts?: { disableNotifs?: boolean; disableLabels?: boolean }, - ) { - this.db.assertNotTransaction() - await this.db.transaction(async (txn) => { - const indexingTx = this.transact(txn) - const indexer = indexingTx.findIndexerForCollection(uri.collection) - if (!indexer) return - if (action === WriteOpAction.Create) { - await indexer.insertRecord(uri, cid, obj, timestamp, opts) - } else { - await indexer.updateRecord(uri, cid, obj, timestamp) - } - }) - if (!opts?.disableLabels) { - this.autoMod.processRecord(uri, cid, obj) - } - } - - async deleteRecord(uri: AtUri, cascading = false) { - this.db.assertNotTransaction() - await this.db.transaction(async (txn) => { - const indexingTx = this.transact(txn) - const indexer = indexingTx.findIndexerForCollection(uri.collection) - if (!indexer) return - await indexer.deleteRecord(uri, cascading) - }) - } - - async indexHandle(did: string, timestamp: string, force = false) { - this.db.assertNotTransaction() - const actor = await this.db.db - .selectFrom('actor') - .where('did', '=', did) - .selectAll() - .executeTakeFirst() - if (!force && !needsHandleReindex(actor, timestamp)) { - return - } - const atpData = await this.idResolver.did.resolveAtprotoData(did, true) - const handleToDid = await this.idResolver.handle.resolve(atpData.handle) - - const handle: string | null = - did === handleToDid ? atpData.handle.toLowerCase() : null - - const actorWithHandle = - handle !== null - ? await this.db.db - .selectFrom('actor') - .where('handle', '=', handle) - .selectAll() - .executeTakeFirst() - : null - - // handle contention - if (handle && actorWithHandle && did !== actorWithHandle.did) { - await this.db.db - .updateTable('actor') - .where('actor.did', '=', actorWithHandle.did) - .set({ handle: null }) - .execute() - } - - const actorInfo = { handle, indexedAt: timestamp } - await this.db.db - .insertInto('actor') - .values({ did, ...actorInfo }) - .onConflict((oc) => oc.column('did').doUpdateSet(actorInfo)) - .returning('did') - .executeTakeFirst() - - if (handle) { - this.autoMod.processHandle(handle, did) - } - } - - async indexRepo(did: string, commit?: string) { - this.db.assertNotTransaction() - const now = new Date().toISOString() - const { pds, signingKey } = await this.idResolver.did.resolveAtprotoData( - did, - true, - ) - const { api } = new AtpAgent({ service: pds }) - - const { data: car } = await retryHttp(() => - api.com.atproto.sync.getRepo({ did }), - ) - const { root, blocks } = await readCarWithRoot(car) - const verifiedRepo = await verifyRepo(blocks, root, did, signingKey) - - const currRecords = await this.getCurrentRecords(did) - const repoRecords = formatCheckout(did, verifiedRepo) - const diff = findDiffFromCheckout(currRecords, repoRecords) - - await Promise.all( - diff.map(async (op) => { - const { uri, cid } = op - try { - if (op.op === 'delete') { - await this.deleteRecord(uri) - } else { - const parsed = await getAndParseRecord(blocks, cid) - await this.indexRecord( - uri, - cid, - parsed.record, - op.op === 'create' ? WriteOpAction.Create : WriteOpAction.Update, - now, - ) - } - } catch (err) { - if (err instanceof ValidationError) { - subLogger.warn( - { did, commit, uri: uri.toString(), cid: cid.toString() }, - 'skipping indexing of invalid record', - ) - } else { - subLogger.error( - { err, did, commit, uri: uri.toString(), cid: cid.toString() }, - 'skipping indexing due to error processing record', - ) - } - } - }), - ) - } - - async getCurrentRecords(did: string) { - const res = await this.db.db - .selectFrom('record') - .where('did', '=', did) - .select(['uri', 'cid']) - .execute() - return res.reduce((acc, cur) => { - acc[cur.uri] = { - uri: new AtUri(cur.uri), - cid: CID.parse(cur.cid), - } - return acc - }, {} as Record) - } - - async setCommitLastSeen( - commit: Commit, - details: { commit: CID; rebase: boolean; tooBig: boolean }, - ) { - const { ref } = this.db.db.dynamic - await this.db.db - .insertInto('actor_sync') - .values({ - did: commit.did, - commitCid: details.commit.toString(), - commitDataCid: commit.data.toString(), - repoRev: commit.rev ?? null, - rebaseCount: details.rebase ? 1 : 0, - tooBigCount: details.tooBig ? 1 : 0, - }) - .onConflict((oc) => { - const sync = (col: string) => ref(`actor_sync.${col}`) - const excluded = (col: string) => ref(`excluded.${col}`) - return oc.column('did').doUpdateSet({ - commitCid: sql`${excluded('commitCid')}`, - commitDataCid: sql`${excluded('commitDataCid')}`, - repoRev: sql`${excluded('repoRev')}`, - rebaseCount: sql`${sync('rebaseCount')} + ${excluded('rebaseCount')}`, - tooBigCount: sql`${sync('tooBigCount')} + ${excluded('tooBigCount')}`, - }) - }) - .execute() - } - - async checkCommitNeedsIndexing(commit: Commit) { - const sync = await this.db.db - .selectFrom('actor_sync') - .select('commitDataCid') - .where('did', '=', commit.did) - .executeTakeFirst() - if (!sync) return true - return sync.commitDataCid !== commit.data.toString() - } - - findIndexerForCollection(collection: string) { - const indexers = Object.values( - this.records as Record>, - ) - return indexers.find((indexer) => indexer.collection === collection) - } - - async tombstoneActor(did: string) { - this.db.assertNotTransaction() - const actorIsHosted = await this.getActorIsHosted(did) - if (actorIsHosted === false) { - await this.db.db.deleteFrom('actor').where('did', '=', did).execute() - await this.unindexActor(did) - await this.db.db - .deleteFrom('notification') - .where('did', '=', did) - .execute() - } - } - - private async getActorIsHosted(did: string) { - const doc = await this.idResolver.did.resolve(did, true) - const pds = doc && getPds(doc) - if (!pds) return false - const { api } = new AtpAgent({ service: pds }) - try { - await retryHttp(() => api.com.atproto.sync.getLatestCommit({ did })) - return true - } catch (err) { - if (err instanceof ComAtprotoSyncGetLatestCommit.RepoNotFoundError) { - return false - } - return null - } - } - - async unindexActor(did: string) { - this.db.assertNotTransaction() - // per-record-type indexes - await this.db.db.deleteFrom('profile').where('creator', '=', did).execute() - await this.db.db.deleteFrom('follow').where('creator', '=', did).execute() - await this.db.db.deleteFrom('repost').where('creator', '=', did).execute() - await this.db.db.deleteFrom('like').where('creator', '=', did).execute() - await this.db.db - .deleteFrom('feed_generator') - .where('creator', '=', did) - .execute() - // lists - await this.db.db - .deleteFrom('list_item') - .where('creator', '=', did) - .execute() - await this.db.db.deleteFrom('list').where('creator', '=', did).execute() - // blocks - await this.db.db - .deleteFrom('actor_block') - .where('creator', '=', did) - .execute() - await this.db.db - .deleteFrom('list_block') - .where('creator', '=', did) - .execute() - // posts - const postByUser = (qb) => - qb - .selectFrom('post') - .where('post.creator', '=', did) - .select('post.uri as uri') - await this.db.db - .deleteFrom('post_embed_image') - .where('post_embed_image.postUri', 'in', postByUser) - .execute() - await this.db.db - .deleteFrom('post_embed_external') - .where('post_embed_external.postUri', 'in', postByUser) - .execute() - await this.db.db - .deleteFrom('post_embed_record') - .where('post_embed_record.postUri', 'in', postByUser) - .execute() - await this.db.db.deleteFrom('post').where('creator', '=', did).execute() - await this.db.db - .deleteFrom('thread_gate') - .where('creator', '=', did) - .execute() - // notifications - await this.db.db - .deleteFrom('notification') - .where('notification.author', '=', did) - .execute() - // generic record indexes - await this.db.db - .deleteFrom('duplicate_record') - .where('duplicate_record.duplicateOf', 'in', (qb) => - qb - .selectFrom('record') - .where('record.did', '=', did) - .select('record.uri as uri'), - ) - .execute() - await this.db.db.deleteFrom('record').where('did', '=', did).execute() - } -} - -type UriAndCid = { - uri: AtUri - cid: CID -} - -type IndexOp = - | ({ - op: 'create' | 'update' - } & UriAndCid) - | ({ op: 'delete' } & UriAndCid) - -const findDiffFromCheckout = ( - curr: Record, - checkout: Record, -): IndexOp[] => { - const ops: IndexOp[] = [] - for (const uri of Object.keys(checkout)) { - const record = checkout[uri] - if (!curr[uri]) { - ops.push({ op: 'create', ...record }) - } else { - if (curr[uri].cid.equals(record.cid)) { - // no-op - continue - } - ops.push({ op: 'update', ...record }) - } - } - for (const uri of Object.keys(curr)) { - const record = curr[uri] - if (!checkout[uri]) { - ops.push({ op: 'delete', ...record }) - } - } - return ops -} - -const formatCheckout = ( - did: string, - verifiedRepo: VerifiedRepo, -): Record => { - const records: Record = {} - for (const create of verifiedRepo.creates) { - const uri = AtUri.make(did, create.collection, create.rkey) - records[uri.toString()] = { - uri, - cid: create.cid, - } - } - return records -} - -const needsHandleReindex = (actor: Actor | undefined, timestamp: string) => { - if (!actor) return true - const timeDiff = - new Date(timestamp).getTime() - new Date(actor.indexedAt).getTime() - // revalidate daily - if (timeDiff > DAY) return true - // revalidate more aggressively for invalidated handles - if (actor.handle === null && timeDiff > HOUR) return true - return false -} diff --git a/packages/mod-service/src/services/indexing/plugins/block.ts b/packages/mod-service/src/services/indexing/plugins/block.ts deleted file mode 100644 index 88e62b6f5ac..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/block.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Selectable } from 'kysely' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as Block from '../../../lexicon/types/app/bsky/graph/block' -import * as lex from '../../../lexicon/lexicons' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { PrimaryDatabase } from '../../../db' -import { BackgroundQueue } from '../../../background' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyGraphBlock -type IndexedBlock = Selectable - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Block.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('actor_block') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subjectDid: obj.subject, - createdAt: normalizeDatetimeAlways(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - uri: AtUri, - obj: Block.Record, -): Promise => { - const found = await db - .selectFrom('actor_block') - .where('creator', '=', uri.host) - .where('subjectDid', '=', obj.subject) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('actor_block') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/plugins/feed-generator.ts b/packages/mod-service/src/services/indexing/plugins/feed-generator.ts deleted file mode 100644 index be5435966f1..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/feed-generator.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { Selectable } from 'kysely' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as FeedGenerator from '../../../lexicon/types/app/bsky/feed/generator' -import * as lex from '../../../lexicon/lexicons' -import { PrimaryDatabase } from '../../../db' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import { BackgroundQueue } from '../../../background' -import RecordProcessor from '../processor' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyFeedGenerator -type IndexedFeedGenerator = Selectable - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: FeedGenerator.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('feed_generator') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - feedDid: obj.did, - displayName: obj.displayName, - description: obj.description, - descriptionFacets: obj.descriptionFacets - ? JSON.stringify(obj.descriptionFacets) - : undefined, - avatarCid: obj.avatar?.ref.toString(), - createdAt: normalizeDatetimeAlways(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async (): Promise => { - return null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('feed_generator') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor< - FeedGenerator.Record, - IndexedFeedGenerator -> - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/plugins/follow.ts b/packages/mod-service/src/services/indexing/plugins/follow.ts deleted file mode 100644 index 8655c7eba71..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/follow.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { Selectable } from 'kysely' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as Follow from '../../../lexicon/types/app/bsky/graph/follow' -import * as lex from '../../../lexicon/lexicons' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { PrimaryDatabase } from '../../../db' -import { countAll, excluded } from '../../../db/util' -import { BackgroundQueue } from '../../../background' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyGraphFollow -type IndexedFollow = Selectable - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Follow.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('follow') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subjectDid: obj.subject, - createdAt: normalizeDatetimeAlways(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - uri: AtUri, - obj: Follow.Record, -): Promise => { - const found = await db - .selectFrom('follow') - .where('creator', '=', uri.host) - .where('subjectDid', '=', obj.subject) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = (obj: IndexedFollow) => { - return [ - { - did: obj.subjectDid, - author: obj.creator, - recordUri: obj.uri, - recordCid: obj.cid, - reason: 'follow' as const, - reasonSubject: null, - sortAt: obj.sortAt, - }, - ] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('follow') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = ( - deleted: IndexedFollow, - replacedBy: IndexedFollow | null, -) => { - const toDelete = replacedBy ? [] : [deleted.uri] - return { notifs: [], toDelete } -} - -const updateAggregates = async (db: DatabaseSchema, follow: IndexedFollow) => { - const followersCountQb = db - .insertInto('profile_agg') - .values({ - did: follow.subjectDid, - followersCount: db - .selectFrom('follow') - .where('follow.subjectDid', '=', follow.subjectDid) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ - followersCount: excluded(db, 'followersCount'), - }), - ) - const followsCountQb = db - .insertInto('profile_agg') - .values({ - did: follow.creator, - followsCount: db - .selectFrom('follow') - .where('follow.creator', '=', follow.creator) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ - followsCount: excluded(db, 'followsCount'), - }), - ) - await Promise.all([followersCountQb.execute(), followsCountQb.execute()]) -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - updateAggregates, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/plugins/like.ts b/packages/mod-service/src/services/indexing/plugins/like.ts deleted file mode 100644 index 703800f67c8..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/like.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { Selectable } from 'kysely' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as Like from '../../../lexicon/types/app/bsky/feed/like' -import * as lex from '../../../lexicon/lexicons' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { countAll, excluded } from '../../../db/util' -import { PrimaryDatabase } from '../../../db' -import { BackgroundQueue } from '../../../background' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyFeedLike -type IndexedLike = Selectable - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Like.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('like') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subject: obj.subject.uri, - subjectCid: obj.subject.cid, - createdAt: normalizeDatetimeAlways(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - uri: AtUri, - obj: Like.Record, -): Promise => { - const found = await db - .selectFrom('like') - .where('creator', '=', uri.host) - .where('subject', '=', obj.subject.uri) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = (obj: IndexedLike) => { - const subjectUri = new AtUri(obj.subject) - // prevent self-notifications - const isSelf = subjectUri.host === obj.creator - return isSelf - ? [] - : [ - { - did: subjectUri.host, - author: obj.creator, - recordUri: obj.uri, - recordCid: obj.cid, - reason: 'like' as const, - reasonSubject: subjectUri.toString(), - sortAt: obj.sortAt, - }, - ] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('like') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = ( - deleted: IndexedLike, - replacedBy: IndexedLike | null, -) => { - const toDelete = replacedBy ? [] : [deleted.uri] - return { notifs: [], toDelete } -} - -const updateAggregates = async (db: DatabaseSchema, like: IndexedLike) => { - const likeCountQb = db - .insertInto('post_agg') - .values({ - uri: like.subject, - likeCount: db - .selectFrom('like') - .where('like.subject', '=', like.subject) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc.column('uri').doUpdateSet({ likeCount: excluded(db, 'likeCount') }), - ) - await likeCountQb.execute() -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - updateAggregates, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/plugins/list-block.ts b/packages/mod-service/src/services/indexing/plugins/list-block.ts deleted file mode 100644 index 3040f1aa3f9..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/list-block.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { Selectable } from 'kysely' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as ListBlock from '../../../lexicon/types/app/bsky/graph/listblock' -import * as lex from '../../../lexicon/lexicons' -import { PrimaryDatabase } from '../../../db' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { BackgroundQueue } from '../../../background' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyGraphListblock -type IndexedListBlock = Selectable - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: ListBlock.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('list_block') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subjectUri: obj.subject, - createdAt: normalizeDatetimeAlways(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - uri: AtUri, - obj: ListBlock.Record, -): Promise => { - const found = await db - .selectFrom('list_block') - .where('creator', '=', uri.host) - .where('subjectUri', '=', obj.subject) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('list_block') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/plugins/list-item.ts b/packages/mod-service/src/services/indexing/plugins/list-item.ts deleted file mode 100644 index 9e08145b23e..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/list-item.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { Selectable } from 'kysely' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as ListItem from '../../../lexicon/types/app/bsky/graph/listitem' -import * as lex from '../../../lexicon/lexicons' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { PrimaryDatabase } from '../../../db' -import { BackgroundQueue } from '../../../background' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyGraphListitem -type IndexedListItem = Selectable - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: ListItem.Record, - timestamp: string, -): Promise => { - const listUri = new AtUri(obj.list) - if (listUri.hostname !== uri.hostname) { - throw new InvalidRequestError( - 'Creator of listitem does not match creator of list', - ) - } - const inserted = await db - .insertInto('list_item') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subjectDid: obj.subject, - listUri: obj.list, - createdAt: normalizeDatetimeAlways(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - _uri: AtUri, - obj: ListItem.Record, -): Promise => { - const found = await db - .selectFrom('list_item') - .where('listUri', '=', obj.list) - .where('subjectDid', '=', obj.subject) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('list_item') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/plugins/list.ts b/packages/mod-service/src/services/indexing/plugins/list.ts deleted file mode 100644 index 0d078572501..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/list.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { Selectable } from 'kysely' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as List from '../../../lexicon/types/app/bsky/graph/list' -import * as lex from '../../../lexicon/lexicons' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { PrimaryDatabase } from '../../../db' -import { BackgroundQueue } from '../../../background' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyGraphList -type IndexedList = Selectable - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: List.Record, - timestamp: string, -): Promise => { - const inserted = await db - .insertInto('list') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - name: obj.name, - purpose: obj.purpose, - description: obj.description, - descriptionFacets: obj.descriptionFacets - ? JSON.stringify(obj.descriptionFacets) - : undefined, - avatarCid: obj.avatar?.ref.toString(), - createdAt: normalizeDatetimeAlways(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async (): Promise => { - return null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('list') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/plugins/post.ts b/packages/mod-service/src/services/indexing/plugins/post.ts deleted file mode 100644 index af581b3bdff..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/post.ts +++ /dev/null @@ -1,475 +0,0 @@ -import { Insertable, Selectable, sql } from 'kysely' -import { CID } from 'multiformats/cid' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { jsonStringToLex } from '@atproto/lexicon' -import { - Record as PostRecord, - ReplyRef, -} from '../../../lexicon/types/app/bsky/feed/post' -import { Record as GateRecord } from '../../../lexicon/types/app/bsky/feed/threadgate' -import { isMain as isEmbedImage } from '../../../lexicon/types/app/bsky/embed/images' -import { isMain as isEmbedExternal } from '../../../lexicon/types/app/bsky/embed/external' -import { isMain as isEmbedRecord } from '../../../lexicon/types/app/bsky/embed/record' -import { isMain as isEmbedRecordWithMedia } from '../../../lexicon/types/app/bsky/embed/recordWithMedia' -import { - isMention, - isLink, -} from '../../../lexicon/types/app/bsky/richtext/facet' -import * as lex from '../../../lexicon/lexicons' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { Notification } from '../../../db/tables/notification' -import { PrimaryDatabase } from '../../../db' -import { countAll, excluded } from '../../../db/util' -import { BackgroundQueue } from '../../../background' -import { getAncestorsAndSelfQb, getDescendentsQb } from '../../util/post' -import { NotificationServer } from '../../../notifications' -import * as feedutil from '../../feed/util' -import { postToThreadgateUri } from '../../feed/util' - -type Notif = Insertable -type Post = Selectable -type PostEmbedImage = DatabaseSchemaType['post_embed_image'] -type PostEmbedExternal = DatabaseSchemaType['post_embed_external'] -type PostEmbedRecord = DatabaseSchemaType['post_embed_record'] -type PostAncestor = { - uri: string - height: number -} -type PostDescendent = { - uri: string - depth: number - cid: string - creator: string - sortAt: string -} -type IndexedPost = { - post: Post - facets?: { type: 'mention' | 'link'; value: string }[] - embeds?: (PostEmbedImage[] | PostEmbedExternal | PostEmbedRecord)[] - ancestors?: PostAncestor[] - descendents?: PostDescendent[] -} - -const lexId = lex.ids.AppBskyFeedPost - -const REPLY_NOTIF_DEPTH = 5 - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: PostRecord, - timestamp: string, -): Promise => { - const post = { - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - text: obj.text, - createdAt: normalizeDatetimeAlways(obj.createdAt), - replyRoot: obj.reply?.root?.uri || null, - replyRootCid: obj.reply?.root?.cid || null, - replyParent: obj.reply?.parent?.uri || null, - replyParentCid: obj.reply?.parent?.cid || null, - langs: obj.langs?.length - ? sql`${JSON.stringify(obj.langs)}` // sidesteps kysely's array serialization, which is non-jsonb - : null, - tags: obj.tags?.length - ? sql`${JSON.stringify(obj.tags)}` // sidesteps kysely's array serialization, which is non-jsonb - : null, - indexedAt: timestamp, - } - const [insertedPost] = await Promise.all([ - db - .insertInto('post') - .values(post) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst(), - db - .insertInto('feed_item') - .values({ - type: 'post', - uri: post.uri, - cid: post.cid, - postUri: post.uri, - originatorDid: post.creator, - sortAt: - post.indexedAt < post.createdAt ? post.indexedAt : post.createdAt, - }) - .onConflict((oc) => oc.doNothing()) - .executeTakeFirst(), - ]) - if (!insertedPost) { - return null // Post already indexed - } - - if (obj.reply) { - const { invalidReplyRoot, violatesThreadGate } = await validateReply( - db, - uri.host, - obj.reply, - ) - if (invalidReplyRoot || violatesThreadGate) { - Object.assign(insertedPost, { invalidReplyRoot, violatesThreadGate }) - await db - .updateTable('post') - .where('uri', '=', post.uri) - .set({ invalidReplyRoot, violatesThreadGate }) - .executeTakeFirst() - } - } - - const facets = (obj.facets || []) - .flatMap((facet) => facet.features) - .flatMap((feature) => { - if (isMention(feature)) { - return { - type: 'mention' as const, - value: feature.did, - } - } - if (isLink(feature)) { - return { - type: 'link' as const, - value: feature.uri, - } - } - return [] - }) - // Embed indices - const embeds: (PostEmbedImage[] | PostEmbedExternal | PostEmbedRecord)[] = [] - const postEmbeds = separateEmbeds(obj.embed) - for (const postEmbed of postEmbeds) { - if (isEmbedImage(postEmbed)) { - const { images } = postEmbed - const imagesEmbed = images.map((img, i) => ({ - postUri: uri.toString(), - position: i, - imageCid: img.image.ref.toString(), - alt: img.alt, - })) - embeds.push(imagesEmbed) - await db.insertInto('post_embed_image').values(imagesEmbed).execute() - } else if (isEmbedExternal(postEmbed)) { - const { external } = postEmbed - const externalEmbed = { - postUri: uri.toString(), - uri: external.uri, - title: external.title, - description: external.description, - thumbCid: external.thumb?.ref.toString() || null, - } - embeds.push(externalEmbed) - await db.insertInto('post_embed_external').values(externalEmbed).execute() - } else if (isEmbedRecord(postEmbed)) { - const { record } = postEmbed - const recordEmbed = { - postUri: uri.toString(), - embedUri: record.uri, - embedCid: record.cid, - } - embeds.push(recordEmbed) - await db.insertInto('post_embed_record').values(recordEmbed).execute() - } - } - - const ancestors = await getAncestorsAndSelfQb(db, { - uri: post.uri, - parentHeight: REPLY_NOTIF_DEPTH, - }) - .selectFrom('ancestor') - .selectAll() - .execute() - const descendents = await getDescendentsQb(db, { - uri: post.uri, - depth: REPLY_NOTIF_DEPTH, - }) - .selectFrom('descendent') - .innerJoin('post', 'post.uri', 'descendent.uri') - .selectAll('descendent') - .select(['cid', 'creator', 'sortAt']) - .execute() - return { - post: insertedPost, - facets, - embeds, - ancestors, - descendents, - } -} - -const findDuplicate = async (): Promise => { - return null -} - -const notifsForInsert = (obj: IndexedPost) => { - const notifs: Notif[] = [] - const notified = new Set([obj.post.creator]) - const maybeNotify = (notif: Notif) => { - if (!notified.has(notif.did)) { - notified.add(notif.did) - notifs.push(notif) - } - } - for (const facet of obj.facets ?? []) { - if (facet.type === 'mention') { - maybeNotify({ - did: facet.value, - reason: 'mention', - author: obj.post.creator, - recordUri: obj.post.uri, - recordCid: obj.post.cid, - sortAt: obj.post.sortAt, - }) - } - } - for (const embed of obj.embeds ?? []) { - if ('embedUri' in embed) { - const embedUri = new AtUri(embed.embedUri) - if (embedUri.collection === lex.ids.AppBskyFeedPost) { - maybeNotify({ - did: embedUri.host, - reason: 'quote', - reasonSubject: embedUri.toString(), - author: obj.post.creator, - recordUri: obj.post.uri, - recordCid: obj.post.cid, - sortAt: obj.post.sortAt, - }) - } - } - } - - if (obj.post.violatesThreadGate) { - // don't generate reply notifications when post violates threadgate - return notifs - } - - // reply notifications - - for (const ancestor of obj.ancestors ?? []) { - if (ancestor.uri === obj.post.uri) continue // no need to notify for own post - if (ancestor.height < REPLY_NOTIF_DEPTH) { - const ancestorUri = new AtUri(ancestor.uri) - maybeNotify({ - did: ancestorUri.host, - reason: 'reply', - reasonSubject: ancestorUri.toString(), - author: obj.post.creator, - recordUri: obj.post.uri, - recordCid: obj.post.cid, - sortAt: obj.post.sortAt, - }) - } - } - - // descendents indicate out-of-order indexing: need to notify - // the current post and upwards. - for (const descendent of obj.descendents ?? []) { - for (const ancestor of obj.ancestors ?? []) { - const totalHeight = descendent.depth + ancestor.height - if (totalHeight < REPLY_NOTIF_DEPTH) { - const ancestorUri = new AtUri(ancestor.uri) - maybeNotify({ - did: ancestorUri.host, - reason: 'reply', - reasonSubject: ancestorUri.toString(), - author: descendent.creator, - recordUri: descendent.uri, - recordCid: descendent.cid, - sortAt: descendent.sortAt, - }) - } - } - } - - return notifs -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const uriStr = uri.toString() - const [deleted] = await Promise.all([ - db - .deleteFrom('post') - .where('uri', '=', uriStr) - .returningAll() - .executeTakeFirst(), - db.deleteFrom('feed_item').where('postUri', '=', uriStr).executeTakeFirst(), - ]) - const deletedEmbeds: ( - | PostEmbedImage[] - | PostEmbedExternal - | PostEmbedRecord - )[] = [] - const [deletedImgs, deletedExternals, deletedPosts] = await Promise.all([ - db - .deleteFrom('post_embed_image') - .where('postUri', '=', uriStr) - .returningAll() - .execute(), - db - .deleteFrom('post_embed_external') - .where('postUri', '=', uriStr) - .returningAll() - .executeTakeFirst(), - db - .deleteFrom('post_embed_record') - .where('postUri', '=', uriStr) - .returningAll() - .executeTakeFirst(), - ]) - if (deletedImgs.length) { - deletedEmbeds.push(deletedImgs) - } - if (deletedExternals) { - deletedEmbeds.push(deletedExternals) - } - if (deletedPosts) { - deletedEmbeds.push(deletedPosts) - } - return deleted - ? { - post: deleted, - facets: [], // Not used - embeds: deletedEmbeds, - } - : null -} - -const notifsForDelete = ( - deleted: IndexedPost, - replacedBy: IndexedPost | null, -) => { - const notifs = replacedBy ? notifsForInsert(replacedBy) : [] - return { - notifs, - toDelete: [deleted.post.uri], - } -} - -const updateAggregates = async (db: DatabaseSchema, postIdx: IndexedPost) => { - const replyCountQb = postIdx.post.replyParent - ? db - .insertInto('post_agg') - .values({ - uri: postIdx.post.replyParent, - replyCount: db - .selectFrom('post') - .where('post.replyParent', '=', postIdx.post.replyParent) - .where((qb) => - qb - .where('post.violatesThreadGate', 'is', null) - .orWhere('post.violatesThreadGate', '=', false), - ) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc - .column('uri') - .doUpdateSet({ replyCount: excluded(db, 'replyCount') }), - ) - : null - const postsCountQb = db - .insertInto('profile_agg') - .values({ - did: postIdx.post.creator, - postsCount: db - .selectFrom('post') - .where('post.creator', '=', postIdx.post.creator) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc.column('did').doUpdateSet({ postsCount: excluded(db, 'postsCount') }), - ) - await Promise.all([replyCountQb?.execute(), postsCountQb.execute()]) -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - updateAggregates, - }) -} - -export default makePlugin - -function separateEmbeds(embed: PostRecord['embed']) { - if (!embed) { - return [] - } - if (isEmbedRecordWithMedia(embed)) { - return [{ $type: lex.ids.AppBskyEmbedRecord, ...embed.record }, embed.media] - } - return [embed] -} - -async function validateReply( - db: DatabaseSchema, - creator: string, - reply: ReplyRef, -) { - const replyRefs = await getReplyRefs(db, reply) - // check reply - const invalidReplyRoot = - !replyRefs.parent || feedutil.invalidReplyRoot(reply, replyRefs.parent) - // check interaction - const violatesThreadGate = await feedutil.violatesThreadGate( - db, - creator, - new AtUri(reply.root.uri).hostname, - replyRefs.root?.record ?? null, - replyRefs.gate?.record ?? null, - ) - return { - invalidReplyRoot, - violatesThreadGate, - } -} - -async function getReplyRefs(db: DatabaseSchema, reply: ReplyRef) { - const replyRoot = reply.root.uri - const replyParent = reply.parent.uri - const replyGate = postToThreadgateUri(replyRoot) - const results = await db - .selectFrom('record') - .where('record.uri', 'in', [replyRoot, replyGate, replyParent]) - .leftJoin('post', 'post.uri', 'record.uri') - .selectAll('post') - .select(['record.uri', 'json']) - .execute() - const root = results.find((ref) => ref.uri === replyRoot) - const parent = results.find((ref) => ref.uri === replyParent) - const gate = results.find((ref) => ref.uri === replyGate) - return { - root: root && { - uri: root.uri, - invalidReplyRoot: root.invalidReplyRoot, - record: jsonStringToLex(root.json) as PostRecord, - }, - parent: parent && { - uri: parent.uri, - invalidReplyRoot: parent.invalidReplyRoot, - record: jsonStringToLex(parent.json) as PostRecord, - }, - gate: gate && { - uri: gate.uri, - record: jsonStringToLex(gate.json) as GateRecord, - }, - } -} diff --git a/packages/mod-service/src/services/indexing/plugins/profile.ts b/packages/mod-service/src/services/indexing/plugins/profile.ts deleted file mode 100644 index ea0c8f07f98..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/profile.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { AtUri } from '@atproto/syntax' -import { CID } from 'multiformats/cid' -import * as Profile from '../../../lexicon/types/app/bsky/actor/profile' -import * as lex from '../../../lexicon/lexicons' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { PrimaryDatabase } from '../../../db' -import { BackgroundQueue } from '../../../background' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyActorProfile -type IndexedProfile = DatabaseSchemaType['profile'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Profile.Record, - timestamp: string, -): Promise => { - if (uri.rkey !== 'self') return null - const inserted = await db - .insertInto('profile') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - displayName: obj.displayName, - description: obj.description, - avatarCid: obj.avatar?.ref.toString(), - bannerCid: obj.banner?.ref.toString(), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async (): Promise => { - return null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('profile') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/plugins/repost.ts b/packages/mod-service/src/services/indexing/plugins/repost.ts deleted file mode 100644 index ea8d517dc52..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/repost.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { Selectable } from 'kysely' -import { CID } from 'multiformats/cid' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import * as Repost from '../../../lexicon/types/app/bsky/feed/repost' -import * as lex from '../../../lexicon/lexicons' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { PrimaryDatabase } from '../../../db' -import { countAll, excluded } from '../../../db/util' -import { BackgroundQueue } from '../../../background' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyFeedRepost -type IndexedRepost = Selectable - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Repost.Record, - timestamp: string, -): Promise => { - const repost = { - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - subject: obj.subject.uri, - subjectCid: obj.subject.cid, - createdAt: normalizeDatetimeAlways(obj.createdAt), - indexedAt: timestamp, - } - const [inserted] = await Promise.all([ - db - .insertInto('repost') - .values(repost) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst(), - db - .insertInto('feed_item') - .values({ - type: 'repost', - uri: repost.uri, - cid: repost.cid, - postUri: repost.subject, - originatorDid: repost.creator, - sortAt: - repost.indexedAt < repost.createdAt - ? repost.indexedAt - : repost.createdAt, - }) - .onConflict((oc) => oc.doNothing()) - .executeTakeFirst(), - ]) - - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - uri: AtUri, - obj: Repost.Record, -): Promise => { - const found = await db - .selectFrom('repost') - .where('creator', '=', uri.host) - .where('subject', '=', obj.subject.uri) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = (obj: IndexedRepost) => { - const subjectUri = new AtUri(obj.subject) - // prevent self-notifications - const isSelf = subjectUri.host === obj.creator - return isSelf - ? [] - : [ - { - did: subjectUri.host, - author: obj.creator, - recordUri: obj.uri, - recordCid: obj.cid, - reason: 'repost' as const, - reasonSubject: subjectUri.toString(), - sortAt: obj.sortAt, - }, - ] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const uriStr = uri.toString() - const [deleted] = await Promise.all([ - db - .deleteFrom('repost') - .where('uri', '=', uriStr) - .returningAll() - .executeTakeFirst(), - db.deleteFrom('feed_item').where('uri', '=', uriStr).executeTakeFirst(), - ]) - return deleted || null -} - -const notifsForDelete = ( - deleted: IndexedRepost, - replacedBy: IndexedRepost | null, -) => { - const toDelete = replacedBy ? [] : [deleted.uri] - return { notifs: [], toDelete } -} - -const updateAggregates = async (db: DatabaseSchema, repost: IndexedRepost) => { - const repostCountQb = db - .insertInto('post_agg') - .values({ - uri: repost.subject, - repostCount: db - .selectFrom('repost') - .where('repost.subject', '=', repost.subject) - .select(countAll.as('count')), - }) - .onConflict((oc) => - oc - .column('uri') - .doUpdateSet({ repostCount: excluded(db, 'repostCount') }), - ) - await repostCountQb.execute() -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - updateAggregates, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/plugins/thread-gate.ts b/packages/mod-service/src/services/indexing/plugins/thread-gate.ts deleted file mode 100644 index 9a58547f2da..00000000000 --- a/packages/mod-service/src/services/indexing/plugins/thread-gate.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { InvalidRequestError } from '@atproto/xrpc-server' -import { CID } from 'multiformats/cid' -import * as Threadgate from '../../../lexicon/types/app/bsky/feed/threadgate' -import * as lex from '../../../lexicon/lexicons' -import { DatabaseSchema, DatabaseSchemaType } from '../../../db/database-schema' -import RecordProcessor from '../processor' -import { PrimaryDatabase } from '../../../db' -import { BackgroundQueue } from '../../../background' -import { NotificationServer } from '../../../notifications' - -const lexId = lex.ids.AppBskyFeedThreadgate -type IndexedGate = DatabaseSchemaType['thread_gate'] - -const insertFn = async ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: Threadgate.Record, - timestamp: string, -): Promise => { - const postUri = new AtUri(obj.post) - if (postUri.host !== uri.host || postUri.rkey !== uri.rkey) { - throw new InvalidRequestError( - 'Creator and rkey of thread gate does not match its post', - ) - } - const inserted = await db - .insertInto('thread_gate') - .values({ - uri: uri.toString(), - cid: cid.toString(), - creator: uri.host, - postUri: obj.post, - createdAt: normalizeDatetimeAlways(obj.createdAt), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .returningAll() - .executeTakeFirst() - return inserted || null -} - -const findDuplicate = async ( - db: DatabaseSchema, - _uri: AtUri, - obj: Threadgate.Record, -): Promise => { - const found = await db - .selectFrom('thread_gate') - .where('postUri', '=', obj.post) - .selectAll() - .executeTakeFirst() - return found ? new AtUri(found.uri) : null -} - -const notifsForInsert = () => { - return [] -} - -const deleteFn = async ( - db: DatabaseSchema, - uri: AtUri, -): Promise => { - const deleted = await db - .deleteFrom('thread_gate') - .where('uri', '=', uri.toString()) - .returningAll() - .executeTakeFirst() - return deleted || null -} - -const notifsForDelete = () => { - return { notifs: [], toDelete: [] } -} - -export type PluginType = RecordProcessor - -export const makePlugin = ( - db: PrimaryDatabase, - backgroundQueue: BackgroundQueue, - notifServer?: NotificationServer, -): PluginType => { - return new RecordProcessor(db, backgroundQueue, notifServer, { - lexId, - insertFn, - findDuplicate, - deleteFn, - notifsForInsert, - notifsForDelete, - }) -} - -export default makePlugin diff --git a/packages/mod-service/src/services/indexing/processor.ts b/packages/mod-service/src/services/indexing/processor.ts deleted file mode 100644 index 2a02c61125e..00000000000 --- a/packages/mod-service/src/services/indexing/processor.ts +++ /dev/null @@ -1,299 +0,0 @@ -import { Insertable } from 'kysely' -import { CID } from 'multiformats/cid' -import { AtUri } from '@atproto/syntax' -import { jsonStringToLex, stringifyLex } from '@atproto/lexicon' -import DatabaseSchema from '../../db/database-schema' -import { lexicons } from '../../lexicon/lexicons' -import { Notification } from '../../db/tables/notification' -import { chunkArray } from '@atproto/common' -import { PrimaryDatabase } from '../../db' -import { BackgroundQueue } from '../../background' -import { NotificationServer } from '../../notifications' -import { dbLogger } from '../../logger' - -// @NOTE re: insertions and deletions. Due to how record updates are handled, -// (insertFn) should have the same effect as (insertFn -> deleteFn -> insertFn). -type RecordProcessorParams = { - lexId: string - insertFn: ( - db: DatabaseSchema, - uri: AtUri, - cid: CID, - obj: T, - timestamp: string, - ) => Promise - findDuplicate: ( - db: DatabaseSchema, - uri: AtUri, - obj: T, - ) => Promise - deleteFn: (db: DatabaseSchema, uri: AtUri) => Promise - notifsForInsert: (obj: S) => Notif[] - notifsForDelete: ( - prev: S, - replacedBy: S | null, - ) => { notifs: Notif[]; toDelete: string[] } - updateAggregates?: (db: DatabaseSchema, obj: S) => Promise -} - -type Notif = Insertable - -export class RecordProcessor { - collection: string - db: DatabaseSchema - constructor( - private appDb: PrimaryDatabase, - private backgroundQueue: BackgroundQueue, - private notifServer: NotificationServer | undefined, - private params: RecordProcessorParams, - ) { - this.db = appDb.db - this.collection = this.params.lexId - } - - matchesSchema(obj: unknown): obj is T { - try { - this.assertValidRecord(obj) - return true - } catch { - return false - } - } - - assertValidRecord(obj: unknown): asserts obj is T { - lexicons.assertValidRecord(this.params.lexId, obj) - } - - async insertRecord( - uri: AtUri, - cid: CID, - obj: unknown, - timestamp: string, - opts?: { disableNotifs?: boolean }, - ) { - this.assertValidRecord(obj) - await this.db - .insertInto('record') - .values({ - uri: uri.toString(), - cid: cid.toString(), - did: uri.host, - json: stringifyLex(obj), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .execute() - const inserted = await this.params.insertFn( - this.db, - uri, - cid, - obj, - timestamp, - ) - if (inserted) { - this.aggregateOnCommit(inserted) - if (!opts?.disableNotifs) { - await this.handleNotifs({ inserted }) - } - return - } - // if duplicate, insert into duplicates table with no events - const found = await this.params.findDuplicate(this.db, uri, obj) - if (found && found.toString() !== uri.toString()) { - await this.db - .insertInto('duplicate_record') - .values({ - uri: uri.toString(), - cid: cid.toString(), - duplicateOf: found.toString(), - indexedAt: timestamp, - }) - .onConflict((oc) => oc.doNothing()) - .execute() - } - } - - // Currently using a very simple strategy for updates: purge the existing index - // for the uri then replace it. The main upside is that this allows the indexer - // for each collection to avoid bespoke logic for in-place updates, which isn't - // straightforward in the general case. We still get nice control over notifications. - async updateRecord( - uri: AtUri, - cid: CID, - obj: unknown, - timestamp: string, - opts?: { disableNotifs?: boolean }, - ) { - this.assertValidRecord(obj) - await this.db - .updateTable('record') - .where('uri', '=', uri.toString()) - .set({ - cid: cid.toString(), - json: stringifyLex(obj), - indexedAt: timestamp, - }) - .execute() - // If the updated record was a dupe, update dupe info for it - const dupe = await this.params.findDuplicate(this.db, uri, obj) - if (dupe) { - await this.db - .updateTable('duplicate_record') - .where('uri', '=', uri.toString()) - .set({ - cid: cid.toString(), - duplicateOf: dupe.toString(), - indexedAt: timestamp, - }) - .execute() - } else { - await this.db - .deleteFrom('duplicate_record') - .where('uri', '=', uri.toString()) - .execute() - } - - const deleted = await this.params.deleteFn(this.db, uri) - if (!deleted) { - // If a record was updated but hadn't been indexed yet, treat it like a plain insert. - return this.insertRecord(uri, cid, obj, timestamp) - } - this.aggregateOnCommit(deleted) - const inserted = await this.params.insertFn( - this.db, - uri, - cid, - obj, - timestamp, - ) - if (!inserted) { - throw new Error( - 'Record update failed: removed from index but could not be replaced', - ) - } - this.aggregateOnCommit(inserted) - if (!opts?.disableNotifs) { - await this.handleNotifs({ inserted, deleted }) - } - } - - async deleteRecord(uri: AtUri, cascading = false) { - await this.db - .deleteFrom('record') - .where('uri', '=', uri.toString()) - .execute() - await this.db - .deleteFrom('duplicate_record') - .where('uri', '=', uri.toString()) - .execute() - const deleted = await this.params.deleteFn(this.db, uri) - if (!deleted) return - this.aggregateOnCommit(deleted) - if (cascading) { - await this.db - .deleteFrom('duplicate_record') - .where('duplicateOf', '=', uri.toString()) - .execute() - return this.handleNotifs({ deleted }) - } else { - const found = await this.db - .selectFrom('duplicate_record') - .innerJoin('record', 'record.uri', 'duplicate_record.uri') - .where('duplicateOf', '=', uri.toString()) - .orderBy('duplicate_record.indexedAt', 'asc') - .limit(1) - .selectAll() - .executeTakeFirst() - - if (!found) { - return this.handleNotifs({ deleted }) - } - const record = jsonStringToLex(found.json) - if (!this.matchesSchema(record)) { - return this.handleNotifs({ deleted }) - } - const inserted = await this.params.insertFn( - this.db, - new AtUri(found.uri), - CID.parse(found.cid), - record, - found.indexedAt, - ) - if (inserted) { - this.aggregateOnCommit(inserted) - } - await this.handleNotifs({ deleted, inserted: inserted ?? undefined }) - } - } - - async handleNotifs(op: { deleted?: S; inserted?: S }) { - let notifs: Notif[] = [] - const runOnCommit: ((db: PrimaryDatabase) => Promise)[] = [] - const sendOnCommit: (() => Promise)[] = [] - if (op.deleted) { - const forDelete = this.params.notifsForDelete( - op.deleted, - op.inserted ?? null, - ) - if (forDelete.toDelete.length > 0) { - // Notifs can be deleted in background: they are expensive to delete and - // listNotifications already excludes notifs with missing records. - runOnCommit.push(async (db) => { - await db.db - .deleteFrom('notification') - .where('recordUri', 'in', forDelete.toDelete) - .execute() - }) - } - notifs = forDelete.notifs - } else if (op.inserted) { - notifs = this.params.notifsForInsert(op.inserted) - } - for (const chunk of chunkArray(notifs, 500)) { - runOnCommit.push(async (db) => { - await db.db.insertInto('notification').values(chunk).execute() - }) - if (this.notifServer) { - const notifServer = this.notifServer - sendOnCommit.push(async () => { - try { - const preparedNotifs = await notifServer.prepareNotifsToSend(chunk) - await notifServer.processNotifications(preparedNotifs) - } catch (error) { - dbLogger.error({ error }, 'error sending push notifications') - } - }) - } - } - if (runOnCommit.length) { - // Need to ensure notif deletion always happens before creation, otherwise delete may clobber in a race. - this.appDb.onCommit(() => { - this.backgroundQueue.add(async (db) => { - for (const fn of runOnCommit) { - await fn(db) - } - }) - }) - } - if (sendOnCommit.length) { - // Need to ensure notif deletion always happens before creation, otherwise delete may clobber in a race. - this.appDb.onCommit(() => { - this.backgroundQueue.add(async () => { - for (const fn of sendOnCommit) { - await fn() - } - }) - }) - } - } - - aggregateOnCommit(indexed: S) { - const { updateAggregates } = this.params - if (!updateAggregates) return - this.appDb.onCommit(() => { - this.backgroundQueue.add((db) => updateAggregates(db.db, indexed)) - }) - } -} - -export default RecordProcessor diff --git a/packages/mod-service/src/services/label/index.ts b/packages/mod-service/src/services/label/index.ts deleted file mode 100644 index f4c11295da7..00000000000 --- a/packages/mod-service/src/services/label/index.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { sql } from 'kysely' -import { AtUri, normalizeDatetimeAlways } from '@atproto/syntax' -import { Database } from '../../db' -import { Label, isSelfLabels } from '../../lexicon/types/com/atproto/label/defs' -import { ids } from '../../lexicon/lexicons' -import { ReadThroughCache } from '../../cache/read-through' -import { Redis } from '../../redis' - -export type Labels = Record - -export type LabelCacheOpts = { - redis: Redis - staleTTL: number - maxTTL: number -} - -export class LabelService { - public cache: ReadThroughCache | null - - constructor(public db: Database, cacheOpts: LabelCacheOpts | null) { - if (cacheOpts) { - this.cache = new ReadThroughCache(cacheOpts.redis, { - ...cacheOpts, - fetchMethod: async (subject: string) => { - const res = await fetchLabelsForSubjects(db, [subject]) - return res[subject] ?? [] - }, - fetchManyMethod: (subjects: string[]) => - fetchLabelsForSubjects(db, subjects), - }) - } - } - - static creator(cacheOpts: LabelCacheOpts | null) { - return (db: Database) => new LabelService(db, cacheOpts) - } - - async formatAndCreate( - src: string, - uri: string, - cid: string | null, - labels: { create?: string[]; negate?: string[] }, - ): Promise { - const { create = [], negate = [] } = labels - const toCreate = create.map((val) => ({ - src, - uri, - cid: cid ?? undefined, - val, - neg: false, - cts: new Date().toISOString(), - })) - const toNegate = negate.map((val) => ({ - src, - uri, - cid: cid ?? undefined, - val, - neg: true, - cts: new Date().toISOString(), - })) - const formatted = [...toCreate, ...toNegate] - await this.createLabels(formatted) - return formatted - } - - async createLabels(labels: Label[]) { - if (labels.length < 1) return - const dbVals = labels.map((l) => ({ - ...l, - cid: l.cid ?? '', - neg: !!l.neg, - })) - const { ref } = this.db.db.dynamic - const excluded = (col: string) => ref(`excluded.${col}`) - await this.db - .asPrimary() - .db.insertInto('label') - .values(dbVals) - .onConflict((oc) => - oc.columns(['src', 'uri', 'cid', 'val']).doUpdateSet({ - neg: sql`${excluded('neg')}`, - cts: sql`${excluded('cts')}`, - }), - ) - .execute() - } - - async getLabelsForUris( - subjects: string[], - opts?: { - includeNeg?: boolean - skipCache?: boolean - }, - ): Promise { - if (subjects.length < 1) return {} - const res = this.cache - ? await this.cache.getMany(subjects, { revalidate: opts?.skipCache }) - : await fetchLabelsForSubjects(this.db, subjects) - - if (opts?.includeNeg) { - return res - } - - const noNegs: Labels = {} - for (const [key, val] of Object.entries(res)) { - noNegs[key] = val.filter((label) => !label.neg) - } - return noNegs - } - - // gets labels for any record. when did is present, combine labels for both did & profile record. - async getLabelsForSubjects( - subjects: string[], - opts?: { - includeNeg?: boolean - skipCache?: boolean - }, - labels: Labels = {}, - ): Promise { - if (subjects.length < 1) return labels - const expandedSubjects = subjects.flatMap((subject) => { - if (labels[subject]) return [] // skip over labels we already have fetched - if (subject.startsWith('did:')) { - return [ - subject, - AtUri.make(subject, ids.AppBskyActorProfile, 'self').toString(), - ] - } - return subject - }) - const labelsByUri = await this.getLabelsForUris(expandedSubjects, opts) - return Object.keys(labelsByUri).reduce((acc, cur) => { - const uri = cur.startsWith('at://') ? new AtUri(cur) : null - if ( - uri && - uri.collection === ids.AppBskyActorProfile && - uri.rkey === 'self' - ) { - // combine labels for profile + did - const did = uri.hostname - acc[did] ??= [] - acc[did].push(...labelsByUri[cur]) - } - acc[cur] ??= [] - acc[cur].push(...labelsByUri[cur]) - return acc - }, labels) - } - - async getLabels( - subject: string, - opts?: { - includeNeg?: boolean - skipCache?: boolean - }, - ): Promise { - const labels = await this.getLabelsForUris([subject], opts) - return labels[subject] ?? [] - } - - async getLabelsForProfile( - did: string, - opts?: { - includeNeg?: boolean - skipCache?: boolean - }, - ): Promise { - const labels = await this.getLabelsForSubjects([did], opts) - return labels[did] ?? [] - } -} - -export function getSelfLabels(details: { - uri: string | null - cid: string | null - record: Record | null -}): Label[] { - const { uri, cid, record } = details - if (!uri || !cid || !record) return [] - if (!isSelfLabels(record.labels)) return [] - const src = new AtUri(uri).host // record creator - const cts = - typeof record.createdAt === 'string' - ? normalizeDatetimeAlways(record.createdAt) - : new Date(0).toISOString() - return record.labels.values.map(({ val }) => { - return { src, uri, cid, val, cts, neg: false } - }) -} - -const fetchLabelsForSubjects = async ( - db: Database, - subjects: string[], -): Promise> => { - if (subjects.length === 0) { - return {} - } - const res = await db.db - .selectFrom('label') - .where('label.uri', 'in', subjects) - .selectAll() - .execute() - const labelMap = res.reduce((acc, cur) => { - acc[cur.uri] ??= [] - acc[cur.uri].push({ - ...cur, - cid: cur.cid === '' ? undefined : cur.cid, - neg: cur.neg, - }) - return acc - }, {} as Record) - // ensure we cache negatives - for (const subject of subjects) { - labelMap[subject] ??= [] - } - return labelMap -} diff --git a/packages/mod-service/src/services/moderation/index.ts b/packages/mod-service/src/services/moderation/index.ts index 3b656e36461..622fb41be0f 100644 --- a/packages/mod-service/src/services/moderation/index.ts +++ b/packages/mod-service/src/services/moderation/index.ts @@ -3,9 +3,7 @@ import { AtUri } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' import { Database } from '../../db' import { ModerationViews } from './views' -import { ImageUriBuilder } from '../../image/uri' import { Main as StrongRef } from '../../lexicon/types/com/atproto/repo/strongRef' -import { ImageInvalidator } from '../../image/invalidator' import { isModEventComment, isModEventLabel, @@ -34,20 +32,10 @@ import { StatusKeyset, TimeIdKeyset } from './pagination' import AtpAgent from '@atproto/api' export class ModerationService { - constructor( - public db: Database, - public appviewAgent: AtpAgent, - public imgUriBuilder: ImageUriBuilder, - public imgInvalidator: ImageInvalidator, - ) {} - - static creator( - appviewAgent: AtpAgent, - imgUriBuilder: ImageUriBuilder, - imgInvalidator: ImageInvalidator, - ) { - return (db: Database) => - new ModerationService(db, appviewAgent, imgUriBuilder, imgInvalidator) + constructor(public db: Database, public appviewAgent: AtpAgent) {} + + static creator(appviewAgent: AtpAgent) { + return (db: Database) => new ModerationService(db, appviewAgent) } views = new ModerationViews(this.db, this.appviewAgent) diff --git a/packages/mod-service/src/services/moderation/status.ts b/packages/mod-service/src/services/moderation/status.ts index 15a698d4d3a..2ecb640e484 100644 --- a/packages/mod-service/src/services/moderation/status.ts +++ b/packages/mod-service/src/services/moderation/status.ts @@ -1,7 +1,7 @@ // This may require better organization but for now, just dumping functions here containing DB queries for moderation status import { AtUri } from '@atproto/syntax' -import { Database, PrimaryDatabase } from '../../db' +import { Database } from '../../db' import { ModerationSubjectStatus } from '../../db/schema/moderation_subject_status' import { REVIEWOPEN, @@ -200,7 +200,7 @@ type ModerationSubjectStatusFilter = | Pick | Pick export const getModerationSubjectStatus = async ( - db: PrimaryDatabase, + db: Database, filters: ModerationSubjectStatusFilter, ) => { let builder = db.db diff --git a/packages/mod-service/src/subscription/util.ts b/packages/mod-service/src/subscription/util.ts deleted file mode 100644 index fe367bcc24c..00000000000 --- a/packages/mod-service/src/subscription/util.ts +++ /dev/null @@ -1,148 +0,0 @@ -import PQueue from 'p-queue' -import { OutputSchema as RepoMessage } from '../lexicon/types/com/atproto/sync/subscribeRepos' -import * as message from '../lexicon/types/com/atproto/sync/subscribeRepos' -import assert from 'node:assert' - -// A queue with arbitrarily many partitions, each processing work sequentially. -// Partitions are created lazily and taken out of memory when they go idle. -export class PartitionedQueue { - main: PQueue - partitions = new Map() - - constructor(opts: { concurrency: number }) { - this.main = new PQueue({ concurrency: opts.concurrency }) - } - - async add(partitionId: string, task: () => Promise) { - if (this.main.isPaused) return - return this.main.add(() => { - return this.getPartition(partitionId).add(task) - }) - } - - async destroy() { - this.main.pause() - this.main.clear() - this.partitions.forEach((p) => p.clear()) - await this.main.onIdle() // All in-flight work completes - } - - private getPartition(partitionId: string) { - let partition = this.partitions.get(partitionId) - if (!partition) { - partition = new PQueue({ concurrency: 1 }) - partition.once('idle', () => this.partitions.delete(partitionId)) - this.partitions.set(partitionId, partition) - } - return partition - } -} - -export class LatestQueue { - queue = new PQueue({ concurrency: 1 }) - - async add(task: () => Promise) { - if (this.queue.isPaused) return - this.queue.clear() // Only queue the latest task, invalidate any previous ones - return this.queue.add(task) - } - - async destroy() { - this.queue.pause() - this.queue.clear() - await this.queue.onIdle() // All in-flight work completes - } -} - -/** - * Add items to a list, and mark those items as - * completed. Upon item completion, get list of consecutive - * items completed at the head of the list. Example: - * - * const consecutive = new ConsecutiveList() - * const item1 = consecutive.push(1) - * const item2 = consecutive.push(2) - * const item3 = consecutive.push(3) - * item2.complete() // [] - * item1.complete() // [1, 2] - * item3.complete() // [3] - * - */ -export class ConsecutiveList { - list: ConsecutiveItem[] = [] - - push(value: T) { - const item = new ConsecutiveItem(this, value) - this.list.push(item) - return item - } - - complete(): T[] { - let i = 0 - while (this.list[i]?.isComplete) { - i += 1 - } - return this.list.splice(0, i).map((item) => item.value) - } -} - -export class ConsecutiveItem { - isComplete = false - constructor(private consecutive: ConsecutiveList, public value: T) {} - - complete() { - this.isComplete = true - return this.consecutive.complete() - } -} - -export class PerfectMap extends Map { - get(key: K): V { - const val = super.get(key) - assert(val !== undefined, `Key not found in PerfectMap: ${key}`) - return val - } -} - -// These are the message types that have a sequence number and a repo -export type ProcessableMessage = - | message.Commit - | message.Handle - | message.Migrate - | message.Tombstone - -export function loggableMessage(msg: RepoMessage) { - if (message.isCommit(msg)) { - const { seq, rebase, prev, repo, commit, time, tooBig, blobs } = msg - return { - $type: msg.$type, - seq, - rebase, - prev: prev?.toString(), - repo, - commit: commit.toString(), - time, - tooBig, - hasBlobs: blobs.length > 0, - } - } else if (message.isHandle(msg)) { - return msg - } else if (message.isMigrate(msg)) { - return msg - } else if (message.isTombstone(msg)) { - return msg - } else if (message.isInfo(msg)) { - return msg - } - return msg -} - -export function jitter(maxMs) { - return Math.round((Math.random() - 0.5) * maxMs * 2) -} - -export function strToInt(str: string) { - const int = parseInt(str, 10) - assert(!isNaN(int), 'string could not be parsed to an integer') - return int -}