diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index 318d1c33b5a..0d57da18ae8 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -248,7 +248,8 @@ } }, "invitesDisabled": { "type": "boolean" }, - "inviteNote": { "type": "string" } + "inviteNote": { "type": "string" }, + "emailConfirmedAt": { "type": "string", "format": "datetime" } } }, "accountView": { @@ -271,6 +272,7 @@ } }, "invitesDisabled": { "type": "boolean" }, + "emailConfirmedAt": { "type": "string", "format": "datetime" }, "inviteNote": { "type": "string" } } }, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index d3e78cff850..3a9bafb8b12 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -434,6 +434,10 @@ export const schemaDict = { inviteNote: { type: 'string', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, }, }, accountView: { @@ -469,6 +473,10 @@ export const schemaDict = { invitesDisabled: { type: 'boolean', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, inviteNote: { type: 'string', }, diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index 7c48fa87a3c..5ab3e3482ec 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -241,6 +241,7 @@ export interface RepoViewDetail { invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean inviteNote?: string + emailConfirmedAt?: string [k: string]: unknown } @@ -264,6 +265,7 @@ export interface AccountView { invitedBy?: ComAtprotoServerDefs.InviteCode invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean + emailConfirmedAt?: string inviteNote?: string [k: string]: unknown } diff --git a/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts b/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts index 18ab99debe2..f68ba68eb66 100644 --- a/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts +++ b/packages/bsky/src/api/app/bsky/actor/getSuggestions.ts @@ -42,12 +42,12 @@ const skeleton = async ( ctx: Context, ): Promise => { const { db } = ctx - const { limit, cursor, viewer } = params + const { viewer } = params + const alreadyIncluded = parseCursor(params.cursor) const { ref } = db.db.dynamic - let suggestionsQb = db.db + const suggestions = await db.db .selectFrom('suggested_follow') .innerJoin('actor', 'actor.did', 'suggested_follow.did') - .innerJoin('profile_agg', 'profile_agg.did', 'actor.did') .where(notSoftDeletedClause(ref('actor'))) .where('suggested_follow.did', '!=', viewer ?? '') .whereNotExists((qb) => @@ -57,27 +57,30 @@ const skeleton = async ( .where('creator', '=', viewer ?? '') .whereRef('subjectDid', '=', ref('actor.did')), ) + .if(alreadyIncluded.length > 0, (qb) => + qb.where('suggested_follow.order', 'not in', alreadyIncluded), + ) .selectAll() - .select('profile_agg.postsCount as postsCount') - .limit(limit) .orderBy('suggested_follow.order', 'asc') + .execute() - if (cursor) { - const cursorRow = await db.db - .selectFrom('suggested_follow') - .where('did', '=', cursor) - .selectAll() - .executeTakeFirst() - if (cursorRow) { - suggestionsQb = suggestionsQb.where( - 'suggested_follow.order', - '>', - cursorRow.order, - ) - } - } - const suggestions = await suggestionsQb.execute() - return { params, suggestions, cursor: suggestions.at(-1)?.did } + // always include first two + const firstTwo = suggestions.filter( + (row) => row.order === 1 || row.order === 2, + ) + const rest = suggestions.filter((row) => row.order !== 1 && row.order !== 2) + const limited = firstTwo.concat(shuffle(rest)).slice(0, params.limit) + + // if the result set ends up getting larger, consider using a seed included in the cursor for for the randomized shuffle + const cursor = + limited.length > 0 + ? limited + .map((row) => row.order.toString()) + .concat(alreadyIncluded.map((id) => id.toString())) + .join(':') + : undefined + + return { params, suggestions: limited, cursor } } const hydration = async (state: SkeletonState, ctx: Context) => { @@ -110,6 +113,27 @@ const presentation = (state: HydrationState) => { return { actors: suggestedActors, cursor } } +const parseCursor = (cursor?: string): number[] => { + if (!cursor) { + return [] + } + try { + return cursor + .split(':') + .map((id) => parseInt(id, 10)) + .filter((id) => !isNaN(id)) + } catch { + return [] + } +} + +const shuffle = (arr: T[]): T[] => { + return arr + .map((value) => ({ value, sort: Math.random() })) + .sort((a, b) => a.sort - b.sort) + .map(({ value }) => value) +} + type Context = { db: Database actorService: ActorService diff --git a/packages/bsky/src/api/app/bsky/feed/searchPosts.ts b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts new file mode 100644 index 00000000000..2edc9e2f138 --- /dev/null +++ b/packages/bsky/src/api/app/bsky/feed/searchPosts.ts @@ -0,0 +1,123 @@ +import AppContext from '../../../../context' +import { Server } from '../../../../lexicon' +import { InvalidRequestError } from '@atproto/xrpc-server' +import AtpAgent from '@atproto/api' +import { AtUri } from '@atproto/syntax' +import { mapDefined } from '@atproto/common' +import { QueryParams } from '../../../../lexicon/types/app/bsky/feed/searchPosts' +import { Database } from '../../../../db' +import { FeedHydrationState, FeedService } from '../../../../services/feed' +import { ActorService } from '../../../../services/actor' +import { createPipeline } from '../../../../pipeline' + +export default function (server: Server, ctx: AppContext) { + const searchPosts = createPipeline( + skeleton, + hydration, + noBlocks, + presentation, + ) + server.app.bsky.feed.searchPosts({ + auth: ctx.authOptionalVerifier, + handler: async ({ auth, params }) => { + const viewer = auth.credentials.did + const db = ctx.db.getReplica('search') + const feedService = ctx.services.feed(db) + const actorService = ctx.services.actor(db) + const searchAgent = ctx.searchAgent + if (!searchAgent) { + throw new InvalidRequestError('Search not available') + } + + const results = await searchPosts( + { ...params, viewer }, + { db, feedService, actorService, searchAgent }, + ) + + return { + encoding: 'application/json', + body: results, + } + }, + }) +} + +const skeleton = async ( + params: Params, + ctx: Context, +): Promise => { + const res = await ctx.searchAgent.api.app.bsky.unspecced.searchPostsSkeleton( + params, + ) + return { + params, + postUris: res.data.posts.map((a) => a.uri), + cursor: res.data.cursor, + } +} + +const hydration = async ( + state: SkeletonState, + ctx: Context, +): Promise => { + const { feedService } = ctx + const { params, postUris } = state + const uris = new Set(postUris) + const dids = new Set(postUris.map((uri) => new AtUri(uri).hostname)) + const hydrated = await feedService.feedHydration({ + uris, + dids, + viewer: params.viewer, + }) + return { ...state, ...hydrated } +} + +const noBlocks = (state: HydrationState): HydrationState => { + const { viewer } = state.params + state.postUris = state.postUris.filter((uri) => { + const post = state.posts[uri] + if (!viewer || !post) return true + return !state.bam.block([viewer, post.creator]) + }) + return state +} + +const presentation = (state: HydrationState, ctx: Context) => { + const { feedService, actorService } = ctx + const { postUris, profiles, params } = state + const actors = actorService.views.profileBasicPresentation( + Object.keys(profiles), + state, + { viewer: params.viewer }, + ) + + const postViews = mapDefined(postUris, (uri) => + feedService.views.formatPostView( + uri, + actors, + state.posts, + state.threadgates, + state.embeds, + state.labels, + state.lists, + ), + ) + return { posts: postViews } +} + +type Context = { + db: Database + feedService: FeedService + actorService: ActorService + searchAgent: AtpAgent +} + +type Params = QueryParams & { viewer: string | null } + +type SkeletonState = { + params: Params + postUris: string[] + cursor?: string +} + +type HydrationState = SkeletonState & FeedHydrationState diff --git a/packages/bsky/src/api/com/atproto/admin/util.ts b/packages/bsky/src/api/com/atproto/admin/util.ts index 6217e71eb0f..7dfd10cce5c 100644 --- a/packages/bsky/src/api/com/atproto/admin/util.ts +++ b/packages/bsky/src/api/com/atproto/admin/util.ts @@ -32,6 +32,7 @@ export const addAccountInfoToRepoViewDetail = ( invitesDisabled: accountInfo.invitesDisabled, inviteNote: accountInfo.inviteNote, invites: accountInfo.invites, + emailConfirmedAt: accountInfo.emailConfirmedAt, } } diff --git a/packages/bsky/src/api/index.ts b/packages/bsky/src/api/index.ts index 3768ed4da0b..cf2121b7792 100644 --- a/packages/bsky/src/api/index.ts +++ b/packages/bsky/src/api/index.ts @@ -13,6 +13,7 @@ import getLikes from './app/bsky/feed/getLikes' import getListFeed from './app/bsky/feed/getListFeed' import getPostThread from './app/bsky/feed/getPostThread' import getPosts from './app/bsky/feed/getPosts' +import searchPosts from './app/bsky/feed/searchPosts' import getActorLikes from './app/bsky/feed/getActorLikes' import getProfile from './app/bsky/actor/getProfile' import getProfiles from './app/bsky/actor/getProfiles' @@ -74,6 +75,7 @@ export default function (server: Server, ctx: AppContext) { getListFeed(server, ctx) getPostThread(server, ctx) getPosts(server, ctx) + searchPosts(server, ctx) getActorLikes(server, ctx) getProfile(server, ctx) getProfiles(server, ctx) diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index d3e78cff850..3a9bafb8b12 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -434,6 +434,10 @@ export const schemaDict = { inviteNote: { type: 'string', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, }, }, accountView: { @@ -469,6 +473,10 @@ export const schemaDict = { invitesDisabled: { type: 'boolean', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, inviteNote: { type: 'string', }, diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index ea463368f8e..7c74f6b8b98 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -241,6 +241,7 @@ export interface RepoViewDetail { invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean inviteNote?: string + emailConfirmedAt?: string [k: string]: unknown } @@ -264,6 +265,7 @@ export interface AccountView { invitedBy?: ComAtprotoServerDefs.InviteCode invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean + emailConfirmedAt?: string inviteNote?: string [k: string]: unknown } diff --git a/packages/bsky/tests/admin/get-repo.test.ts b/packages/bsky/tests/admin/get-repo.test.ts index 3c1e909a4ab..9b4f6690ccd 100644 --- a/packages/bsky/tests/admin/get-repo.test.ts +++ b/packages/bsky/tests/admin/get-repo.test.ts @@ -91,6 +91,39 @@ describe('admin get repo view', () => { expect(triage).toEqual({ ...admin, email: undefined }) }) + it('includes emailConfirmedAt timestamp', async () => { + const { data: beforeEmailVerification } = + await agent.api.com.atproto.admin.getRepo( + { did: sc.dids.bob }, + { headers: network.pds.adminAuthHeaders() }, + ) + + expect(beforeEmailVerification.emailConfirmedAt).toBeUndefined() + const timestampBeforeVerification = Date.now() + const bobsAccount = sc.accounts[sc.dids.bob] + const verificationToken = await network.pds.ctx.services + .account(network.pds.ctx.db) + .createEmailToken(sc.dids.bob, 'confirm_email') + await agent.api.com.atproto.server.confirmEmail( + { email: bobsAccount.email, token: verificationToken }, + { + encoding: 'application/json', + + headers: sc.getHeaders(sc.dids.bob), + }, + ) + const { data: afterEmailVerification } = + await agent.api.com.atproto.admin.getRepo( + { did: sc.dids.bob }, + { headers: network.pds.adminAuthHeaders() }, + ) + + expect(afterEmailVerification.emailConfirmedAt).toBeTruthy() + expect( + new Date(afterEmailVerification.emailConfirmedAt as string).getTime(), + ).toBeGreaterThan(timestampBeforeVerification) + }) + it('fails when repo does not exist.', async () => { const promise = agent.api.com.atproto.admin.getRepo( { did: 'did:plc:doesnotexist' }, diff --git a/packages/bsky/tests/views/suggestions.test.ts b/packages/bsky/tests/views/suggestions.test.ts index 2dcadf9e6ad..4253f528b13 100644 --- a/packages/bsky/tests/views/suggestions.test.ts +++ b/packages/bsky/tests/views/suggestions.test.ts @@ -19,10 +19,12 @@ describe('pds user search views', () => { await network.bsky.processAll() const suggestions = [ - { did: sc.dids.bob, order: 1 }, - { did: sc.dids.carol, order: 2 }, - { did: sc.dids.dan, order: 3 }, + { did: sc.dids.alice, order: 1 }, + { did: sc.dids.bob, order: 2 }, + { did: sc.dids.carol, order: 3 }, + { did: sc.dids.dan, order: 4 }, ] + await network.bsky.ctx.db .getPrimary() .db.insertInto('suggested_follow') @@ -63,16 +65,22 @@ describe('pds user search views', () => { { limit: 1 }, { headers: await network.serviceHeaders(sc.dids.carol) }, ) + expect(result1.data.actors.length).toBe(1) + expect(result1.data.actors[0].handle).toEqual('bob.test') + const result2 = await agent.api.app.bsky.actor.getSuggestions( { limit: 1, cursor: result1.data.cursor }, { headers: await network.serviceHeaders(sc.dids.carol) }, ) - - expect(result1.data.actors.length).toBe(1) - expect(result1.data.actors[0].handle).toEqual('bob.test') - expect(result2.data.actors.length).toBe(1) expect(result2.data.actors[0].handle).toEqual('dan.test') + + const result3 = await agent.api.app.bsky.actor.getSuggestions( + { limit: 1, cursor: result2.data.cursor }, + { headers: await network.serviceHeaders(sc.dids.carol) }, + ) + expect(result3.data.actors.length).toBe(0) + expect(result3.data.cursor).toBeUndefined() }) it('fetches suggestions unauthed', async () => { diff --git a/packages/pds/src/api/app/bsky/feed/index.ts b/packages/pds/src/api/app/bsky/feed/index.ts index 8c4cfaa8b5f..026ce86f612 100644 --- a/packages/pds/src/api/app/bsky/feed/index.ts +++ b/packages/pds/src/api/app/bsky/feed/index.ts @@ -13,6 +13,7 @@ import getPostThread from './getPostThread' import getRepostedBy from './getRepostedBy' import getSuggestedFeeds from './getSuggestedFeeds' import getTimeline from './getTimeline' +import searchPosts from './searchPosts' export default function (server: Server, ctx: AppContext) { getActorFeeds(server, ctx) @@ -28,4 +29,5 @@ export default function (server: Server, ctx: AppContext) { getRepostedBy(server, ctx) getSuggestedFeeds(server, ctx) getTimeline(server, ctx) + searchPosts(server, ctx) } diff --git a/packages/pds/src/api/app/bsky/feed/searchPosts.ts b/packages/pds/src/api/app/bsky/feed/searchPosts.ts new file mode 100644 index 00000000000..85384751ea1 --- /dev/null +++ b/packages/pds/src/api/app/bsky/feed/searchPosts.ts @@ -0,0 +1,19 @@ +import { Server } from '../../../../lexicon' +import AppContext from '../../../../context' + +export default function (server: Server, ctx: AppContext) { + server.app.bsky.feed.searchPosts({ + auth: ctx.authVerifier.access, + handler: async ({ params, auth }) => { + const requester = auth.credentials.did + const res = await ctx.appViewAgent.api.app.bsky.feed.searchPosts( + params, + await ctx.serviceAuthHeaders(requester), + ) + return { + encoding: 'application/json', + body: res.data, + } + }, + }) +} diff --git a/packages/pds/src/api/com/atproto/admin/util.ts b/packages/pds/src/api/com/atproto/admin/util.ts index 841d3b2b9f2..ee862236b4f 100644 --- a/packages/pds/src/api/com/atproto/admin/util.ts +++ b/packages/pds/src/api/com/atproto/admin/util.ts @@ -1,17 +1,24 @@ -import { - RepoView, - RepoViewDetail, -} from '../../../../lexicon/types/com/atproto/admin/defs' +import express from 'express' -// @NOTE mutates. -// merges-in details that the pds knows about the repo. -export function mergeRepoViewPdsDetails( - other: T, - pds: T, -) { - other.email ??= pds.email - other.invites ??= pds.invites - other.invitedBy ??= pds.invitedBy - other.invitesDisabled ??= pds.invitesDisabled - return other +// Output designed to passed as second arg to AtpAgent methods. +// The encoding field here is a quirk of the AtpAgent. +export function authPassthru( + req: express.Request, + withEncoding?: false, +): { headers: { authorization: string }; encoding: undefined } | undefined + +export function authPassthru( + req: express.Request, + withEncoding: true, +): + | { headers: { authorization: string }; encoding: 'application/json' } + | undefined + +export function authPassthru(req: express.Request, withEncoding?: boolean) { + if (req.headers.authorization) { + return { + headers: { authorization: req.headers.authorization }, + encoding: withEncoding ? 'application/json' : undefined, + } + } } diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index d3e78cff850..3a9bafb8b12 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -434,6 +434,10 @@ export const schemaDict = { inviteNote: { type: 'string', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, }, }, accountView: { @@ -469,6 +473,10 @@ export const schemaDict = { invitesDisabled: { type: 'boolean', }, + emailConfirmedAt: { + type: 'string', + format: 'datetime', + }, inviteNote: { type: 'string', }, diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index ea463368f8e..7c74f6b8b98 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -241,6 +241,7 @@ export interface RepoViewDetail { invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean inviteNote?: string + emailConfirmedAt?: string [k: string]: unknown } @@ -264,6 +265,7 @@ export interface AccountView { invitedBy?: ComAtprotoServerDefs.InviteCode invites?: ComAtprotoServerDefs.InviteCode[] invitesDisabled?: boolean + emailConfirmedAt?: string inviteNote?: string [k: string]: unknown } diff --git a/packages/pds/src/services/account/index.ts b/packages/pds/src/services/account/index.ts index e5479e3af65..83a6b3f87a8 100644 --- a/packages/pds/src/services/account/index.ts +++ b/packages/pds/src/services/account/index.ts @@ -403,6 +403,7 @@ export class AccountService { 'did_handle.did', 'did_handle.handle', 'user_account.email', + 'user_account.emailConfirmedAt', 'user_account.invitesDisabled', 'user_account.inviteNote', 'user_account.createdAt as indexedAt', @@ -421,6 +422,7 @@ export class AccountService { handle: account?.handle ?? INVALID_HANDLE, invitesDisabled: account.invitesDisabled === 1, inviteNote: account.inviteNote ?? undefined, + emailConfirmedAt: account.emailConfirmedAt ?? undefined, invites, invitedBy: invitedBy[did], } diff --git a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap index fcf1063954c..0dbe9b5498d 100644 --- a/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/views.test.ts.snap @@ -114,7 +114,7 @@ Object { }, }, ], - "cursor": "user(2)", + "cursor": "1:3", } `;